# slurm.conf file generated by configurator.html.
# Put this file on all nodes of your cluster.
# See the slurm.conf man page for more information.

ControlMachine=slurm
#BackupController=
#
AuthType=auth/munge
CacheGroups=0
#CheckpointType=checkpoint/none
CryptoType=crypto/munge
#DisableRootJobs=NO
#EnforcePartLimits=NO
#Epilog=
#EpilogSlurmctld=
#FirstJobId=1
#MaxJobId=999999
GresTypes=gpu,scratch,nvenc-streams
#GroupUpdateForce=0
#GroupUpdateTime=600
#JobCheckpointDir=/var/slurm/checkpoint
#JobCredentialPrivateKey=
#JobCredentialPublicCertificate=
JobFileAppend=1
JobRequeue=0
#JobSubmitPlugins=1
#KillOnBadExit=0
#LaunchType=launch/slurm
Licenses=bandwidth_dx:40
#MailProg=/bin/mail
MaxArraySize=75000
MaxJobCount=500000
#MaxStepCount=40000
#MaxTasksPerNode=128
MpiDefault=none
MpiParams=ports=12000-12099
#PluginDir=
#PlugStackConfig=
#PrivateData=jobs
ProctrackType=proctrack/pgid
#Prolog=
#PrologFlags=
#PrologSlurmctld=
#PropagatePrioProcess=0
#PropagateResourceLimits=
#PropagateResourceLimitsExcept=
#RebootProgram=
ReturnToService=1
#SallocDefaultCommand=
SlurmctldPidFile=/var/run/slurm/slurmctld.pid
SlurmctldPort=6800-6817
SlurmdPidFile=/var/run/slurm/slurmd.pid
SlurmdPort=6818
SlurmdSpoolDir=/var/tmp/slurmd.spool
SlurmUser=slurm
#SlurmdUser=root
#SrunEpilog=
#SrunProlog=
StateSaveLocation=/var/tmp/slurm.state
SwitchType=switch/none
#TaskEpilog=
TaskPlugin=task/none
#TaskPluginParam=
#TaskProlog=
#TopologyPlugin=topology/tree
#TmpFS=/tmp
#TrackWCKey=no
#TreeWidth=
#UnkillableStepProgram=
#UsePAM=0
#
#
# TIMERS
#BatchStartTimeout=10
#CompleteWait=0
#EpilogMsgTime=2000
#GetEnvTimeout=2
#HealthCheckInterval=0
#HealthCheckProgram=
InactiveLimit=0
KillWait=30
MessageTimeout=20
#ResvOverRun=0
MinJobAge=120
#OverTimeLimit=0
SlurmctldTimeout=120
SlurmdTimeout=3600
#UnkillableStepTimeout=60
#VSizeFactor=0
Waittime=0
#
#
# SCHEDULING
#
# Explicitly set DefMem variables
# If unset, they default to 0 (unlimited memory) and negatively impacts scheduling.
DefMemPerCPU=1
DefMemPerNode=1
FastSchedule=1
#MaxMemPerCPU=0
#SchedulerRootFilter=1
#SchedulerTimeSlice=30
SchedulerType=sched/backfill
SchedulerPort=7321

# We specify the following settings to mitigate Slurmctld getting overwhelmed by incoming requests.
#
# max_rpc_cnt - If the number of active threads in the slurmctld daemon is equal to or larger than this value, defer scheduling of jobs.
# batch_sched_delay - how long the scheduling of batch jobs can be delayed.
# sched_min_interval - the minimum time between the end of one scheduling cycle and the beginning of the next scheduling cycle.
#   The scheduler runs in a limited fashion every time that any event happens which could enable a job to start (e.g. job submit, job terminate, etc.)
#
# We need the backfill scheduler to parse deeply into the queue to prevent scenarios where overall cluster utilization is unncessarily low.
# We accomplish this with bf_continue and a generous bf_max_time. We also specify yield settings so backfill does not overrun other processing.
#
# bf_interval - The number of seconds between backfill iterations
# bf_max_time - The maximum time in seconds the backfill scheduler can spend (including time spent sleeping when locks are released) before discontinuing.
# bf_max_job_test - The maximum number of jobs to attempt backfill scheduling for
# bf_yield_interval - This specifies the times when the locks are relinquish in microseconds.
# bf_yield_sleep - This specifies the length of time for which the locks are relinquish in microseconds.
#
# Further tuning of the backfill scheduler:
#
# bf_max_job_array_resv - The maximum number of tasks from a job array for which the backfill scheduler will reserve resources in the future.
#   These jobs are typically blocked on array throttling so reserving resources is wasteful. Observed setting to 0 makes backfill not work.
# There will be 30 seconds of other processing, followed by 480 seconds of flipping between backfill and other processing every 2 seconds.
SchedulerParameters=kill_invalid_depend,max_rpc_cnt=150,batch_sched_delay=20,sched_interval=20,sched_min_interval=2000000,bf_continue,bf_interval=30,bf_max_time=480,bf_max_job_test=2000,bf_yield_interval=2000000,bf_yield_sleep=2000000,bf_max_job_array_resv=1
SelectType=select/cons_res
SelectTypeParameters=CR_LLN,CR_CPU_Memory
#
#
# JOB PRIORITY
#PriorityFlags=
PriorityType=priority/multifactor
#PriorityDecayHalfLife=
#PriorityCalcPeriod=
#PriorityFavorSmall=
#PriorityMaxAge=
#PriorityUsageResetPeriod=
#PriorityWeightAge=
PriorityWeightPartition=1000
PriorityWeightJobSize=1000
PriorityWeightFairshare=100
#
#
# PREEMPTION
PreemptType=preempt/qos
PreemptMode=CANCEL
#
#
# LOGGING AND ACCOUNTING
AccountingStorageEnforce=associations
AccountingStorageHost=localhost
AccountingStorageLoc=slurm_acct_db
#AccountingStoragePass=
AccountingStoragePort=6819
AccountingStorageType=accounting_storage/slurmdbd
AccountingStorageUser=slurm
AccountingStoreJobComment=YES
ClusterName=slurm_cluster
DebugFlags=Backfill
#JobCompHost=
#JobCompLoc=
#JobCompPass=
#JobCompPort=3306
JobCompType=jobcomp/none
#JobCompUser=slurm
#JobContainerType=job_container/none
#JobAcctGatherFrequency=5
#JobAcctGatherType=jobacct_gather/linux
SlurmctldDebug=debug3
SlurmctldLogFile=/var/log/slurm/slurmctld.logfile
SlurmdDebug=3
SlurmdLogFile=/var/log/slurm/slurmd.logfile
SlurmSchedLogFile=/var/log/slurm/slurmsched.logfile
SlurmSchedLogLevel=0
#
#
# POWER SAVE SUPPORT FOR IDLE NODES (optional)
#SuspendProgram=
#ResumeProgram=
#SuspendTimeout=
#ResumeTimeout=
#ResumeRate=
#SuspendExcNodes=
#SuspendExcParts=
#SuspendRate=
#SuspendTime=
#
#
# COMPUTE NODES
NodeName=slurm$ Gres=nvenc-streams:a:2
NodeName=slurm-worker$ Gres=gpu:1,nvenc-streams:b:2
NodeName=aws-slurm-worker$
PartitionName=main Nodes=slurm,slurm-worker Default=YES MaxTime=INFINITE State=UP
PartitionName=preemptible Nodes=slurm,slurm-worker MaxTime=INFINITE State=UP
PartitionName=cloud_burst_test Nodes=aws-slurm-worker MaxTime=INFINITE State=UP
