# NHC Configuration File (sample)
#
# Lines are in the form "<hostmask>||<check>"
# Hostmask is a glob, /regexp/, or {noderange}
# Comments begin with '#'
#

#######################################################################
###
### NHC Configuration Variables
###
# Explicitly instruct NHC to assume slurm is the Resource Manager
   * || export NHC_RM=slurm
   * || export SLURM_SINFO=/usr/local/bin/sinfo
   * || export SLURM_SCONTROL=/usr/local/bin/scontrol
   * || export DF_FLAGS=-TklBG
   * || export DFI_FLAGS=-Til

# Mark nodes that fail health checks offline when set to 1
   * || export MARK_OFFLINE=1

# Set watchdog timer to 15 seconds
   * || export TIMEOUT=60

# In out-of-band contexts, enable all checks
#   * || export NHC_CHECK_ALL=1

#######################################################################
###
### Hardware checks
###
# Set these to your correct socket, core, and thread counts.
{% if ec2_private_dns_name is not defined  %}
     # Dont do this check on EC2 because:
     # 1. We'll get correct cpu resources based on instance type.
     # 2. Determining sockets on EC2 is tricky: for example for r5.16xlarge we get 32 cores and 2 threads per core
     #    so total 64vCPU, but on /proc/cpuinfo  shows 64 cpus & there are two physical ids (0 & 1) which NHC sees as 2 sockets and nhc check fails.
     #    Hence, we can rely on (1) and skip this check.
     {{ ansible_hostname }}  || check_hw_cpuinfo {{ sockets }} {{ sockets * cores }} {{ sockets * cores * threads }}
{% endif %}

# Set these to the amount of physical RAM you have (leave the fudge factor).
   {{ ansible_hostname }}  || check_hw_physmem {{ mem_mb }}mb {{ mem_mb }}mb 15%

# Check for some sort of free memory of either type.
   * || check_hw_mem_free 1mb
# This should work if you place it after one of the check_hw_*() checks.
   * || check_ps_loadavg $((100*HW_CORES)) $((15*HW_CORES))


#######################################################################
###
### Filesystem checks
###
# All nodes should have their root filesystem mounted read/write.
   * || check_fs_mount_rw -f /
   * || check_fs_mount_rw -f /mnt/sun-pcs01
   * || check_file_test   -f /mnt/sun-pcs01/test.file
   * || check_fs_mount    -f /mnt/nautilus
   * || check_file_test   -f /mnt/nautilus/test.file
   * || check_fs_mount_rw -f /mnt/nautilus_rw
   * || check_file_test   -f /mnt/nautilus_rw/test.file
   * || check_fs_mount_rw -f /mnt/sun-pcp01
{% if ansible_distribution_version != '14.04' %}
   * || check_fs_mount_rw -f /mnt/sun-tcs01
   * || check_file_test -d /mnt/sun-tcs01/it
{% endif %}
   * || check_fs_mount_rw -f /mnt/data_cache
   * || check_fs_mount_rw -f /mnt/freenas
{% if 'perception-ci-volta' not in ansible_hostname and ( gpus > 0 or '/prestage/docker' in docker_dir | default('')) %}
   {% if ec2_private_dns_name is not defined  %}
     {{ ansible_hostname }} || check_fs_mount_rw -t "ext4" -f "/prestage"
   {% endif %}
   {{ ansible_hostname }} || check_fs_ifree /prestage 10k
   {{ ansible_hostname }} || check_fs_free /prestage 10%
{% endif %}

# Make sure the filesystem doesn't get too full.
   * || check_fs_free / 10%

# Free inodes are also important.
   * || check_fs_ifree / 10k


{# these tests are particular to our typical gpu nodes configuration #}
{% if gpus > 0 and not disable_nhc_gpu_checks %}
#######################################################################
###
### GPU host checks
###
   {{ ansible_hostname }} || check_file_contents /proc/driver/nvidia/version '*{{ nvidia_driver_version }}*'
   {{ ansible_hostname }} || check_file_test -f -x /usr/bin/nvidia-smi
   # nvidia-smi should return without error
   {{ ansible_hostname }} || check_cmd_status -t 5 -r 0 /usr/bin/nvidia-smi
   {% if ec2_private_dns_name is not defined  %}
   {{ ansible_hostname }} || check_cmd_status -t 5 -r 0 /opt/zoox/check-for-x {{ gpus }}
   {% endif %}
{% endif %}


#######################################################################
###
### File/metadata checks
###
# check that automount directorys 'exist' when stated.
{% for item in autofs %}
{% for mount in item.mounts %}
   * || check_file_test -d {{item.path}}/{{mount.name}}
{% endfor %}
{% endfor %}

#######################################################################
###
### Process checks
###
# Everybody needs sshd running, right?  But don't use -r (restart)!
   * || check_ps_service -u root -S sshd
   * || check_ps_service -u root dockerd
   * || check_ps_service -u root -d slurmd -S slurm
   * || check_ps_service -u root -d *automount -S autofs
{% if ansible_distribution_version != '14.04' %}
   * || check_ps_service -u root mount.quobyte
{% elif ansible_distribution_version != '18.04' %}
   * || check_ps_service -u root -d mount.quobyte -S quobyte-client
{% endif %}

{% if 'timing_metrics' in group_names %}
   {{ ansible_hostname }} || check_cmd_output -m '/performance/' /usr/bin/cpufreq-info -p
{% endif %}
{% if 'swapon' in group_names %}
   {{ ansible_hostname }} || check_hw_swap 200GB 599GB
{% endif %}
