---

- block:
  - name: NHC * install nhc
    # https://github.com/mej/nhc node health check tool "thanks lnbl!"
    apt: deb=https://s3-us-west-1.amazonaws.com/zoox-third-party/lbnl-nhc/lbnl-nhc_1.4.2-1_all.deb

  - name: ensure '/opt/zoox' present
    file:
      path: /opt/zoox
      group: slurm
      state: directory
      mode: 0755

  - name: NHC * slack reporter
    template:
        src: slack_reporter.py.j2
        dest: /opt/zoox/slack_reporter.py
        owner: root
        group: root
        mode: "0755"

  - name: copy check for Xserver script
    copy:
        dest: /opt/zoox/check-for-x
        src: check-for-x.sh
        owner: root
        group: root
        mode: "0775"

  - name: copy check for gpu health check
    copy:
        dest: /opt/zoox/check-gpu-health
        src: check-gpu-health.sh
        owner: root
        group: root
        mode: "0775"

  # slurm-reconfigure for ec2: because AMI could have been
  # built for a different instance type, so threads, cores & memory could be different,
  # so nhc configuration needs to be updated only on EC2 instances.
  - name: NHC * configure
    template:
      src: nhc.conf
      dest: /etc/nhc/nhc.conf
      owner: root
      group: root
      mode: "0644"
    when: ("slurm-reconfigure" not in ansible_run_tags) or ("slurm-reconfigure" in ansible_run_tags and ec2_private_dns_name is defined)
    tags: slurm-reconfigure

  - name: NHC * set up node health check cron job
    cron:
      name: nhc
      cron_file: nhc
      user: root
      job: bash -c "(/usr/sbin/nhc-wrapper -X 24h NHC_RM=slurm) | /usr/bin/python /opt/zoox/slack_reporter.py"
      minute: "*/5"
  become: yes
