---
# Order of operations
# 1. Common setup for all slurm hosts
# 2. Set up slurm master (so its up when workers come up)
# 3. Set up slurm workers

- hosts: slurm:&ec2:!slurm_production_masters:!slurm_masters
  # Slurm puts EC2 instances to stopped state when not in use.
  # We need to bring them up to run playbook or slurm-reconfigure
  name: Start EC2 instances
  gather_facts: False
  roles:
    - role: start-ec2
      tags: [start-ec2, slurm-reconfigure]

# Upgrade kernel on all nodes in our test inventory that are EC2
- hosts: slurm:&ec2:!slurm_production_masters
  name: upgrade kernel
  roles:
    - role: kernel-version
      tags: kernel-version

- hosts: slurm:!ec2:!rds
  name: setup on prem
  roles:
    - role: ansible-role-dns
      become: yes
      dns_domain: 'zooxlabs.com'
      dns_searchs:
        - 'zooxlabs.com'
      dns_nameservers: ["10.65.4.25", "10.65.4.26"]
      tags: dns

    - role: prestage
      tags: prestage
      when: "gpus|int > 0 or '/prestage/docker' in docker_dir"

- hosts: slurm_mysql:!rds
  name: install mysql server
  roles:
    - role: mysql
      mysql_server_root_password: "{{ lookup('hashi_vault', 'secret=secret/mysql/slurm').password }}"
      tags: mysql

- hosts: slurm
  tasks:
  - name: wait for automatic updates to complete
    become: yes
    shell: while sudo fuser /var/lib/dpkg/lock-frontend >/dev/null 2>&1; do sleep 1; done;

- hosts: slurm
  name: base slurm setup
  roles:
    - role: apt_sources
      tags: apt_sources
    - role: os-file-limits
      tags: os-file-limits
    - role: ntp
      tags: ntp
    - role: nfsclient
      tags: nfsclient
      when: ansible_distribution_version == '18.04'
    - role: ../../scripts/ansible/roles/autofs
      tags: autofs
    - role: data_cache
      tags:
        - data_cache
        - quobyte
    - role: storage_pcs
      tags: storage_pcs
    - role: pip
      tags: pip
    - role: ../../scripts/ansible/roles/python_pip_conf
      tags: pip
    - role: python-secure-deps
      tags: python-secure-deps
    - role: munge
      tags: munge
    - role: docker-host
      tags:
        - docker
        - docker-host
    - role: docker-gc
      tags:
        - docker
        - docker-gc
    - role: temporary_accounts
      temporary_users: "{{ slurm_users }}"
      tags: accounts
    - role: filebeat
      tags: filebeat
    - role: ../../scripts/ansible/roles/updatedb
      tags: updatedb
    - role: cpu-governor
      tags: cpu-governor
    - role: lz4
      tags: lz4

# TODO: Enable AD on test cluster for parity. https://jira.zooxlabs.com/browse/INFRA-6992
- hosts: slurm:!vagrant:!slurm_test
  name: setup AD and sudo
  roles:
    - role: likewise
      tags: likewise
    - role: sudo
      sudoers_groups:
        - name: linuxadmins
        - name: domain^admins
        # Allow anyone at all logging in through AD to use sudo.
        - name: domain^users
      tags: sudo
    - role: prometheus-exporter
      tags: [prometheus-exporter, prometheus]

- hosts: slurm_masters:&ec2
  name: setup efs mount
  tasks:
    - name: install nfs-common
      apt:
        pkg: nfs-common
        state: present
      become: yes
      tags: efs
    - name: make directory for efs mount
      file:
        path: "{{ slurm_state_save_location | dirname }}"
        state: directory
        mode: "0777"
      become: yes
      tags: efs
    - name: mount efs
      mount:
        name: "{{ slurm_state_save_location | dirname }}"
        src: "{{ slurm_efs_ip }}:/"
        fstype: nfs4
        opts: "nfsvers=4.1,rsize=1048576,wsize=1048576,hard,timeo=600,retrans=2"
        state: mounted
      become: yes
      tags: efs

- hosts: slurm_workers
  name: slurm worker setup
  # This limits the number of hosts to run each play on at a time. It puts an
  # upper limit of 1000 by default but can be overriden by adding
  # `-e "serial=5"` to the ansible-playbook command.
  serial: "{{ serial | default('1000') | int }}"
  roles:
    - role: cgroup-setup
      tags: cgroup-setup
    - role: slurm-worker
      tags: worker
    - role: gpu-server
      tags: gpu
    - role: xserver
      tags: [xserver, gpu]
    - role: SimpliField.transparent-huge-pages
      tags: transparent_huge_pages
      become: yes
    - role: rdma-driver
      tags: rdma-driver
    - role: "pipedream-collectors"
      tags: "collectors"
    - role: nhc
      tags: [nhc, node-health-check, gpu]
    - role: traffic-control
      tags: traffic-control

- hosts: slurm_production
  name: production slurm setup
  roles:
    - role: slurm-prometheus
      tags: [prometheus, slurm-prometheus]

- hosts: slurm_masters
  name: slurm master setup
  roles:
    - role: slurm-master
      tags: master
    - role: slurm-accounts
      tags:
        - slurm-accounts
        - accounts

- hosts: swapon
  roles:
    - role: swap
      tags: swap

- hosts: slurm_primary_masters:!slurm_test
  roles:
    - role: slurm-primary-crons
      tags: slurm-primary-crons
