---

- name: Set slurm user, grant necessary privileges
  mysql_user:
    name: slurm
    password: "{{ mysql_password }}"
    priv: "{{ slurm_db }}.*:ALL"
    host: "{{ item }}"
    login_host: "{{ hostvars[groups['slurm_mysql'][0]]['ansible_ssh_host'] }}"
    login_user: root
    login_password: "{{ lookup('hashi_vault', 'secret=secret/mysql/slurm').password }}"
  with_items:
    - "localhost"
    - "%"

- name: create database
  mysql_db:
    name: "{{ slurm_db }}"
    state: present
    login_user: slurm
    login_host: "{{ hostvars[groups['slurm_mysql'][0]]['ansible_ssh_host'] }}"
    login_password: "{{ mysql_password }}"

- name: Create master state directory
  file:
    path: "{{ slurm_state_save_location }}"
    state: directory
    mode: 0755
    owner: slurm
    group: slurm

- name: Copy slurmdbd.init
  copy:
    src: slurmdbd.init
    dest: /etc/init.d/slurmdbd
    mode: 0755
    owner: root
    group: root

- name: Configure slurmdb
  template:
    src: "{{item.src}}"
    dest: "{{item.dest}}"
    mode: 0644
    owner: slurm
    group: slurm
  with_items:
    - { src: 'slurmdbd.conf.template', dest: /usr/local/etc/slurmdbd.conf }
  notify: restart slurmdbd
  tags: slurm-reconfigure

- name: Start slurmdbd
  service:
    name: slurmdbd
    state: started
    enabled: yes
  notify:
    - restart slurm

- name: slurm add cluster
  shell: sacctmgr --immediate add cluster {{ cluster }}
  register: slurm_result
  changed_when: slurm_result.rc == 0
  failed_when: "'already exists.' not in slurm_result.stdout and slurm_result.rc != 0"

- name: slurm add account
  shell: sacctmgr --immediate add account slurm Cluster={{ cluster }}
  register: slurm_result
  changed_when: slurm_result.rc == 0
  failed_when: "'Nothing new added.' not in slurm_result.stdout and slurm_result.rc != 0"

- name: slurm add default users
  shell: sacctmgr --immediate add user {{item}} DefaultAccount=slurm
  register: slurm_result
  changed_when: slurm_result.rc == 0
  failed_when: "'Nothing new added.' not in slurm_result.stdout and slurm_result.rc != 0"
  with_items:
    - slurm
    - "{{ansible_user_id}}"

- name: slurm add preemptible QoS
  shell: sacctmgr --immediate add QoS preemptible
  register: slurm_result
  changed_when: slurm_result.rc == 0
  failed_when: "'Nothing new added.' not in slurm_result.stdout and slurm_result.rc != 0"

- name: slurm add critical QoS
  shell: sacctmgr --immediate add QoS critical Preempt=preemptible
  register: slurm_result
  changed_when: slurm_result.rc == 0
  failed_when: "'Nothing new added.' not in slurm_result.stdout and slurm_result.rc != 0"

# FIXME(INFRA-534): this should be removed in favor of passing
# authentication credentials to scheduler and through the
# HTTP API
- name: set up passwordless_sudo for pipedream user
  copy:
    src: passwordless_sudo
    dest: /etc/sudoers.d/passwordless_sudo
    mode: 0600
    owner: root
    group: root
    validate: "visudo -cf %s"

# Configure logrotate for slurm log files
- name: configure logrotate
  template:
    src: "{{item.src}}"
    dest: "{{item.dest}}"
    mode: 0644
    owner: root
    group: root
  with_items:
    - { src: 'slurm.logrotate.j2', dest: /etc/logrotate.d/slurm }
  tags:
    - logrotate

# Configure sshd
- name: configure sshd
  lineinfile:
    path: /etc/ssh/sshd_config
    regexp: "{{ item.regexp }}"
    line: "{{ item.line }}"
    state: "{{ item.state }}"
    insertafter: "{{ item.insertafter }}"
  with_items:
    - {regexp: '^MaxSessions' , line: 'MaxSessions {{sshd_max_sessions}}', state: present, insertafter: EOF}
  notify:
    - reload sshd
  tags:
    - sshd

# Copy ssh host keys on the slurm-master nodes so
# that a ssh load balancer can work without throwing
# a man in the middle attack warning
- name: copy ssh host keys
  copy: content={{ item.content }} dest={{ item.dest }} mode={{ item.mode }} owner=root
  with_items:
    - { content: "{{ lookup('hashi_vault', 'secret=secret/ssh/pipedream/master_node/ssh_host_ecdsa_key').value }}",
        dest: '/etc/ssh/ssh_host_ecdsa_key', mode: u=rw,g=,o= }
    - { content: "{{ lookup('hashi_vault', 'secret=secret/ssh/pipedream/master_node/ssh_host_ecdsa_key.pub').value }}",
        dest: '/etc/ssh/ssh_host_ecdsa_key.pub', mode: u=rw,g=r,o=r }
    - { content: "{{ lookup('hashi_vault', 'secret=secret/ssh/pipedream/master_node/ssh_host_rsa_key').value }}",
        dest: '/etc/ssh/ssh_host_rsa_key', mode: u=rw,g=,o= }
    - { content: "{{ lookup('hashi_vault', 'secret=secret/ssh/pipedream/master_node/ssh_host_rsa_key.pub').value }}",
        dest: '/etc/ssh/ssh_host_rsa_key.pub', mode: u=rw,g=r,o=r }

# Copy the files required for Pipedream cloud burst
# the 'dest' variables are defined in //pipedream/cloud_burst/main.yml
- name: copy cloud burst files
  copy: src={{ item.src }} dest={{ item.dest }} mode=0755 owner=slurm group=slurm
  with_items:
    - { src: 'cloud_burst_suspend_program.py', dest: "{{ cloud_burst_suspend_program }}" }
    - { src: 'cloud_burst_resume_program.py', dest: "{{ cloud_burst_resume_program }}" }
    - { src: 'config.py', dest: "{{ cloud_burst_config }}" }
    - { src: 'framework.py', dest: "{{ cloud_burst_framework }}" }
    - { src: 'cloud_burst_state_graph.py', dest: "{{ cloud_burst_state_graph }}" }
  tags:
    - slurm-reconfigure
    - cloud-burst

# Generate local settings for cloud burst
- name: Configure local settings, AD secret
  vars:
    text: |
      AD_SLURM_ENROLL_KEY = '{{ lookup('hashi_vault', 'secret=secret/ad/slurm_enroll_key').value }}'
      ENVIRONMENT = '{{ pipedream_env }}'

  copy:
    content: "{{ text }}"
    dest: '{{ cloud_burst_local_config }}'
    mode: u=rw,g=r,o=r
    owner: slurm
    group: slurm

  tags:
    - slurm-reconfigure
    - cloud-burst

# Cloud burst requires newer aws cli to support FSX
- name: Install required pip packages for cloud burst
  pip:
    executable: pip2
    name:
      - awscli==1.18.135
      - boto3==1.14.58
    state: present
  tags:
    - slurm-reconfigure
    - cloud-burst

-
# Autotune files and cron job
- name: ensure '/opt/zoox' present
  file:
    path: /opt/zoox
    group: slurm
    state: directory
    mode: 0755
  tags: autotune

- name: Autotune - Create directory
  file:
    path: /opt/zoox/autotune
    group: slurm
    state: directory
    mode: 0755
  tags: autotune

- name: Autotune - Deploy
  copy: src={{ item.src }} dest={{ item.dest }} mode=0655 owner=slurm group=slurm
  with_items:
    - { src: "../../../../experimental/cherrmann/autotune/autotune.py", dest: "/opt/zoox/autotune/" }
    - { src: "../../../../base/metrics/__init__.py", dest: "/opt/zoox/autotune/base/metrics/" }
    - { src: "../../../../base/__init__.py", dest: "/opt/zoox/autotune/base/" }
    - { src: "../../../../vehicle/__init__.py", dest: "/opt/zoox/autotune/vehicle/" }
    - { src: "../../../../vehicle/common/__init__.py", dest: "/opt/zoox/autotune/vehicle/common/" }
    - { src: "../../../../vehicle/common/environment/__init__.py", dest: "/opt/zoox/autotune/vehicle/common/environment//" }
    - { src: "../../../../vehicle/common/environment/robot_name/__init__.py", dest: "/opt/zoox/autotune/vehicle/common/environment/robot_name/" }
    - { src: "../../../../vehicle/common/environment/robot_name/robot_name.py", dest: "/opt/zoox/autotune/vehicle/common/environment/robot_name/" }
  tags: autotune

- name: Install required pip packages
  pip:
    executable: pip2
    name:
      - requests==2.21.0
      - six==1.12.0
    state: present
  tags: autotune

- name: Slurm Controller Metrics - copy files
  copy: src={{ item.src }} dest={{ item.dest }} mode=0755 owner=slurm group=slurm
  with_items:
    - { src: "sdiag_metrics.py", dest: "/opt/zoox/" }
    - { src: "slurm_ping_metrics.py", dest: "/opt/zoox/" }
  tags: controller_metrics

- name: Slurm Controller Metrics - setup sdiag cron jobs
  cron:
    user: root
    name: "sdiag metrics"
    minute: "*/2"
    job: "bash -c 'python /opt/zoox/sdiag_metrics.py > /opt/prometheus/metrics/sdiag_metrics.prom'"
    cron_file: sdiag_metrics
  tags: controller_metrics

- name: Slurm Controller Metrics - setup slurm ping cron jobs
  cron:
    user: root
    name: "slurm ping metrics"
    minute: "*/2"
    job: "bash -c '/opt/zoox/slurm_ping_metrics.py > /opt/prometheus/metrics/slurm_ping_metrics.prom'"
    cron_file: slurm_ping_metrics
  tags: controller_metrics

- name: Update slurm controller hostname
  hostname:
    name: "{{ slurm_hostname }}"
  when: slurm_hostname is defined
  tags: slurm-reconfigure

- name: Update /etc/hosts file with hostname
  lineinfile:
    path: /etc/hosts
    regexp: "{{ hostvars[inventory_hostname]['ansible_default_ipv4']['address'] }}"
    line: "{{ hostvars[inventory_hostname]['ansible_default_ipv4']['address'] }} {{ slurm_hostname }}"
  when: slurm_hostname is defined
  tags: slurm-reconfigure
