---

- name: Install required apt packages
  apt:
    name:
      - libmunge2
      - libmunge-dev
      - libmysqlclient-dev
      - libhwloc-dev
      - libncurses5-dev
      - mysql-client
      - python-mysqldb
      - python-pip
      - python-prctl
      - python-dev
      - libcap-dev
    state: present
  tags:
    - "pip-packages"
    - "pipecleaner"

# Need to remove this package before installing retrying.
- name: remove ubuntu-packaged python libraries
  apt:
    name: python-six
    state: absent
  tags:
    - "pip-packages"
    - "pipecleaner"

- name: install required packages for PipeDream
  pip:
    name: "{{item.name}}"
    version: "{{ item.version | default(omit) }}"
    state: present
  with_items:
    - { name: "future", version: "0.18.2" }
    - { name: "pyfiglet", version: "0.7.5"}
    - { name: "python-redis-lock", version: "3.2.0" }
    - { name: "pytz", version: "2012rc0" }
    - { name: "requests", version: "2.11.1" }
    - { name: "retrying", version: "1.3.3"}
    - { name: "subprocess32", version: "3.2.7"}
    - { name: "zclient", version: "1.1.0" }
  tags:
    - "pip-packages"
    - "pipecleaner"

- name: remove old SLURM
  apt: name={{ old_slurm_version }} state=absent

# - name: remove old SLURM
#   apt: name=slurm state=absent

- name: install SLURM
  apt: deb=https://s3-us-west-1.amazonaws.com/zoox-third-party/slurm/{{ ansible_distribution_release }}/{{ slurm_version }}

- name: make slurm user group
  group:
    name: slurm
    state: present
    gid: 1101

- name: make slurm user
  user:
    name: slurm
    uid: 1101
    groups: slurm,syslog
    state: present

#- file: path=/etc/slurm/ state=directory mode=0755 owner=root group=root
- name: Make necessary directories
  file:
    path: "{{item}}"
    state: directory
    mode: 0755
    owner: slurm
    group: slurm
  with_items:
    - /var/log/slurm
    - /var/run/slurm

- name: Ensure pipedream_env directory exists
  file:
    path: /nfs/pipedream/config/{{ pipedream_env }}
    state: directory
    mode: 0755
    owner: slurm
    group: slurm

# pipedream_env = staging | production | test
# Specified in infra/ansible/inventory/slurm/production/production
- name: Copy slurm configuration
  copy:
    src: "{{item.src}}"
    dest: "{{item.dest}}"
    mode: 0644
    owner: slurm
    group: slurm

  with_items:
    - { src: /tmp/slurm.conf, dest: /usr/local/etc/slurm.conf }
    - { src: /tmp/slurm.conf, dest: "/nfs/pipedream/config/{{ pipedream_env }}/slurm.conf" }
    - { src: /tmp/gres.conf, dest: /usr/local/etc/gres.conf }
    - { src: /tmp/gres.conf, dest: "/nfs/pipedream/config/{{ pipedream_env }}/gres.conf" }

  tags: slurm-reconfigure
  notify: restart slurm

- name: Copy slurm.init
  copy:
    src: slurm.init
    dest: /etc/init.d/slurm
    mode: 0755
    owner: root
    group: root
  when: ansible_distribution_version == '14.04'
  notify: restart slurm

- name: Copy slurm.service
  template:
    src: slurm.service.j2
    dest: /etc/systemd/system/slurm.service
    mode: 0644
    owner: root
    group: root
  when: ansible_distribution_version == '18.04'
  notify: restart systemd slurm

- name: install sbump
  copy:
    src: sbump.py
    dest: /usr/local/sbin/sbump
    mode: 0755
    owner: root
    group: root

# Currently disabled so that we can start it from rc.local after having run
# nvidia-smi. Revisit when we upgrade to 16.04/systemd
- name: Start slurm
  service:
    name: slurm
    state: started
    enabled: no
  when: ansible_distribution_version == '14.04'

- name: Start slurm
  service:
    name: slurm
    state: started
    enabled: yes
  when: ansible_distribution_version == '18.04'

- name: start slurm on boot after nvidia-smi
  lineinfile:
    dest: /etc/rc.local
    insertbefore: "exit 0"
    line: "/etc/init.d/slurm start"
    state: present
  when: ansible_distribution_version == '14.04'

# Configure dynamic linker run-time bindings so Slurm libraries can be found
# by applications that intend to use Slurm APIs directly.
- name: configure dynamic linker run-time bindings
  command: /sbin/ldconfig

- name: add passwordless sudo for scancel only
  copy:
    src: scancel
    dest: /etc/sudoers.d/scancel
    owner: root
    group: root
    mode: u=r,g=r,o=
    validate: visudo --check --file=%s

- name: create nvidia jit cache directory
  file:
    path: "/prestage/nvCompilerCache"
    state: directory
    mode: 0777
  tags: nvcompilercache

- name: ensure '/opt/zoox' present
  file:
    path: /opt/zoox
    group: slurm
    state: directory
    mode: 0755

- name: install restart program
  template:
    src: restart.sh.j2
    dest: /opt/zoox/restart
    mode: 0755
    owner: root
    group: root

- name: PipeCleaner - remove hourly cron job
  cron:
    name: pipecleaner
    special_time: hourly
    job: "/opt/zoox/pipecleaner/pipecleaner.py"
    state : absent
  tags: pipecleaner

- tags: core_pattern
  block:
    - name: Core Pattern - Create /coredumps directory in case the host generates any dumps.
      file:
        path: /coredumps
        group: slurm
        state: directory
        mode: 0777

    - name: Core Pattern - Disable apport (so it doesn't rewrite core_pattern on boot).
      copy:
        content: "enabled=0"
        dest: /etc/default/apport

    - name: Core Pattern - Set /proc/sys/kernel/core_pattern.
      sysctl:
        name: "kernel.core_pattern"
        value: "/coredumps/exec-%e.pid-%p.signum-%s.hostname-%h.time-%t.core"
        sysctl_set: yes
        state: present
        reload: yes

# Allow non-root users to resolve kernel symbols
# Without it , `perf record` prints
#
#   Couldn't record kernel reference relocation symbol
#   Symbol resolution may be skewed if relocation was used (e.g. kexec).
#   Check /proc/kallsyms permission or run as root.
#
#   Kernel address maps (/proc/{kallsyms,modules}) were restricted.
#   Check /proc/sys/kernel/kptr_restrict before running 'perf record'
#   If some relocation was applied (e.g. kexec) symbols may be misresolved.
#   Samples in kernel modules can't be resolved as well.
- name: Set /proc/sys/kernel/kptr_restrict.
  sysctl:
    name: "kernel.kptr_restrict"
    value: "0"
    sysctl_set: yes
    state: present
    reload: yes

# Default is 1, which says processes can only PTRACE_ATTACH to their descendants.
# Setting it to 0 allows processes owned by the same uid attach to each other.
# See https://www.kernel.org/doc/Documentation/security/Yama.txt
- name: Set /proc/sys/kernel/yama/ptrace_scope to 0
  sysctl:
    name: "kernel.yama.ptrace_scope"
    value: "0"
    sysctl_set: yes
    state: present
    reload: yes

