---
- block:
  - cron:
      name: Missing GPU restarter cron
      state: absent

  - name: Remove old drivers
    apt:
      update_cache: yes
      state: absent
      purge: yes
      name:
        - 'cuda-*'
        - 'libnvidia-*'
        - 'nvidia-*'
        - 'xserver-xorg-video-nvidia-*'

  - name: Download Nvidia driver deb
    command: curl -LO https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/cuda-repo-ubuntu1804_10.0.130-1_amd64.deb

  - name: Install driver
    command: dpkg -i cuda-repo-ubuntu1804_10.0.130-1_amd64.deb

  - name: Get key
    command: apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/7fa2af80.pub

  - name: Install new drivers
    apt:
      update_cache: yes
      name:
        - 'cuda-drivers-460=460.32.03-1'
        - 'libnvidia-cfg1-460=460.32.03-0ubuntu1'
        - 'libnvidia-compute-460=460.32.03-0ubuntu1'
        - 'libnvidia-decode-460=460.32.03-0ubuntu1'
        - 'libnvidia-encode-460=460.32.03-0ubuntu1'
        - 'libnvidia-fbc1-460=460.32.03-0ubuntu1'
        - 'libnvidia-gl-460=460.32.03-0ubuntu1'
        - 'libnvidia-ifr1-460=460.32.03-0ubuntu1'
        - 'nvidia-compute-utils-460=460.32.03-0ubuntu1'
        - 'nvidia-container-runtime'
        - 'nvidia-dkms-460=460.32.03-0ubuntu1'
        - 'nvidia-driver-460=460.32.03-0ubuntu1'
        - 'libnvidia-extra-460=460.32.03-0ubuntu1'
        - 'nvidia-kernel-source-460=460.32.03-0ubuntu1'
        - 'nvidia-utils-460=460.32.03-0ubuntu1'
        - 'nvidia-kernel-common-460=460.32.03-0ubuntu1'
        - 'xserver-xorg-video-nvidia-460=460.32.03-0ubuntu1'

  - name: reboot nodes
    reboot:
      post_reboot_delay: 30
      reboot_timeout: 600

  - cron:
      name: Missing GPU restarter cron
      job:  "{{ kraken_gpu_checker_cron_path }} --metrics_file={{ kraken_gpu_checker_metrics_file }} >> {{ kraken_gpu_checker_log_file }} 2>&1"
      minute: "*/5"
      state: present

  become: yes
  tags: nvidia-driver
  when: install_nvidia_driver is defined

- block:
  - name: set up passwordless_sudo for zoox user
    copy:
      src: passwordless_sudo
      dest: /etc/sudoers.d/passwordless_sudo
      mode: 0600
      owner: root
      group: root
      validate: "visudo --check --file %s"

  become: yes
  tags: passwordless-sudo

- block:
  - name: ensure /etc/systemd/logind.conf.d
    file:
      path: /etc/systemd/logind.conf.d
      state: directory
      mode: 0755

  - name: disable removing of /dev/shm files upon logout
    copy:
      src: dont-remove-ipc.conf
      dest: /etc/systemd/logind.conf.d/dont-remove-ipc.conf
    notify: restart systemd-logind

  become: yes
  tags: systemd

- block:
  - name: uninstall previously installed zrte
    apt:
      state: absent
      name:
        - zrte-base-{{ kraken_zrte_base_version }}
        - zrte-nvidia-{{ kraken_zrte_nvidia_version }}
    when: force_reinstall_zrte | default(False)

  - name: ensure total removal of previously installed zrte
    file:
      state: absent
      path: "{{ item }}"
    with_items:
    - /opt/zoox/zrte/base/{{ kraken_zrte_base_version }}
    - /opt/zoox/zrte/nvidia/{{ kraken_zrte_nvidia_version }}
    when: force_reinstall_zrte | default(False)

  - name: install zrte-base
    apt:
      deb: https://z3.zooxlabs.com/zoox-third-party/afc59263c017f2ed731c613537b6d9e2d3d690afa1f0d8d228d6cb5e8cb14067/zrte-base-test_1_all.deb

  - name: update alternatives for zrte-base
    # Using shell because the `alternatives` module doesn't support the --altdir flag.
    shell: update-alternatives --altdir /opt/zoox/zrte/alternatives --set zrte /opt/zoox/zrte/base/{{ kraken_zrte_base_version }}

  # TODO (INFRA-11640): This will be taken care of in that ticket
  # - name: install zrte-nvidia
  #   apt:
  #     deb: https://z3.zooxlabs.com/zoox-third-party/fe676ae07b8468ff283e58abb9acbfbe57f407e98c0047f79bd93da93fb0748a/zrte-nvidia-440.64.00-0ubuntu1_1_all.deb
  #   when: worker_type == "gpu"

  # - name: update alternatives for zrte-nvidia
  #   # Using shell because the `alternatives` module doesn't support the --altdir flag.
  #   shell: update-alternatives --altdir /opt/zoox/zrte/alternatives --set zrte_nvidia /opt/zoox/zrte/nvidia/{{ kraken_zrte_nvidia_version }}
  #   when: worker_type == "gpu"

  become: yes
  tags: zrte

- block:
  - name: Install chronyd
    apt:
      state: present
      name:
        - chrony

  become: yes
  tags: install-chrony

# Upgrade docker-compose from the version installed by the `docker-host` role.
- block:
  - name: check current versions
    pip_package_info: {}
    register: existing_packages

  - set_fact:
      existing_compose_version: "{{ existing_packages['packages']['pip']['docker-compose'][0]['version'] }}"
    ignore_errors: yes

  - name: remove /usr/local/bin/docker-compose
    file:
      path: /usr/local/bin/docker-compose
      state: absent
    when: existing_compose_version | default("") != "1.21.0"

  # docker-py conflicts with docker
  - name: remove existing pip packages
    pip:
      name:
      - docker-compose
      - docker-py
      - docker
      state: absent
    when: existing_compose_version | default("") != "1.21.0"

  - name: remove existing apt packages
    apt:
      name:
        - python-docker
        - docker-compose
      state: absent

  - name: install docker-compose
    pip:
      name:
      - docker-compose==1.21.0
      - docker==3.7.3
      - requests==2.11.1

  become: yes
  tags: upgrade-docker-compose

- block:
  - name: Create docker daemon config dir
    file:
      path: /lib/systemd/system/docker.service.d
      state: directory

  - name: Copy config file to make weka mount first before running docker daemon
    copy:
      src: 10-after-weka.conf
      dest: /lib/systemd/system/docker.service.d/10-after-weka.conf

  - name: Copy auto create kraken work path service config
    template:
      src: kraken-work-path.service
      dest: /etc/systemd/system/kraken-work-path.service

  - name: Copy config file to make work directory before running docker daemon
    copy:
      src: 10-after-worker-dir.conf
      dest: /lib/systemd/system/docker.service.d/10-after-worker-dir.conf

  - name: Run auto create kraken work path service
    systemd:
      name: kraken-work-path.service
      daemon_reload: yes
      enabled: yes
      state: restarted

  - name: create docker config directory
    file:
      path: /etc/docker
      state: directory
      mode: u=rwx,g=rx,o=rx

  # This file is just a copy of default docker daemon. We just add nvidia
  # runtime to make containers have access to GPUs by default for GPU workers.
  # We also set 'init' to 'true' so that orphaned processes could be reaped.
  - name: Copy worker type specific docker daemon settings
    template:
      src: "{{ docker_daemon_filename }}"
      dest: /etc/docker/daemon.json
      mode: 0644
      owner: root
      group: root
    notify: "{{ 'restart dockerd' if not building_ami else 'nop' }}"
  become: yes
  tags: configure-docker-daemon

- block:
  # Disable apport from taking over kernel.core_pattern variable
  # that we are about to set. This is a bug in ubuntu and discussion on
  # this is here: https://bugs.launchpad.net/ubuntu/+source/apport/+bug/1080978
  - name: Disable apport from taking over kernel core pattern
    copy:
      dest: /etc/default/apport
      mode: "0644"
      content: |
        # This value is set by Kraken ansible
        # set this to 0 to disable apport, or to 1 to enable it
        # you can temporarily override this with
        # sudo service apport start force_start=1
        enabled=0

  # Disable Weka hijacking core_pattern variable. They said they would
  # not be taking over core_pattern by default starting version 3.6.
  # For now, we will disable it.
  - name: Disable Weka from hijacking kernel core pattern
    lineinfile:
      dest: /etc/wekaio/service.conf
      regexp: '^override_core_pattern='
      line: override_core_pattern=false
      backrefs: yes
    when: not building_ami
    notify: restart weka-agent

  # This change disables core dumps on all the Kraken V3 workers.
  # This is done because tests have code where they explicitly crash,
  # thus generating core dump as a part of test. Core dumps were disabled
  # on Kraken v2, so in order to make it consistent, we disable core
  # dump on Kraken v3 nodes as well.
  - name: Make Kraken workers not write core dump
    sysctl:
      name: "kernel.core_pattern"
      sysctl_set: yes
      value: "|/opt/bin/core_handler.sh %e %p %h %t"
      # From `man 5 core`:
      # %%  a single % character
      # %c  core file size soft resource limit of crashing process (since Linux 2.6.24)
      # %d  dump mode-same as value returned by prctl(2) PR_GET_DUMPABLE (since Linux 3.7)
      # %e  executable filename (without path prefix)
      # %E  pathname of executable, with slashes ('/') replaced by exclamation marks ('!') (since Linux 3.0).
      # %g  (numeric) real GID of dumped process
      # %h  hostname (same as nodename returned by uname(2))
      # %i  TID of thread that triggered core dump, as seen in the PID namespace in which the thread resides (since Linux 3.18)
      # %I  TID of thread that triggered core dump, as seen in the initial PID namespace (since Linux 3.18)
      # %p  PID of dumped process, as seen in the PID namespace in which the process resides
      # %P  PID of dumped process, as seen in the initial PID namespace (since Linux 3.12)
      # %s  number of signal causing dump
      # %t  time of dump, expressed as seconds since the Epoch, 1970-01-01 00:00:00 +0000 (UTC)
      # %u  (numeric) real UID of dumped process

  - name: Create /opt/bin directory
    file: path=/opt/bin/ state=directory
    become: yes

  - name: Copy core_handler.sh
    copy:
      dest: /opt/bin/core_handler.sh
      mode: "0755"
      content: |
        #!/bin/bash
        set -euo pipefail
        # only save core dumps for clang
        if [[ $1 == "clang" ]]; then
            exec /usr/bin/gzcore /var/crash "$@"
        else
            exit 1
        fi
  become: yes
  tags: disable-core-dump

- block:
  - name: Create directory for kraken worker docker compose
    file:
      path: "{{ kraken_worker_docker_compose_dir }}"
      state: directory
      mode: u=rwx,g=r,o=r

  - name: Create directory for kraken worker cache
    file:
      path: "{{ kraken_worker_work_path }}"
      state: directory
      mode: u=rwx,g=rwx,o=rwx

  - name: Touch Kraken metrics file
    copy:
      content: ""
      dest: "{{ kraken_metrics_file }}"
      force: no
      group: sys
      owner: "{{ kraken_user }}"
      mode: 0777

  - name: Create dir for action log
    file:
      state: directory
      path: "{{ kraken_action_logs_dir_host }}"
      group: sys
      owner: "{{ kraken_user }}"
      mode: "1777"

  - name: Copy Kraken worker docker compose
    template:
      src: "{{ docker_compose_template }}"
      dest: "{{ kraken_worker_docker_compose_dir }}/docker-compose.yml"
  become: yes
  tags: kraken-docker-files

- block:
  - name: Pull Kraken docker image
    docker_image:
      name: "{{ weka_kraken_docker_image }}"
      pull: yes
      state: present

  - name: Unleash the Kraken
    docker_service:
      project_src: "{{ kraken_worker_docker_compose_dir }}"
      pull: yes
      state: present

  - name: Pause
    pause:
      seconds: "{{ pause_seconds_after_deploy | default(0) }}"

  become: yes
  tags: pull-restart-kraken-worker

- block:
  - name: create project directory
    file:
      path: /etc/docker-compose/filebeat
      state: directory
      mode: u=rwx,g=r,o=r

  - name: create project directory
    file:
      path: "{{ filebeat_data_dir }}"
      state: directory
      mode: u=rwx,g=rwx,o=rx
      owner: "{{ kraken_user }}"

  - name: copy docker-compose.yml
    template:
      src: filebeat-docker-compose.yml.tmpl
      dest: /etc/docker-compose/filebeat/docker-compose.yml

  - name: copy filebeat template
    template:
      src: filebeat
      dest: /etc/docker-compose/filebeat/filebeat.yml

  - name: start filebeat
    docker_service:
      project_src: /etc/docker-compose/filebeat
      pull: yes
      restarted: yes
      state: present

  become: yes
  tags: filebeat

- block:
  - name: "logstash: create project directory"
    file:
      path: /etc/docker-compose/logstash
      state: directory
      mode: u=rwx,g=rx,o=rx
    check_mode: False

  - name: "logstash: create data directory"
    file:
      path: "{{ logstash_data_dir }}"
      state: directory
      mode: u=rwx,g=rwx,o=rx
      owner: "{{ kraken_user }}"
    check_mode: False

  - name: "logstash: copy docker-compose.yml"
    template:
      src: logstash-docker-compose.yml.tmpl
      dest: /etc/docker-compose/logstash/docker-compose.yml

  - name: "logstash: copy config"
    copy:
      src: "{{ item }}"
      dest: /etc/docker-compose/logstash/{{ item }}
    with_items:
      - logstash.conf
      - logstash.yml

  - name: start logstash
    docker_service:
      project_src: /etc/docker-compose/logstash
      pull: yes
      restarted: yes
      state: present

  become: yes
  tags: logstash

- block:
  - name: create project directory
    file:
      path: /etc/docker-compose/jaeger-agent
      state: directory
      mode: u=rwx,g=r,o=r

  - name: copy docker-compose.yml
    copy:
      src: jaeger_agent.yml
      dest: /etc/docker-compose/jaeger-agent/docker-compose.yml

  - name: start jaeger_agent
    docker_compose:
      project_src: /etc/docker-compose/jaeger-agent
      state: present
  become: yes
  tags: jaeger-agent

# Some GPU tests require an xserver running on Kraken server.
# This accomplishes that.
- block:
  - name: install supervisor and xvfb
    apt:
      name: "{{ packages }}"
      update_cache: yes
    vars:
      packages:
        - supervisor
        - xvfb

  - name: supervisord config
    copy:
      src: supervisord.conf
      dest: /etc/supervisor/conf.d/supervisord.conf
      mode: 0644
      owner: root
      group: root
    notify: restart supervisor

  - name: register xserver service definition
    template:
      src: weka_xserver.conf
      dest: /etc/supervisor/conf.d/xserver.conf
      mode: 0644
      owner: root
      group: root
    notify: supervisor update

  - name: Run xserver
    supervisorctl:
      name: xserver
      state: restarted

  become: yes
  tags: weka-xserver
  when: worker_type == "gpu"

- block:
  - name: Create crons directory
    file:
      path: "{{ kraken_crons_path }}"
      state: directory
      mode: u=rwx,g=r,o=r

  - name: Create crons logs directory
    file:
      path: "{{ kraken_crons_log_path }}"
      state: directory
      mode: u=rwx,g=r,o=r

  - name: Touch GPU checker metrics file
    file:
      path: "{{ kraken_gpu_checker_metrics_file }}"
      state: touch
      mode: 0777

  - name: Touch GPU checker log file
    file:
      path: "{{ kraken_gpu_checker_log_file }}"
      state: touch
      mode: 0777

  - name: Copy missing GPU restarter script
    copy:
      src: gpu_checker.py
      dest: "{{ kraken_gpu_checker_cron_path }}"
      mode: 0755
      owner: root
      group: root

  - cron:
      name: Missing GPU restarter cron
      job:  "{{ kraken_gpu_checker_cron_path }} --metrics_file={{ kraken_gpu_checker_metrics_file }} >> {{ kraken_gpu_checker_log_file }} 2>&1"
      minute: "*/5"
      state: present

  become: yes
  tags: missing-gpu-restarter
  when: worker_type == "gpu"

- block:
  - name: Download QNX binaries tar
    get_url:
      url: "https://s3-us-west-1.amazonaws.com/zoox-third-party/{{ qnx_binaries_sha }}/qnx_7.0.0.tar.gz"
      dest: "{{ qnx_binaries_tar_path }}"
      timeout: 1000

  - name: Delete QNX binary path if present
    file:
      path: "{{ qnx_binary_path }}"
      state: absent

  - name: Extract QNX binaries tar
    unarchive:
      src: "{{ qnx_binaries_tar_path }}"
      dest: "{{ qnx_binary_dir }}"
      remote_src: yes

  - name: Remove QNX binaries tar
    file:
      name: "{{ qnx_binaries_tar_path }}"
      state: absent

  - name: Create QNX config directory
    file:
      path: "{{ qnx_config_path }}"
      state: directory
      mode: u=rwx,g=rwx,o=rx

  - name: Make QNX directory for toolchain
    file:
      path: "{{ qnx_toolchain_path }}"
      state: directory
      mode: u=rwx,g=rx,o=rx
    become: yes

  - name: Create a symlink in toolchain path for QNX binaries
    file:
      state: link
      path: "{{ qnx_symlink_path }}"
      src: "{{ qnx_binary_path }}"
    become: yes

  - name: Create QNX config dir in toolchain path
    file:
      path: "{{ qnx_symlink_path }}/.qnx"
      state: directory
      mode: u=rwx,g=rx,o=rx

  - name: Create a symlink in toolchain path for QNX binaries
    file:
      state: link
      path: "{{ qnx_symlink_path }}/.qnx/license"
      src: "{{ qnx_config_path }}/license"
      mode: u=rwx,g=rwx,o=rwx
      force: yes # This is done because license file will appear later
    become: yes

  tags: qnx-install
  when: worker_type == "firmware" and install_qnx is defined

- block:
  - name: Download QNX setup run file
    get_url:
      url: "https://s3-us-west-1.amazonaws.com/zoox-third-party/{{ qnx_run_file_sha }}/qnx-setup-lin.run"
      dest: "{{ qnx_run_file_path }}"
  - name: Make run file executable
    file:
      path: "{{ qnx_run_file_path }}"
      mode: u=rwx,g=rwx,o=rx
  tags: qnx-setup-file
  when: worker_type == "firmware" and install_qnx is defined

- block:
  - name: Setup QNX license
    shell: (echo y; echo "{{ qnx_install_dir }}") | LESS="-FERX +G" "{{ qnx_run_file_path }}" --nox11 disable-auto-start

  - name: Add QNX license information script
    shell: "{{ qnx_install_dir }}/qnxsoftwarecenter/qnxsoftwarecenter_clt -url https://qnx.com/swcenter -syncLicenseKeys -myqnx.user={{ lookup('hashi_vault', 'secret=secret/qnx').username }} -myqnx.password={{ lookup('hashi_vault', 'secret=secret/qnx').password }} -addLicenseKey {{ lookup('hashi_vault', 'secret=secret/qnx')[inventory_hostname] }} -listLicenseKeys"

  - name: Clean up QNX install location
    file:
      path: "{{ qnx_install_dir }}"
      state: absent

  - name: Delete QNX setup run file
    file:
      name: "{{ qnx_run_file_path }}"
      state: absent

  tags: qnx-activate
  when: worker_type == "firmware" and install_qnx is defined

- block:
  - name: create project directory
    file:
      path: /etc/docker-compose/weka-cache-evictor
      state: directory
      mode: u=rwx,g=r,o=r

  - name: create logs directory
    file:
      path: "{{ kraken_logs_path }}"
      state: directory
      mode: u=rwx,g=rwx,o=rwx

  - name: copy cache evictor docker compose
    template:
      src: weka-cache-evictor.yml
      dest: /etc/docker-compose/weka-cache-evictor/docker-compose.yml

  - name: start weka cache evictor
    docker_compose:
      project_src: /etc/docker-compose/weka-cache-evictor
      pull: yes
      state: present

  - name: Delete old weka evictor logs
    cron:
      name: delete weka logs older than 8 days
      minute: "0"
      hour: "10"
      job: find /prestage/kraken/logs -type f -name "weka_*.log*" -mtime +8 -delete
      state: present

  become: yes
  tags: weka-cache-evictor
  when: worker_type == "file_evictor"

- block:
  - name: create project directory
    file:
      path: /etc/docker-compose/weka-metrics-collector
      state: directory
      mode: u=rwx,g=r,o=r

  - name: copy metrics collector docker compose
    template:
      src: weka-metrics-collector-docker-compose.yml
      dest: /etc/docker-compose/weka-metrics-collector/docker-compose.yml

  - name: start weka metrics collector
    docker_compose:
      project_src: /etc/docker-compose/weka-metrics-collector
      pull: yes
      state: present

  become: yes
  tags: weka-metrics-collector
  when: worker_type == "file_evictor"

- block:
  - name: Cron the weka build directory cleaner
    cron:
      name: weka build_directory_cleaner cron
      job: docker exec krakenworker_kraken-worker_1 java -jar /opt/bin/weka_work_directory_cleaner_deploy.jar --work_path=/dev/shm/worker --metrics_log_mode=always >> {{ kraken_build_directory_cleaner_log_file }} 2>&1
      minute: "*/5"
      state: present
  tags: wbdc-cron

- block:
  - name: Cron the weka process killer
    cron:
      name: weka kill orphan process
      job:  docker exec --privileged krakenworker_kraken-worker_1 /opt/bin/kill_process.sh > /dev/null 2>&1
      minute: "*/1"
      state: present
  tags: wpk-cron

- block:
  - name: Cron the docker pruner
    cron:
      name: docker system prune
      job:  docker system prune -f -a --volumes
      minute: "0"
      hour: "0"
      state: present
    become: yes
  tags: weka-docker-prune-cron

- block:
  - name: Create /tmp cleaner cron
    cron:
      name: tmpfs cleaner cron
      job: find /tmp -type f -atime +1 -delete
      special_time: daily
      state: present
    become: yes

  tags: tmp-cleaner-cron

# This is added so that we can run a cron job that kills
# zombie simple_sim processes. This needs to run in the
# kraken docker container because running killall from outside the
# container doesn't clean up those processes.
- block:
  - name: Create simple_sim process reaper
    cron:
      name: simple_sim process reaper
      job: "docker exec -it krakenworker_kraken-worker_1 killall --older-than 30m --signal SIGKILL play_private >> ~/sim_reaper_log 2>&1"
      minute: "*/30"
      state: present
  become: yes
  tags: weka-sim-reaper

- block:
  - name: Create /dev/shm/worker cleaner cron
    cron:
      name: tmpfs cleaner cron
      job: find /dev/shm/ -type f -mmin +180 -delete
      minute: "*/30"
      state: present
    become: yes
    tags: dev_shm_cleaner

- name: Reboot on hung kernel tasks
  sysctl:
    name: "{{ item.name }}"
    sysctl_set: yes
    value: "{{ item.value }}"
  with_items:
  - name: kernel.panic
    value: "5"  # Reboot 5 seconds after panic
  - name: kernel.hung_task_panic
    value: "1"  # Panic if a hung task was found
  - name: kernel.hung_task_timeout_secs
    value: "300"  # Set timeout for hung task to 300 seconds
  become: yes
  tags: reboot-on-hung-tasks

- block:
  - name: Create rootfs mountpoints
    file:
      path: "{{ kraken_rootfs_dir }}/{{ item.name }}"
      state: directory
    with_items: "{{ rootfs_images }}"

  - name: Create directory to put packed rootfs images
    file:
      path: "{{ kraken_packed_rootfs_dir }}"
      state: directory
      mode: 0777

  - name: Download rootfs images
    get_url:
      checksum: sha256:{{ item.sha256 }}
      dest: "{{ kraken_packed_rootfs_dir }}/{{ item.name }}.squashfs"
      url: https://z3.zooxlabs.com/zoox-third-party/{{ item.sha256 }}/{{ item.name }}.squashfs
    with_items: "{{ rootfs_images }}"

  - name: create rootfs mount config
    template:
      src: rootfs.mount.tmpl
      dest: /etc/systemd/system/mnt-kraken-rootfs-{{ item.name }}.mount
    with_items: "{{ rootfs_images }}"

  - name: Mount rootfs images
    systemd:
      name: mnt-kraken-rootfs-{{ item.name }}.mount
      daemon_reload: yes
      enabled: yes
      state: started
    with_items: "{{ rootfs_images }}"

  become: yes
  tags: mount-rootfs-images
