version: "{{ docker_compose_format | default('2.4') }}"
services:
  kraken-worker:
    image: "{{ weka_kraken_docker_image }}"
{% if (docker_compose_format | default('2.0')) >= '2.3' %}
    runtime: nvidia
{% endif %}
    ulimits:
      nofile:
        soft: 524288
        hard: 524288
    volumes:
      - {{ kraken_work_path_root }}:{{ kraken_work_path_root }}
      - {{ kraken_metrics_file }}:/home/zoox/kraken_metrics.log
      - {{ kraken_action_logs_dir_host }}:{{ kraken_action_logs_dir_container }}
      - type: bind
        source: /mnt
        target: /mnt
        bind:
          propagation: rslave
      - /sys/fs/cgroup:/sys/fs/cgroup:ro
      - /opt/zoox/zrte:/opt/zoox/zrte:ro
      # We need to add all the files listed below in order to have cuda library in docker container
      - /usr/bin/nvidia-cuda-mps-control:/usr/bin/nvidia-cuda-mps-control:ro
      - /usr/lib/x86_64-linux-gnu/libcuda.so:/usr/lib/x86_64-linux-gnu/libcuda.so:ro
      - /usr/lib/x86_64-linux-gnu/libcuda.so.1:/usr/lib/x86_64-linux-gnu/libcuda.so.1:ro
      - /usr/lib/x86_64-linux-gnu/libcuda.so.460.32.03:/usr/lib/x86_64-linux-gnu/libcuda.so.460.32.03:ro
      - /usr/lib/x86_64-linux-gnu/libEGL_nvidia.so.0:/usr/lib/x86_64-linux-gnu/libEGL_nvidia.so.0:ro
      - /usr/lib/x86_64-linux-gnu/libEGL_nvidia.so.460.32.03:/usr/lib/x86_64-linux-gnu/libEGL_nvidia.so.460.32.03:ro
      - /usr/lib/x86_64-linux-gnu/libGLESv1_CM_nvidia.so.1:/usr/lib/x86_64-linux-gnu/libGLESv1_CM_nvidia.so.1:ro
      - /usr/lib/x86_64-linux-gnu/libGLESv1_CM_nvidia.so.460.32.03:/usr/lib/x86_64-linux-gnu/libGLESv1_CM_nvidia.so.460.32.03:ro
      - /usr/lib/x86_64-linux-gnu/libGLESv2_nvidia.so.2:/usr/lib/x86_64-linux-gnu/libGLESv2_nvidia.so.2:ro
      - /usr/lib/x86_64-linux-gnu/libGLESv2_nvidia.so.460.32.03:/usr/lib/x86_64-linux-gnu/libGLESv2_nvidia.so.460.32.03:ro
      - /usr/lib/x86_64-linux-gnu/libGLX_nvidia.so.0:/usr/lib/x86_64-linux-gnu/libGLX_nvidia.so.0:ro
      - /usr/lib/x86_64-linux-gnu/libGLX_nvidia.so.460.32.03:/usr/lib/x86_64-linux-gnu/libGLX_nvidia.so.460.32.03:ro
      - /usr/lib/x86_64-linux-gnu/libnvcuvid.so:/usr/lib/x86_64-linux-gnu/libnvcuvid.so:ro
      - /usr/lib/x86_64-linux-gnu/libnvcuvid.so.1:/usr/lib/x86_64-linux-gnu/libnvcuvid.so.1:ro
      - /usr/lib/x86_64-linux-gnu/libnvcuvid.so.460.32.03:/usr/lib/x86_64-linux-gnu/libnvcuvid.so.460.32.03:ro
      - /usr/lib/x86_64-linux-gnu/libnvidia-allocator.so:/usr/lib/x86_64-linux-gnu/libnvidia-allocator.so:ro
      - /usr/lib/x86_64-linux-gnu/libnvidia-allocator.so.1:/usr/lib/x86_64-linux-gnu/libnvidia-allocator.so.1:ro
      - /usr/lib/x86_64-linux-gnu/libnvidia-allocator.so.460.32.03:/usr/lib/x86_64-linux-gnu/libnvidia-allocator.so.460.32.03:ro
      - /usr/lib/x86_64-linux-gnu/libnvidia-cbl.so.460.32.03:/usr/lib/x86_64-linux-gnu/libnvidia-cbl.so.460.32.03:ro
      - /usr/lib/x86_64-linux-gnu/libnvidia-cfg.so:/usr/lib/x86_64-linux-gnu/libnvidia-cfg.so:ro
      - /usr/lib/x86_64-linux-gnu/libnvidia-cfg.so.1:/usr/lib/x86_64-linux-gnu/libnvidia-cfg.so.1:ro
      - /usr/lib/x86_64-linux-gnu/libnvidia-cfg.so.460.32.03:/usr/lib/x86_64-linux-gnu/libnvidia-cfg.so.460.32.03:ro
      - /usr/lib/x86_64-linux-gnu/libnvidia-compiler.so.460.32.03:/usr/lib/x86_64-linux-gnu/libnvidia-compiler.so.460.32.03:ro
      - /usr/lib/x86_64-linux-gnu/libnvidia-container.so.1:/usr/lib/x86_64-linux-gnu/libnvidia-container.so.1:ro
      - /usr/lib/x86_64-linux-gnu/libnvidia-container.so.1.4.0:/usr/lib/x86_64-linux-gnu/libnvidia-container.so.1.4.0:ro
      - /usr/lib/x86_64-linux-gnu/libnvidia-eglcore.so.460.32.03:/usr/lib/x86_64-linux-gnu/libnvidia-eglcore.so.460.32.03:ro
      - /usr/lib/x86_64-linux-gnu/libnvidia-encode.so:/usr/lib/x86_64-linux-gnu/libnvidia-encode.so:ro
      - /usr/lib/x86_64-linux-gnu/libnvidia-encode.so.1:/usr/lib/x86_64-linux-gnu/libnvidia-encode.so.1:ro
      - /usr/lib/x86_64-linux-gnu/libnvidia-encode.so.460.32.03:/usr/lib/x86_64-linux-gnu/libnvidia-encode.so.460.32.03:ro
      - /usr/lib/x86_64-linux-gnu/libnvidia-fbc.so:/usr/lib/x86_64-linux-gnu/libnvidia-fbc.so:ro
      - /usr/lib/x86_64-linux-gnu/libnvidia-fbc.so.1:/usr/lib/x86_64-linux-gnu/libnvidia-fbc.so.1:ro
      - /usr/lib/x86_64-linux-gnu/libnvidia-fbc.so.460.32.03:/usr/lib/x86_64-linux-gnu/libnvidia-fbc.so.460.32.03:ro
      - /usr/lib/x86_64-linux-gnu/libnvidia-glcore.so.460.32.03:/usr/lib/x86_64-linux-gnu/libnvidia-glcore.so.460.32.03:ro
      - /usr/lib/x86_64-linux-gnu/libnvidia-glsi.so.460.32.03:/usr/lib/x86_64-linux-gnu/libnvidia-glsi.so.460.32.03:ro
      - /usr/lib/x86_64-linux-gnu/libnvidia-glvkspirv.so.460.32.03:/usr/lib/x86_64-linux-gnu/libnvidia-glvkspirv.so.460.32.03:ro
      - /usr/lib/x86_64-linux-gnu/libnvidia-ifr.so:/usr/lib/x86_64-linux-gnu/libnvidia-ifr.so:ro
      - /usr/lib/x86_64-linux-gnu/libnvidia-ifr.so.1:/usr/lib/x86_64-linux-gnu/libnvidia-ifr.so.1:ro
      - /usr/lib/x86_64-linux-gnu/libnvidia-ifr.so.460.32.03:/usr/lib/x86_64-linux-gnu/libnvidia-ifr.so.460.32.03:ro
      - /usr/lib/x86_64-linux-gnu/libnvidia-ml.so:/usr/lib/x86_64-linux-gnu/libnvidia-ml.so:ro
      - /usr/lib/x86_64-linux-gnu/libnvidia-ml.so.1:/usr/lib/x86_64-linux-gnu/libnvidia-ml.so.1:ro
      - /usr/lib/x86_64-linux-gnu/libnvidia-ml.so.460.32.03:/usr/lib/x86_64-linux-gnu/libnvidia-ml.so.460.32.03:ro
      - /usr/lib/x86_64-linux-gnu/libnvidia-ngx.so.1:/usr/lib/x86_64-linux-gnu/libnvidia-ngx.so.1:ro
      - /usr/lib/x86_64-linux-gnu/libnvidia-ngx.so.460.32.03:/usr/lib/x86_64-linux-gnu/libnvidia-ngx.so.460.32.03:ro
      - /usr/lib/x86_64-linux-gnu/libnvidia-opencl.so.1:/usr/lib/x86_64-linux-gnu/libnvidia-opencl.so.1:ro
      - /usr/lib/x86_64-linux-gnu/libnvidia-opencl.so.460.32.03:/usr/lib/x86_64-linux-gnu/libnvidia-opencl.so.460.32.03:ro
      - /usr/lib/x86_64-linux-gnu/libnvidia-opticalflow.so:/usr/lib/x86_64-linux-gnu/libnvidia-opticalflow.so:ro
      - /usr/lib/x86_64-linux-gnu/libnvidia-opticalflow.so.1:/usr/lib/x86_64-linux-gnu/libnvidia-opticalflow.so.1:ro
      - /usr/lib/x86_64-linux-gnu/libnvidia-opticalflow.so.460.32.03:/usr/lib/x86_64-linux-gnu/libnvidia-opticalflow.so.460.32.03:ro
      - /usr/lib/x86_64-linux-gnu/libnvidia-ptxjitcompiler.so:/usr/lib/x86_64-linux-gnu/libnvidia-ptxjitcompiler.so:ro
      - /usr/lib/x86_64-linux-gnu/libnvidia-ptxjitcompiler.so.1:/usr/lib/x86_64-linux-gnu/libnvidia-ptxjitcompiler.so.1:ro
      - /usr/lib/x86_64-linux-gnu/libnvidia-ptxjitcompiler.so.460.32.03:/usr/lib/x86_64-linux-gnu/libnvidia-ptxjitcompiler.so.460.32.03:ro
      - /usr/lib/x86_64-linux-gnu/libnvidia-rtcore.so.460.32.03:/usr/lib/x86_64-linux-gnu/libnvidia-rtcore.so.460.32.03:ro
      - /usr/lib/x86_64-linux-gnu/libnvidia-tls.so.460.32.03:/usr/lib/x86_64-linux-gnu/libnvidia-tls.so.460.32.03:ro
      - /usr/lib/x86_64-linux-gnu/libnvoptix.so.1:/usr/lib/x86_64-linux-gnu/libnvoptix.so.1:ro
      - /usr/lib/x86_64-linux-gnu/libnvoptix.so.460.32.03:/usr/lib/x86_64-linux-gnu/libnvoptix.so.460.32.03:ro
      - /usr/bin/nvidia-smi:/usr/bin/nvidia-smi:ro
    security_opt:
      - apparmor:unconfined # This is needed to address the apparmor issue on Ubuntu: https://github.com/moby/moby/issues/16429
    network_mode: host
    pid: host # Needed for process name to be shown when running nvidia-smi in docker
    cap_add:
      - SYS_ADMIN
      - SYS_PTRACE # Needed so that we can do ptrace when debugging code from client side
    restart: unless-stopped
    environment:
      - JAEGER_SERVICE_NAME={{ jaeger_service_name }}
      - NVIDIA_VISIBLE_DEVICES=all
    devices:
      - "/dev/nvidia-uvm:/dev/nvidia-uvm" # Needed for GPUs to be accessible
    command:
      - "-XX:MaxDirectMemorySize=64000000000"
      - com.google.devtools.build.remote.worker.RemoteWorker
      - --action_log_path={{ kraken_action_logs_dir_container }}/action_log
      - --gpu_acquire_queue_timeout_seconds={{ kraken_gpu_acquire_queue_timeout_seconds }}
      - --gpu_memory_usage_polling_ms={{ kraken_gpu_memory_usage_polling_ms }}
      - --jobs={{ gpu_max_concurrent_actions }}
      - --kraken_cluster_name=onprem-weka-kraken
      - --listen_port={{ kraken_worker_port_number }}
      - --max_permissible_worker_timeout={{ kraken_max_permissible_worker_timeout }}
      - --metrics_file_path=/home/zoox/kraken_metrics.log
      - --metrics_log_mode=selected
      - --mtime_modification_chance={{ mtime_modification_chance }}
      - --remote_filesystem_cache_root={{ weka_remote_filesystem_cache_root }}
      - --run_in_cgroup
      - --max_assignable_gpu_count={{ kraken_max_assignable_gpu_count }}
      - --total_nvenc_streams={{ kraken_gpu_worker_total_nvenc_streams }}
      - --work_path={{ kraken_worker_work_path }}
      - --worker_timeout={{ kraken_gpu_worker_execute_timeout }}
      - --worker_type=gpu
