#! /bin/bash

# GPU Stress Transient Suite for C3.6 (Main AI + Vision)
#
# These waveforms are selected based on observed waveforms on this generation of
# GPU. NVQual runs most tests on a 100ms cycle. Zoox Driving SW runs primarily
# on a 50ms (50hz) cycle.
#
# These tests pass if the PCU node is still powered on and communication with
# GPUs is funcitonal (nvidia-smi will error if any nodes are lost). The matrix
# computation correctness is not currently checked.
#
# Test time: approx 1 hr.

# config
GPU_STRESS_BIN="./bazel-bin/vehicle/tools/gpu_stress/gpu_stress"  # relative to bcd
GPU_STRESS_OPTIONS="--alsologtostderr --gpus 0,1,2,3 --max_clocks --test_time_s 900 --report_reps 10"

# compile
bazel build //vehicle/tools/gpu_stress/... --config=cuda

# run - assumes cd is repo top dir
nvidia-smi
sudo bash <<EOF # here-file-token not quoted so $vars are expanded below
set -o xtrace # show running commands
bcd
killall nvidia-cuda-mps-server  # old mps server seems to avoid our loads
$GPU_STRESS_BIN $GPU_STRESS_OPTIONS --dim 8192 --transient_period_ms 300 # close to nvqual
$GPU_STRESS_BIN $GPU_STRESS_OPTIONS --dim 6144 --transient_period_ms 100 # lower duty cycle to stay under thermal cap
$GPU_STRESS_BIN $GPU_STRESS_OPTIONS --dim 4096 --transient_period_ms 50 # closer to camera frames
$GPU_STRESS_BIN $GPU_STRESS_OPTIONS --dim 2048 --transient_period_ms 20 # faster transients
EOF

# and check
nvidia-smi
