# !/bin/bash
# The script is meant to be run on both Main AI and Vision. (For vision, cgconfig can be commented out)
# Remember to pull temperature logs at the end from both Main AI and Vision.

# Ctrl-C Handler
function ctrl_c() {
    echo "Trapped CTRL-C: END TESTING"
    # end CPU stress test
    kill %4;
    # end all GPU stress tests
    kill %5;
    kill %6;
    kill %7;
    kill %8;
    # end temperature logging
    kill %3;
    # restore cgconfig
    sudo cp /etc/cgconfig.bak /etc/cgconfig.conf
    sudo service cgconfig restart
}

# Allow general tasks to run on all cores
sudo cp /etc/cgconfig.conf /etc/cgconfig.bak
sudo sed -i '/general cgroup/ { n; n; n; n; n; s/.*cpuset.cpus.*/    cpuset.cpus="0-71";/; }' /etc/cgconfig.conf
sudo service cgconfig restart
# Build the GPU Stress Script
bazel build -c opt --config=cuda experimental/gma/paulius/gpu_power_test_sgemm:gpu_power_test_sgemm
# Open monitors for CPU/GPU temperature
xterm -hold -e watch sensors &
xterm -hold -e watch nvidia-smi &
# Log temperature, run CPU stress and GPU stress
trap ctrl_c INT
./experimental/gma/compute_stress_test/temp_logging.sh 7200 &
stress-ng --cpu 72 --cpu-method all --verify -t 120m --metrics-brief &
./bazel-bin/experimental/gma/paulius/gpu_power_test_sgemm/gpu_power_test_sgemm --dim 4096 --magnitude 10 --num_reps 1000000 --gpu_id 0 &
./bazel-bin/experimental/gma/paulius/gpu_power_test_sgemm/gpu_power_test_sgemm --dim 4096 --magnitude 10 --num_reps 1000000 --gpu_id 1 &
./bazel-bin/experimental/gma/paulius/gpu_power_test_sgemm/gpu_power_test_sgemm --dim 4096 --magnitude 10 --num_reps 1000000 --gpu_id 2 &
./bazel-bin/experimental/gma/paulius/gpu_power_test_sgemm/gpu_power_test_sgemm --dim 4096 --magnitude 10 --num_reps 1000000 --gpu_id 3
