#!/usr/bin/env bash
#
# monitor_cpu.sh: This script runs under cron on Linux and assesses the processes most heavily
#                 utilizing the CPU. If the highest-use process is exceeding the hardcoded
#                 threshold then that process is killed and the process name is reported
#                 out to Slack for recording purposes
#
# @copyright Highly Confidential and Proprietary Zoox Information. Copyright (c) Zoox
#
# Author: trawat
# Date Created: March 24, 2020
#

# Create directory for logs if it doesn't already exist
mkdir -p logs

# When set to 1, no action is taken for stopping processes but the metrics are logged to a file
# When set to 0, the normal action for stopping processes is taken
shadow_mode=0

# Variable definitions
cpu_max_threshold=360.0  # for a "4-thread" job w/ 90% CPU utilization per thread
channel="#fw-tester-health"
username="fwval-mon"
icon_emoji=":skull_and_crossbones:"
hostname=`hostname`

# Check that this script is running on linux only
platform=`uname | tr '[:upper:]' '[:lower:]'`

if [ "$platform" != "linux" ]; then
    echo "[ERROR] Script is for Linux operating systems only (detected: '$platform')... aborting"
    exit 1
fi

# Get the process line from ps for the process with highest CPU percentage
process_line=`ps aux --sort=-%cpu | head -2 | tail -1`

# Get the process ID, percent CPU and command being run
# E.g.
#
# USER    PID %CPU %MEM    VSZ   RSS TTY      STAT START   TIME COMMAND
# robot 19229 14.0  0.3 230536 52092 ?        Sl   22:40   0:01 /usr/bin/python <--- get this line
#
process_started_by=`echo "$process_line" | awk '{print $1}'`  # USER
process_id=`echo "$process_line" | awk '{print $2}'`  # PID
process_cpu=`echo "$process_line" | awk '{print $3}'`  # %CPU
process_command=`echo "$process_line" | awk '{print $11}'`  # COMMAND

# If shadow mode is on we want to log to a file
d8=`date +%H%M`  # used to facilitate "roll over" of logs by reusing the filename
echo "%CPU: ${process_cpu}, Limit: ${cpu_max_threshold}, Proc: ${process_command}" >> logs/cpu_${d8}.log

# Check if threshold has been exceeded
if (( $(echo "$process_cpu > $cpu_max_threshold" |bc -l) )); then
    # Kill that process
    if [ $shadow_mode -eq 0 ]; then
        sudo kill -9 $process_id
        # Notify Slack channel #fw-tester-health
        threshold_exceeded_msg="${process_cpu}% CPU exceeds threshold of ${cpu_max_threshold}%"
        process_stopped_msg="Stopping process w/ ID: ${process_id}"
        command_killed_msg="Killed job: ${process_command}, started by ${process_started_by}"
        text="On $hostname\n$threshold_exceeded_msg\n$process_stopped_msg\n$command_killed_msg"
        # Apparently in bash you can't indent lines following a line continuation character \ ???
        # who knew...
        curl -X POST \
-H 'Content-Type: application/json; charset=utf-8' \
--data "{\"channel\":\"$channel\", \"icon_emoji\":\"$icon_emoji\", \"username\":\"$username\",
\"text\":\"$text\"}" \
https://hooks.slack.com/services/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
        logger "$process_stopped_msg"
        logger "$command_killed_msg"
    fi
    # Also dump into syslog
    logger "$threshold_exceeded_msg"
fi
