#!/usr/bin/env bash
#
# monitor_bamboo.sh: This script runs under cron on Linux and inspects syslog for Bamboo process
#                    status. If a job has a status of "execution started" and the timestamp on
#                    that process is older than a hardcoded time limit threshold, then the
#                    Bamboo agent is restarted on this system, which terminates potentially
#                    hanging jobs
#
# @copyright Highly Confidential and Proprietary Zoox Information. Copyright (c) Zoox
#
# Author: trawat
# Date Created: March 24, 2020
#

# Create directory for logs if it doesn't already exist
mkdir -p logs

# When set to 1, no action is taken for stopping processes but the metrics are logged to a file
# When set to 0, the normal action for stopping processes is taken
shadow_mode=0

# Variable definitions
job_wait_time_minutes=15  # [minutes]
channel="#fw-tester-health"
username="fwval-mon"
icon_emoji=":pr-slight-red:"
hostname=`hostname`
minutes_in_hour=60  # [minutes] in an hour

# Check that this script is running on linux only
platform=`uname | tr '[:upper:]' '[:lower:]'`

if [ "$platform" != "linux" ]; then
    echo "[ERROR] Script is for Linux operating systems only (detected: '$platform')... aborting"
    exit 1
fi

# Get the status line from /var/log/syslog for the most recently active Bamboo job status
status_line=`sudo cat /var/log/syslog | grep "execution started\|execution finished" | tail -1`

# If the status line is empty, that means that no Bamboo job has started OR finished.
# Therefore, no action is required
if [ "$status_line" = "" ]; then
    exit 0
fi

# If the status_line does have a non-empty string, check if it is reporting a job as started
# If this is the case, then $job_state should not be empty. If it is empty, then the status_line
# does not contain started and we can exit here without taking any action
job_state=`echo $status_line | grep "started"`
if [ "$job_state" = "" ]; then
    exit 0
fi

# At this point, we know that a Bamboo job has started and not finished. We need to check the
# timestamp and evaluate that against the current date and see if more than 15 minutes have
# elapsed
# If it has not yet been 15 minutes, again take no action
status_month=`echo $job_state | awk '{print $1}'`
status_day=`echo $job_state | awk '{print $2}'`
status_time=`echo $job_state | awk '{print $3}'`

# Get current date as epoch string
right_now=`date +%s`

# Get Bamboo job date as epoch string
right_then=`date +%s -d "$status_month $status_day $status_time"`

# We may already have taken action on this exact job. See if anything was written
# to the temp cache and if there is a value there that matches then take no action
seen_before=`cat /tmp/monitor_bamboo.sentinel 2> /dev/null`
if [ "$seen_before" = "$right_then" ]; then
    exit 0
fi

# Calculate time elapsed (this returns an integer rather than a float and can be marginally off)
minutes_passed=`echo "($right_now - $right_then) / $minutes_in_hour" | bc`

# Get the unique job identifier
job_id=`echo "$job_state" | awk '{print $(NF-2)}'`

d8=`date +%H%M`  # used to facilitate "roll over" of logs by reusing the filename
echo "Time Elapsed: ${minutes_passed}, Time Limit: ${job_wait_time_minutes}, ID: ${job_id}" >> logs/bamboo_${d8}.log

# Check if threshold has been exceeded
if (( $(echo "$minutes_passed > $job_wait_time_minutes" |bc -l) )); then
    if [ $shadow_mode -eq 0 ]; then
        # Restart Bamboo agent
        sudo -u svc-citeststation /home/svc-citeststation/bamboo-agent-home/bin/bamboo-agent.sh restart
        # Notify Slack channel #fw-tester-health
        courtesy_msg="Bamboo job has exceeded allowable runtime envelope"
        threshold_exceeded_msg="${minutes_passed} minutes > ${job_wait_time_minutes} minutes"
        process_stopped_msg="Stopping: ${job_id} and restarting Bamboo agent"
        text="On $hostname\n$courtesy_msg\n$threshold_exceeded_msg\n$process_stopped_msg"
        # Apparently in bash you can't indent lines following a line continuation character \ ???
        # for curl who knew...
        curl -X POST \
-H 'Content-Type: application/json; charset=utf-8' \
--data "{\"channel\":\"$channel\", \"icon_emoji\":\"$icon_emoji\", \"username\":\"$username\",
\"text\":\"$text\"}" \
https://hooks.slack.com/services/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
        logger "$courtesy_msg"
        logger "$process_stopped_msg"
    fi
    # Also dump into syslog
    logger "$threshold_exceeded_msg"
    echo $right_then > /tmp/monitor_bamboo.sentinel
fi
