apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  labels:
    app: teleops-prometheus
    release: teleops-prometheus
  name: teleops-prometheus-rule
  namespace: teleops-prometheus
spec:
  groups:
    - name: node_alerts
      rules:
      - alert: InstanceDown
        expr: up < 1
        for: 3m
        labels:
          severity: page
        annotations:
          summary: "Instance {{ $labels.instance }} down"
          description: "{{ $labels.instance }} of job {{ $labels.job }} has been down for more than 3 minutes."
      - alert: HighCPU
        expr: (100 - (avg by (instance) (irate(node_cpu_seconds_total{mode="idle"}[1m])) * 100)) > 80
        for: 10m
        labels:
          severity: page
        annotations:
          summary: "High CPU utilization for {{ $labels.instance }}"
          description: "CPU utilization of {{ $labels.instance }} has been >= 80% for more than 10 minutes."
      - alert: HighMemory
        expr: (sum(1.0 - ((node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) or ((node_memory_MemFree_bytes + node_memory_Buffers_bytes + node_memory_Cached_bytes) / node_memory_MemTotal_bytes))) by (instance)) > 0.8
        for: 30m
        labels:
          severity: page
        annotations:
          summary: "High memory utilization for {{ $labels.instance }}"
          description: "Memory utilization of {{ $labels.instance }} has been >= 80% for more than 30 minutes."
      - alert: HighDisk
        expr: 1.0 - (node_filesystem_avail_bytes / node_filesystem_free_bytes) > 0.8
        for: 30m
        labels:
          severity: page
        annotations:
          summary: "High disk utilization for {{ $labels.instance }}, device {{ $labels.device }}"
          description: "Disk utilization of {{ $labels.instance }}, device {{ $labels.device }} has been >= 80% for more than 30 minutes."
      - alert: HighSwap
        expr: (((node_memory_SwapTotal_bytes - node_memory_SwapFree_bytes) / (node_memory_SwapTotal_bytes)) * 100) > 80
        for: 30m
        labels:
          severity: page
        annotations:
          summary: "High swap utilization for {{ $labels.instance }}"
          description: "Swap utilization of {{ $labels.instance }} has been >= 80% for more than 30 minutes."
