apiVersion: v1
kind: ConfigMap
metadata:
  namespace: monitoring
  name: prometheus-rules
data:
  kraken.rules.yaml: |
    groups:
    - name: kraken
      rules:
      - expr: |
          sum(irate(node_cpu_seconds_total{job=~"kraken-.*-workers", mode!="idle"}[5m])) by (job, instance)
        record: instance:node_cpu_non_idle_seconds_total:sum_rate
      - expr: |
          count(node_cpu_frequency_hertz{job=~"kraken-.*-workers"}) by (job, instance)
        record: instance:node_cpu_frequency_hertz:count
      - expr: |
          histogram_quantile(
            0.95,
            rate(grpc_server_handled_latency_seconds_bucket{job=~"kraken-.*-workers", grpc_method="Execute"}[5m])
          )
        record: instance:grpc_server_Execute_handled_latency_seconds:quantile95
      - alert: ExecuteLatencyOutlier
        expr: |
          (
            topk by (job)(
              1,
              instance:grpc_server_Execute_handled_latency_seconds:quantile95
            )
              / on(job) group_left()
            avg by (job)(
              instance:grpc_server_Execute_handled_latency_seconds:quantile95
            )
          ) > 1.5
        for: 10m
      - alert: LoadAvgOutlier
        expr: |
          (topk(1, node_load1{job=~"kraken-.*-workers"}) by (job) > 100)
            and
          (
            (
              topk(1, node_load1{job=~"kraken-.*-workers"}) by (job)
                / on (job) group_left
              quantile(0.5, node_load1{job=~"kraken-.*-workers"}) by (job)
            ) > 2
          )
        for: 5m
