input {
  beats {
    port => 5044
  }
}

filter {
  if [type] == "pd_job_log" {
    if [message] =~ /^[A-Z][0-9]{4}/ {
      grok {
        match => { "message" => "(?<level>^[A-Z])(?<date>[0-9]{4} [0-9]{2}\:[0-9]{2}:[0-9]{2}\.[0-9]{6})\s{1,5}(?<thread_id>[0-9].+?)\s(?<source_file>.+?)\:(?<source_line>[0-9].+?)\]\s(?<msg>.*)" }
      }

      grok {
        match => { "msg" => "(?<msg_short>^.{1,128})" }
      }

      fingerprint {
        source => "msg"
        target => "msg_sha"
      }
    }
  }
}

output {
    if [type] in ["slurmd_log", "slurmdbd_log", "slurmctl_log", "cloud_burst_py_log", "cloud_burst_mon_log", "cloud_burst_state_log", "sync_acct_log", "cloud_init_output_log", "cloud_init_log"] {
        elasticsearch {
          hosts => ["https://search-metrics-production-c65mdhdibmbmkl4pfiqsj5yhmq.us-west-2.es.amazonaws.com:443"]
          index => "filebeat-%{+YYYY.MM.dd}"
          ilm_enabled => false
          # document_id => "%{[@metadata][fingerprint]}"
        }
    } else if [realm] in ["slurm_host_logs", "cb_node_log"] {
        elasticsearch {
          hosts => ["https://search-pipedream-logs-5d5rxl2lanwrap2fnzte3mzwti.us-west-2.es.amazonaws.com:443"]
          index => "log-slurm-host-%{+YYYY.MM.dd}"
          ilm_enabled => false
          # document_id => "%{[@metadata][fingerprint]}"
        }
    } else {
        elasticsearch {
          hosts => ["https://search-pipedream-logs-5d5rxl2lanwrap2fnzte3mzwti.us-west-2.es.amazonaws.com:443"]
          index => "log-pipedream-%{+YYYY.MM.dd}"
          ilm_enabled => false
          # document_id => "%{[@metadata][fingerprint]}"
        }
    }
}

