ouranos/ansible/alloy/prospero/config.alloy.j2

// Prospero Alloy Configuration
// Red Panda Approved 🐼
// Services: PPLG stack (Grafana, Prometheus, Loki, Alertmanager, PgAdmin, OAuth2-Proxy)

logging {
  level = "{{alloy_log_level}}"
}

// ============================================================================
// LOG COLLECTION - Loki Forwarding
// ============================================================================

// System log files
loki.source.file "system_logs" {
  targets = [
    {__path__ = "/var/log/syslog", job = "syslog"},
    {__path__ = "/var/log/auth.log", job = "auth"},
  ]
  forward_to = [loki.write.default.receiver]
}

// Journal relabeling - assign dedicated job labels per systemd unit
loki.relabel "journal" {
  forward_to = []

  // Expose the systemd unit as a label
  rule {
    source_labels = ["__journal__systemd_unit"]
    target_label  = "unit"
  }

  // Grafana
  rule {
    source_labels = ["__journal__systemd_unit"]
    regex         = "grafana-server\\.service"
    target_label  = "job"
    replacement   = "grafana"
  }

  // Prometheus
  rule {
    source_labels = ["__journal__systemd_unit"]
    regex         = "prometheus\\.service"
    target_label  = "job"
    replacement   = "prometheus"
  }

  // Loki
  rule {
    source_labels = ["__journal__systemd_unit"]
    regex         = "loki\\.service"
    target_label  = "job"
    replacement   = "loki"
  }

  // Alertmanager
  rule {
    source_labels = ["__journal__systemd_unit"]
    regex         = "alertmanager\\.service"
    target_label  = "job"
    replacement   = "alertmanager"
  }

  // PgAdmin
  rule {
    source_labels = ["__journal__systemd_unit"]
    regex         = "pgadmin\\.service"
    target_label  = "job"
    replacement   = "pgadmin"
  }

  // OAuth2-Proxy (Prometheus UI)
  rule {
    source_labels = ["__journal__systemd_unit"]
    regex         = "oauth2-proxy-prometheus\\.service"
    target_label  = "job"
    replacement   = "oauth2-proxy-prometheus"
  }

  // Alloy
  rule {
    source_labels = ["__journal__systemd_unit"]
    regex         = "alloy\\.service"
    target_label  = "job"
    replacement   = "alloy"
  }

  // Default job for unmatched units
  rule {
    source_labels = ["__journal__systemd_unit"]
    regex         = ".+"
    target_label  = "job"
    replacement   = "systemd"
  }
}

// Systemd journal logs with per-service job labels
loki.source.journal "systemd_logs" {
  forward_to    = [loki.write.default.receiver]
  relabel_rules = loki.relabel.journal.rules
  labels = {
    hostname    = "{{inventory_hostname}}",
    environment = "{{deployment_environment}}",
  }
}

// Loki endpoint
loki.write "default" {
  endpoint {
    url = "{{loki_url}}"
  }
}

// ============================================================================
// METRICS COLLECTION - Prometheus Remote Write
// ============================================================================

// Unix/Node metrics - Incus-safe collectors only
// Disabled collectors that don't work in containers: hwmon, thermal, mdadm, powersupplyclass, nvme
prometheus.exporter.unix "default" {
  include_exporter_metrics = true
  disable_collectors = [
    "arp",
    "bcache",
    "bonding",
    "btrfs",
    "hwmon",
    "infiniband",
    "ipvs",
    "mdadm",
    "nfs",
    "nfsd",
    "nvme",
    "powersupplyclass",
    "rapl",
    "thermal_zone",
    "zfs",
  ]
}

// Process exporter - Track all processes by command name
// Provides: namedprocess_namegroup_* metrics
prometheus.exporter.process "default" {
  track_children = true
  track_threads = true
  gather_smaps = false
  recheck_on_scrape = true

  matcher {
    name = "{% raw %}{{.Comm}}{% endraw %}"
    cmdline = [".+"]
  }
}

// Scrape local exporters
prometheus.scrape "local_exporters" {
  targets = concat(
    prometheus.exporter.unix.default.targets,
    prometheus.exporter.process.default.targets,
  )
  forward_to = [prometheus.relabel.add_instance.receiver]
  scrape_interval = "15s"
  job_name = "prospero"
}

// Add instance label for Prometheus compatibility
prometheus.relabel "add_instance" {
  forward_to = [prometheus.remote_write.default.receiver]

  rule {
    target_label = "instance"
    replacement = "{{inventory_hostname}}"
  }
}

// Remote write to Prospero Prometheus
prometheus.remote_write "default" {
  endpoint {
    url = "{{prometheus_remote_write_url}}"
  }
}