Files
ouranos/ansible/alloy/miranda/config.alloy.j2
Robert Helewka 3c2f8c57ca feat(observability): add SearXNG, Argos, and Pallas monitoring
- Add SearXNG syslog ingestion and blackbox health probes on miranda
  and rosalind for per-host attributable failure detection
- Scrape Argos MCP application metrics from miranda
- Add Pallas dashboard panels for downstream availability and turn
  error ratios
2026-05-24 23:52:53 -04:00

171 lines
4.3 KiB
Django/Jinja

logging {
level = "{{alloy_log_level}}"
}
loki.source.file "system_logs" {
targets = [
{__path__ = "/var/log/syslog", job = "syslog"},
{__path__ = "/var/log/auth.log", job = "auth"},
]
forward_to = [loki.write.default.receiver]
}
loki.source.journal "systemd_logs" {
forward_to = [loki.relabel.journal_apps.receiver]
labels = {
job = "systemd",
hostname = "{{inventory_hostname}}",
environment = "{{deployment_environment}}",
}
}
loki.relabel "journal_apps" {
forward_to = [loki.write.default.receiver]
rule {
source_labels = ["__journal__systemd_unit"]
regex = "mcpo\\.service"
target_label = "job"
replacement = "mcpo"
}
rule {
source_labels = ["__journal__systemd_unit"]
regex = "mcpo\\.service"
target_label = "app"
replacement = "mcpo"
}
}
loki.source.syslog "argos_logs" {
listener {
address = "127.0.0.1:{{argos_syslog_port}}"
protocol = "tcp"
syslog_format = "{{ syslog_format }}"
labels = {
job = "argos",
hostname = "{{inventory_hostname}}",
environment = "{{deployment_environment}}",
}
}
forward_to = [loki.write.default.receiver]
}
loki.source.syslog "grafana_mcp_logs" {
listener {
address = "127.0.0.1:{{grafana_mcp_syslog_port}}"
protocol = "tcp"
syslog_format = "{{ syslog_format }}"
labels = {
job = "grafana_mcp",
hostname = "{{inventory_hostname}}",
environment = "{{deployment_environment}}",
}
}
forward_to = [loki.write.default.receiver]
}
loki.source.syslog "neo4j_cypher_logs" {
listener {
address = "127.0.0.1:{{neo4j_cypher_syslog_port}}"
protocol = "tcp"
syslog_format = "{{ syslog_format }}"
labels = {
job = "neo4j-cypher",
hostname = "{{inventory_hostname}}",
environment = "{{deployment_environment}}",
}
}
forward_to = [loki.write.default.receiver]
}
loki.source.syslog "gitea_mcp_logs" {
listener {
address = "127.0.0.1:{{gitea_mcp_syslog_port}}"
protocol = "tcp"
syslog_format = "{{ syslog_format }}"
labels = {
job = "gitea-mcp",
hostname = "{{inventory_hostname}}",
environment = "{{deployment_environment}}",
}
}
forward_to = [loki.write.default.receiver]
}
loki.source.syslog "searxng_logs" {
listener {
address = "127.0.0.1:{{searxng_syslog_port}}"
protocol = "tcp"
syslog_format = "{{ syslog_format }}"
labels = {
job = "searxng",
hostname = "{{inventory_hostname}}",
environment = "{{deployment_environment}}",
}
}
forward_to = [loki.write.default.receiver]
}
prometheus.exporter.unix "default" {
include_exporter_metrics = true
disable_collectors = ["mdadm"]
}
prometheus.scrape "default" {
targets = prometheus.exporter.unix.default.targets
forward_to = [prometheus.remote_write.default.receiver]
job_name = "mcp_docker_host"
}
// Argos MCP application metrics (/metrics is exposed by argos itself; see
// argos/argos_searxng/metrics.py).
prometheus.scrape "argos" {
targets = [{
__address__ = "127.0.0.1:{{argos_port}}",
job = "argos",
instance = "{{inventory_hostname}}",
hostname = "{{inventory_hostname}}",
environment = "{{deployment_environment}}",
}]
forward_to = [prometheus.remote_write.default.receiver]
scrape_interval = "30s"
metrics_path = "/metrics"
}
// Independent verification that this host's SearXNG instance answers /healthz
// (Argos's own per-instance gauge can lie — argos itself could be sick).
prometheus.exporter.blackbox "searxng" {
config = "{ modules: { http_2xx: { prober: http, timeout: 5s, http: { valid_status_codes: [200] } } } }"
target {
name = "{{inventory_hostname}}"
address = "http://127.0.0.1:{{searxng_port}}/healthz"
module = "http_2xx"
labels = {
service = "searxng",
hostname = "{{inventory_hostname}}",
environment = "{{deployment_environment}}",
}
}
}
prometheus.scrape "searxng_blackbox" {
targets = prometheus.exporter.blackbox.searxng.targets
forward_to = [prometheus.remote_write.default.receiver]
scrape_interval = "30s"
job_name = "searxng_blackbox"
}
prometheus.remote_write "default" {
endpoint {
url = "{{prometheus_remote_write_url}}"
}
}
loki.write "default" {
endpoint {
url = "{{loki_url}}"
}
}