feat(observability): add SearXNG, Argos, and Pallas monitoring

- Add SearXNG syslog ingestion and blackbox health probes on miranda
  and rosalind for per-host attributable failure detection
- Scrape Argos MCP application metrics from miranda
- Add Pallas dashboard panels for downstream availability and turn
  error ratios
This commit is contained in:
2026-05-24 23:52:53 -04:00
parent 43fae203d1
commit 3c2f8c57ca
24 changed files with 1968 additions and 938 deletions

View File

@@ -190,6 +190,31 @@ prometheus.scrape "gitea" {
bearer_token = "{{gitea_metrics_token}}"
}
// Independent verification that this host's SearXNG instance answers /healthz.
// Argos (on miranda) load-balances across this instance and miranda's own;
// each host's Alloy probes its local SearXNG so failures are attributable.
prometheus.exporter.blackbox "searxng" {
config = "{ modules: { http_2xx: { prober: http, timeout: 5s, http: { valid_status_codes: [200] } } } }"
target {
name = "{{inventory_hostname}}"
address = "http://127.0.0.1:{{searxng_port}}/healthz"
module = "http_2xx"
labels = {
service = "searxng",
hostname = "{{inventory_hostname}}",
environment = "{{deployment_environment}}",
}
}
}
prometheus.scrape "searxng_blackbox" {
targets = prometheus.exporter.blackbox.searxng.targets
forward_to = [prometheus.remote_write.default.receiver]
scrape_interval = "30s"
job_name = "searxng_blackbox"
}
// Prometheus remote write endpoint
prometheus.remote_write "default" {
endpoint {