- Add SearXNG syslog ingestion and blackbox health probes on miranda and rosalind for per-host attributable failure detection - Scrape Argos MCP application metrics from miranda - Add Pallas dashboard panels for downstream availability and turn error ratios
171 lines
4.3 KiB
Django/Jinja
171 lines
4.3 KiB
Django/Jinja
logging {
|
|
level = "{{alloy_log_level}}"
|
|
}
|
|
|
|
loki.source.file "system_logs" {
|
|
targets = [
|
|
{__path__ = "/var/log/syslog", job = "syslog"},
|
|
{__path__ = "/var/log/auth.log", job = "auth"},
|
|
]
|
|
forward_to = [loki.write.default.receiver]
|
|
}
|
|
|
|
loki.source.journal "systemd_logs" {
|
|
forward_to = [loki.relabel.journal_apps.receiver]
|
|
labels = {
|
|
job = "systemd",
|
|
hostname = "{{inventory_hostname}}",
|
|
environment = "{{deployment_environment}}",
|
|
}
|
|
}
|
|
|
|
loki.relabel "journal_apps" {
|
|
forward_to = [loki.write.default.receiver]
|
|
|
|
rule {
|
|
source_labels = ["__journal__systemd_unit"]
|
|
regex = "mcpo\\.service"
|
|
target_label = "job"
|
|
replacement = "mcpo"
|
|
}
|
|
|
|
rule {
|
|
source_labels = ["__journal__systemd_unit"]
|
|
regex = "mcpo\\.service"
|
|
target_label = "app"
|
|
replacement = "mcpo"
|
|
}
|
|
}
|
|
|
|
loki.source.syslog "argos_logs" {
|
|
listener {
|
|
address = "127.0.0.1:{{argos_syslog_port}}"
|
|
protocol = "tcp"
|
|
syslog_format = "{{ syslog_format }}"
|
|
labels = {
|
|
job = "argos",
|
|
hostname = "{{inventory_hostname}}",
|
|
environment = "{{deployment_environment}}",
|
|
}
|
|
}
|
|
forward_to = [loki.write.default.receiver]
|
|
}
|
|
|
|
loki.source.syslog "grafana_mcp_logs" {
|
|
listener {
|
|
address = "127.0.0.1:{{grafana_mcp_syslog_port}}"
|
|
protocol = "tcp"
|
|
syslog_format = "{{ syslog_format }}"
|
|
labels = {
|
|
job = "grafana_mcp",
|
|
hostname = "{{inventory_hostname}}",
|
|
environment = "{{deployment_environment}}",
|
|
}
|
|
}
|
|
forward_to = [loki.write.default.receiver]
|
|
}
|
|
|
|
loki.source.syslog "neo4j_cypher_logs" {
|
|
listener {
|
|
address = "127.0.0.1:{{neo4j_cypher_syslog_port}}"
|
|
protocol = "tcp"
|
|
syslog_format = "{{ syslog_format }}"
|
|
labels = {
|
|
job = "neo4j-cypher",
|
|
hostname = "{{inventory_hostname}}",
|
|
environment = "{{deployment_environment}}",
|
|
}
|
|
}
|
|
forward_to = [loki.write.default.receiver]
|
|
}
|
|
|
|
loki.source.syslog "gitea_mcp_logs" {
|
|
listener {
|
|
address = "127.0.0.1:{{gitea_mcp_syslog_port}}"
|
|
protocol = "tcp"
|
|
syslog_format = "{{ syslog_format }}"
|
|
labels = {
|
|
job = "gitea-mcp",
|
|
hostname = "{{inventory_hostname}}",
|
|
environment = "{{deployment_environment}}",
|
|
}
|
|
}
|
|
forward_to = [loki.write.default.receiver]
|
|
}
|
|
|
|
loki.source.syslog "searxng_logs" {
|
|
listener {
|
|
address = "127.0.0.1:{{searxng_syslog_port}}"
|
|
protocol = "tcp"
|
|
syslog_format = "{{ syslog_format }}"
|
|
labels = {
|
|
job = "searxng",
|
|
hostname = "{{inventory_hostname}}",
|
|
environment = "{{deployment_environment}}",
|
|
}
|
|
}
|
|
forward_to = [loki.write.default.receiver]
|
|
}
|
|
|
|
prometheus.exporter.unix "default" {
|
|
include_exporter_metrics = true
|
|
disable_collectors = ["mdadm"]
|
|
}
|
|
|
|
prometheus.scrape "default" {
|
|
targets = prometheus.exporter.unix.default.targets
|
|
forward_to = [prometheus.remote_write.default.receiver]
|
|
job_name = "mcp_docker_host"
|
|
}
|
|
|
|
// Argos MCP application metrics (/metrics is exposed by argos itself; see
|
|
// argos/argos_searxng/metrics.py).
|
|
prometheus.scrape "argos" {
|
|
targets = [{
|
|
__address__ = "127.0.0.1:{{argos_port}}",
|
|
job = "argos",
|
|
instance = "{{inventory_hostname}}",
|
|
hostname = "{{inventory_hostname}}",
|
|
environment = "{{deployment_environment}}",
|
|
}]
|
|
forward_to = [prometheus.remote_write.default.receiver]
|
|
scrape_interval = "30s"
|
|
metrics_path = "/metrics"
|
|
}
|
|
|
|
// Independent verification that this host's SearXNG instance answers /healthz
|
|
// (Argos's own per-instance gauge can lie — argos itself could be sick).
|
|
prometheus.exporter.blackbox "searxng" {
|
|
config = "{ modules: { http_2xx: { prober: http, timeout: 5s, http: { valid_status_codes: [200] } } } }"
|
|
|
|
target {
|
|
name = "{{inventory_hostname}}"
|
|
address = "http://127.0.0.1:{{searxng_port}}/healthz"
|
|
module = "http_2xx"
|
|
labels = {
|
|
service = "searxng",
|
|
hostname = "{{inventory_hostname}}",
|
|
environment = "{{deployment_environment}}",
|
|
}
|
|
}
|
|
}
|
|
|
|
prometheus.scrape "searxng_blackbox" {
|
|
targets = prometheus.exporter.blackbox.searxng.targets
|
|
forward_to = [prometheus.remote_write.default.receiver]
|
|
scrape_interval = "30s"
|
|
job_name = "searxng_blackbox"
|
|
}
|
|
|
|
prometheus.remote_write "default" {
|
|
endpoint {
|
|
url = "{{prometheus_remote_write_url}}"
|
|
}
|
|
}
|
|
|
|
loki.write "default" {
|
|
endpoint {
|
|
url = "{{loki_url}}"
|
|
}
|
|
}
|