feat(observability): add SearXNG, Argos, and Pallas monitoring
- Add SearXNG syslog ingestion and blackbox health probes on miranda and rosalind for per-host attributable failure detection - Scrape Argos MCP application metrics from miranda - Add Pallas dashboard panels for downstream availability and turn error ratios
This commit is contained in:
202
dashboards/searxng.json
Normal file
202
dashboards/searxng.json
Normal file
@@ -0,0 +1,202 @@
|
||||
{
|
||||
"title": "SearXNG",
|
||||
"uid": "searxng",
|
||||
"tags": ["searxng", "argos", "ouranos"],
|
||||
"timezone": "browser",
|
||||
"schemaVersion": 39,
|
||||
"version": 1,
|
||||
"editable": true,
|
||||
"fiscalYearStartMonth": 0,
|
||||
"weekStart": "",
|
||||
"refresh": "30s",
|
||||
"time": {"from": "now-1h", "to": "now"},
|
||||
"links": [
|
||||
{
|
||||
"asDropdown": false,
|
||||
"icon": "external link",
|
||||
"includeVars": true,
|
||||
"keepTime": true,
|
||||
"tags": [],
|
||||
"targetBlank": true,
|
||||
"title": "Argos dashboard",
|
||||
"tooltip": "Argos MCP server using these SearXNG instances",
|
||||
"type": "link",
|
||||
"url": "/d/argos"
|
||||
},
|
||||
{
|
||||
"asDropdown": false,
|
||||
"icon": "doc",
|
||||
"includeVars": true,
|
||||
"keepTime": true,
|
||||
"tags": [],
|
||||
"targetBlank": true,
|
||||
"title": "SearXNG logs",
|
||||
"tooltip": "Loki: {job=\"searxng\"}",
|
||||
"type": "link",
|
||||
"url": "/explore?orgId=1&left=%7B%22datasource%22:%22Loki%22,%22queries%22:%5B%7B%22refId%22:%22A%22,%22expr%22:%22%7Bjob%3D%5C%22searxng%5C%22%7D%22%7D%5D%7D"
|
||||
}
|
||||
],
|
||||
"templating": {
|
||||
"list": [
|
||||
{
|
||||
"name": "prom",
|
||||
"type": "datasource",
|
||||
"query": "prometheus",
|
||||
"current": {"selected": false, "text": "Prometheus", "value": "Prometheus"},
|
||||
"hide": 0,
|
||||
"label": "Prometheus datasource"
|
||||
},
|
||||
{
|
||||
"name": "loki",
|
||||
"type": "datasource",
|
||||
"query": "loki",
|
||||
"current": {"selected": false, "text": "Loki", "value": "Loki"},
|
||||
"hide": 0,
|
||||
"label": "Loki datasource"
|
||||
},
|
||||
{
|
||||
"name": "host",
|
||||
"type": "query",
|
||||
"datasource": {"type": "prometheus", "uid": "${prom}"},
|
||||
"query": "label_values(probe_success{service=\"searxng\"}, hostname)",
|
||||
"refresh": 1,
|
||||
"includeAll": true,
|
||||
"multi": true,
|
||||
"current": {"selected": true, "text": "All", "value": "$__all"},
|
||||
"label": "SearXNG host"
|
||||
}
|
||||
]
|
||||
},
|
||||
"panels": [
|
||||
{
|
||||
"id": 1,
|
||||
"type": "row",
|
||||
"title": "Independent probe (Alloy blackbox /healthz)",
|
||||
"collapsed": false,
|
||||
"gridPos": {"h": 1, "w": 24, "x": 0, "y": 0}
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"type": "stat",
|
||||
"title": "SearXNG /healthz",
|
||||
"datasource": {"type": "prometheus", "uid": "${prom}"},
|
||||
"gridPos": {"h": 5, "w": 8, "x": 0, "y": 1},
|
||||
"targets": [
|
||||
{"refId": "A", "expr": "probe_success{service=\"searxng\", hostname=~\"$host\"}", "legendFormat": "{{hostname}}"}
|
||||
],
|
||||
"options": {"reduceOptions": {"calcs": ["lastNotNull"]}, "colorMode": "background", "textMode": "value_and_name"},
|
||||
"fieldConfig": {"defaults": {"mappings": [{"type": "value", "options": {"0": {"text": "DOWN", "color": "red"}, "1": {"text": "UP", "color": "green"}}}], "thresholds": {"mode": "absolute", "steps": [{"color": "red"}, {"color": "green", "value": 1}]}}}
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"type": "stat",
|
||||
"title": "Last probe HTTP status",
|
||||
"datasource": {"type": "prometheus", "uid": "${prom}"},
|
||||
"gridPos": {"h": 5, "w": 8, "x": 8, "y": 1},
|
||||
"targets": [
|
||||
{"refId": "A", "expr": "probe_http_status_code{service=\"searxng\", hostname=~\"$host\"}", "legendFormat": "{{hostname}}"}
|
||||
],
|
||||
"options": {"reduceOptions": {"calcs": ["lastNotNull"]}, "colorMode": "value", "textMode": "value_and_name"},
|
||||
"fieldConfig": {"defaults": {"unit": "short", "thresholds": {"mode": "absolute", "steps": [{"color": "red"}, {"color": "green", "value": 200}, {"color": "orange", "value": 300}, {"color": "red", "value": 400}]}}}
|
||||
},
|
||||
{
|
||||
"id": 4,
|
||||
"type": "stat",
|
||||
"title": "Probe duration (last)",
|
||||
"datasource": {"type": "prometheus", "uid": "${prom}"},
|
||||
"gridPos": {"h": 5, "w": 8, "x": 16, "y": 1},
|
||||
"targets": [
|
||||
{"refId": "A", "expr": "probe_duration_seconds{service=\"searxng\", hostname=~\"$host\"}", "legendFormat": "{{hostname}}"}
|
||||
],
|
||||
"options": {"reduceOptions": {"calcs": ["lastNotNull"]}, "colorMode": "value", "textMode": "value_and_name", "graphMode": "area"},
|
||||
"fieldConfig": {"defaults": {"unit": "s", "thresholds": {"mode": "absolute", "steps": [{"color": "green"}, {"color": "orange", "value": 1}, {"color": "red", "value": 3}]}}}
|
||||
},
|
||||
{
|
||||
"id": 5,
|
||||
"type": "timeseries",
|
||||
"title": "Probe success over time",
|
||||
"datasource": {"type": "prometheus", "uid": "${prom}"},
|
||||
"gridPos": {"h": 8, "w": 12, "x": 0, "y": 6},
|
||||
"targets": [
|
||||
{"refId": "A", "expr": "probe_success{service=\"searxng\", hostname=~\"$host\"}", "legendFormat": "{{hostname}}"}
|
||||
],
|
||||
"fieldConfig": {"defaults": {"unit": "short", "min": 0, "max": 1, "custom": {"drawStyle": "line", "lineWidth": 2, "fillOpacity": 20}}},
|
||||
"options": {"legend": {"displayMode": "list", "placement": "bottom", "showLegend": true}}
|
||||
},
|
||||
{
|
||||
"id": 6,
|
||||
"type": "timeseries",
|
||||
"title": "Probe duration over time",
|
||||
"datasource": {"type": "prometheus", "uid": "${prom}"},
|
||||
"gridPos": {"h": 8, "w": 12, "x": 12, "y": 6},
|
||||
"targets": [
|
||||
{"refId": "A", "expr": "probe_duration_seconds{service=\"searxng\", hostname=~\"$host\"}", "legendFormat": "{{hostname}}"}
|
||||
],
|
||||
"fieldConfig": {"defaults": {"unit": "s"}},
|
||||
"options": {"legend": {"displayMode": "list", "placement": "bottom", "showLegend": true}}
|
||||
},
|
||||
|
||||
{
|
||||
"id": 10,
|
||||
"type": "row",
|
||||
"title": "Argos's view of these instances",
|
||||
"collapsed": false,
|
||||
"gridPos": {"h": 1, "w": 24, "x": 0, "y": 14}
|
||||
},
|
||||
{
|
||||
"id": 11,
|
||||
"type": "timeseries",
|
||||
"title": "argos_searxng_instance_up by SearXNG instance",
|
||||
"datasource": {"type": "prometheus", "uid": "${prom}"},
|
||||
"gridPos": {"h": 8, "w": 12, "x": 0, "y": 15},
|
||||
"targets": [
|
||||
{"refId": "A", "expr": "argos_searxng_instance_up", "legendFormat": "{{searxng_instance}}"}
|
||||
],
|
||||
"fieldConfig": {"defaults": {"unit": "short", "min": 0, "max": 1, "custom": {"drawStyle": "line", "lineWidth": 2, "fillOpacity": 20}}},
|
||||
"options": {"legend": {"displayMode": "list", "placement": "bottom", "showLegend": true}}
|
||||
},
|
||||
{
|
||||
"id": 12,
|
||||
"type": "timeseries",
|
||||
"title": "Search latency p95 from Argos (5m)",
|
||||
"datasource": {"type": "prometheus", "uid": "${prom}"},
|
||||
"gridPos": {"h": 8, "w": 12, "x": 12, "y": 15},
|
||||
"targets": [
|
||||
{"refId": "A", "expr": "histogram_quantile(0.95, sum by (le, searxng_instance) (rate(argos_searxng_request_duration_seconds_bucket[5m])))", "legendFormat": "{{searxng_instance}} p95"}
|
||||
],
|
||||
"fieldConfig": {"defaults": {"unit": "s"}},
|
||||
"options": {"legend": {"displayMode": "list", "placement": "bottom", "showLegend": true}}
|
||||
},
|
||||
{
|
||||
"id": 13,
|
||||
"type": "timeseries",
|
||||
"title": "Search request error ratio from Argos (5m)",
|
||||
"datasource": {"type": "prometheus", "uid": "${prom}"},
|
||||
"gridPos": {"h": 8, "w": 24, "x": 0, "y": 23},
|
||||
"targets": [
|
||||
{"refId": "A", "expr": "sum by (searxng_instance) (rate(argos_searxng_requests_total{status=\"error\"}[5m])) / clamp_min(sum by (searxng_instance) (rate(argos_searxng_requests_total[5m])), 0.0001)", "legendFormat": "{{searxng_instance}}"}
|
||||
],
|
||||
"fieldConfig": {"defaults": {"unit": "percentunit"}},
|
||||
"options": {"legend": {"displayMode": "list", "placement": "bottom", "showLegend": true}}
|
||||
},
|
||||
|
||||
{
|
||||
"id": 90,
|
||||
"type": "row",
|
||||
"title": "Logs",
|
||||
"collapsed": false,
|
||||
"gridPos": {"h": 1, "w": 24, "x": 0, "y": 31}
|
||||
},
|
||||
{
|
||||
"id": 91,
|
||||
"type": "logs",
|
||||
"title": "searxng (Loki)",
|
||||
"datasource": {"type": "loki", "uid": "${loki}"},
|
||||
"gridPos": {"h": 12, "w": 24, "x": 0, "y": 32},
|
||||
"targets": [
|
||||
{"refId": "A", "expr": "{job=\"searxng\"}"}
|
||||
],
|
||||
"options": {"showTime": true, "wrapLogMessage": true, "enableLogDetails": true, "dedupStrategy": "none"}
|
||||
}
|
||||
]
|
||||
}
|
||||
Reference in New Issue
Block a user