- Add SearXNG syslog ingestion and blackbox health probes on miranda and rosalind for per-host attributable failure detection - Scrape Argos MCP application metrics from miranda - Add Pallas dashboard panels for downstream availability and turn error ratios
203 lines
7.7 KiB
JSON
203 lines
7.7 KiB
JSON
{
|
|
"title": "SearXNG",
|
|
"uid": "searxng",
|
|
"tags": ["searxng", "argos", "ouranos"],
|
|
"timezone": "browser",
|
|
"schemaVersion": 39,
|
|
"version": 1,
|
|
"editable": true,
|
|
"fiscalYearStartMonth": 0,
|
|
"weekStart": "",
|
|
"refresh": "30s",
|
|
"time": {"from": "now-1h", "to": "now"},
|
|
"links": [
|
|
{
|
|
"asDropdown": false,
|
|
"icon": "external link",
|
|
"includeVars": true,
|
|
"keepTime": true,
|
|
"tags": [],
|
|
"targetBlank": true,
|
|
"title": "Argos dashboard",
|
|
"tooltip": "Argos MCP server using these SearXNG instances",
|
|
"type": "link",
|
|
"url": "/d/argos"
|
|
},
|
|
{
|
|
"asDropdown": false,
|
|
"icon": "doc",
|
|
"includeVars": true,
|
|
"keepTime": true,
|
|
"tags": [],
|
|
"targetBlank": true,
|
|
"title": "SearXNG logs",
|
|
"tooltip": "Loki: {job=\"searxng\"}",
|
|
"type": "link",
|
|
"url": "/explore?orgId=1&left=%7B%22datasource%22:%22Loki%22,%22queries%22:%5B%7B%22refId%22:%22A%22,%22expr%22:%22%7Bjob%3D%5C%22searxng%5C%22%7D%22%7D%5D%7D"
|
|
}
|
|
],
|
|
"templating": {
|
|
"list": [
|
|
{
|
|
"name": "prom",
|
|
"type": "datasource",
|
|
"query": "prometheus",
|
|
"current": {"selected": false, "text": "Prometheus", "value": "Prometheus"},
|
|
"hide": 0,
|
|
"label": "Prometheus datasource"
|
|
},
|
|
{
|
|
"name": "loki",
|
|
"type": "datasource",
|
|
"query": "loki",
|
|
"current": {"selected": false, "text": "Loki", "value": "Loki"},
|
|
"hide": 0,
|
|
"label": "Loki datasource"
|
|
},
|
|
{
|
|
"name": "host",
|
|
"type": "query",
|
|
"datasource": {"type": "prometheus", "uid": "${prom}"},
|
|
"query": "label_values(probe_success{service=\"searxng\"}, hostname)",
|
|
"refresh": 1,
|
|
"includeAll": true,
|
|
"multi": true,
|
|
"current": {"selected": true, "text": "All", "value": "$__all"},
|
|
"label": "SearXNG host"
|
|
}
|
|
]
|
|
},
|
|
"panels": [
|
|
{
|
|
"id": 1,
|
|
"type": "row",
|
|
"title": "Independent probe (Alloy blackbox /healthz)",
|
|
"collapsed": false,
|
|
"gridPos": {"h": 1, "w": 24, "x": 0, "y": 0}
|
|
},
|
|
{
|
|
"id": 2,
|
|
"type": "stat",
|
|
"title": "SearXNG /healthz",
|
|
"datasource": {"type": "prometheus", "uid": "${prom}"},
|
|
"gridPos": {"h": 5, "w": 8, "x": 0, "y": 1},
|
|
"targets": [
|
|
{"refId": "A", "expr": "probe_success{service=\"searxng\", hostname=~\"$host\"}", "legendFormat": "{{hostname}}"}
|
|
],
|
|
"options": {"reduceOptions": {"calcs": ["lastNotNull"]}, "colorMode": "background", "textMode": "value_and_name"},
|
|
"fieldConfig": {"defaults": {"mappings": [{"type": "value", "options": {"0": {"text": "DOWN", "color": "red"}, "1": {"text": "UP", "color": "green"}}}], "thresholds": {"mode": "absolute", "steps": [{"color": "red"}, {"color": "green", "value": 1}]}}}
|
|
},
|
|
{
|
|
"id": 3,
|
|
"type": "stat",
|
|
"title": "Last probe HTTP status",
|
|
"datasource": {"type": "prometheus", "uid": "${prom}"},
|
|
"gridPos": {"h": 5, "w": 8, "x": 8, "y": 1},
|
|
"targets": [
|
|
{"refId": "A", "expr": "probe_http_status_code{service=\"searxng\", hostname=~\"$host\"}", "legendFormat": "{{hostname}}"}
|
|
],
|
|
"options": {"reduceOptions": {"calcs": ["lastNotNull"]}, "colorMode": "value", "textMode": "value_and_name"},
|
|
"fieldConfig": {"defaults": {"unit": "short", "thresholds": {"mode": "absolute", "steps": [{"color": "red"}, {"color": "green", "value": 200}, {"color": "orange", "value": 300}, {"color": "red", "value": 400}]}}}
|
|
},
|
|
{
|
|
"id": 4,
|
|
"type": "stat",
|
|
"title": "Probe duration (last)",
|
|
"datasource": {"type": "prometheus", "uid": "${prom}"},
|
|
"gridPos": {"h": 5, "w": 8, "x": 16, "y": 1},
|
|
"targets": [
|
|
{"refId": "A", "expr": "probe_duration_seconds{service=\"searxng\", hostname=~\"$host\"}", "legendFormat": "{{hostname}}"}
|
|
],
|
|
"options": {"reduceOptions": {"calcs": ["lastNotNull"]}, "colorMode": "value", "textMode": "value_and_name", "graphMode": "area"},
|
|
"fieldConfig": {"defaults": {"unit": "s", "thresholds": {"mode": "absolute", "steps": [{"color": "green"}, {"color": "orange", "value": 1}, {"color": "red", "value": 3}]}}}
|
|
},
|
|
{
|
|
"id": 5,
|
|
"type": "timeseries",
|
|
"title": "Probe success over time",
|
|
"datasource": {"type": "prometheus", "uid": "${prom}"},
|
|
"gridPos": {"h": 8, "w": 12, "x": 0, "y": 6},
|
|
"targets": [
|
|
{"refId": "A", "expr": "probe_success{service=\"searxng\", hostname=~\"$host\"}", "legendFormat": "{{hostname}}"}
|
|
],
|
|
"fieldConfig": {"defaults": {"unit": "short", "min": 0, "max": 1, "custom": {"drawStyle": "line", "lineWidth": 2, "fillOpacity": 20}}},
|
|
"options": {"legend": {"displayMode": "list", "placement": "bottom", "showLegend": true}}
|
|
},
|
|
{
|
|
"id": 6,
|
|
"type": "timeseries",
|
|
"title": "Probe duration over time",
|
|
"datasource": {"type": "prometheus", "uid": "${prom}"},
|
|
"gridPos": {"h": 8, "w": 12, "x": 12, "y": 6},
|
|
"targets": [
|
|
{"refId": "A", "expr": "probe_duration_seconds{service=\"searxng\", hostname=~\"$host\"}", "legendFormat": "{{hostname}}"}
|
|
],
|
|
"fieldConfig": {"defaults": {"unit": "s"}},
|
|
"options": {"legend": {"displayMode": "list", "placement": "bottom", "showLegend": true}}
|
|
},
|
|
|
|
{
|
|
"id": 10,
|
|
"type": "row",
|
|
"title": "Argos's view of these instances",
|
|
"collapsed": false,
|
|
"gridPos": {"h": 1, "w": 24, "x": 0, "y": 14}
|
|
},
|
|
{
|
|
"id": 11,
|
|
"type": "timeseries",
|
|
"title": "argos_searxng_instance_up by SearXNG instance",
|
|
"datasource": {"type": "prometheus", "uid": "${prom}"},
|
|
"gridPos": {"h": 8, "w": 12, "x": 0, "y": 15},
|
|
"targets": [
|
|
{"refId": "A", "expr": "argos_searxng_instance_up", "legendFormat": "{{searxng_instance}}"}
|
|
],
|
|
"fieldConfig": {"defaults": {"unit": "short", "min": 0, "max": 1, "custom": {"drawStyle": "line", "lineWidth": 2, "fillOpacity": 20}}},
|
|
"options": {"legend": {"displayMode": "list", "placement": "bottom", "showLegend": true}}
|
|
},
|
|
{
|
|
"id": 12,
|
|
"type": "timeseries",
|
|
"title": "Search latency p95 from Argos (5m)",
|
|
"datasource": {"type": "prometheus", "uid": "${prom}"},
|
|
"gridPos": {"h": 8, "w": 12, "x": 12, "y": 15},
|
|
"targets": [
|
|
{"refId": "A", "expr": "histogram_quantile(0.95, sum by (le, searxng_instance) (rate(argos_searxng_request_duration_seconds_bucket[5m])))", "legendFormat": "{{searxng_instance}} p95"}
|
|
],
|
|
"fieldConfig": {"defaults": {"unit": "s"}},
|
|
"options": {"legend": {"displayMode": "list", "placement": "bottom", "showLegend": true}}
|
|
},
|
|
{
|
|
"id": 13,
|
|
"type": "timeseries",
|
|
"title": "Search request error ratio from Argos (5m)",
|
|
"datasource": {"type": "prometheus", "uid": "${prom}"},
|
|
"gridPos": {"h": 8, "w": 24, "x": 0, "y": 23},
|
|
"targets": [
|
|
{"refId": "A", "expr": "sum by (searxng_instance) (rate(argos_searxng_requests_total{status=\"error\"}[5m])) / clamp_min(sum by (searxng_instance) (rate(argos_searxng_requests_total[5m])), 0.0001)", "legendFormat": "{{searxng_instance}}"}
|
|
],
|
|
"fieldConfig": {"defaults": {"unit": "percentunit"}},
|
|
"options": {"legend": {"displayMode": "list", "placement": "bottom", "showLegend": true}}
|
|
},
|
|
|
|
{
|
|
"id": 90,
|
|
"type": "row",
|
|
"title": "Logs",
|
|
"collapsed": false,
|
|
"gridPos": {"h": 1, "w": 24, "x": 0, "y": 31}
|
|
},
|
|
{
|
|
"id": 91,
|
|
"type": "logs",
|
|
"title": "searxng (Loki)",
|
|
"datasource": {"type": "loki", "uid": "${loki}"},
|
|
"gridPos": {"h": 12, "w": 24, "x": 0, "y": 32},
|
|
"targets": [
|
|
{"refId": "A", "expr": "{job=\"searxng\"}"}
|
|
],
|
|
"options": {"showTime": true, "wrapLogMessage": true, "enableLogDetails": true, "dedupStrategy": "none"}
|
|
}
|
|
]
|
|
}
|