{ "annotations": { "list": [ { "builtIn": 1, "datasource": { "type": "grafana", "uid": "-- Grafana --" }, "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", "name": "Annotations & Alerts", "type": "dashboard" } ] }, "description": "Process-level monitoring for puck.incus - Red Panda Approved 🐼", "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 0, "id": null, "links": [ { "asDropdown": false, "icon": "external link", "includeVars": true, "keepTime": true, "tags": [], "targetBlank": true, "title": "Explore Logs", "tooltip": "View logs in Loki for puck.incus", "type": "link", "url": "/explore?orgId=1&left=%7B%22datasource%22:%22prospero-loki%22,%22queries%22:%5B%7B%22refId%22:%22A%22,%22expr%22:%22%7Bhostname%3D%5C%22puck.incus%5C%22%7D%22%7D%5D%7D" } ], "panels": [ { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, "id": 100, "panels": [], "title": "Overview", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "prospero-prometheus" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 50 }, { "color": "red", "value": 100 } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 4, "w": 4, "x": 0, "y": 1 }, "id": 1, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "textMode": "auto" }, "pluginVersion": "10.2.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "prospero-prometheus" }, "expr": "count(namedprocess_namegroup_num_procs{instance=~\"puck.*\"})", "legendFormat": "Total Processes", "refId": "A" } ], "title": "Monitored Processes", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "prospero-prometheus" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 50 }, { "color": "red", "value": 80 } ] }, "unit": "percent" }, "overrides": [] }, "gridPos": { "h": 4, "w": 5, "x": 4, "y": 1 }, "id": 2, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "textMode": "auto" }, "pluginVersion": "10.2.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "prospero-prometheus" }, "expr": "100 - (avg(rate(node_cpu_seconds_total{instance=~\"puck.*\", mode=\"idle\"}[5m])) * 100)", "legendFormat": "CPU Usage", "refId": "A" } ], "title": "Overall CPU Usage", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "prospero-prometheus" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 50 }, { "color": "red", "value": 80 } ] }, "unit": "percent" }, "overrides": [] }, "gridPos": { "h": 4, "w": 5, "x": 9, "y": 1 }, "id": 3, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "textMode": "auto" }, "pluginVersion": "10.2.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "prospero-prometheus" }, "expr": "(1 - (node_memory_MemAvailable_bytes{instance=~\"puck.*\"} / node_memory_MemTotal_bytes{instance=~\"puck.*\"})) * 100", "legendFormat": "Memory Usage", "refId": "A" } ], "title": "Overall Memory Usage", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "prospero-prometheus" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "bytes" }, "overrides": [] }, "gridPos": { "h": 4, "w": 5, "x": 14, "y": 1 }, "id": 4, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "textMode": "auto" }, "pluginVersion": "10.2.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "prospero-prometheus" }, "expr": "node_memory_MemTotal_bytes{instance=~\"puck.*\"}", "legendFormat": "Total Memory", "refId": "A" } ], "title": "Total Memory", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "prospero-prometheus" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 4, "w": 5, "x": 19, "y": 1 }, "id": 5, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "textMode": "auto" }, "pluginVersion": "10.2.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "prospero-prometheus" }, "expr": "time() - node_boot_time_seconds{instance=~\"puck.*\"}", "legendFormat": "Uptime", "refId": "A" } ], "title": "Uptime", "type": "stat" }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 5 }, "id": 101, "panels": [], "title": "Top CPU Consumers", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "prospero-prometheus" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "smooth", "lineWidth": 2, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "line" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 80 }, { "color": "red", "value": 95 } ] }, "unit": "percent" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 6 }, "id": 10, "options": { "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "right", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "prospero-prometheus" }, "expr": "topk(10, sum by (groupname) (rate(namedprocess_namegroup_cpu_seconds_total{instance=~\"puck.*\"}[2m])) * 100)", "legendFormat": "{{groupname}}", "refId": "A" } ], "title": "Top 10 CPU Processes (Over Time)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "prospero-prometheus" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { "align": "auto", "cellOptions": { "type": "auto" }, "inspect": false }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 50 }, { "color": "red", "value": 80 } ] }, "unit": "percent" }, "overrides": [ { "matcher": { "id": "byName", "options": "CPU %" }, "properties": [ { "id": "custom.cellOptions", "value": { "mode": "gradient", "type": "gauge" } } ] } ] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 6 }, "id": 11, "options": { "cellHeight": "sm", "footer": { "countRows": false, "fields": "", "reducer": ["sum"], "show": false }, "showHeader": true, "sortBy": [ { "desc": true, "displayName": "CPU %" } ] }, "pluginVersion": "10.2.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "prospero-prometheus" }, "expr": "topk(10, sum by (groupname) (rate(namedprocess_namegroup_cpu_seconds_total{instance=~\"puck.*\"}[2m])) * 100)", "format": "table", "instant": true, "legendFormat": "__auto", "refId": "A" } ], "title": "Top 10 CPU Processes (Current)", "transformations": [ { "id": "organize", "options": { "excludeByName": { "Time": true, "instance": true }, "indexByName": {}, "renameByName": { "Value": "CPU %", "groupname": "Process" } } } ], "type": "table" }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 14 }, "id": 102, "panels": [], "title": "Top Memory Consumers", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "prospero-prometheus" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "smooth", "lineWidth": 2, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "line" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 1073741824 }, { "color": "red", "value": 2147483648 } ] }, "unit": "bytes" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 15 }, "id": 20, "options": { "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "right", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "prospero-prometheus" }, "expr": "topk(10, namedprocess_namegroup_memory_bytes{instance=~\"puck.*\", memtype=\"resident\"})", "legendFormat": "{{groupname}}", "refId": "A" } ], "title": "Top 10 Memory Processes (Over Time)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "prospero-prometheus" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { "align": "auto", "cellOptions": { "type": "auto" }, "inspect": false }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 536870912 }, { "color": "red", "value": 1073741824 } ] }, "unit": "bytes" }, "overrides": [ { "matcher": { "id": "byName", "options": "Resident Memory" }, "properties": [ { "id": "custom.cellOptions", "value": { "mode": "gradient", "type": "gauge" } } ] } ] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 15 }, "id": 21, "options": { "cellHeight": "sm", "footer": { "countRows": false, "fields": "", "reducer": ["sum"], "show": false }, "showHeader": true, "sortBy": [ { "desc": true, "displayName": "Resident Memory" } ] }, "pluginVersion": "10.2.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "prospero-prometheus" }, "expr": "topk(10, namedprocess_namegroup_memory_bytes{instance=~\"puck.*\", memtype=\"resident\"})", "format": "table", "instant": true, "legendFormat": "__auto", "refId": "A" } ], "title": "Top 10 Memory Processes (Current)", "transformations": [ { "id": "organize", "options": { "excludeByName": { "Time": true, "instance": true, "job": true, "memtype": true }, "indexByName": {}, "renameByName": { "Value": "Resident Memory", "groupname": "Process" } } } ], "type": "table" }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 23 }, "id": 103, "panels": [], "title": "Process Details", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "prospero-prometheus" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { "align": "auto", "cellOptions": { "type": "auto" }, "inspect": false }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] } }, "overrides": [ { "matcher": { "id": "byName", "options": "CPU %" }, "properties": [ { "id": "unit", "value": "percent" }, { "id": "custom.cellOptions", "value": { "mode": "basic", "type": "gauge" } }, { "id": "thresholds", "value": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 50 }, { "color": "red", "value": 80 } ] } } ] }, { "matcher": { "id": "byName", "options": "Memory" }, "properties": [ { "id": "unit", "value": "bytes" } ] }, { "matcher": { "id": "byName", "options": "Threads" }, "properties": [ { "id": "unit", "value": "short" } ] } ] }, "gridPos": { "h": 10, "w": 24, "x": 0, "y": 24 }, "id": 30, "options": { "cellHeight": "sm", "footer": { "countRows": false, "fields": "", "reducer": ["sum"], "show": false }, "showHeader": true, "sortBy": [ { "desc": true, "displayName": "CPU %" } ] }, "pluginVersion": "10.2.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "prospero-prometheus" }, "expr": "sum by (groupname) (rate(namedprocess_namegroup_cpu_seconds_total{instance=~\"puck.*\"}[2m])) * 100", "format": "table", "instant": true, "legendFormat": "__auto", "refId": "CPU" }, { "datasource": { "type": "prometheus", "uid": "prospero-prometheus" }, "expr": "namedprocess_namegroup_memory_bytes{instance=~\"puck.*\", memtype=\"resident\"}", "format": "table", "hide": false, "instant": true, "legendFormat": "__auto", "refId": "Memory" }, { "datasource": { "type": "prometheus", "uid": "prospero-prometheus" }, "expr": "namedprocess_namegroup_num_threads{instance=~\"puck.*\"}", "format": "table", "hide": false, "instant": true, "legendFormat": "__auto", "refId": "Threads" } ], "title": "All Processes", "transformations": [ { "id": "seriesToColumns", "options": { "byField": "groupname" } }, { "id": "organize", "options": { "excludeByName": { "Time 1": true, "Time 2": true, "Time 3": true, "instance 1": true, "instance 2": true, "instance 3": true, "job 1": true, "job 2": true, "job 3": true, "memtype": true }, "indexByName": {}, "renameByName": { "Value #CPU": "CPU %", "Value #Memory": "Memory", "Value #Threads": "Threads", "groupname": "Process" } } } ], "type": "table" } ], "refresh": "30s", "schemaVersion": 38, "tags": ["puck", "processes", "monitoring", "red-panda-approved"], "templating": { "list": [] }, "time": { "from": "now-1h", "to": "now" }, "timepicker": {}, "timezone": "", "title": "Puck Process Monitoring", "uid": "puck-process-monitoring", "version": 1, "weekStart": "" }