enhance dashboarding

This commit is contained in:
Clément Désiles
2026-09-11 21:32:00 +02:00
parent 9440d78cd9
commit 20cbf01040
4 changed files with 93 additions and 53 deletions
+89 -30
View File
@@ -5,7 +5,7 @@
"schemaVersion": 39, "schemaVersion": 39,
"version": 1, "version": 1,
"refresh": "1m", "refresh": "1m",
"time": { "from": "now-30d", "to": "now" }, "time": { "from": "now-7d", "to": "now" },
"timezone": "browser", "timezone": "browser",
"editable": true, "editable": true,
"graphTooltip": 1, "graphTooltip": 1,
@@ -13,49 +13,108 @@
"list": [ "list": [
{ {
"name": "instance", "name": "instance",
"label": "Host", "label": "Instance",
"type": "query", "type": "query",
"datasource": { "type": "prometheus", "uid": "prometheus" }, "datasource": { "type": "prometheus", "uid": "prometheus" },
"query": { "query": "query_result(count by (instance) (backup_last_success_timestamp_seconds))", "refId": "A" }, "query": { "query": "label_values(node_systemd_timer_last_trigger_seconds{name=~\"backup@.*\"}, instance)", "refId": "A" },
"regex": "/.*instance=\"([^\"]+)\".*/", "refresh": 1,
"refresh": 2, "multi": false,
"multi": true, "includeAll": false,
"includeAll": true,
"allValue": ".*",
"sort": 1, "sort": 1,
"current": { "selected": true, "text": "All", "value": "$__all" } "current": { "selected": false, "text": "", "value": "" }
} }
] ]
}, },
"panels": [ "panels": [
{ {
"id": 1, "id": 5,
"type": "stat", "type": "timeseries",
"title": "Last backup age", "title": "Backup runs (successes up, failures down)",
"gridPos": { "h": 6, "w": 12, "x": 0, "y": 0 }, "description": "Runs counted per bucket with changes() on the last-run timestamp: every run stamps a new value, so a value change is a run. Successes and failures are separate series rather than a signed sum, so a bucket containing both stays readable. Bucket width follows the zoom level with a 12h floor. Rendered as a timeseries rather than a barchart panel: barchart joins all series into one frame and returns no data at all when some of them are empty, which is the normal state here (no failures, no skips). Caveat: the first run after a job/repo first appears registers no change and is not counted.",
"gridPos": { "h": 8, "w": 24, "x": 0, "y": 0 },
"datasource": { "type": "prometheus", "uid": "prometheus" }, "datasource": { "type": "prometheus", "uid": "prometheus" },
"fieldConfig": { "defaults": { "unit": "dtdhms", "color": { "mode": "thresholds" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "orange", "value": 172800 }, { "color": "red", "value": 1209600 } ] } }, "overrides": [] }, "interval": "12h",
"options": { "reduceOptions": { "values": false, "calcs": ["lastNotNull"], "fields": "" }, "colorMode": "background", "graphMode": "none", "textMode": "auto", "justifyMode": "auto", "orientation": "auto" }, "fieldConfig": {
"targets": [ { "refId": "A", "expr": "time() - backup_last_success_timestamp_seconds{instance=~\"$instance\"}", "legendFormat": "{{backup_job}} / {{repo}}" } ] "defaults": {
"unit": "short",
"decimals": 0,
"color": { "mode": "palette-classic" },
"custom": {
"drawStyle": "bars",
"barAlignment": 0,
"lineWidth": 1,
"fillOpacity": 80,
"gradientMode": "none",
"showPoints": "never",
"spanNulls": false,
"axisSoftMin": -1,
"axisSoftMax": 1,
"stacking": { "mode": "none", "group": "A" },
"thresholdsStyle": { "mode": "off" }
}
}, },
{ "overrides": [
"id": 2, { "matcher": { "id": "byName", "options": "successful runs" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "green" } } ] },
"type": "stat", { "matcher": { "id": "byName", "options": "failed runs" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "red" } } ] },
"title": "Failing backups (14d)", { "matcher": { "id": "byName", "options": "skipped runs" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "orange" } } ] }
"gridPos": { "h": 6, "w": 12, "x": 12, "y": 0 }, ]
"datasource": { "type": "prometheus", "uid": "prometheus" }, },
"fieldConfig": { "defaults": { "unit": "short", "color": { "mode": "thresholds" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 1 } ] } }, "overrides": [] }, "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "none" } },
"options": { "reduceOptions": { "values": false, "calcs": ["lastNotNull"], "fields": "" }, "colorMode": "background", "graphMode": "none", "textMode": "auto", "justifyMode": "auto", "orientation": "auto" }, "targets": [
"targets": [ { "refId": "A", "expr": "count(backup_last_run_timestamp_seconds{instance=~\"$instance\",result=\"fail\"} > (time() - 14 * 86400)) or vector(0)" } ] { "refId": "A", "expr": "sum(changes(backup_last_run_timestamp_seconds{instance=\"$instance\",result=\"success\"}[$__interval])) or vector(0)", "legendFormat": "successful runs" },
{ "refId": "B", "expr": "-(sum(changes(backup_last_run_timestamp_seconds{instance=\"$instance\",result=\"fail\"}[$__interval])) or vector(0))", "legendFormat": "failed runs" },
{ "refId": "C", "expr": "-(sum(changes(backup_last_run_timestamp_seconds{instance=\"$instance\",result=\"skip\"}[$__interval])) or vector(0))", "legendFormat": "skipped runs" }
]
}, },
{ {
"id": 3, "id": 3,
"type": "timeseries", "type": "stat",
"title": "Backup success history", "title": "Timer last triggered",
"gridPos": { "h": 8, "w": 24, "x": 0, "y": 6 }, "description": "Time since each backup@*.timer last fired. Timers that have never fired report 0 and are filtered out.",
"gridPos": { "h": 6, "w": 24, "x": 0, "y": 8 },
"datasource": { "type": "prometheus", "uid": "prometheus" }, "datasource": { "type": "prometheus", "uid": "prometheus" },
"fieldConfig": { "defaults": { "unit": "dateTimeAsIso", "color": { "mode": "palette-classic" }, "custom": { "fillOpacity": 0, "lineWidth": 1, "drawStyle": "line", "showPoints": "always", "pointSize": 5 } }, "overrides": [] }, "fieldConfig": { "defaults": { "unit": "dtdhms", "noValue": "never triggered", "color": { "mode": "thresholds" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 1209600 } ] } }, "overrides": [] },
"targets": [ { "refId": "A", "expr": "backup_last_success_timestamp_seconds{instance=~\"$instance\"}", "legendFormat": "{{backup_job}} / {{repo}}" } ] "options": { "reduceOptions": { "values": false, "calcs": ["lastNotNull"], "fields": "" }, "colorMode": "background", "graphMode": "none", "textMode": "auto", "justifyMode": "auto", "orientation": "auto" },
"targets": [ { "refId": "A", "expr": "time() - (node_systemd_timer_last_trigger_seconds{instance=\"$instance\", name=~\"backup@.*\"} > 0)", "legendFormat": "{{name}}" } ]
},
{
"id": 4,
"type": "table",
"title": "Last run per job / repository",
"description": "Most recent run of every job/repository pair, whatever the outcome. Age is time since that run, so a repository going stale is visible here even while the graph above stays empty.",
"gridPos": { "h": 10, "w": 24, "x": 0, "y": 14 },
"datasource": { "type": "prometheus", "uid": "prometheus" },
"fieldConfig": {
"defaults": { "unit": "dtdhms", "custom": { "align": "auto", "cellOptions": { "type": "auto" } } },
"overrides": [
{
"matcher": { "id": "byName", "options": "Result" },
"properties": [
{ "id": "custom.cellOptions", "value": { "type": "color-background" } },
{ "id": "mappings", "value": [
{ "type": "value", "options": {
"success": { "text": "success", "color": "green", "index": 0 },
"fail": { "text": "fail", "color": "red", "index": 1 },
"skip": { "text": "skip", "color": "orange", "index": 2 }
} }
] }
]
}
]
},
"options": { "showHeader": true, "cellHeight": "sm", "footer": { "show": false, "reducer": ["sum"], "fields": "" } },
"targets": [ { "refId": "A", "expr": "time() - backup_last_run_timestamp_seconds{instance=\"$instance\"}", "format": "table", "instant": true } ],
"transformations": [
{
"id": "organize",
"options": {
"excludeByName": { "Time": true, "__name__": true, "job": true },
"renameByName": { "Value": "Age", "backup_job": "Job", "repo": "Repository", "result": "Result", "instance": "Instance" },
"indexByName": { "instance": 0, "backup_job": 1, "repo": 2, "result": 3, "Value": 4 }
}
},
{ "id": "sortBy", "options": { "fields": {}, "sort": [ { "field": "Result", "desc": false } ] } }
]
} }
] ]
} }
@@ -822,25 +822,6 @@
], ],
"templating": { "templating": {
"list": [ "list": [
{
"current": {
"selected": false,
"text": "Prometheus",
"value": "Prometheus"
},
"hide": 0,
"includeAll": false,
"label": "Data Source",
"multi": false,
"name": "DataSource",
"options": [],
"query": "prometheus",
"queryValue": "",
"refresh": 1,
"regex": "",
"skipUrlSync": false,
"type": "datasource"
},
{ {
"current": {}, "current": {},
"datasource": { "datasource": {
+1 -1
View File
@@ -16,7 +16,7 @@
"label": "Instance", "label": "Instance",
"type": "query", "type": "query",
"datasource": { "type": "prometheus", "uid": "prometheus" }, "datasource": { "type": "prometheus", "uid": "prometheus" },
"query": { "query": "label_values(smartctl_device, instance)", "refId": "A" }, "query": { "query": "label_values(smartctl_device_smart_status, instance)", "refId": "A" },
"refresh": 2, "refresh": 2,
"sort": 1, "sort": 1,
"multi": false, "multi": false,
+2 -2
View File
@@ -44,9 +44,9 @@
"title": "Failed unit", "title": "Failed unit",
"gridPos": { "h": 4, "w": 18, "x": 6, "y": 0 }, "gridPos": { "h": 4, "w": 18, "x": 6, "y": 0 },
"datasource": { "type": "prometheus", "uid": "prometheus" }, "datasource": { "type": "prometheus", "uid": "prometheus" },
"fieldConfig": { "defaults": { "unit": "none", "color": { "mode": "fixed" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 1 } ] }, "mappings": [ { "type": "value", "options": { "1": { "text": "FAILED", "color": "red" } } } ] }, "overrides": [] }, "fieldConfig": { "defaults": { "unit": "none", "color": { "mode": "thresholds" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 1 } ] }, "mappings": [ { "type": "value", "options": { "0": { "text": "OK", "color": "green" }, "1": { "text": "FAILED", "color": "red" } } } ] }, "overrides": [] },
"options": { "reduceOptions": { "values": false, "calcs": ["lastNotNull"], "fields": "" }, "colorMode": "background", "graphMode": "none", "textMode": "auto", "justifyMode": "auto", "orientation": "auto" }, "options": { "reduceOptions": { "values": false, "calcs": ["lastNotNull"], "fields": "" }, "colorMode": "background", "graphMode": "none", "textMode": "auto", "justifyMode": "auto", "orientation": "auto" },
"targets": [ { "refId": "A", "expr": "node_systemd_unit_state{instance=~\"$instance\",state=\"failed\"} == 1", "legendFormat": "{{name}}" } ] "targets": [ { "refId": "A", "expr": "node_systemd_unit_state{instance=~\"$instance\",state=\"failed\"} == 1 or vector(0)", "legendFormat": "{{name}}" } ]
}, },
{ {
"id": 3, "id": 3,