enhance dashboarding

roles/grafana/files/backup-dashboard.json

@@ -5,7 +5,7 @@
   "schemaVersion": 39,
   "version": 1,
   "refresh": "1m",
-  "time": { "from": "now-30d", "to": "now" },
+  "time": { "from": "now-7d", "to": "now" },
   "timezone": "browser",
   "editable": true,
   "graphTooltip": 1,
@@ -13,49 +13,108 @@
     "list": [
       {
         "name": "instance",
-        "label": "Host",
+        "label": "Instance",
         "type": "query",
         "datasource": { "type": "prometheus", "uid": "prometheus" },
-        "query": { "query": "query_result(count by (instance) (backup_last_success_timestamp_seconds))", "refId": "A" },
+        "query": { "query": "label_values(node_systemd_timer_last_trigger_seconds{name=~\"backup@.*\"}, instance)", "refId": "A" },
-        "regex": "/.*instance=\"([^\"]+)\".*/",
+        "refresh": 1,
-        "refresh": 2,
+        "multi": false,
-        "multi": true,
+        "includeAll": false,
-        "includeAll": true,
-        "allValue": ".*",
         "sort": 1,
-        "current": { "selected": true, "text": "All", "value": "$__all" }
+        "current": { "selected": false, "text": "", "value": "" }
       }
     ]
   },
   "panels": [
     {
-      "id": 1,
+      "id": 5,
-      "type": "stat",
+      "type": "timeseries",
-      "title": "Last backup age",
+      "title": "Backup runs (successes up, failures down)",
-      "gridPos": { "h": 6, "w": 12, "x": 0, "y": 0 },
+      "description": "Runs counted per bucket with changes() on the last-run timestamp: every run stamps a new value, so a value change is a run. Successes and failures are separate series rather than a signed sum, so a bucket containing both stays readable. Bucket width follows the zoom level with a 12h floor. Rendered as a timeseries rather than a barchart panel: barchart joins all series into one frame and returns no data at all when some of them are empty, which is the normal state here (no failures, no skips). Caveat: the first run after a job/repo first appears registers no change and is not counted.",
+      "gridPos": { "h": 8, "w": 24, "x": 0, "y": 0 },
       "datasource": { "type": "prometheus", "uid": "prometheus" },
-      "fieldConfig": { "defaults": { "unit": "dtdhms", "color": { "mode": "thresholds" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "orange", "value": 172800 }, { "color": "red", "value": 1209600 } ] } }, "overrides": [] },
+      "interval": "12h",
-      "options": { "reduceOptions": { "values": false, "calcs": ["lastNotNull"], "fields": "" }, "colorMode": "background", "graphMode": "none", "textMode": "auto", "justifyMode": "auto", "orientation": "auto" },
+      "fieldConfig": {
-      "targets": [ { "refId": "A", "expr": "time() - backup_last_success_timestamp_seconds{instance=~\"$instance\"}", "legendFormat": "{{backup_job}} / {{repo}}" } ]
+        "defaults": {
-    },
+          "unit": "short",
-    {
+          "decimals": 0,
-      "id": 2,
+          "color": { "mode": "palette-classic" },
-      "type": "stat",
+          "custom": {
-      "title": "Failing backups (14d)",
+            "drawStyle": "bars",
-      "gridPos": { "h": 6, "w": 12, "x": 12, "y": 0 },
+            "barAlignment": 0,
-      "datasource": { "type": "prometheus", "uid": "prometheus" },
+            "lineWidth": 1,
-      "fieldConfig": { "defaults": { "unit": "short", "color": { "mode": "thresholds" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 1 } ] } }, "overrides": [] },
+            "fillOpacity": 80,
-      "options": { "reduceOptions": { "values": false, "calcs": ["lastNotNull"], "fields": "" }, "colorMode": "background", "graphMode": "none", "textMode": "auto", "justifyMode": "auto", "orientation": "auto" },
+            "gradientMode": "none",
-      "targets": [ { "refId": "A", "expr": "count(backup_last_run_timestamp_seconds{instance=~\"$instance\",result=\"fail\"} > (time() - 14 * 86400)) or vector(0)" } ]
+            "showPoints": "never",
+            "spanNulls": false,
+            "axisSoftMin": -1,
+            "axisSoftMax": 1,
+            "stacking": { "mode": "none", "group": "A" },
+            "thresholdsStyle": { "mode": "off" }
+          }
+        },
+        "overrides": [
+          { "matcher": { "id": "byName", "options": "successful runs" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "green" } } ] },
+          { "matcher": { "id": "byName", "options": "failed runs" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "red" } } ] },
+          { "matcher": { "id": "byName", "options": "skipped runs" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "orange" } } ] }
+        ]
+      },
+      "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "none" } },
+      "targets": [
+        { "refId": "A", "expr": "sum(changes(backup_last_run_timestamp_seconds{instance=\"$instance\",result=\"success\"}[$__interval])) or vector(0)", "legendFormat": "successful runs" },
+        { "refId": "B", "expr": "-(sum(changes(backup_last_run_timestamp_seconds{instance=\"$instance\",result=\"fail\"}[$__interval])) or vector(0))", "legendFormat": "failed runs" },
+        { "refId": "C", "expr": "-(sum(changes(backup_last_run_timestamp_seconds{instance=\"$instance\",result=\"skip\"}[$__interval])) or vector(0))", "legendFormat": "skipped runs" }
+      ]
     },
     {
       "id": 3,
-      "type": "timeseries",
+      "type": "stat",
-      "title": "Backup success history",
+      "title": "Timer last triggered",
-      "gridPos": { "h": 8, "w": 24, "x": 0, "y": 6 },
+      "description": "Time since each backup@*.timer last fired. Timers that have never fired report 0 and are filtered out.",
+      "gridPos": { "h": 6, "w": 24, "x": 0, "y": 8 },
       "datasource": { "type": "prometheus", "uid": "prometheus" },
-      "fieldConfig": { "defaults": { "unit": "dateTimeAsIso", "color": { "mode": "palette-classic" }, "custom": { "fillOpacity": 0, "lineWidth": 1, "drawStyle": "line", "showPoints": "always", "pointSize": 5 } }, "overrides": [] },
+      "fieldConfig": { "defaults": { "unit": "dtdhms", "noValue": "never triggered", "color": { "mode": "thresholds" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 1209600 } ] } }, "overrides": [] },
-      "targets": [ { "refId": "A", "expr": "backup_last_success_timestamp_seconds{instance=~\"$instance\"}", "legendFormat": "{{backup_job}} / {{repo}}" } ]
+      "options": { "reduceOptions": { "values": false, "calcs": ["lastNotNull"], "fields": "" }, "colorMode": "background", "graphMode": "none", "textMode": "auto", "justifyMode": "auto", "orientation": "auto" },
+      "targets": [ { "refId": "A", "expr": "time() - (node_systemd_timer_last_trigger_seconds{instance=\"$instance\", name=~\"backup@.*\"} > 0)", "legendFormat": "{{name}}" } ]
+    },
+    {
+      "id": 4,
+      "type": "table",
+      "title": "Last run per job / repository",
+      "description": "Most recent run of every job/repository pair, whatever the outcome. Age is time since that run, so a repository going stale is visible here even while the graph above stays empty.",
+      "gridPos": { "h": 10, "w": 24, "x": 0, "y": 14 },
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "fieldConfig": {
+        "defaults": { "unit": "dtdhms", "custom": { "align": "auto", "cellOptions": { "type": "auto" } } },
+        "overrides": [
+          {
+            "matcher": { "id": "byName", "options": "Result" },
+            "properties": [
+              { "id": "custom.cellOptions", "value": { "type": "color-background" } },
+              { "id": "mappings", "value": [
+                { "type": "value", "options": {
+                  "success": { "text": "success", "color": "green", "index": 0 },
+                  "fail": { "text": "fail", "color": "red", "index": 1 },
+                  "skip": { "text": "skip", "color": "orange", "index": 2 }
+                } }
+              ] }
+            ]
+          }
+        ]
+      },
+      "options": { "showHeader": true, "cellHeight": "sm", "footer": { "show": false, "reducer": ["sum"], "fields": "" } },
+      "targets": [ { "refId": "A", "expr": "time() - backup_last_run_timestamp_seconds{instance=\"$instance\"}", "format": "table", "instant": true } ],
+      "transformations": [
+        {
+          "id": "organize",
+          "options": {
+            "excludeByName": { "Time": true, "__name__": true, "job": true },
+            "renameByName": { "Value": "Age", "backup_job": "Job", "repo": "Repository", "result": "Result", "instance": "Instance" },
+            "indexByName": { "instance": 0, "backup_job": 1, "repo": 2, "result": 3, "Value": 4 }
+          }
+        },
+        { "id": "sortBy", "options": { "fields": {}, "sort": [ { "field": "Result", "desc": false } ] } }
+      ]
     }
   ]
 }

roles/grafana/files/fail2ban-dashboard.json

@@ -822,25 +822,6 @@
   ],
   "templating": {
     "list": [
-      {
-        "current": {
-          "selected": false,
-          "text": "Prometheus",
-          "value": "Prometheus"
-        },
-        "hide": 0,
-        "includeAll": false,
-        "label": "Data Source",
-        "multi": false,
-        "name": "DataSource",
-        "options": [],
-        "query": "prometheus",
-        "queryValue": "",
-        "refresh": 1,
-        "regex": "",
-        "skipUrlSync": false,
-        "type": "datasource"
-      },
       {
         "current": {},
         "datasource": {

roles/grafana/files/smartctl-dashboard.json

@@ -16,7 +16,7 @@
         "label": "Instance",
         "type": "query",
         "datasource": { "type": "prometheus", "uid": "prometheus" },
-        "query": { "query": "label_values(smartctl_device, instance)", "refId": "A" },
+        "query": { "query": "label_values(smartctl_device_smart_status, instance)", "refId": "A" },
         "refresh": 2,
         "sort": 1,
         "multi": false,

roles/grafana/files/systemd-dashboard.json

@@ -44,9 +44,9 @@
       "title": "Failed unit",
       "gridPos": { "h": 4, "w": 18, "x": 6, "y": 0 },
       "datasource": { "type": "prometheus", "uid": "prometheus" },
-      "fieldConfig": { "defaults": { "unit": "none", "color": { "mode": "fixed" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 1 } ] }, "mappings": [ { "type": "value", "options": { "1": { "text": "FAILED", "color": "red" } } } ] }, "overrides": [] },
+      "fieldConfig": { "defaults": { "unit": "none", "color": { "mode": "thresholds" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 1 } ] }, "mappings": [ { "type": "value", "options": { "0": { "text": "OK", "color": "green" }, "1": { "text": "FAILED", "color": "red" } } } ] }, "overrides": [] },
       "options": { "reduceOptions": { "values": false, "calcs": ["lastNotNull"], "fields": "" }, "colorMode": "background", "graphMode": "none", "textMode": "auto", "justifyMode": "auto", "orientation": "auto" },
-      "targets": [ { "refId": "A", "expr": "node_systemd_unit_state{instance=~\"$instance\",state=\"failed\"} == 1", "legendFormat": "{{name}}" } ]
+      "targets": [ { "refId": "A", "expr": "node_systemd_unit_state{instance=~\"$instance\",state=\"failed\"} == 1 or vector(0)", "legendFormat": "{{name}}" } ]
     },
     {
       "id": 3,