feat(monitoring): add backup and systemd metrics, alerts, and dashboards

- backup role emits last-success/run metrics into node_exporter's textfile collector
- node_exporter enables systemd and textfile collectors
- prometheus deploys BackupStale/BackupTimerSilent/SystemdUnitFailed alert rules
- grafana gains Backups - Restic and Systemd dashboards
- fix prometheus.yml params rendering (quote values so {job=...} stays a string)

roles/backup/README.md

@@ -258,6 +258,24 @@ Every snapshot is a logically complete view; "incremental" describes only how it
 
 The role always excludes restic's own cache, and the staging directories of *other* jobs — otherwise a job backing up `/var` would sweep up both.
 
+### Monitoring
+
+After each run the script writes Prometheus metrics into node_exporter's
+textfile directory (`backup_metrics_dir`), re-exposed by the exporter:
+
+```
+backup_last_success_timestamp_seconds{backup_job="ntfy", repo="next_s3"}
+backup_last_run_timestamp_seconds{backup_job="ntfy", repo="next_s3", result="success|fail|skip"}
+```
+
+This is what turns "silence" into a signal: ntfy alerts on failures, but a
+timer that stopped firing (or a job that's been failing for a week) looks the
+same as success unless you track `backup_last_success_timestamp_seconds`. The
+Grafana "Backups - Restic" dashboard and the `BackupStale` /
+`BackupTimerSilent` Prometheus rules (deployed by the prometheus role) consume
+these. Metrics are written only when `backup_metrics_dir` exists, i.e. when
+node_exporter is deployed on the host.
+
 ### Failure notification
 
 Each repository can carry a `notify` channel: `{ topic, server, token, notify_success }`. A repository with `notify.topic` alerts on that destination's own result — an offsite failure is `urgent`, a missing USB disk is a low-priority `warning`, and a destination without `notify` is silent. Success alerts are opt-in per channel (`notify_success`), off by default because routine chatter trains you to ignore the failure alerts.
@@ -270,4 +288,3 @@ Every configured channel gets a test notification during the play. ntfy runs wit
 
 - **No automated restore test.** An untested backup is a hypothesis. Restore one by hand, on a schedule.
 - **No append-only protection.** The host holds credentials that can delete its own snapshots, so an attacker with root can destroy the backups too. Mitigate with object versioning or an object-lock policy on the bucket, which is outside this role.
-- **No metrics.** Failures are visible through `systemctl --failed`, the journal, and ntfy.

roles/backup/defaults/main.yml

@@ -13,6 +13,13 @@ backup_dumps_dir: /var/lib/backup/dumps
 # be excluded from snapshots rather than silently backing itself up.
 backup_cache_dir: /var/cache/restic
 
+# Directory node_exporter's textfile collector reads `*.prom` files from. The
+# backup script drops one file per job here on every run, so Prometheus can
+# track last-success timestamps and alert when backups go silent. Must match
+# node_exporter_textfile_dir in roles/node_exporter/defaults/main.yml. Metrics
+# are only written when the directory exists (i.e. node_exporter is deployed).
+backup_metrics_dir: /var/lib/node_exporter/textfile_collector
+
 # ---------------------------------------------------------------------------
 # Schedule (systemd OnCalendar syntax) - per-job `schedule` overrides this
 # ---------------------------------------------------------------------------

roles/backup/templates/backup.sh.j2

@@ -369,6 +369,62 @@ run_repository() {
     return "$rc"
 }
 
+# ---------------------------------------------------------------------------
+# Metrics
+# ---------------------------------------------------------------------------
+
+# Escape a value for use as a Prometheus label value.
+escape_label() {
+    local v="$1"
+    v="${v//\\/\\\\}"
+    v="${v//\"/\\\"}"
+    printf '%s' "$v"
+}
+
+# Write the job's result as Prometheus textfile metrics, consumed by
+# node_exporter's textfile collector. Only emitted when that directory exists
+# (i.e. when the node_exporter role is deployed on this host).
+#
+# One file per job, written atomically (tmp + mv) so the collector never reads
+# a half-written file. The `backup_job` label is used rather than `job` to
+# avoid colliding with Prometheus's own scrape-job label.
+emit_metrics() {
+    local metrics_dir="{{ backup_metrics_dir }}"
+    [[ -d "$metrics_dir" ]] || return 0
+
+    local now_ts
+    now_ts="$(date +%s)"
+
+    local job_label repo_label
+    job_label="$(escape_label "$JOB")"
+
+    local out="$metrics_dir/backup-$JOB.prom"
+    local tmp="$out.tmp"
+
+    {
+        echo "# HELP backup_last_success_timestamp_seconds Unix timestamp of the last successful backup to this repository."
+        echo "# TYPE backup_last_success_timestamp_seconds gauge"
+        echo "# HELP backup_last_run_timestamp_seconds Unix timestamp of the last run against this repository, by result."
+        echo "# TYPE backup_last_run_timestamp_seconds gauge"
+
+        local repo
+        for repo in ${succeeded[@]+"${succeeded[@]}"}; do
+            repo_label="$(escape_label "$repo")"
+            echo "backup_last_success_timestamp_seconds{backup_job=\"$job_label\",repo=\"$repo_label\"} $now_ts"
+            echo "backup_last_run_timestamp_seconds{backup_job=\"$job_label\",repo=\"$repo_label\",result=\"success\"} $now_ts"
+        done
+        for repo in ${failed[@]+"${failed[@]}"}; do
+            repo_label="$(escape_label "$repo")"
+            echo "backup_last_run_timestamp_seconds{backup_job=\"$job_label\",repo=\"$repo_label\",result=\"fail\"} $now_ts"
+        done
+        for repo in ${skipped[@]+"${skipped[@]}"}; do
+            repo_label="$(escape_label "$repo")"
+            echo "backup_last_run_timestamp_seconds{backup_job=\"$job_label\",repo=\"$repo_label\",result=\"skip\"} $now_ts"
+        done
+    } > "$tmp"
+    mv "$tmp" "$out"
+}
+
 # ---------------------------------------------------------------------------
 # Main
 # ---------------------------------------------------------------------------
@@ -438,6 +494,8 @@ main() {
     log "=== Backup job '$JOB' complete ==="
     log "$summary"
 
+    emit_metrics
+
     # Notifications are per-repository above; the exit code still tells systemd
     # whether any destination failed.
     if [[ "$n_failed" -gt 0 ]]; then

roles/grafana/files/backup-dashboard.json

@@ -0,0 +1,61 @@
+{
+  "title": "Backups - Restic",
+  "uid": "backup-restic",
+  "tags": ["backup", "restic"],
+  "schemaVersion": 39,
+  "version": 1,
+  "refresh": "1m",
+  "time": { "from": "now-30d", "to": "now" },
+  "timezone": "browser",
+  "editable": true,
+  "graphTooltip": 1,
+  "templating": {
+    "list": [
+      {
+        "name": "instance",
+        "label": "Host",
+        "type": "query",
+        "datasource": { "type": "prometheus", "uid": "prometheus" },
+        "query": { "query": "query_result(count by (instance) (backup_last_success_timestamp_seconds))", "refId": "A" },
+        "regex": "/.*instance=\"([^\"]+)\".*/",
+        "refresh": 2,
+        "multi": true,
+        "includeAll": true,
+        "allValue": ".*",
+        "sort": 1,
+        "current": { "selected": true, "text": "All", "value": "$__all" }
+      }
+    ]
+  },
+  "panels": [
+    {
+      "id": 1,
+      "type": "stat",
+      "title": "Last backup age",
+      "gridPos": { "h": 6, "w": 12, "x": 0, "y": 0 },
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "fieldConfig": { "defaults": { "unit": "dtdhms", "color": { "mode": "thresholds" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "orange", "value": 172800 }, { "color": "red", "value": 1209600 } ] } }, "overrides": [] },
+      "options": { "reduceOptions": { "values": false, "calcs": ["lastNotNull"], "fields": "" }, "colorMode": "background", "graphMode": "none", "textMode": "auto", "justifyMode": "auto", "orientation": "auto" },
+      "targets": [ { "refId": "A", "expr": "time() - backup_last_success_timestamp_seconds{instance=~\"$instance\"}", "legendFormat": "{{backup_job}} / {{repo}}" } ]
+    },
+    {
+      "id": 2,
+      "type": "stat",
+      "title": "Failing backups (14d)",
+      "gridPos": { "h": 6, "w": 12, "x": 12, "y": 0 },
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "fieldConfig": { "defaults": { "unit": "short", "color": { "mode": "thresholds" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 1 } ] } }, "overrides": [] },
+      "options": { "reduceOptions": { "values": false, "calcs": ["lastNotNull"], "fields": "" }, "colorMode": "background", "graphMode": "none", "textMode": "auto", "justifyMode": "auto", "orientation": "auto" },
+      "targets": [ { "refId": "A", "expr": "count(backup_last_run_timestamp_seconds{instance=~\"$instance\",result=\"fail\"} > (time() - 14 * 86400)) or vector(0)" } ]
+    },
+    {
+      "id": 3,
+      "type": "timeseries",
+      "title": "Backup success history",
+      "gridPos": { "h": 8, "w": 24, "x": 0, "y": 6 },
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "fieldConfig": { "defaults": { "unit": "dateTimeAsIso", "color": { "mode": "palette-classic" }, "custom": { "fillOpacity": 0, "lineWidth": 1, "drawStyle": "line", "showPoints": "always", "pointSize": 5 } }, "overrides": [] },
+      "targets": [ { "refId": "A", "expr": "backup_last_success_timestamp_seconds{instance=~\"$instance\"}", "legendFormat": "{{backup_job}} / {{repo}}" } ]
+    }
+  ]
+}

roles/grafana/files/systemd-dashboard.json

@@ -0,0 +1,71 @@
+{
+  "title": "Systemd",
+  "uid": "systemd",
+  "tags": ["systemd", "services", "timers"],
+  "schemaVersion": 39,
+  "version": 1,
+  "refresh": "1m",
+  "time": { "from": "now-7d", "to": "now" },
+  "timezone": "browser",
+  "editable": true,
+  "graphTooltip": 1,
+  "templating": {
+    "list": [
+      {
+        "name": "instance",
+        "label": "Host",
+        "type": "query",
+        "datasource": { "type": "prometheus", "uid": "prometheus" },
+        "query": { "query": "query_result(count by (instance) (node_systemd_unit_state))", "refId": "A" },
+        "regex": "/.*instance=\"([^\"]+)\".*/",
+        "refresh": 2,
+        "multi": true,
+        "includeAll": true,
+        "allValue": ".*",
+        "sort": 1,
+        "current": { "selected": true, "text": "All", "value": "$__all" }
+      }
+    ]
+  },
+  "panels": [
+    {
+      "id": 1,
+      "type": "stat",
+      "title": "Failed units",
+      "gridPos": { "h": 4, "w": 6, "x": 0, "y": 0 },
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "fieldConfig": { "defaults": { "unit": "short", "color": { "mode": "thresholds" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 1 } ] } }, "overrides": [] },
+      "options": { "reduceOptions": { "values": false, "calcs": ["lastNotNull"], "fields": "" }, "colorMode": "background", "graphMode": "none", "textMode": "auto", "justifyMode": "auto", "orientation": "auto" },
+      "targets": [ { "refId": "A", "expr": "sum(node_systemd_unit_state{instance=~\"$instance\",state=\"failed\"}) or vector(0)" } ]
+    },
+    {
+      "id": 2,
+      "type": "stat",
+      "title": "Failed unit",
+      "gridPos": { "h": 4, "w": 18, "x": 6, "y": 0 },
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "fieldConfig": { "defaults": { "unit": "none", "color": { "mode": "fixed" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 1 } ] }, "mappings": [ { "type": "value", "options": { "1": { "text": "FAILED", "color": "red" } } } ] }, "overrides": [] },
+      "options": { "reduceOptions": { "values": false, "calcs": ["lastNotNull"], "fields": "" }, "colorMode": "background", "graphMode": "none", "textMode": "auto", "justifyMode": "auto", "orientation": "auto" },
+      "targets": [ { "refId": "A", "expr": "node_systemd_unit_state{instance=~\"$instance\",state=\"failed\"} == 1", "legendFormat": "{{name}}" } ]
+    },
+    {
+      "id": 3,
+      "type": "stat",
+      "title": "Timer last trigger age",
+      "gridPos": { "h": 6, "w": 24, "x": 0, "y": 4 },
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "fieldConfig": { "defaults": { "unit": "dtdhms", "color": { "mode": "thresholds" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "orange", "value": 172800 }, { "color": "red", "value": 1209600 } ] } }, "overrides": [] },
+      "options": { "reduceOptions": { "values": false, "calcs": ["lastNotNull"], "fields": "" }, "colorMode": "background", "graphMode": "none", "textMode": "auto", "justifyMode": "auto", "orientation": "auto" },
+      "targets": [ { "refId": "A", "expr": "time() - node_systemd_timer_last_trigger_seconds{instance=~\"$instance\"}", "legendFormat": "{{name}}" } ]
+    },
+    {
+      "id": 4,
+      "type": "timeseries",
+      "title": "Unit states",
+      "gridPos": { "h": 8, "w": 24, "x": 0, "y": 10 },
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "fieldConfig": { "defaults": { "unit": "short", "color": { "mode": "palette-classic" }, "custom": { "fillOpacity": 20, "lineWidth": 1, "drawStyle": "line", "showPoints": "never" } }, "overrides": [] },
+      "targets": [ { "refId": "A", "expr": "sum by (state) (node_systemd_unit_state{instance=~\"$instance\"})", "legendFormat": "{{state}}" } ]
+    }
+  ]
+}

roles/grafana/tasks/main.yml

@@ -248,6 +248,18 @@
     - wireguard-dashboard.json
   notify: Restart grafana
 
+- name: Deploy backup and systemd dashboards
+  ansible.builtin.copy:
+    src: "{{ item }}"
+    dest: "{{ grafana_provisioning_path }}/dashboards/{{ item }}"
+    owner: "{{ grafana_user }}"
+    group: "{{ grafana_group }}"
+    mode: "0644"
+  loop:
+    - backup-dashboard.json
+    - systemd-dashboard.json
+  notify: Restart grafana
+
 # Service management
 - name: Enable and start Grafana service
   ansible.builtin.systemd:

roles/node_exporter/README.md

@@ -21,6 +21,19 @@ Keep localhost-only — Prometheus scrapes it locally on the same host. To add
 collectors (e.g. `--collector.systemd`, `--collector.textfile.directory`), extend
 `node_exporter_listen_address` or override the args in host_vars.
 
+## Collectors
+
+The role enables, on top of the defaults:
+
+- `--collector.zfs` — ZFS pool/dataset metrics.
+- `--collector.systemd` — unit state (`node_systemd_unit_state`), restart
+  counts (`node_systemd_restart_count_total`) and timer trigger times
+  (`node_systemd_timer_last_trigger_seconds` / `..._next_trigger_seconds`).
+  Powers the systemd dashboard (failed units, timers that stopped firing).
+- `--collector.textfile.directory={{ node_exporter_textfile_dir }}` — re-exposes
+  `*.prom` files dropped into that directory. The backup role writes
+  `backup_*` metrics there; other cron/timer jobs can do the same.
+
 ## Pairing with Prometheus
 
 ```yaml

roles/node_exporter/defaults/main.yml

@@ -4,3 +4,8 @@
 # Address the exporter listens on. Keep localhost-only — Prometheus scrapes it
 # locally on the same host; no need to expose on the network.
 node_exporter_listen_address: "127.0.0.1:9100"
+
+# Directory the textfile collector reads `*.prom` files from. Jobs (e.g. the
+# backup role) drop metrics here; node_exporter re-exposes them to Prometheus.
+# Must match backup_metrics_dir in roles/backup/defaults/main.yml.
+node_exporter_textfile_dir: /var/lib/node_exporter/textfile_collector

roles/node_exporter/tasks/main.yml

@@ -10,6 +10,14 @@
     name: "{{ node_exporter_package }}"
     state: present
 
+- name: Create textfile collector directory
+  ansible.builtin.file:
+    path: "{{ node_exporter_textfile_dir }}"
+    state: directory
+    owner: root
+    group: root
+    mode: "0755"
+
 - name: Deploy node_exporter configuration
   ansible.builtin.template:
     src: env.conf.j2

roles/node_exporter/templates/env.conf.j2

@@ -1,2 +1,2 @@
 # Managed by Ansible - DO NOT EDIT MANUALLY
-{{ node_exporter_args_var }}="--web.listen-address={{ node_exporter_listen_address }} --collector.zfs"
+{{ node_exporter_args_var }}="--web.listen-address={{ node_exporter_listen_address }} --collector.zfs --collector.systemd --collector.textfile.directory={{ node_exporter_textfile_dir }}"

roles/prometheus/README.md

@@ -49,6 +49,21 @@ prometheus_nginx_ssl_certificate: /path/to/cert.pem
 prometheus_nginx_ssl_certificate_key: /path/to/key.pem
 ```
 
+## Alert Rules
+
+The role deploys two rule files into `/etc/prometheus/rules/` (loaded via the
+`prometheus_rule_files` glob):
+
+- `backup-alerts.yml` — `BackupStale` (no successful backup in 14 days) and
+  `BackupTimerSilent` (a `backup@*.timer` that stopped firing). Metrics come
+  from the backup role via node_exporter's textfile collector.
+- `systemd-alerts.yml` — `SystemdUnitFailed`, fired when any unit is in the
+  failed state.
+
+There is no Alertmanager in this setup: rules evaluate into the `ALERTS`
+series (visible in Grafana) but do not page. Push notifications are handled
+per-service via ntfy (see each role's own notify config).
+
 ## Usage
 
 ```yaml

roles/prometheus/defaults/main.yml

@@ -43,10 +43,11 @@ prometheus_scrape_configs:
 #       - targets: ['localhost:9093']
 prometheus_alertmanager_configs: []
 
-# Rule files (optional)
+# Rule files (optional). The rules directory is always created; drop .yml rule
-# prometheus_rule_files:
+# files into /etc/prometheus/rules/ and they are picked up by this glob. The
-#   - '/etc/prometheus/rules/*.yml'
+# role deploys backup-alerts.yml and systemd-alerts.yml there by default.
-prometheus_rule_files: []
+prometheus_rule_files:
+  - '/etc/prometheus/rules/*.yml'
 
 # Nginx reverse proxy (optional)
 prometheus_nginx_enabled: false

roles/prometheus/files/backup-alerts.yml

@@ -0,0 +1,31 @@
+# Backup monitoring alerts. Managed by Ansible (roles/prometheus).
+#
+# Metrics are emitted by the backup role (roles/backup) into node_exporter's
+# textfile collector. The default backup schedule is weekly, so a successful
+# backup is expected at least once a week; alert only when nothing succeeded
+# for twice that. Tune the threshold if backup_schedule changes.
+groups:
+  - name: backup
+    rules:
+      - alert: BackupStale
+        expr: time() - backup_last_success_timestamp_seconds > 14 * 24 * 3600
+        for: 1h
+        labels:
+          severity: warning
+        annotations:
+          summary: "Backup stale ({{ $labels.backup_job }} / {{ $labels.repo }})"
+          description: >-
+            Job '{{ $labels.backup_job }}' has not backed up repository
+            '{{ $labels.repo }}' successfully on {{ $labels.instance }} in over
+            14 days.
+
+      - alert: BackupTimerSilent
+        expr: time() - node_systemd_timer_last_trigger_seconds{name=~"backup@.*"} > 14 * 24 * 3600
+        for: 1h
+        labels:
+          severity: warning
+        annotations:
+          summary: "Backup timer {{ $labels.name }} has not fired on {{ $labels.instance }}"
+          description: >-
+            Timer {{ $labels.name }} last triggered {{ $value | humanizeDuration }}
+            ago.

roles/prometheus/files/systemd-alerts.yml

@@ -0,0 +1,17 @@
+# Systemd state alerts. Managed by Ansible (roles/prometheus).
+#
+# node_exporter's systemd collector exposes node_systemd_unit_state. A unit in
+# the failed state is exactly the kind of thing that needs paging at 03:00.
+groups:
+  - name: systemd
+    rules:
+      - alert: SystemdUnitFailed
+        expr: node_systemd_unit_state{state="failed"} == 1
+        for: 5m
+        labels:
+          severity: critical
+        annotations:
+          summary: "systemd unit {{ $labels.name }} is in failed state ({{ $labels.instance }})"
+          description: >-
+            Unit {{ $labels.name }} (type {{ $labels.type }}) has been failed
+            for more than 5 minutes on {{ $labels.instance }}.

roles/prometheus/tasks/main.yml

@@ -35,6 +35,18 @@
       mode: "0755"
   when: prometheus_rule_files | length > 0
 
+- name: Deploy backup and systemd alert rules
+  ansible.builtin.copy:
+      src: "{{ item }}"
+      dest: "{{ prometheus_config_dir }}/rules/{{ item }}"
+      owner: "{{ prometheus_user }}"
+      group: "{{ prometheus_group }}"
+      mode: "0644"
+  loop:
+      - backup-alerts.yml
+      - systemd-alerts.yml
+  notify: Restart Prometheus
+
 - name: Deploy Prometheus configuration
   ansible.builtin.template:
       src: prometheus.yml.j2

roles/prometheus/templates/prometheus.yml.j2

@@ -44,7 +44,7 @@ scrape_configs:
 {% for param_key, param_values in scrape_config.params.items() %}
       {{ param_key }}:
 {% for param_value in param_values %}
-        - {{ param_value }}
+        - {{ param_value | to_json }}
 {% endfor %}
 {% endfor %}
 {% endif %}