feat(monitoring): add backup and systemd metrics, alerts, and dashboards

- backup role emits last-success/run metrics into node_exporter's textfile collector
- node_exporter enables systemd and textfile collectors
- prometheus deploys BackupStale/BackupTimerSilent/SystemdUnitFailed alert rules
- grafana gains Backups - Restic and Systemd dashboards
- fix prometheus.yml params rendering (quote values so {job=...} stays a string)

roles/backup/README.md

@@ -258,6 +258,24 @@ Every snapshot is a logically complete view; "incremental" describes only how it
 
 The role always excludes restic's own cache, and the staging directories of *other* jobs — otherwise a job backing up `/var` would sweep up both.
 
+### Monitoring
+
+After each run the script writes Prometheus metrics into node_exporter's
+textfile directory (`backup_metrics_dir`), re-exposed by the exporter:
+
+```
+backup_last_success_timestamp_seconds{backup_job="ntfy", repo="next_s3"}
+backup_last_run_timestamp_seconds{backup_job="ntfy", repo="next_s3", result="success|fail|skip"}
+```
+
+This is what turns "silence" into a signal: ntfy alerts on failures, but a
+timer that stopped firing (or a job that's been failing for a week) looks the
+same as success unless you track `backup_last_success_timestamp_seconds`. The
+Grafana "Backups - Restic" dashboard and the `BackupStale` /
+`BackupTimerSilent` Prometheus rules (deployed by the prometheus role) consume
+these. Metrics are written only when `backup_metrics_dir` exists, i.e. when
+node_exporter is deployed on the host.
+
 ### Failure notification
 
 Each repository can carry a `notify` channel: `{ topic, server, token, notify_success }`. A repository with `notify.topic` alerts on that destination's own result — an offsite failure is `urgent`, a missing USB disk is a low-priority `warning`, and a destination without `notify` is silent. Success alerts are opt-in per channel (`notify_success`), off by default because routine chatter trains you to ignore the failure alerts.
@@ -270,4 +288,3 @@ Every configured channel gets a test notification during the play. ntfy runs wit
 
 - **No automated restore test.** An untested backup is a hypothesis. Restore one by hand, on a schedule.
 - **No append-only protection.** The host holds credentials that can delete its own snapshots, so an attacker with root can destroy the backups too. Mitigate with object versioning or an object-lock policy on the bucket, which is outside this role.
-- **No metrics.** Failures are visible through `systemctl --failed`, the journal, and ntfy.

roles/backup/defaults/main.yml

@@ -13,6 +13,13 @@ backup_dumps_dir: /var/lib/backup/dumps
 # be excluded from snapshots rather than silently backing itself up.
 backup_cache_dir: /var/cache/restic
 
+# Directory node_exporter's textfile collector reads `*.prom` files from. The
+# backup script drops one file per job here on every run, so Prometheus can
+# track last-success timestamps and alert when backups go silent. Must match
+# node_exporter_textfile_dir in roles/node_exporter/defaults/main.yml. Metrics
+# are only written when the directory exists (i.e. node_exporter is deployed).
+backup_metrics_dir: /var/lib/node_exporter/textfile_collector
+
 # ---------------------------------------------------------------------------
 # Schedule (systemd OnCalendar syntax) - per-job `schedule` overrides this
 # ---------------------------------------------------------------------------

roles/backup/templates/backup.sh.j2

@@ -369,6 +369,62 @@ run_repository() {
     return "$rc"
 }
 
+# ---------------------------------------------------------------------------
+# Metrics
+# ---------------------------------------------------------------------------
+
+# Escape a value for use as a Prometheus label value.
+escape_label() {
+    local v="$1"
+    v="${v//\\/\\\\}"
+    v="${v//\"/\\\"}"
+    printf '%s' "$v"
+}
+
+# Write the job's result as Prometheus textfile metrics, consumed by
+# node_exporter's textfile collector. Only emitted when that directory exists
+# (i.e. when the node_exporter role is deployed on this host).
+#
+# One file per job, written atomically (tmp + mv) so the collector never reads
+# a half-written file. The `backup_job` label is used rather than `job` to
+# avoid colliding with Prometheus's own scrape-job label.
+emit_metrics() {
+    local metrics_dir="{{ backup_metrics_dir }}"
+    [[ -d "$metrics_dir" ]] || return 0
+
+    local now_ts
+    now_ts="$(date +%s)"
+
+    local job_label repo_label
+    job_label="$(escape_label "$JOB")"
+
+    local out="$metrics_dir/backup-$JOB.prom"
+    local tmp="$out.tmp"
+
+    {
+        echo "# HELP backup_last_success_timestamp_seconds Unix timestamp of the last successful backup to this repository."
+        echo "# TYPE backup_last_success_timestamp_seconds gauge"
+        echo "# HELP backup_last_run_timestamp_seconds Unix timestamp of the last run against this repository, by result."
+        echo "# TYPE backup_last_run_timestamp_seconds gauge"
+
+        local repo
+        for repo in ${succeeded[@]+"${succeeded[@]}"}; do
+            repo_label="$(escape_label "$repo")"
+            echo "backup_last_success_timestamp_seconds{backup_job=\"$job_label\",repo=\"$repo_label\"} $now_ts"
+            echo "backup_last_run_timestamp_seconds{backup_job=\"$job_label\",repo=\"$repo_label\",result=\"success\"} $now_ts"
+        done
+        for repo in ${failed[@]+"${failed[@]}"}; do
+            repo_label="$(escape_label "$repo")"
+            echo "backup_last_run_timestamp_seconds{backup_job=\"$job_label\",repo=\"$repo_label\",result=\"fail\"} $now_ts"
+        done
+        for repo in ${skipped[@]+"${skipped[@]}"}; do
+            repo_label="$(escape_label "$repo")"
+            echo "backup_last_run_timestamp_seconds{backup_job=\"$job_label\",repo=\"$repo_label\",result=\"skip\"} $now_ts"
+        done
+    } > "$tmp"
+    mv "$tmp" "$out"
+}
+
 # ---------------------------------------------------------------------------
 # Main
 # ---------------------------------------------------------------------------
@@ -438,6 +494,8 @@ main() {
     log "=== Backup job '$JOB' complete ==="
     log "$summary"
 
+    emit_metrics
+
     # Notifications are per-repository above; the exit code still tells systemd
     # whether any destination failed.
     if [[ "$n_failed" -gt 0 ]]; then