feat(monitoring): add backup and systemd metrics, alerts, and dashboards

- backup role emits last-success/run metrics into node_exporter's textfile collector
- node_exporter enables systemd and textfile collectors
- prometheus deploys BackupStale/BackupTimerSilent/SystemdUnitFailed alert rules
- grafana gains Backups - Restic and Systemd dashboards
- fix prometheus.yml params rendering (quote values so {job=...} stays a string)

roles/prometheus/files/backup-alerts.yml

@@ -0,0 +1,31 @@
+# Backup monitoring alerts. Managed by Ansible (roles/prometheus).
+#
+# Metrics are emitted by the backup role (roles/backup) into node_exporter's
+# textfile collector. The default backup schedule is weekly, so a successful
+# backup is expected at least once a week; alert only when nothing succeeded
+# for twice that. Tune the threshold if backup_schedule changes.
+groups:
+  - name: backup
+    rules:
+      - alert: BackupStale
+        expr: time() - backup_last_success_timestamp_seconds > 14 * 24 * 3600
+        for: 1h
+        labels:
+          severity: warning
+        annotations:
+          summary: "Backup stale ({{ $labels.backup_job }} / {{ $labels.repo }})"
+          description: >-
+            Job '{{ $labels.backup_job }}' has not backed up repository
+            '{{ $labels.repo }}' successfully on {{ $labels.instance }} in over
+            14 days.
+
+      - alert: BackupTimerSilent
+        expr: time() - node_systemd_timer_last_trigger_seconds{name=~"backup@.*"} > 14 * 24 * 3600
+        for: 1h
+        labels:
+          severity: warning
+        annotations:
+          summary: "Backup timer {{ $labels.name }} has not fired on {{ $labels.instance }}"
+          description: >-
+            Timer {{ $labels.name }} last triggered {{ $value | humanizeDuration }}
+            ago.

roles/prometheus/files/systemd-alerts.yml

@@ -0,0 +1,17 @@
+# Systemd state alerts. Managed by Ansible (roles/prometheus).
+#
+# node_exporter's systemd collector exposes node_systemd_unit_state. A unit in
+# the failed state is exactly the kind of thing that needs paging at 03:00.
+groups:
+  - name: systemd
+    rules:
+      - alert: SystemdUnitFailed
+        expr: node_systemd_unit_state{state="failed"} == 1
+        for: 5m
+        labels:
+          severity: critical
+        annotations:
+          summary: "systemd unit {{ $labels.name }} is in failed state ({{ $labels.instance }})"
+          description: >-
+            Unit {{ $labels.name }} (type {{ $labels.type }}) has been failed
+            for more than 5 minutes on {{ $labels.instance }}.