feat(monitoring): add backup and systemd metrics, alerts, and dashboards

- backup role emits last-success/run metrics into node_exporter's textfile collector
- node_exporter enables systemd and textfile collectors
- prometheus deploys BackupStale/BackupTimerSilent/SystemdUnitFailed alert rules
- grafana gains Backups - Restic and Systemd dashboards
- fix prometheus.yml params rendering (quote values so {job=...} stays a string)

roles/prometheus/README.md

@@ -49,6 +49,21 @@ prometheus_nginx_ssl_certificate: /path/to/cert.pem
 prometheus_nginx_ssl_certificate_key: /path/to/key.pem
 ```
 
+## Alert Rules
+
+The role deploys two rule files into `/etc/prometheus/rules/` (loaded via the
+`prometheus_rule_files` glob):
+
+- `backup-alerts.yml` — `BackupStale` (no successful backup in 14 days) and
+  `BackupTimerSilent` (a `backup@*.timer` that stopped firing). Metrics come
+  from the backup role via node_exporter's textfile collector.
+- `systemd-alerts.yml` — `SystemdUnitFailed`, fired when any unit is in the
+  failed state.
+
+There is no Alertmanager in this setup: rules evaluate into the `ALERTS`
+series (visible in Grafana) but do not page. Push notifications are handled
+per-service via ntfy (see each role's own notify config).
+
 ## Usage
 
 ```yaml

roles/prometheus/defaults/main.yml

@@ -43,10 +43,11 @@ prometheus_scrape_configs:
 #       - targets: ['localhost:9093']
 prometheus_alertmanager_configs: []
 
-# Rule files (optional)
-# prometheus_rule_files:
-#   - '/etc/prometheus/rules/*.yml'
-prometheus_rule_files: []
+# Rule files (optional). The rules directory is always created; drop .yml rule
+# files into /etc/prometheus/rules/ and they are picked up by this glob. The
+# role deploys backup-alerts.yml and systemd-alerts.yml there by default.
+prometheus_rule_files:
+  - '/etc/prometheus/rules/*.yml'
 
 # Nginx reverse proxy (optional)
 prometheus_nginx_enabled: false

roles/prometheus/files/backup-alerts.yml

@@ -0,0 +1,31 @@
+# Backup monitoring alerts. Managed by Ansible (roles/prometheus).
+#
+# Metrics are emitted by the backup role (roles/backup) into node_exporter's
+# textfile collector. The default backup schedule is weekly, so a successful
+# backup is expected at least once a week; alert only when nothing succeeded
+# for twice that. Tune the threshold if backup_schedule changes.
+groups:
+  - name: backup
+    rules:
+      - alert: BackupStale
+        expr: time() - backup_last_success_timestamp_seconds > 14 * 24 * 3600
+        for: 1h
+        labels:
+          severity: warning
+        annotations:
+          summary: "Backup stale ({{ $labels.backup_job }} / {{ $labels.repo }})"
+          description: >-
+            Job '{{ $labels.backup_job }}' has not backed up repository
+            '{{ $labels.repo }}' successfully on {{ $labels.instance }} in over
+            14 days.
+
+      - alert: BackupTimerSilent
+        expr: time() - node_systemd_timer_last_trigger_seconds{name=~"backup@.*"} > 14 * 24 * 3600
+        for: 1h
+        labels:
+          severity: warning
+        annotations:
+          summary: "Backup timer {{ $labels.name }} has not fired on {{ $labels.instance }}"
+          description: >-
+            Timer {{ $labels.name }} last triggered {{ $value | humanizeDuration }}
+            ago.

roles/prometheus/files/systemd-alerts.yml

@@ -0,0 +1,17 @@
+# Systemd state alerts. Managed by Ansible (roles/prometheus).
+#
+# node_exporter's systemd collector exposes node_systemd_unit_state. A unit in
+# the failed state is exactly the kind of thing that needs paging at 03:00.
+groups:
+  - name: systemd
+    rules:
+      - alert: SystemdUnitFailed
+        expr: node_systemd_unit_state{state="failed"} == 1
+        for: 5m
+        labels:
+          severity: critical
+        annotations:
+          summary: "systemd unit {{ $labels.name }} is in failed state ({{ $labels.instance }})"
+          description: >-
+            Unit {{ $labels.name }} (type {{ $labels.type }}) has been failed
+            for more than 5 minutes on {{ $labels.instance }}.

roles/prometheus/tasks/main.yml

@@ -35,6 +35,18 @@
       mode: "0755"
   when: prometheus_rule_files | length > 0
 
+- name: Deploy backup and systemd alert rules
+  ansible.builtin.copy:
+      src: "{{ item }}"
+      dest: "{{ prometheus_config_dir }}/rules/{{ item }}"
+      owner: "{{ prometheus_user }}"
+      group: "{{ prometheus_group }}"
+      mode: "0644"
+  loop:
+      - backup-alerts.yml
+      - systemd-alerts.yml
+  notify: Restart Prometheus
+
 - name: Deploy Prometheus configuration
   ansible.builtin.template:
       src: prometheus.yml.j2

roles/prometheus/templates/prometheus.yml.j2

@@ -44,7 +44,7 @@ scrape_configs:
 {% for param_key, param_values in scrape_config.params.items() %}
       {{ param_key }}:
 {% for param_value in param_values %}
-        - {{ param_value }}
+        - {{ param_value | to_json }}
 {% endfor %}
 {% endfor %}
 {% endif %}