diff --git a/roles/backup/README.md b/roles/backup/README.md index 60ef427..40b5ee6 100644 --- a/roles/backup/README.md +++ b/roles/backup/README.md @@ -258,6 +258,24 @@ Every snapshot is a logically complete view; "incremental" describes only how it The role always excludes restic's own cache, and the staging directories of *other* jobs — otherwise a job backing up `/var` would sweep up both. +### Monitoring + +After each run the script writes Prometheus metrics into node_exporter's +textfile directory (`backup_metrics_dir`), re-exposed by the exporter: + +``` +backup_last_success_timestamp_seconds{backup_job="ntfy", repo="next_s3"} +backup_last_run_timestamp_seconds{backup_job="ntfy", repo="next_s3", result="success|fail|skip"} +``` + +This is what turns "silence" into a signal: ntfy alerts on failures, but a +timer that stopped firing (or a job that's been failing for a week) looks the +same as success unless you track `backup_last_success_timestamp_seconds`. The +Grafana "Backups - Restic" dashboard and the `BackupStale` / +`BackupTimerSilent` Prometheus rules (deployed by the prometheus role) consume +these. Metrics are written only when `backup_metrics_dir` exists, i.e. when +node_exporter is deployed on the host. + ### Failure notification Each repository can carry a `notify` channel: `{ topic, server, token, notify_success }`. A repository with `notify.topic` alerts on that destination's own result — an offsite failure is `urgent`, a missing USB disk is a low-priority `warning`, and a destination without `notify` is silent. Success alerts are opt-in per channel (`notify_success`), off by default because routine chatter trains you to ignore the failure alerts. @@ -270,4 +288,3 @@ Every configured channel gets a test notification during the play. ntfy runs wit - **No automated restore test.** An untested backup is a hypothesis. Restore one by hand, on a schedule. - **No append-only protection.** The host holds credentials that can delete its own snapshots, so an attacker with root can destroy the backups too. Mitigate with object versioning or an object-lock policy on the bucket, which is outside this role. -- **No metrics.** Failures are visible through `systemctl --failed`, the journal, and ntfy. diff --git a/roles/backup/defaults/main.yml b/roles/backup/defaults/main.yml index 07b6ec6..5c3ec9a 100644 --- a/roles/backup/defaults/main.yml +++ b/roles/backup/defaults/main.yml @@ -13,6 +13,13 @@ backup_dumps_dir: /var/lib/backup/dumps # be excluded from snapshots rather than silently backing itself up. backup_cache_dir: /var/cache/restic +# Directory node_exporter's textfile collector reads `*.prom` files from. The +# backup script drops one file per job here on every run, so Prometheus can +# track last-success timestamps and alert when backups go silent. Must match +# node_exporter_textfile_dir in roles/node_exporter/defaults/main.yml. Metrics +# are only written when the directory exists (i.e. node_exporter is deployed). +backup_metrics_dir: /var/lib/node_exporter/textfile_collector + # --------------------------------------------------------------------------- # Schedule (systemd OnCalendar syntax) - per-job `schedule` overrides this # --------------------------------------------------------------------------- diff --git a/roles/backup/templates/backup.sh.j2 b/roles/backup/templates/backup.sh.j2 index 519544b..f17c638 100644 --- a/roles/backup/templates/backup.sh.j2 +++ b/roles/backup/templates/backup.sh.j2 @@ -369,6 +369,62 @@ run_repository() { return "$rc" } +# --------------------------------------------------------------------------- +# Metrics +# --------------------------------------------------------------------------- + +# Escape a value for use as a Prometheus label value. +escape_label() { + local v="$1" + v="${v//\\/\\\\}" + v="${v//\"/\\\"}" + printf '%s' "$v" +} + +# Write the job's result as Prometheus textfile metrics, consumed by +# node_exporter's textfile collector. Only emitted when that directory exists +# (i.e. when the node_exporter role is deployed on this host). +# +# One file per job, written atomically (tmp + mv) so the collector never reads +# a half-written file. The `backup_job` label is used rather than `job` to +# avoid colliding with Prometheus's own scrape-job label. +emit_metrics() { + local metrics_dir="{{ backup_metrics_dir }}" + [[ -d "$metrics_dir" ]] || return 0 + + local now_ts + now_ts="$(date +%s)" + + local job_label repo_label + job_label="$(escape_label "$JOB")" + + local out="$metrics_dir/backup-$JOB.prom" + local tmp="$out.tmp" + + { + echo "# HELP backup_last_success_timestamp_seconds Unix timestamp of the last successful backup to this repository." + echo "# TYPE backup_last_success_timestamp_seconds gauge" + echo "# HELP backup_last_run_timestamp_seconds Unix timestamp of the last run against this repository, by result." + echo "# TYPE backup_last_run_timestamp_seconds gauge" + + local repo + for repo in ${succeeded[@]+"${succeeded[@]}"}; do + repo_label="$(escape_label "$repo")" + echo "backup_last_success_timestamp_seconds{backup_job=\"$job_label\",repo=\"$repo_label\"} $now_ts" + echo "backup_last_run_timestamp_seconds{backup_job=\"$job_label\",repo=\"$repo_label\",result=\"success\"} $now_ts" + done + for repo in ${failed[@]+"${failed[@]}"}; do + repo_label="$(escape_label "$repo")" + echo "backup_last_run_timestamp_seconds{backup_job=\"$job_label\",repo=\"$repo_label\",result=\"fail\"} $now_ts" + done + for repo in ${skipped[@]+"${skipped[@]}"}; do + repo_label="$(escape_label "$repo")" + echo "backup_last_run_timestamp_seconds{backup_job=\"$job_label\",repo=\"$repo_label\",result=\"skip\"} $now_ts" + done + } > "$tmp" + mv "$tmp" "$out" +} + # --------------------------------------------------------------------------- # Main # --------------------------------------------------------------------------- @@ -438,6 +494,8 @@ main() { log "=== Backup job '$JOB' complete ===" log "$summary" + emit_metrics + # Notifications are per-repository above; the exit code still tells systemd # whether any destination failed. if [[ "$n_failed" -gt 0 ]]; then diff --git a/roles/grafana/files/backup-dashboard.json b/roles/grafana/files/backup-dashboard.json new file mode 100644 index 0000000..42b987e --- /dev/null +++ b/roles/grafana/files/backup-dashboard.json @@ -0,0 +1,61 @@ +{ + "title": "Backups - Restic", + "uid": "backup-restic", + "tags": ["backup", "restic"], + "schemaVersion": 39, + "version": 1, + "refresh": "1m", + "time": { "from": "now-30d", "to": "now" }, + "timezone": "browser", + "editable": true, + "graphTooltip": 1, + "templating": { + "list": [ + { + "name": "instance", + "label": "Host", + "type": "query", + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "query": { "query": "query_result(count by (instance) (backup_last_success_timestamp_seconds))", "refId": "A" }, + "regex": "/.*instance=\"([^\"]+)\".*/", + "refresh": 2, + "multi": true, + "includeAll": true, + "allValue": ".*", + "sort": 1, + "current": { "selected": true, "text": "All", "value": "$__all" } + } + ] + }, + "panels": [ + { + "id": 1, + "type": "stat", + "title": "Last backup age", + "gridPos": { "h": 6, "w": 12, "x": 0, "y": 0 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { "defaults": { "unit": "dtdhms", "color": { "mode": "thresholds" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "orange", "value": 172800 }, { "color": "red", "value": 1209600 } ] } }, "overrides": [] }, + "options": { "reduceOptions": { "values": false, "calcs": ["lastNotNull"], "fields": "" }, "colorMode": "background", "graphMode": "none", "textMode": "auto", "justifyMode": "auto", "orientation": "auto" }, + "targets": [ { "refId": "A", "expr": "time() - backup_last_success_timestamp_seconds{instance=~\"$instance\"}", "legendFormat": "{{backup_job}} / {{repo}}" } ] + }, + { + "id": 2, + "type": "stat", + "title": "Failing backups (14d)", + "gridPos": { "h": 6, "w": 12, "x": 12, "y": 0 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { "defaults": { "unit": "short", "color": { "mode": "thresholds" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 1 } ] } }, "overrides": [] }, + "options": { "reduceOptions": { "values": false, "calcs": ["lastNotNull"], "fields": "" }, "colorMode": "background", "graphMode": "none", "textMode": "auto", "justifyMode": "auto", "orientation": "auto" }, + "targets": [ { "refId": "A", "expr": "count(backup_last_run_timestamp_seconds{instance=~\"$instance\",result=\"fail\"} > (time() - 14 * 86400)) or vector(0)" } ] + }, + { + "id": 3, + "type": "timeseries", + "title": "Backup success history", + "gridPos": { "h": 8, "w": 24, "x": 0, "y": 6 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { "defaults": { "unit": "dateTimeAsIso", "color": { "mode": "palette-classic" }, "custom": { "fillOpacity": 0, "lineWidth": 1, "drawStyle": "line", "showPoints": "always", "pointSize": 5 } }, "overrides": [] }, + "targets": [ { "refId": "A", "expr": "backup_last_success_timestamp_seconds{instance=~\"$instance\"}", "legendFormat": "{{backup_job}} / {{repo}}" } ] + } + ] +} diff --git a/roles/grafana/files/systemd-dashboard.json b/roles/grafana/files/systemd-dashboard.json new file mode 100644 index 0000000..15268dc --- /dev/null +++ b/roles/grafana/files/systemd-dashboard.json @@ -0,0 +1,71 @@ +{ + "title": "Systemd", + "uid": "systemd", + "tags": ["systemd", "services", "timers"], + "schemaVersion": 39, + "version": 1, + "refresh": "1m", + "time": { "from": "now-7d", "to": "now" }, + "timezone": "browser", + "editable": true, + "graphTooltip": 1, + "templating": { + "list": [ + { + "name": "instance", + "label": "Host", + "type": "query", + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "query": { "query": "query_result(count by (instance) (node_systemd_unit_state))", "refId": "A" }, + "regex": "/.*instance=\"([^\"]+)\".*/", + "refresh": 2, + "multi": true, + "includeAll": true, + "allValue": ".*", + "sort": 1, + "current": { "selected": true, "text": "All", "value": "$__all" } + } + ] + }, + "panels": [ + { + "id": 1, + "type": "stat", + "title": "Failed units", + "gridPos": { "h": 4, "w": 6, "x": 0, "y": 0 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { "defaults": { "unit": "short", "color": { "mode": "thresholds" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 1 } ] } }, "overrides": [] }, + "options": { "reduceOptions": { "values": false, "calcs": ["lastNotNull"], "fields": "" }, "colorMode": "background", "graphMode": "none", "textMode": "auto", "justifyMode": "auto", "orientation": "auto" }, + "targets": [ { "refId": "A", "expr": "sum(node_systemd_unit_state{instance=~\"$instance\",state=\"failed\"}) or vector(0)" } ] + }, + { + "id": 2, + "type": "stat", + "title": "Failed unit", + "gridPos": { "h": 4, "w": 18, "x": 6, "y": 0 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { "defaults": { "unit": "none", "color": { "mode": "fixed" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 1 } ] }, "mappings": [ { "type": "value", "options": { "1": { "text": "FAILED", "color": "red" } } } ] }, "overrides": [] }, + "options": { "reduceOptions": { "values": false, "calcs": ["lastNotNull"], "fields": "" }, "colorMode": "background", "graphMode": "none", "textMode": "auto", "justifyMode": "auto", "orientation": "auto" }, + "targets": [ { "refId": "A", "expr": "node_systemd_unit_state{instance=~\"$instance\",state=\"failed\"} == 1", "legendFormat": "{{name}}" } ] + }, + { + "id": 3, + "type": "stat", + "title": "Timer last trigger age", + "gridPos": { "h": 6, "w": 24, "x": 0, "y": 4 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { "defaults": { "unit": "dtdhms", "color": { "mode": "thresholds" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "orange", "value": 172800 }, { "color": "red", "value": 1209600 } ] } }, "overrides": [] }, + "options": { "reduceOptions": { "values": false, "calcs": ["lastNotNull"], "fields": "" }, "colorMode": "background", "graphMode": "none", "textMode": "auto", "justifyMode": "auto", "orientation": "auto" }, + "targets": [ { "refId": "A", "expr": "time() - node_systemd_timer_last_trigger_seconds{instance=~\"$instance\"}", "legendFormat": "{{name}}" } ] + }, + { + "id": 4, + "type": "timeseries", + "title": "Unit states", + "gridPos": { "h": 8, "w": 24, "x": 0, "y": 10 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { "defaults": { "unit": "short", "color": { "mode": "palette-classic" }, "custom": { "fillOpacity": 20, "lineWidth": 1, "drawStyle": "line", "showPoints": "never" } }, "overrides": [] }, + "targets": [ { "refId": "A", "expr": "sum by (state) (node_systemd_unit_state{instance=~\"$instance\"})", "legendFormat": "{{state}}" } ] + } + ] +} diff --git a/roles/grafana/tasks/main.yml b/roles/grafana/tasks/main.yml index f7b86cb..c625218 100644 --- a/roles/grafana/tasks/main.yml +++ b/roles/grafana/tasks/main.yml @@ -248,6 +248,18 @@ - wireguard-dashboard.json notify: Restart grafana +- name: Deploy backup and systemd dashboards + ansible.builtin.copy: + src: "{{ item }}" + dest: "{{ grafana_provisioning_path }}/dashboards/{{ item }}" + owner: "{{ grafana_user }}" + group: "{{ grafana_group }}" + mode: "0644" + loop: + - backup-dashboard.json + - systemd-dashboard.json + notify: Restart grafana + # Service management - name: Enable and start Grafana service ansible.builtin.systemd: diff --git a/roles/node_exporter/README.md b/roles/node_exporter/README.md index 89254c8..413fbeb 100644 --- a/roles/node_exporter/README.md +++ b/roles/node_exporter/README.md @@ -21,6 +21,19 @@ Keep localhost-only — Prometheus scrapes it locally on the same host. To add collectors (e.g. `--collector.systemd`, `--collector.textfile.directory`), extend `node_exporter_listen_address` or override the args in host_vars. +## Collectors + +The role enables, on top of the defaults: + +- `--collector.zfs` — ZFS pool/dataset metrics. +- `--collector.systemd` — unit state (`node_systemd_unit_state`), restart + counts (`node_systemd_restart_count_total`) and timer trigger times + (`node_systemd_timer_last_trigger_seconds` / `..._next_trigger_seconds`). + Powers the systemd dashboard (failed units, timers that stopped firing). +- `--collector.textfile.directory={{ node_exporter_textfile_dir }}` — re-exposes + `*.prom` files dropped into that directory. The backup role writes + `backup_*` metrics there; other cron/timer jobs can do the same. + ## Pairing with Prometheus ```yaml diff --git a/roles/node_exporter/defaults/main.yml b/roles/node_exporter/defaults/main.yml index 8be3366..697a8cd 100644 --- a/roles/node_exporter/defaults/main.yml +++ b/roles/node_exporter/defaults/main.yml @@ -4,3 +4,8 @@ # Address the exporter listens on. Keep localhost-only — Prometheus scrapes it # locally on the same host; no need to expose on the network. node_exporter_listen_address: "127.0.0.1:9100" + +# Directory the textfile collector reads `*.prom` files from. Jobs (e.g. the +# backup role) drop metrics here; node_exporter re-exposes them to Prometheus. +# Must match backup_metrics_dir in roles/backup/defaults/main.yml. +node_exporter_textfile_dir: /var/lib/node_exporter/textfile_collector diff --git a/roles/node_exporter/tasks/main.yml b/roles/node_exporter/tasks/main.yml index 490be02..f8f7556 100644 --- a/roles/node_exporter/tasks/main.yml +++ b/roles/node_exporter/tasks/main.yml @@ -10,6 +10,14 @@ name: "{{ node_exporter_package }}" state: present +- name: Create textfile collector directory + ansible.builtin.file: + path: "{{ node_exporter_textfile_dir }}" + state: directory + owner: root + group: root + mode: "0755" + - name: Deploy node_exporter configuration ansible.builtin.template: src: env.conf.j2 diff --git a/roles/node_exporter/templates/env.conf.j2 b/roles/node_exporter/templates/env.conf.j2 index 624ab67..24a0671 100644 --- a/roles/node_exporter/templates/env.conf.j2 +++ b/roles/node_exporter/templates/env.conf.j2 @@ -1,2 +1,2 @@ # Managed by Ansible - DO NOT EDIT MANUALLY -{{ node_exporter_args_var }}="--web.listen-address={{ node_exporter_listen_address }} --collector.zfs" +{{ node_exporter_args_var }}="--web.listen-address={{ node_exporter_listen_address }} --collector.zfs --collector.systemd --collector.textfile.directory={{ node_exporter_textfile_dir }}" diff --git a/roles/prometheus/README.md b/roles/prometheus/README.md index 1bca79a..bb2158a 100644 --- a/roles/prometheus/README.md +++ b/roles/prometheus/README.md @@ -49,6 +49,21 @@ prometheus_nginx_ssl_certificate: /path/to/cert.pem prometheus_nginx_ssl_certificate_key: /path/to/key.pem ``` +## Alert Rules + +The role deploys two rule files into `/etc/prometheus/rules/` (loaded via the +`prometheus_rule_files` glob): + +- `backup-alerts.yml` — `BackupStale` (no successful backup in 14 days) and + `BackupTimerSilent` (a `backup@*.timer` that stopped firing). Metrics come + from the backup role via node_exporter's textfile collector. +- `systemd-alerts.yml` — `SystemdUnitFailed`, fired when any unit is in the + failed state. + +There is no Alertmanager in this setup: rules evaluate into the `ALERTS` +series (visible in Grafana) but do not page. Push notifications are handled +per-service via ntfy (see each role's own notify config). + ## Usage ```yaml diff --git a/roles/prometheus/defaults/main.yml b/roles/prometheus/defaults/main.yml index 00fc338..f992da7 100644 --- a/roles/prometheus/defaults/main.yml +++ b/roles/prometheus/defaults/main.yml @@ -43,10 +43,11 @@ prometheus_scrape_configs: # - targets: ['localhost:9093'] prometheus_alertmanager_configs: [] -# Rule files (optional) -# prometheus_rule_files: -# - '/etc/prometheus/rules/*.yml' -prometheus_rule_files: [] +# Rule files (optional). The rules directory is always created; drop .yml rule +# files into /etc/prometheus/rules/ and they are picked up by this glob. The +# role deploys backup-alerts.yml and systemd-alerts.yml there by default. +prometheus_rule_files: + - '/etc/prometheus/rules/*.yml' # Nginx reverse proxy (optional) prometheus_nginx_enabled: false diff --git a/roles/prometheus/files/backup-alerts.yml b/roles/prometheus/files/backup-alerts.yml new file mode 100644 index 0000000..3ecff93 --- /dev/null +++ b/roles/prometheus/files/backup-alerts.yml @@ -0,0 +1,31 @@ +# Backup monitoring alerts. Managed by Ansible (roles/prometheus). +# +# Metrics are emitted by the backup role (roles/backup) into node_exporter's +# textfile collector. The default backup schedule is weekly, so a successful +# backup is expected at least once a week; alert only when nothing succeeded +# for twice that. Tune the threshold if backup_schedule changes. +groups: + - name: backup + rules: + - alert: BackupStale + expr: time() - backup_last_success_timestamp_seconds > 14 * 24 * 3600 + for: 1h + labels: + severity: warning + annotations: + summary: "Backup stale ({{ $labels.backup_job }} / {{ $labels.repo }})" + description: >- + Job '{{ $labels.backup_job }}' has not backed up repository + '{{ $labels.repo }}' successfully on {{ $labels.instance }} in over + 14 days. + + - alert: BackupTimerSilent + expr: time() - node_systemd_timer_last_trigger_seconds{name=~"backup@.*"} > 14 * 24 * 3600 + for: 1h + labels: + severity: warning + annotations: + summary: "Backup timer {{ $labels.name }} has not fired on {{ $labels.instance }}" + description: >- + Timer {{ $labels.name }} last triggered {{ $value | humanizeDuration }} + ago. diff --git a/roles/prometheus/files/systemd-alerts.yml b/roles/prometheus/files/systemd-alerts.yml new file mode 100644 index 0000000..7f9d7eb --- /dev/null +++ b/roles/prometheus/files/systemd-alerts.yml @@ -0,0 +1,17 @@ +# Systemd state alerts. Managed by Ansible (roles/prometheus). +# +# node_exporter's systemd collector exposes node_systemd_unit_state. A unit in +# the failed state is exactly the kind of thing that needs paging at 03:00. +groups: + - name: systemd + rules: + - alert: SystemdUnitFailed + expr: node_systemd_unit_state{state="failed"} == 1 + for: 5m + labels: + severity: critical + annotations: + summary: "systemd unit {{ $labels.name }} is in failed state ({{ $labels.instance }})" + description: >- + Unit {{ $labels.name }} (type {{ $labels.type }}) has been failed + for more than 5 minutes on {{ $labels.instance }}. diff --git a/roles/prometheus/tasks/main.yml b/roles/prometheus/tasks/main.yml index 0dd5b83..31c33f7 100644 --- a/roles/prometheus/tasks/main.yml +++ b/roles/prometheus/tasks/main.yml @@ -35,6 +35,18 @@ mode: "0755" when: prometheus_rule_files | length > 0 +- name: Deploy backup and systemd alert rules + ansible.builtin.copy: + src: "{{ item }}" + dest: "{{ prometheus_config_dir }}/rules/{{ item }}" + owner: "{{ prometheus_user }}" + group: "{{ prometheus_group }}" + mode: "0644" + loop: + - backup-alerts.yml + - systemd-alerts.yml + notify: Restart Prometheus + - name: Deploy Prometheus configuration ansible.builtin.template: src: prometheus.yml.j2 diff --git a/roles/prometheus/templates/prometheus.yml.j2 b/roles/prometheus/templates/prometheus.yml.j2 index e3bde05..00b99d5 100644 --- a/roles/prometheus/templates/prometheus.yml.j2 +++ b/roles/prometheus/templates/prometheus.yml.j2 @@ -44,7 +44,7 @@ scrape_configs: {% for param_key, param_values in scrape_config.params.items() %} {{ param_key }}: {% for param_value in param_values %} - - {{ param_value }} + - {{ param_value | to_json }} {% endfor %} {% endfor %} {% endif %}