feat(monitoring): add backup and systemd metrics, alerts, and dashboards

- backup role emits last-success/run metrics into node_exporter's textfile collector
- node_exporter enables systemd and textfile collectors
- prometheus deploys BackupStale/BackupTimerSilent/SystemdUnitFailed alert rules
- grafana gains Backups - Restic and Systemd dashboards
- fix prometheus.yml params rendering (quote values so {job=...} stays a string)
This commit is contained in:
Clément Désiles
2026-09-09 12:07:50 +02:00
parent 8756eb1183
commit f6c4d36bbe
16 changed files with 335 additions and 7 deletions
+58
View File
@@ -369,6 +369,62 @@ run_repository() {
return "$rc"
}
# ---------------------------------------------------------------------------
# Metrics
# ---------------------------------------------------------------------------
# Escape a value for use as a Prometheus label value.
escape_label() {
local v="$1"
v="${v//\\/\\\\}"
v="${v//\"/\\\"}"
printf '%s' "$v"
}
# Write the job's result as Prometheus textfile metrics, consumed by
# node_exporter's textfile collector. Only emitted when that directory exists
# (i.e. when the node_exporter role is deployed on this host).
#
# One file per job, written atomically (tmp + mv) so the collector never reads
# a half-written file. The `backup_job` label is used rather than `job` to
# avoid colliding with Prometheus's own scrape-job label.
emit_metrics() {
local metrics_dir="{{ backup_metrics_dir }}"
[[ -d "$metrics_dir" ]] || return 0
local now_ts
now_ts="$(date +%s)"
local job_label repo_label
job_label="$(escape_label "$JOB")"
local out="$metrics_dir/backup-$JOB.prom"
local tmp="$out.tmp"
{
echo "# HELP backup_last_success_timestamp_seconds Unix timestamp of the last successful backup to this repository."
echo "# TYPE backup_last_success_timestamp_seconds gauge"
echo "# HELP backup_last_run_timestamp_seconds Unix timestamp of the last run against this repository, by result."
echo "# TYPE backup_last_run_timestamp_seconds gauge"
local repo
for repo in ${succeeded[@]+"${succeeded[@]}"}; do
repo_label="$(escape_label "$repo")"
echo "backup_last_success_timestamp_seconds{backup_job=\"$job_label\",repo=\"$repo_label\"} $now_ts"
echo "backup_last_run_timestamp_seconds{backup_job=\"$job_label\",repo=\"$repo_label\",result=\"success\"} $now_ts"
done
for repo in ${failed[@]+"${failed[@]}"}; do
repo_label="$(escape_label "$repo")"
echo "backup_last_run_timestamp_seconds{backup_job=\"$job_label\",repo=\"$repo_label\",result=\"fail\"} $now_ts"
done
for repo in ${skipped[@]+"${skipped[@]}"}; do
repo_label="$(escape_label "$repo")"
echo "backup_last_run_timestamp_seconds{backup_job=\"$job_label\",repo=\"$repo_label\",result=\"skip\"} $now_ts"
done
} > "$tmp"
mv "$tmp" "$out"
}
# ---------------------------------------------------------------------------
# Main
# ---------------------------------------------------------------------------
@@ -438,6 +494,8 @@ main() {
log "=== Backup job '$JOB' complete ==="
log "$summary"
emit_metrics
# Notifications are per-repository above; the exit code still tells systemd
# whether any destination failed.
if [[ "$n_failed" -gt 0 ]]; then