feat(monitoring): add backup and systemd metrics, alerts, and dashboards
- backup role emits last-success/run metrics into node_exporter's textfile collector
- node_exporter enables systemd and textfile collectors
- prometheus deploys BackupStale/BackupTimerSilent/SystemdUnitFailed alert rules
- grafana gains Backups - Restic and Systemd dashboards
- fix prometheus.yml params rendering (quote values so {job=...} stays a string)
This commit is contained in:
@@ -0,0 +1,31 @@
|
||||
# Backup monitoring alerts. Managed by Ansible (roles/prometheus).
|
||||
#
|
||||
# Metrics are emitted by the backup role (roles/backup) into node_exporter's
|
||||
# textfile collector. The default backup schedule is weekly, so a successful
|
||||
# backup is expected at least once a week; alert only when nothing succeeded
|
||||
# for twice that. Tune the threshold if backup_schedule changes.
|
||||
groups:
|
||||
- name: backup
|
||||
rules:
|
||||
- alert: BackupStale
|
||||
expr: time() - backup_last_success_timestamp_seconds > 14 * 24 * 3600
|
||||
for: 1h
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Backup stale ({{ $labels.backup_job }} / {{ $labels.repo }})"
|
||||
description: >-
|
||||
Job '{{ $labels.backup_job }}' has not backed up repository
|
||||
'{{ $labels.repo }}' successfully on {{ $labels.instance }} in over
|
||||
14 days.
|
||||
|
||||
- alert: BackupTimerSilent
|
||||
expr: time() - node_systemd_timer_last_trigger_seconds{name=~"backup@.*"} > 14 * 24 * 3600
|
||||
for: 1h
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Backup timer {{ $labels.name }} has not fired on {{ $labels.instance }}"
|
||||
description: >-
|
||||
Timer {{ $labels.name }} last triggered {{ $value | humanizeDuration }}
|
||||
ago.
|
||||
Reference in New Issue
Block a user