From be73f3adfa825ee918bf8a9dd134baae029f97d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20D=C3=A9siles?= <1536672+cdesiles@users.noreply.github.com> Date: Fri, 18 Sep 2026 18:32:02 +0200 Subject: [PATCH] feat(grafana): provision disk alerts to ntfy Provision Grafana Unified Alerting as code: ntfy webhook contact point (?template=grafana), notification policy tree, and cumulative disk tiers for HDD temperature (48-60C, <20C) plus ZFS/filesystem capacity. Lower the smartctl exporter interval to 30m for alert responsiveness. --- roles/disks/README.md | 22 +- roles/disks/defaults/main.yml | 2 +- roles/grafana/README.md | 53 ++ roles/grafana/defaults/main.yml | 11 + roles/grafana/files/alerting-policies.yaml | 16 + roles/grafana/files/alerting-rules.yaml | 706 ++++++++++++++++++ roles/grafana/tasks/main.yml | 64 ++ .../templates/alerting-contact-points.yaml.j2 | 15 + 8 files changed, 887 insertions(+), 2 deletions(-) create mode 100644 roles/grafana/files/alerting-policies.yaml create mode 100644 roles/grafana/files/alerting-rules.yaml create mode 100644 roles/grafana/templates/alerting-contact-points.yaml.j2 diff --git a/roles/disks/README.md b/roles/disks/README.md index 58a7064..5561da0 100644 --- a/roles/disks/README.md +++ b/roles/disks/README.md @@ -11,7 +11,7 @@ See [defaults/main.yml](defaults/main.yml). Gated by `smartctl_exporter_enabled`. When true, downloads and runs the `smartctl_exporter` static binary as a root systemd service, polling `smartctl` -at `smartctl_exporter_interval` (default `1h`). +at `smartctl_exporter_interval` (default `30m`). ```yaml smartctl_exporter_enabled: true @@ -27,3 +27,23 @@ prometheus_scrape_configs: labels: instance: andromeda ``` + +### Temperature alerts + +The `grafana` role provisions cumulative temperature thresholds: a rise fires +each newly crossed threshold (lower tiers keep firing), a fall resolves each +crossed-down threshold (ack), and the final ack fires when leaving the lowest +tier. + +| Threshold | `severity` | `for` | +|-----------|-----------|-------| +| ≥48°C | warning | 10m | +| ≥50°C | warning | 10m | +| ≥52°C | warning | 10m | +| ≥54°C | high | 10m | +| ≥56°C | high | 10m | +| ≥58°C | critical | 10m | +| ≥60°C | critical | 5m | +| <20°C | warning | 10m | + +Responsiveness is bounded by `smartctl_exporter_interval` (default `30m`). diff --git a/roles/disks/defaults/main.yml b/roles/disks/defaults/main.yml index d3326e0..a8ea893 100644 --- a/roles/disks/defaults/main.yml +++ b/roles/disks/defaults/main.yml @@ -7,4 +7,4 @@ smartctl_exporter_version: "0.14.0" smartctl_exporter_listen_address: "127.0.0.1:9633" smartctl_exporter_binary_dir: "/usr/local/bin" smartctl_exporter_smartctl_path: "/usr/sbin/smartctl" -smartctl_exporter_interval: "1h" +smartctl_exporter_interval: "30m" diff --git a/roles/grafana/README.md b/roles/grafana/README.md index e2e9793..4e42519 100644 --- a/roles/grafana/README.md +++ b/roles/grafana/README.md @@ -10,6 +10,7 @@ This role: - Creates a dedicated PostgreSQL database and user - Configures Grafana to use PostgreSQL - Optionally configures Nginx reverse proxy +- Optionally provisions disk alerting (temperature + capacity) to ntfy (see [Alerting](#alerting)) - Follows the shared services pattern ## Dependencies @@ -65,6 +66,18 @@ grafana_nginx_enabled: false grafana_nginx_hostname: grafana.local ``` +### Alerting + +```yaml +grafana_alerting_enabled: false +grafana_ntfy_webhook_url: "https://ntfy.example.com/sys-alerts?template=grafana" +grafana_ntfy_token: "{{ vault_ntfy_token }}" # write access to the topic +``` + +`grafana_ntfy_webhook_url` and `grafana_ntfy_token` are required when +`grafana_alerting_enabled: true`. The token is rendered into the contact point +provisioning file, which is deployed `0600`. + ## Example Playbook ```yaml @@ -169,6 +182,46 @@ When `grafana_nginx_enabled: true`, the role: When disabled, the vhost configuration is removed. +## Alerting + +When `grafana_alerting_enabled: true`, the role provisions [Grafana Unified +Alerting](https://grafana.com/docs/grafana/latest/alerting/) as code under +`{{ grafana_provisioning_path }}/alerting/`: + +- `contact-points.yaml` — stock **webhook** receiver pointing at an ntfy topic, + token sent as `Authorization: Bearer`. Deployed `0600` (secret). +- `policies.yaml` — notification policy tree (root route to that receiver). +- `rules.yaml` — disk alert rules: HDD temperature + (`smartctl_device_temperature{temperature_type="current"}`) and capacity + (ZFS pools `zfs_pool_capacity_percent`; local filesystems + `node_filesystem_avail_bytes` / `node_filesystem_size_bytes`, filtered to + `fstype=~"ext4|xfs|btrfs"`). + +No plugin, bridge or Alertmanager is involved: the webhook URL carries +`?template=grafana`, so ntfy's built-in `grafana` template renders Grafana's +webhook JSON into a `🚨`/`✅` title and message. Resolved alerts are sent +automatically (`disableResolveMessage: false`). Adding rules means adding +entries to `rules.yaml`; the receiver and policy stay generic. + +> Provisioning the notification policy tree replaces the entire tree; UI edits +> to policies are overwritten on the next deploy. + +### Capacity tiers + +Cumulative thresholds: a rise fires each newly crossed threshold (lower tiers +keep firing), a fall acks each crossed-down threshold. ZFS capacity is emitted +hourly by the `zfs` role, so its tiers need at least an hourly `for` to see the +new value. + +| Metric | Threshold | `severity` | `for` | +|--------|-----------|-----------|-------| +| `zfs_pool_capacity_percent` / filesystem used % | ≥80% | warning | 6h | +| `zfs_pool_capacity_percent` / filesystem used % | ≥90% | high | 1h | +| `zfs_pool_capacity_percent` / filesystem used % | ≥95% | critical | 15m | + +ZFS datasets are excluded from the filesystem rule: they report pool-wide free +space, not per-dataset usage, so the pool metric is authoritative. + ## OS Support - **Arch Linux:** Installs from community repository diff --git a/roles/grafana/defaults/main.yml b/roles/grafana/defaults/main.yml index db2570b..8195c69 100644 --- a/roles/grafana/defaults/main.yml +++ b/roles/grafana/defaults/main.yml @@ -31,3 +31,14 @@ grafana_check_for_updates: false # Nginx reverse proxy grafana_nginx_enabled: true grafana_nginx_hostname: "{{ grafana_domain }}" + +# Alerting (Grafana Unified Alerting → ntfy webhook) +# When enabled, provisions disk alert rules (HDD temperature + ZFS/filesystem +# capacity), notification policies and an ntfy contact point using ntfy's +# built-in "grafana" webhook template. +grafana_alerting_enabled: false +# Webhook URL of the target ntfy topic. Keep ?template=grafana so ntfy parses +# Grafana's webhook payload into a readable title/message. +# grafana_ntfy_webhook_url: "https://ntfy.example.com/sys-alerts?template=grafana" +# ntfy access token with write access to the topic (set in inventory/vault). +# grafana_ntfy_token: "" diff --git a/roles/grafana/files/alerting-policies.yaml b/roles/grafana/files/alerting-policies.yaml new file mode 100644 index 0000000..d2d1863 --- /dev/null +++ b/roles/grafana/files/alerting-policies.yaml @@ -0,0 +1,16 @@ +apiVersion: 1 + +# Managed by Ansible (roles/grafana). Notification policy tree. +# NOTE: provisioning replaces the entire tree; UI edits are overwritten on deploy. +policies: + - orgId: 1 + receiver: ntfy-sys-alerts + group_by: + - alertname + - instance + - device + - pool + - mountpoint + group_wait: 30s + group_interval: 5m + repeat_interval: 1h diff --git a/roles/grafana/files/alerting-rules.yaml b/roles/grafana/files/alerting-rules.yaml new file mode 100644 index 0000000..bd57844 --- /dev/null +++ b/roles/grafana/files/alerting-rules.yaml @@ -0,0 +1,706 @@ +apiVersion: 1 + +# Managed by Ansible (roles/grafana). Disk alert rules: HDD temperature and +# capacity (ZFS pools + local filesystems). +# +# Temperature metric: smartctl_device_temperature{temperature_type="current"} (job="smartctl"). +# Capacity metrics: zfs_pool_capacity_percent and +# node_filesystem_avail_bytes/node_filesystem_size_bytes (job="node"). +# node_filesystem_* is filtered to real local filesystems (ext4/xfs/btrfs); +# ZFS datasets are excluded because they report pool-wide free space, not +# per-dataset, and are covered by zfs_pool_capacity_percent instead. +# +# Tiers are cumulative thresholds (>= N), not non-overlapping bands: +# rising -> each newly crossed threshold fires; lower tiers keep firing +# falling -> each crossed-down threshold resolves (ack); lower tiers keep firing +# So a rise pages once per tier crossed and a fall acks once per tier, with the +# final ack when leaving the lowest tier. Below the lowest threshold = no series +# = NoData, mapped to OK on purpose. +groups: + - orgId: 1 + name: disk-temperature + folder: Disks + interval: 1m + rules: + - uid: disk_temp_ge_48 + title: "Disk temperature ≥48°C" + condition: C + data: + - refId: A + relativeTimeRange: { from: 3600, to: 0 } + datasourceUid: prometheus + model: + editorMode: code + expr: >- + smartctl_device_temperature{job="smartctl", temperature_type="current"} >= 48 + instant: false + intervalMs: 1000 + legendFormat: "{{device}}" + maxDataPoints: 43200 + range: true + refId: A + - refId: B + datasourceUid: __expr__ + model: + datasource: { type: __expr__, uid: __expr__ } + expression: A + reducer: last + refId: B + type: reduce + settings: { mode: dropNN } + - refId: C + datasourceUid: __expr__ + model: + conditions: + - evaluator: { params: [0], type: gt } + operator: { type: and } + query: { params: [C] } + reducer: { type: last } + type: query + datasource: { type: __expr__, uid: __expr__ } + expression: B + refId: C + type: threshold + for: 10m + noDataState: OK + execErrState: Error + labels: + severity: warning + annotations: + summary: "Disk {{ $labels.device }} at {{ $values.B }}°C ({{ $labels.instance }})" + description: "S.M.A.R.T. temperature is at or above 48°C. Ideal is 20-45°C." + + - uid: disk_temp_ge_50 + title: "Disk temperature ≥50°C" + condition: C + data: + - refId: A + relativeTimeRange: { from: 3600, to: 0 } + datasourceUid: prometheus + model: + editorMode: code + expr: >- + smartctl_device_temperature{job="smartctl", temperature_type="current"} >= 50 + instant: false + intervalMs: 1000 + legendFormat: "{{device}}" + maxDataPoints: 43200 + range: true + refId: A + - refId: B + datasourceUid: __expr__ + model: + datasource: { type: __expr__, uid: __expr__ } + expression: A + reducer: last + refId: B + type: reduce + settings: { mode: dropNN } + - refId: C + datasourceUid: __expr__ + model: + conditions: + - evaluator: { params: [0], type: gt } + operator: { type: and } + query: { params: [C] } + reducer: { type: last } + type: query + datasource: { type: __expr__, uid: __expr__ } + expression: B + refId: C + type: threshold + for: 10m + noDataState: OK + execErrState: Error + labels: + severity: warning + annotations: + summary: "Disk {{ $labels.device }} at {{ $values.B }}°C ({{ $labels.instance }})" + description: "S.M.A.R.T. temperature is at or above 50°C." + + - uid: disk_temp_ge_52 + title: "Disk temperature ≥52°C" + condition: C + data: + - refId: A + relativeTimeRange: { from: 3600, to: 0 } + datasourceUid: prometheus + model: + editorMode: code + expr: >- + smartctl_device_temperature{job="smartctl", temperature_type="current"} >= 52 + instant: false + intervalMs: 1000 + legendFormat: "{{device}}" + maxDataPoints: 43200 + range: true + refId: A + - refId: B + datasourceUid: __expr__ + model: + datasource: { type: __expr__, uid: __expr__ } + expression: A + reducer: last + refId: B + type: reduce + settings: { mode: dropNN } + - refId: C + datasourceUid: __expr__ + model: + conditions: + - evaluator: { params: [0], type: gt } + operator: { type: and } + query: { params: [C] } + reducer: { type: last } + type: query + datasource: { type: __expr__, uid: __expr__ } + expression: B + refId: C + type: threshold + for: 10m + noDataState: OK + execErrState: Error + labels: + severity: warning + annotations: + summary: "Disk {{ $labels.device }} at {{ $values.B }}°C ({{ $labels.instance }})" + description: "S.M.A.R.T. temperature is at or above 52°C." + + - uid: disk_temp_ge_54 + title: "Disk temperature ≥54°C" + condition: C + data: + - refId: A + relativeTimeRange: { from: 3600, to: 0 } + datasourceUid: prometheus + model: + editorMode: code + expr: >- + smartctl_device_temperature{job="smartctl", temperature_type="current"} >= 54 + instant: false + intervalMs: 1000 + legendFormat: "{{device}}" + maxDataPoints: 43200 + range: true + refId: A + - refId: B + datasourceUid: __expr__ + model: + datasource: { type: __expr__, uid: __expr__ } + expression: A + reducer: last + refId: B + type: reduce + settings: { mode: dropNN } + - refId: C + datasourceUid: __expr__ + model: + conditions: + - evaluator: { params: [0], type: gt } + operator: { type: and } + query: { params: [C] } + reducer: { type: last } + type: query + datasource: { type: __expr__, uid: __expr__ } + expression: B + refId: C + type: threshold + for: 10m + noDataState: OK + execErrState: Error + labels: + severity: high + annotations: + summary: "Disk {{ $labels.device }} at {{ $values.B }}°C ({{ $labels.instance }})" + description: "S.M.A.R.T. temperature is at or above 54°C. Check airflow/fan." + + - uid: disk_temp_ge_56 + title: "Disk temperature ≥56°C" + condition: C + data: + - refId: A + relativeTimeRange: { from: 3600, to: 0 } + datasourceUid: prometheus + model: + editorMode: code + expr: >- + smartctl_device_temperature{job="smartctl", temperature_type="current"} >= 56 + instant: false + intervalMs: 1000 + legendFormat: "{{device}}" + maxDataPoints: 43200 + range: true + refId: A + - refId: B + datasourceUid: __expr__ + model: + datasource: { type: __expr__, uid: __expr__ } + expression: A + reducer: last + refId: B + type: reduce + settings: { mode: dropNN } + - refId: C + datasourceUid: __expr__ + model: + conditions: + - evaluator: { params: [0], type: gt } + operator: { type: and } + query: { params: [C] } + reducer: { type: last } + type: query + datasource: { type: __expr__, uid: __expr__ } + expression: B + refId: C + type: threshold + for: 10m + noDataState: OK + execErrState: Error + labels: + severity: high + annotations: + summary: "Disk {{ $labels.device }} at {{ $values.B }}°C ({{ $labels.instance }})" + description: "S.M.A.R.T. temperature is at or above 56°C. Check airflow/fan." + + - uid: disk_temp_ge_58 + title: "Disk temperature ≥58°C" + condition: C + data: + - refId: A + relativeTimeRange: { from: 3600, to: 0 } + datasourceUid: prometheus + model: + editorMode: code + expr: >- + smartctl_device_temperature{job="smartctl", temperature_type="current"} >= 58 + instant: false + intervalMs: 1000 + legendFormat: "{{device}}" + maxDataPoints: 43200 + range: true + refId: A + - refId: B + datasourceUid: __expr__ + model: + datasource: { type: __expr__, uid: __expr__ } + expression: A + reducer: last + refId: B + type: reduce + settings: { mode: dropNN } + - refId: C + datasourceUid: __expr__ + model: + conditions: + - evaluator: { params: [0], type: gt } + operator: { type: and } + query: { params: [C] } + reducer: { type: last } + type: query + datasource: { type: __expr__, uid: __expr__ } + expression: B + refId: C + type: threshold + for: 10m + noDataState: OK + execErrState: Error + labels: + severity: critical + annotations: + summary: "Disk {{ $labels.device }} at {{ $values.B }}°C ({{ $labels.instance }})" + description: "S.M.A.R.T. temperature is at or above 58°C, close to the failure limit." + + - uid: disk_temp_ge_60 + title: "Disk temperature ≥60°C" + condition: C + data: + - refId: A + relativeTimeRange: { from: 3600, to: 0 } + datasourceUid: prometheus + model: + editorMode: code + expr: >- + smartctl_device_temperature{job="smartctl", temperature_type="current"} >= 60 + instant: false + intervalMs: 1000 + legendFormat: "{{device}}" + maxDataPoints: 43200 + range: true + refId: A + - refId: B + datasourceUid: __expr__ + model: + datasource: { type: __expr__, uid: __expr__ } + expression: A + reducer: last + refId: B + type: reduce + settings: { mode: dropNN } + - refId: C + datasourceUid: __expr__ + model: + conditions: + - evaluator: { params: [0], type: gt } + operator: { type: and } + query: { params: [C] } + reducer: { type: last } + type: query + datasource: { type: __expr__, uid: __expr__ } + expression: B + refId: C + type: threshold + for: 5m + noDataState: OK + execErrState: Error + labels: + severity: critical + annotations: + summary: "Disk {{ $labels.device }} at {{ $values.B }}°C ({{ $labels.instance }})" + description: "S.M.A.R.T. temperature is at or above the 60°C failure limit." + + - uid: disk_temp_lt_20 + title: "Disk temperature <20°C" + condition: C + data: + - refId: A + relativeTimeRange: { from: 3600, to: 0 } + datasourceUid: prometheus + model: + editorMode: code + expr: >- + smartctl_device_temperature{job="smartctl", temperature_type="current"} < 20 + instant: false + intervalMs: 1000 + legendFormat: "{{device}}" + maxDataPoints: 43200 + range: true + refId: A + - refId: B + datasourceUid: __expr__ + model: + datasource: { type: __expr__, uid: __expr__ } + expression: A + reducer: last + refId: B + type: reduce + settings: { mode: dropNN } + - refId: C + datasourceUid: __expr__ + model: + conditions: + - evaluator: { params: [0], type: gt } + operator: { type: and } + query: { params: [C] } + reducer: { type: last } + type: query + datasource: { type: __expr__, uid: __expr__ } + expression: B + refId: C + type: threshold + for: 10m + noDataState: OK + execErrState: Error + labels: + severity: warning + annotations: + summary: "Disk {{ $labels.device }} at {{ $values.B }}°C ({{ $labels.instance }})" + description: "S.M.A.R.T. temperature is below the 20°C ideal minimum (sensor/fan/offline disk?)." + + # --- Capacity: ZFS pools (data disk) --- + # zfs_pool_capacity_percent is emitted hourly by the zfs role's + # zfs-metrics.timer via node_exporter's textfile collector. Cumulative + # thresholds; below threshold = no series = NoData, mapped to OK. + - uid: zfs_pool_capacity_ge_80 + title: "ZFS pool ≥80% full" + condition: C + data: + - refId: A + relativeTimeRange: { from: 3600, to: 0 } + datasourceUid: prometheus + model: + editorMode: code + expr: >- + zfs_pool_capacity_percent{job="node"} >= 80 + instant: false + intervalMs: 1000 + legendFormat: "{{pool}}" + maxDataPoints: 43200 + range: true + refId: A + - refId: B + datasourceUid: __expr__ + model: + datasource: { type: __expr__, uid: __expr__ } + expression: A + reducer: last + refId: B + type: reduce + settings: { mode: dropNN } + - refId: C + datasourceUid: __expr__ + model: + conditions: + - evaluator: { params: [0], type: gt } + operator: { type: and } + query: { params: [C] } + reducer: { type: last } + type: query + datasource: { type: __expr__, uid: __expr__ } + expression: B + refId: C + type: threshold + for: 6h + noDataState: OK + execErrState: Error + labels: + severity: warning + annotations: + summary: "ZFS pool {{ $labels.pool }} at {{ $values.B }}% ({{ $labels.instance }})" + description: "ZFS pool is at or above 80% capacity. ZFS performance degrades above 80%." + + - uid: zfs_pool_capacity_ge_90 + title: "ZFS pool ≥90% full" + condition: C + data: + - refId: A + relativeTimeRange: { from: 3600, to: 0 } + datasourceUid: prometheus + model: + editorMode: code + expr: >- + zfs_pool_capacity_percent{job="node"} >= 90 + instant: false + intervalMs: 1000 + legendFormat: "{{pool}}" + maxDataPoints: 43200 + range: true + refId: A + - refId: B + datasourceUid: __expr__ + model: + datasource: { type: __expr__, uid: __expr__ } + expression: A + reducer: last + refId: B + type: reduce + settings: { mode: dropNN } + - refId: C + datasourceUid: __expr__ + model: + conditions: + - evaluator: { params: [0], type: gt } + operator: { type: and } + query: { params: [C] } + reducer: { type: last } + type: query + datasource: { type: __expr__, uid: __expr__ } + expression: B + refId: C + type: threshold + for: 1h + noDataState: OK + execErrState: Error + labels: + severity: high + annotations: + summary: "ZFS pool {{ $labels.pool }} at {{ $values.B }}% ({{ $labels.instance }})" + description: "ZFS pool is at or above 90% capacity. Free space is getting low." + + - uid: zfs_pool_capacity_ge_95 + title: "ZFS pool ≥95% full" + condition: C + data: + - refId: A + relativeTimeRange: { from: 3600, to: 0 } + datasourceUid: prometheus + model: + editorMode: code + expr: >- + zfs_pool_capacity_percent{job="node"} >= 95 + instant: false + intervalMs: 1000 + legendFormat: "{{pool}}" + maxDataPoints: 43200 + range: true + refId: A + - refId: B + datasourceUid: __expr__ + model: + datasource: { type: __expr__, uid: __expr__ } + expression: A + reducer: last + refId: B + type: reduce + settings: { mode: dropNN } + - refId: C + datasourceUid: __expr__ + model: + conditions: + - evaluator: { params: [0], type: gt } + operator: { type: and } + query: { params: [C] } + reducer: { type: last } + type: query + datasource: { type: __expr__, uid: __expr__ } + expression: B + refId: C + type: threshold + for: 15m + noDataState: OK + execErrState: Error + labels: + severity: critical + annotations: + summary: "ZFS pool {{ $labels.pool }} at {{ $values.B }}% ({{ $labels.instance }})" + description: "ZFS pool is at or above 95% capacity. Writes may fail soon." + + # --- Capacity: local filesystems (system disk, db disk) --- + # node_exporter's filesystem collector. fstype filter excludes ZFS + # (pool-wide free space is not per-dataset) and virtual filesystems. + # Cumulative thresholds; below threshold = no series = NoData, OK. + - uid: fs_capacity_ge_80 + title: "Filesystem ≥80% full" + condition: C + data: + - refId: A + relativeTimeRange: { from: 3600, to: 0 } + datasourceUid: prometheus + model: + editorMode: code + expr: >- + 100 * (1 - node_filesystem_avail_bytes{job="node", fstype=~"ext4|xfs|btrfs"} + / node_filesystem_size_bytes{job="node", fstype=~"ext4|xfs|btrfs"}) >= 80 + instant: false + intervalMs: 1000 + legendFormat: "{{mountpoint}}" + maxDataPoints: 43200 + range: true + refId: A + - refId: B + datasourceUid: __expr__ + model: + datasource: { type: __expr__, uid: __expr__ } + expression: A + reducer: last + refId: B + type: reduce + settings: { mode: dropNN } + - refId: C + datasourceUid: __expr__ + model: + conditions: + - evaluator: { params: [0], type: gt } + operator: { type: and } + query: { params: [C] } + reducer: { type: last } + type: query + datasource: { type: __expr__, uid: __expr__ } + expression: B + refId: C + type: threshold + for: 6h + noDataState: OK + execErrState: Error + labels: + severity: warning + annotations: + summary: "Filesystem {{ $labels.mountpoint }} at {{ $values.B }}% ({{ $labels.instance }})" + description: "Filesystem is at or above 80% capacity." + + - uid: fs_capacity_ge_90 + title: "Filesystem ≥90% full" + condition: C + data: + - refId: A + relativeTimeRange: { from: 3600, to: 0 } + datasourceUid: prometheus + model: + editorMode: code + expr: >- + 100 * (1 - node_filesystem_avail_bytes{job="node", fstype=~"ext4|xfs|btrfs"} + / node_filesystem_size_bytes{job="node", fstype=~"ext4|xfs|btrfs"}) >= 90 + instant: false + intervalMs: 1000 + legendFormat: "{{mountpoint}}" + maxDataPoints: 43200 + range: true + refId: A + - refId: B + datasourceUid: __expr__ + model: + datasource: { type: __expr__, uid: __expr__ } + expression: A + reducer: last + refId: B + type: reduce + settings: { mode: dropNN } + - refId: C + datasourceUid: __expr__ + model: + conditions: + - evaluator: { params: [0], type: gt } + operator: { type: and } + query: { params: [C] } + reducer: { type: last } + type: query + datasource: { type: __expr__, uid: __expr__ } + expression: B + refId: C + type: threshold + for: 1h + noDataState: OK + execErrState: Error + labels: + severity: high + annotations: + summary: "Filesystem {{ $labels.mountpoint }} at {{ $values.B }}% ({{ $labels.instance }})" + description: "Filesystem is at or above 90% capacity. Free space is getting low." + + - uid: fs_capacity_ge_95 + title: "Filesystem ≥95% full" + condition: C + data: + - refId: A + relativeTimeRange: { from: 3600, to: 0 } + datasourceUid: prometheus + model: + editorMode: code + expr: >- + 100 * (1 - node_filesystem_avail_bytes{job="node", fstype=~"ext4|xfs|btrfs"} + / node_filesystem_size_bytes{job="node", fstype=~"ext4|xfs|btrfs"}) >= 95 + instant: false + intervalMs: 1000 + legendFormat: "{{mountpoint}}" + maxDataPoints: 43200 + range: true + refId: A + - refId: B + datasourceUid: __expr__ + model: + datasource: { type: __expr__, uid: __expr__ } + expression: A + reducer: last + refId: B + type: reduce + settings: { mode: dropNN } + - refId: C + datasourceUid: __expr__ + model: + conditions: + - evaluator: { params: [0], type: gt } + operator: { type: and } + query: { params: [C] } + reducer: { type: last } + type: query + datasource: { type: __expr__, uid: __expr__ } + expression: B + refId: C + type: threshold + for: 15m + noDataState: OK + execErrState: Error + labels: + severity: critical + annotations: + summary: "Filesystem {{ $labels.mountpoint }} at {{ $values.B }}% ({{ $labels.instance }})" + description: "Filesystem is at or above 95% capacity. Writes may fail soon." diff --git a/roles/grafana/tasks/main.yml b/roles/grafana/tasks/main.yml index f95b423..8d669c9 100644 --- a/roles/grafana/tasks/main.yml +++ b/roles/grafana/tasks/main.yml @@ -269,6 +269,70 @@ mode: "0644" notify: Restart grafana +# Alerting provisioning (Unified Alerting -> ntfy webhook) +- name: Validate ntfy alerting settings are set + ansible.builtin.assert: + that: + - grafana_ntfy_webhook_url is defined + - grafana_ntfy_webhook_url | length > 0 + - grafana_ntfy_token is defined + - grafana_ntfy_token | length > 0 + fail_msg: | + grafana_alerting_enabled is true but grafana_ntfy_webhook_url / grafana_ntfy_token are not set. + See roles/grafana/defaults/main.yml for configuration instructions. + success_msg: "ntfy alerting settings validated" + when: grafana_alerting_enabled + +- name: Ensure Grafana alerting provisioning directory exists + ansible.builtin.file: + path: "{{ grafana_provisioning_path }}/alerting" + state: directory + owner: "{{ grafana_user }}" + group: "{{ grafana_group }}" + mode: "0755" + when: grafana_alerting_enabled + +- name: Deploy Grafana alerting contact points + ansible.builtin.template: + src: alerting-contact-points.yaml.j2 + dest: "{{ grafana_provisioning_path }}/alerting/contact-points.yaml" + owner: "{{ grafana_user }}" + group: "{{ grafana_group }}" + mode: "0600" + when: grafana_alerting_enabled + notify: Restart grafana + +- name: Deploy Grafana alerting notification policies + ansible.builtin.copy: + src: alerting-policies.yaml + dest: "{{ grafana_provisioning_path }}/alerting/policies.yaml" + owner: "{{ grafana_user }}" + group: "{{ grafana_group }}" + mode: "0644" + when: grafana_alerting_enabled + notify: Restart grafana + +- name: Deploy Grafana disk alert rules + ansible.builtin.copy: + src: alerting-rules.yaml + dest: "{{ grafana_provisioning_path }}/alerting/rules.yaml" + owner: "{{ grafana_user }}" + group: "{{ grafana_group }}" + mode: "0644" + when: grafana_alerting_enabled + notify: Restart grafana + +- name: Remove Grafana alerting provisioning when disabled + ansible.builtin.file: + path: "{{ grafana_provisioning_path }}/alerting/{{ item }}" + state: absent + loop: + - contact-points.yaml + - policies.yaml + - rules.yaml + when: not grafana_alerting_enabled + notify: Restart grafana + # Service management - name: Enable and start Grafana service ansible.builtin.systemd: diff --git a/roles/grafana/templates/alerting-contact-points.yaml.j2 b/roles/grafana/templates/alerting-contact-points.yaml.j2 new file mode 100644 index 0000000..aee5562 --- /dev/null +++ b/roles/grafana/templates/alerting-contact-points.yaml.j2 @@ -0,0 +1,15 @@ +apiVersion: 1 + +# Managed by Ansible (roles/grafana). Contains the ntfy access token: mode 0600. +contactPoints: + - orgId: 1 + name: ntfy-sys-alerts + receivers: + - uid: ntfy_sys_alerts + type: webhook + disableResolveMessage: false + settings: + url: "{{ grafana_ntfy_webhook_url }}" + httpMethod: POST + authorization_scheme: Bearer + authorization_credentials: "{{ grafana_ntfy_token }}"