fix(grafana): split NVMe temperature alerts from HDD tiers
NVMe drives run hotter than HDDs and kept firing the 48°C HDD alerts. HDD tiers now exclude nvme devices; NVMe gets its own 65/70/75°C ladder. The device matcher tolerates both "nvme0" and "/dev/nvme0" label forms (smartctl_exporter strips the /dev/ prefix). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Clément Désiles
co-authored by
Claude Opus 5.5
Claude Opus 5.5
parent
5efc0b0d6e
commit
1d37313bd6
+18
-1
@@ -191,7 +191,7 @@ Alerting](https://grafana.com/docs/grafana/latest/alerting/) as code under
|
||||
- `contact-points.yaml` — stock **webhook** receiver pointing at an ntfy topic,
|
||||
token sent as `Authorization: Bearer`. Deployed `0600` (secret).
|
||||
- `policies.yaml` — notification policy tree (root route to that receiver).
|
||||
- `rules.yaml` — disk alert rules: HDD temperature
|
||||
- `rules.yaml` — disk alert rules: HDD and NVMe temperature
|
||||
(`smartctl_device_temperature{temperature_type="current"}`) and capacity
|
||||
(ZFS pools `zfs_pool_capacity_percent`; local filesystems
|
||||
`node_filesystem_avail_bytes` / `node_filesystem_size_bytes`, filtered to
|
||||
@@ -206,6 +206,23 @@ entries to `rules.yaml`; the receiver and policy stay generic.
|
||||
> Provisioning the notification policy tree replaces the entire tree; UI edits
|
||||
> to policies are overwritten on the next deploy.
|
||||
|
||||
### Temperature tiers
|
||||
|
||||
HDD and NVMe have different safe ranges, so they use separate ladders selected
|
||||
by the `device` label (`nvme*` vs everything else). Cumulative thresholds, same
|
||||
rise/fall semantics as capacity.
|
||||
|
||||
| Device | Threshold | `severity` | `for` |
|
||||
|--------|-----------|-----------|-------|
|
||||
| HDD | ≥48 / ≥50 / ≥52 / ≥54 / ≥56 / ≥58 / ≥60°C | warning → critical | 10m (5m at 60) |
|
||||
| HDD | <20°C | warning | 10m |
|
||||
| NVMe | ≥65°C | warning | 15m |
|
||||
| NVMe | ≥70°C | high | 10m |
|
||||
| NVMe | ≥75°C | critical | 5m |
|
||||
|
||||
NVMe runs hotter: 40-60°C is normal, 60-70°C is warm (acceptable under
|
||||
sustained load), throttling typically starts at 70-80°C.
|
||||
|
||||
### Capacity tiers
|
||||
|
||||
Cumulative thresholds: a rise fires each newly crossed threshold (lower tiers
|
||||
|
||||
@@ -4,6 +4,10 @@ apiVersion: 1
|
||||
# capacity (ZFS pools + local filesystems).
|
||||
#
|
||||
# Temperature metric: smartctl_device_temperature{temperature_type="current"} (job="smartctl").
|
||||
# Split by device class: HDDs (sd*) and NVMe (nvme*) have different
|
||||
# safe ranges, so they get separate tier ladders.
|
||||
# HDD: 48..60°C (below 45 ideal) — device!~"(/dev/)?nvme.*"
|
||||
# NVMe: 65..75°C (40-60 normal, 60-70 warm, >70 throttling) — device=~"(/dev/)?nvme.*"
|
||||
# Capacity metrics: zfs_pool_capacity_percent and
|
||||
# node_filesystem_avail_bytes/node_filesystem_size_bytes (job="node").
|
||||
# node_filesystem_* is filtered to real local filesystems (ext4/xfs/btrfs);
|
||||
@@ -32,7 +36,7 @@ groups:
|
||||
model:
|
||||
editorMode: code
|
||||
expr: >-
|
||||
smartctl_device_temperature{job="smartctl", temperature_type="current"} >= 48
|
||||
smartctl_device_temperature{job="smartctl", temperature_type="current", device!~"(/dev/)?nvme.*"} >= 48
|
||||
instant: false
|
||||
intervalMs: 1000
|
||||
legendFormat: "{{device}}"
|
||||
@@ -80,7 +84,7 @@ groups:
|
||||
model:
|
||||
editorMode: code
|
||||
expr: >-
|
||||
smartctl_device_temperature{job="smartctl", temperature_type="current"} >= 50
|
||||
smartctl_device_temperature{job="smartctl", temperature_type="current", device!~"(/dev/)?nvme.*"} >= 50
|
||||
instant: false
|
||||
intervalMs: 1000
|
||||
legendFormat: "{{device}}"
|
||||
@@ -128,7 +132,7 @@ groups:
|
||||
model:
|
||||
editorMode: code
|
||||
expr: >-
|
||||
smartctl_device_temperature{job="smartctl", temperature_type="current"} >= 52
|
||||
smartctl_device_temperature{job="smartctl", temperature_type="current", device!~"(/dev/)?nvme.*"} >= 52
|
||||
instant: false
|
||||
intervalMs: 1000
|
||||
legendFormat: "{{device}}"
|
||||
@@ -176,7 +180,7 @@ groups:
|
||||
model:
|
||||
editorMode: code
|
||||
expr: >-
|
||||
smartctl_device_temperature{job="smartctl", temperature_type="current"} >= 54
|
||||
smartctl_device_temperature{job="smartctl", temperature_type="current", device!~"(/dev/)?nvme.*"} >= 54
|
||||
instant: false
|
||||
intervalMs: 1000
|
||||
legendFormat: "{{device}}"
|
||||
@@ -224,7 +228,7 @@ groups:
|
||||
model:
|
||||
editorMode: code
|
||||
expr: >-
|
||||
smartctl_device_temperature{job="smartctl", temperature_type="current"} >= 56
|
||||
smartctl_device_temperature{job="smartctl", temperature_type="current", device!~"(/dev/)?nvme.*"} >= 56
|
||||
instant: false
|
||||
intervalMs: 1000
|
||||
legendFormat: "{{device}}"
|
||||
@@ -272,7 +276,7 @@ groups:
|
||||
model:
|
||||
editorMode: code
|
||||
expr: >-
|
||||
smartctl_device_temperature{job="smartctl", temperature_type="current"} >= 58
|
||||
smartctl_device_temperature{job="smartctl", temperature_type="current", device!~"(/dev/)?nvme.*"} >= 58
|
||||
instant: false
|
||||
intervalMs: 1000
|
||||
legendFormat: "{{device}}"
|
||||
@@ -320,7 +324,7 @@ groups:
|
||||
model:
|
||||
editorMode: code
|
||||
expr: >-
|
||||
smartctl_device_temperature{job="smartctl", temperature_type="current"} >= 60
|
||||
smartctl_device_temperature{job="smartctl", temperature_type="current", device!~"(/dev/)?nvme.*"} >= 60
|
||||
instant: false
|
||||
intervalMs: 1000
|
||||
legendFormat: "{{device}}"
|
||||
@@ -368,7 +372,7 @@ groups:
|
||||
model:
|
||||
editorMode: code
|
||||
expr: >-
|
||||
smartctl_device_temperature{job="smartctl", temperature_type="current"} < 20
|
||||
smartctl_device_temperature{job="smartctl", temperature_type="current", device!~"(/dev/)?nvme.*"} < 20
|
||||
instant: false
|
||||
intervalMs: 1000
|
||||
legendFormat: "{{device}}"
|
||||
@@ -406,6 +410,154 @@ groups:
|
||||
summary: "Disk {{ $labels.device }} at {{ $values.B }}°C ({{ $labels.instance }})"
|
||||
description: "S.M.A.R.T. temperature is below the 20°C ideal minimum (sensor/fan/offline disk?)."
|
||||
|
||||
# --- NVMe temperature tiers (device=~"(/dev/)?nvme.*") ---
|
||||
# NVMe runs hotter than HDD: 40-60°C is normal, 60-70°C is warm but
|
||||
# acceptable under sustained load, and throttling typically starts in the
|
||||
# 70-80°C range. The HDD ladder above excludes NVMe devices.
|
||||
- uid: nvme_temp_ge_65
|
||||
title: "NVMe temperature ≥65°C"
|
||||
condition: C
|
||||
data:
|
||||
- refId: A
|
||||
relativeTimeRange: { from: 3600, to: 0 }
|
||||
datasourceUid: prometheus
|
||||
model:
|
||||
editorMode: code
|
||||
expr: >-
|
||||
smartctl_device_temperature{job="smartctl", temperature_type="current", device=~"(/dev/)?nvme.*"} >= 65
|
||||
instant: false
|
||||
intervalMs: 1000
|
||||
legendFormat: "{{device}}"
|
||||
maxDataPoints: 43200
|
||||
range: true
|
||||
refId: A
|
||||
- refId: B
|
||||
datasourceUid: __expr__
|
||||
model:
|
||||
datasource: { type: __expr__, uid: __expr__ }
|
||||
expression: A
|
||||
reducer: last
|
||||
refId: B
|
||||
type: reduce
|
||||
settings: { mode: dropNN }
|
||||
- refId: C
|
||||
datasourceUid: __expr__
|
||||
model:
|
||||
conditions:
|
||||
- evaluator: { params: [0], type: gt }
|
||||
operator: { type: and }
|
||||
query: { params: [C] }
|
||||
reducer: { type: last }
|
||||
type: query
|
||||
datasource: { type: __expr__, uid: __expr__ }
|
||||
expression: B
|
||||
refId: C
|
||||
type: threshold
|
||||
for: 15m
|
||||
noDataState: OK
|
||||
execErrState: Error
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "NVMe {{ $labels.device }} at {{ $values.B }}°C ({{ $labels.instance }})"
|
||||
description: "NVMe temperature is at or above 65°C (warm). Normal is 40-60°C; acceptable under sustained load."
|
||||
|
||||
- uid: nvme_temp_ge_70
|
||||
title: "NVMe temperature ≥70°C"
|
||||
condition: C
|
||||
data:
|
||||
- refId: A
|
||||
relativeTimeRange: { from: 3600, to: 0 }
|
||||
datasourceUid: prometheus
|
||||
model:
|
||||
editorMode: code
|
||||
expr: >-
|
||||
smartctl_device_temperature{job="smartctl", temperature_type="current", device=~"(/dev/)?nvme.*"} >= 70
|
||||
instant: false
|
||||
intervalMs: 1000
|
||||
legendFormat: "{{device}}"
|
||||
maxDataPoints: 43200
|
||||
range: true
|
||||
refId: A
|
||||
- refId: B
|
||||
datasourceUid: __expr__
|
||||
model:
|
||||
datasource: { type: __expr__, uid: __expr__ }
|
||||
expression: A
|
||||
reducer: last
|
||||
refId: B
|
||||
type: reduce
|
||||
settings: { mode: dropNN }
|
||||
- refId: C
|
||||
datasourceUid: __expr__
|
||||
model:
|
||||
conditions:
|
||||
- evaluator: { params: [0], type: gt }
|
||||
operator: { type: and }
|
||||
query: { params: [C] }
|
||||
reducer: { type: last }
|
||||
type: query
|
||||
datasource: { type: __expr__, uid: __expr__ }
|
||||
expression: B
|
||||
refId: C
|
||||
type: threshold
|
||||
for: 10m
|
||||
noDataState: OK
|
||||
execErrState: Error
|
||||
labels:
|
||||
severity: high
|
||||
annotations:
|
||||
summary: "NVMe {{ $labels.device }} at {{ $values.B }}°C ({{ $labels.instance }})"
|
||||
description: "NVMe temperature is at or above 70°C. Thermal throttling typically begins in the 70-80°C range."
|
||||
|
||||
- uid: nvme_temp_ge_75
|
||||
title: "NVMe temperature ≥75°C"
|
||||
condition: C
|
||||
data:
|
||||
- refId: A
|
||||
relativeTimeRange: { from: 3600, to: 0 }
|
||||
datasourceUid: prometheus
|
||||
model:
|
||||
editorMode: code
|
||||
expr: >-
|
||||
smartctl_device_temperature{job="smartctl", temperature_type="current", device=~"(/dev/)?nvme.*"} >= 75
|
||||
instant: false
|
||||
intervalMs: 1000
|
||||
legendFormat: "{{device}}"
|
||||
maxDataPoints: 43200
|
||||
range: true
|
||||
refId: A
|
||||
- refId: B
|
||||
datasourceUid: __expr__
|
||||
model:
|
||||
datasource: { type: __expr__, uid: __expr__ }
|
||||
expression: A
|
||||
reducer: last
|
||||
refId: B
|
||||
type: reduce
|
||||
settings: { mode: dropNN }
|
||||
- refId: C
|
||||
datasourceUid: __expr__
|
||||
model:
|
||||
conditions:
|
||||
- evaluator: { params: [0], type: gt }
|
||||
operator: { type: and }
|
||||
query: { params: [C] }
|
||||
reducer: { type: last }
|
||||
type: query
|
||||
datasource: { type: __expr__, uid: __expr__ }
|
||||
expression: B
|
||||
refId: C
|
||||
type: threshold
|
||||
for: 5m
|
||||
noDataState: OK
|
||||
execErrState: Error
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "NVMe {{ $labels.device }} at {{ $values.B }}°C ({{ $labels.instance }})"
|
||||
description: "NVMe temperature is at or above 75°C (hot). Check airflow/heatsink; sustained throttling expected."
|
||||
|
||||
# --- Capacity: ZFS pools (data disk) ---
|
||||
# zfs_pool_capacity_percent is emitted hourly by the zfs role's
|
||||
# zfs-metrics.timer via node_exporter's textfile collector. Cumulative
|
||||
|
||||
Reference in New Issue
Block a user