From 86ad271b237ad55f052267104a0dbaadea647391 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20D=C3=A9siles?= <1536672+cdesiles@users.noreply.github.com> Date: Tue, 29 Sep 2026 12:25:56 +0200 Subject: [PATCH] fix(grafana): align NVMe temperature thresholds with 0-70C rating Dashboard temperature panel now uses NVMe-specific thresholds (orange 60, red 70) instead of HDD ones; alert descriptions reference the rated range. --- roles/grafana/README.md | 3 +-- roles/grafana/files/alerting-rules.yaml | 13 ++++++------- roles/grafana/files/smartctl-dashboard.json | 4 ++-- 3 files changed, 9 insertions(+), 11 deletions(-) diff --git a/roles/grafana/README.md b/roles/grafana/README.md index 5f13361..1626544 100644 --- a/roles/grafana/README.md +++ b/roles/grafana/README.md @@ -220,8 +220,7 @@ rise/fall semantics as capacity. | NVMe | ≥70°C | high | 10m | | NVMe | ≥75°C | critical | 5m | -NVMe runs hotter: 40-60°C is normal, 60-70°C is warm (acceptable under -sustained load), throttling typically starts at 70-80°C. +NVMe tiers follow the 0-70°C operating spec (dashboard: orange ≥60, red ≥70°C). ### Capacity tiers diff --git a/roles/grafana/files/alerting-rules.yaml b/roles/grafana/files/alerting-rules.yaml index 1c775d9..df9a1ba 100644 --- a/roles/grafana/files/alerting-rules.yaml +++ b/roles/grafana/files/alerting-rules.yaml @@ -7,7 +7,7 @@ apiVersion: 1 # Split by device class: HDDs (sd*) and NVMe (nvme*) have different # safe ranges, so they get separate tier ladders. # HDD: 48..60°C (below 45 ideal) — device!~"(/dev/)?nvme.*" -# NVMe: 65..75°C (40-60 normal, 60-70 warm, >70 throttling) — device=~"(/dev/)?nvme.*" +# NVMe: 65..75°C (rated 0-70°C) — device=~"(/dev/)?nvme.*" # Capacity metrics: zfs_pool_capacity_percent and # node_filesystem_avail_bytes/node_filesystem_size_bytes (job="node"). # node_filesystem_* is filtered to real local filesystems (ext4/xfs/btrfs); @@ -411,9 +411,8 @@ groups: description: "S.M.A.R.T. temperature is below the 20°C ideal minimum (sensor/fan/offline disk?)." # --- NVMe temperature tiers (device=~"(/dev/)?nvme.*") --- - # NVMe runs hotter than HDD: 40-60°C is normal, 60-70°C is warm but - # acceptable under sustained load, and throttling typically starts in the - # 70-80°C range. The HDD ladder above excludes NVMe devices. + # Rated 0-70°C: <60 normal, 65 warm, 70 at the rated ceiling, 75 above + # spec. The HDD ladder above excludes NVMe devices. - uid: nvme_temp_ge_65 title: "NVMe temperature ≥65°C" condition: C @@ -460,7 +459,7 @@ groups: severity: warning annotations: summary: "NVMe {{ $labels.device }} at {{ $values.B }}°C ({{ $labels.instance }})" - description: "NVMe temperature is at or above 65°C (warm). Normal is 40-60°C; acceptable under sustained load." + description: "NVMe temperature is at or above 65°C (warm), approaching the 70°C rated maximum." - uid: nvme_temp_ge_70 title: "NVMe temperature ≥70°C" @@ -508,7 +507,7 @@ groups: severity: high annotations: summary: "NVMe {{ $labels.device }} at {{ $values.B }}°C ({{ $labels.instance }})" - description: "NVMe temperature is at or above 70°C. Thermal throttling typically begins in the 70-80°C range." + description: "NVMe temperature is at or above 70°C, the rated maximum operating temperature." - uid: nvme_temp_ge_75 title: "NVMe temperature ≥75°C" @@ -556,7 +555,7 @@ groups: severity: critical annotations: summary: "NVMe {{ $labels.device }} at {{ $values.B }}°C ({{ $labels.instance }})" - description: "NVMe temperature is at or above 75°C (hot). Check airflow/heatsink; sustained throttling expected." + description: "NVMe temperature is at or above 75°C, above the 70°C rated maximum. Check airflow/heatsink." # --- Capacity: ZFS pools (data disk) --- # zfs_pool_capacity_percent is emitted hourly by the zfs role's diff --git a/roles/grafana/files/smartctl-dashboard.json b/roles/grafana/files/smartctl-dashboard.json index 3c3ec53..d6a698e 100644 --- a/roles/grafana/files/smartctl-dashboard.json +++ b/roles/grafana/files/smartctl-dashboard.json @@ -54,9 +54,9 @@ "title": "Temperature", "gridPos": { "h": 4, "w": 6, "x": 6, "y": 0 }, "datasource": { "type": "prometheus", "uid": "prometheus" }, - "fieldConfig": { "defaults": { "unit": "celsius", "color": { "mode": "thresholds" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "orange", "value": 45 }, { "color": "red", "value": 55 } ] } }, "overrides": [] }, + "fieldConfig": { "defaults": { "unit": "celsius", "color": { "mode": "thresholds" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "orange", "value": 45 }, { "color": "red", "value": 55 } ] } }, "overrides": [ { "matcher": { "id": "byRegexp", "options": "^(/dev/)?nvme.*" }, "properties": [ { "id": "thresholds", "value": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "orange", "value": 60 }, { "color": "red", "value": 70 } ] } } ] } ] }, "options": { "reduceOptions": { "values": false, "calcs": ["lastNotNull"], "fields": "" }, "colorMode": "background", "graphMode": "none", "textMode": "auto", "justifyMode": "auto", "orientation": "auto" }, - "targets": [ { "refId": "A", "expr": "smartctl_device_temperature{instance=~\"$instance\", device=~\"$device\", temperature_type=\"current\"}" } ] + "targets": [ { "refId": "A", "expr": "smartctl_device_temperature{instance=~\"$instance\", device=~\"$device\", temperature_type=\"current\"}", "legendFormat": "{{device}}" } ] }, { "id": 3,