From c1273363213727b9ce65cfcb516bdb9d416bcc4d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20D=C3=A9siles?= <1536672+cdesiles@users.noreply.github.com> Date: Fri, 18 Sep 2026 15:25:23 +0200 Subject: [PATCH] feat(zfs): add capacity metrics and Grafana dashboard Deploy an hourly systemd timer that writes pool and dataset capacity to node_exporter's textfile collector, since --collector.zfs exposes only ARC/I-O/state. Add a provisioned 'ZFS - Pools & Datasets' dashboard with per-pool totals, per-dataset used/quota and daily growth panels. --- roles/grafana/files/zfs-dashboard.json | 148 +++++++++++++++++++++ roles/grafana/tasks/main.yml | 9 ++ roles/zfs/README.md | 26 +++- roles/zfs/defaults/main.yml | 18 +++ roles/zfs/handlers/main.yml | 4 + roles/zfs/tasks/main.yml | 4 + roles/zfs/tasks/metrics.yml | 43 ++++++ roles/zfs/templates/zfs-metrics.service.j2 | 7 + roles/zfs/templates/zfs-metrics.sh.j2 | 49 +++++++ roles/zfs/templates/zfs-metrics.timer.j2 | 10 ++ 10 files changed, 317 insertions(+), 1 deletion(-) create mode 100644 roles/grafana/files/zfs-dashboard.json create mode 100644 roles/zfs/handlers/main.yml create mode 100644 roles/zfs/tasks/metrics.yml create mode 100644 roles/zfs/templates/zfs-metrics.service.j2 create mode 100644 roles/zfs/templates/zfs-metrics.sh.j2 create mode 100644 roles/zfs/templates/zfs-metrics.timer.j2 diff --git a/roles/grafana/files/zfs-dashboard.json b/roles/grafana/files/zfs-dashboard.json new file mode 100644 index 0000000..792a422 --- /dev/null +++ b/roles/grafana/files/zfs-dashboard.json @@ -0,0 +1,148 @@ +{ + "title": "ZFS - Pools & Datasets", + "uid": "zfs", + "tags": ["zfs", "storage", "disks"], + "schemaVersion": 39, + "version": 1, + "refresh": "30m", + "time": { "from": "now-90d", "to": "now" }, + "timezone": "browser", + "editable": true, + "graphTooltip": 1, + "templating": { + "list": [ + { + "name": "pool", + "label": "Pool", + "type": "query", + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "query": { "query": "label_values(zfs_pool_size_bytes, pool)", "refId": "A" }, + "refresh": 2, + "sort": 1, + "multi": true, + "includeAll": true, + "allValue": ".*", + "current": { "selected": true, "text": ["All"], "value": ["$__all"] } + }, + { + "name": "dataset", + "label": "Dataset", + "type": "query", + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "query": { "query": "label_values(zfs_dataset_used_bytes, dataset)", "refId": "A" }, + "refresh": 2, + "sort": 1, + "multi": true, + "includeAll": true, + "allValue": ".*", + "current": { "selected": true, "text": ["All"], "value": ["$__all"] } + } + ] + }, + "panels": [ + { + "id": 1, + "type": "stat", + "title": "Pool Size", + "gridPos": { "h": 4, "w": 6, "x": 0, "y": 0 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { "defaults": { "unit": "bytes", "color": { "mode": "none" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] } }, "overrides": [] }, + "options": { "reduceOptions": { "values": false, "calcs": ["lastNotNull"], "fields": "" }, "colorMode": "value", "graphMode": "none", "textMode": "auto", "justifyMode": "auto", "orientation": "auto" }, + "targets": [ { "refId": "A", "expr": "zfs_pool_size_bytes{pool=~\"$pool\"}", "legendFormat": "{{pool}}" } ] + }, + { + "id": 2, + "type": "stat", + "title": "Pool Allocated", + "gridPos": { "h": 4, "w": 6, "x": 6, "y": 0 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { "defaults": { "unit": "bytes", "color": { "mode": "none" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] } }, "overrides": [] }, + "options": { "reduceOptions": { "values": false, "calcs": ["lastNotNull"], "fields": "" }, "colorMode": "value", "graphMode": "none", "textMode": "auto", "justifyMode": "auto", "orientation": "auto" }, + "targets": [ { "refId": "A", "expr": "zfs_pool_alloc_bytes{pool=~\"$pool\"}", "legendFormat": "{{pool}}" } ] + }, + { + "id": 3, + "type": "stat", + "title": "Pool Free", + "gridPos": { "h": 4, "w": 6, "x": 12, "y": 0 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { "defaults": { "unit": "bytes", "color": { "mode": "none" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] } }, "overrides": [] }, + "options": { "reduceOptions": { "values": false, "calcs": ["lastNotNull"], "fields": "" }, "colorMode": "value", "graphMode": "none", "textMode": "auto", "justifyMode": "auto", "orientation": "auto" }, + "targets": [ { "refId": "A", "expr": "zfs_pool_free_bytes{pool=~\"$pool\"}", "legendFormat": "{{pool}}" } ] + }, + { + "id": 4, + "type": "stat", + "title": "Pool Capacity", + "gridPos": { "h": 4, "w": 6, "x": 18, "y": 0 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { "defaults": { "unit": "percent", "min": 0, "max": 100, "color": { "mode": "thresholds" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "orange", "value": 80 }, { "color": "red", "value": 90 } ] } }, "overrides": [] }, + "options": { "reduceOptions": { "values": false, "calcs": ["lastNotNull"], "fields": "" }, "colorMode": "background", "graphMode": "area", "textMode": "auto", "justifyMode": "auto", "orientation": "auto" }, + "targets": [ { "refId": "A", "expr": "zfs_pool_capacity_percent{pool=~\"$pool\"}", "legendFormat": "{{pool}}" } ] + }, + { + "id": 5, + "type": "timeseries", + "title": "Pool usage over time", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 4 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { "defaults": { "unit": "bytes", "color": { "mode": "palette-classic" }, "custom": { "fillOpacity": 15, "lineWidth": 1, "drawStyle": "line", "showPoints": "never" } }, "overrides": [] }, + "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, + "targets": [ + { "refId": "A", "expr": "zfs_pool_size_bytes{pool=~\"$pool\"}", "legendFormat": "{{pool}} size" }, + { "refId": "B", "expr": "zfs_pool_alloc_bytes{pool=~\"$pool\"}", "legendFormat": "{{pool}} alloc" }, + { "refId": "C", "expr": "zfs_pool_free_bytes{pool=~\"$pool\"}", "legendFormat": "{{pool}} free" } + ] + }, + { + "id": 6, + "type": "timeseries", + "title": "Pool capacity %", + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 4 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { "defaults": { "unit": "percent", "min": 0, "max": 100, "color": { "mode": "palette-classic" }, "custom": { "fillOpacity": 15, "lineWidth": 1, "drawStyle": "line", "showPoints": "never" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "orange", "value": 80 }, { "color": "red", "value": 90 } ] } }, "overrides": [] }, + "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, + "targets": [ { "refId": "A", "expr": "zfs_pool_capacity_percent{pool=~\"$pool\"}", "legendFormat": "{{pool}}" } ] + }, + { + "id": 7, + "type": "timeseries", + "title": "Dataset used bytes", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 12 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { "defaults": { "unit": "bytes", "color": { "mode": "palette-classic" }, "custom": { "fillOpacity": 15, "lineWidth": 1, "drawStyle": "line", "showPoints": "never" } }, "overrides": [] }, + "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, + "targets": [ { "refId": "A", "expr": "zfs_dataset_used_bytes{dataset=~\"$dataset\"}", "legendFormat": "{{dataset}}" } ] + }, + { + "id": 8, + "type": "bargauge", + "title": "Dataset used (current)", + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 12 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { "defaults": { "unit": "bytes", "min": 0, "color": { "mode": "continuous-GrYlRd" } }, "overrides": [] }, + "options": { "displayMode": "gradient", "orientation": "horizontal", "reduceOptions": { "values": false, "calcs": ["lastNotNull"], "fields": "" }, "showUnfilled": true }, + "targets": [ { "refId": "A", "expr": "zfs_dataset_used_bytes{dataset=~\"$dataset\"}", "legendFormat": "{{dataset}}" } ] + }, + { + "id": 9, + "type": "timeseries", + "title": "Daily growth - pool allocated (bytes/day)", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 20 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { "defaults": { "unit": "bytes", "color": { "mode": "palette-classic" }, "custom": { "fillOpacity": 15, "lineWidth": 1, "drawStyle": "line", "showPoints": "never" } }, "overrides": [] }, + "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, + "targets": [ { "refId": "A", "expr": "deriv(zfs_pool_alloc_bytes{pool=~\"$pool\"}[1d]) * 86400", "legendFormat": "{{pool}}" } ] + }, + { + "id": 10, + "type": "timeseries", + "title": "Daily growth - datasets (bytes/day)", + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 20 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { "defaults": { "unit": "bytes", "color": { "mode": "palette-classic" }, "custom": { "fillOpacity": 15, "lineWidth": 1, "drawStyle": "line", "showPoints": "never" } }, "overrides": [] }, + "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, + "targets": [ { "refId": "A", "expr": "deriv(zfs_dataset_used_bytes{dataset=~\"$dataset\"}[1d]) * 86400", "legendFormat": "{{dataset}}" } ] + } + ] +} diff --git a/roles/grafana/tasks/main.yml b/roles/grafana/tasks/main.yml index c625218..f95b423 100644 --- a/roles/grafana/tasks/main.yml +++ b/roles/grafana/tasks/main.yml @@ -260,6 +260,15 @@ - systemd-dashboard.json notify: Restart grafana +- name: Deploy ZFS dashboard + ansible.builtin.copy: + src: zfs-dashboard.json + dest: "{{ grafana_provisioning_path }}/dashboards/zfs.json" + owner: "{{ grafana_user }}" + group: "{{ grafana_group }}" + mode: "0644" + notify: Restart grafana + # Service management - name: Enable and start Grafana service ansible.builtin.systemd: diff --git a/roles/zfs/README.md b/roles/zfs/README.md index 4d3ee4b..8c3df94 100644 --- a/roles/zfs/README.md +++ b/roles/zfs/README.md @@ -1,4 +1,4 @@ -# Disks +# ZFS Ansible community support for ZFS is limited to create filesystems, volumes and snapshots. There is no support for managing zpools, so here it is. @@ -36,6 +36,30 @@ zfs_datasets: quota: 500G ``` +## Prometheus metrics + +The role deploys an hourly systemd timer (`zfs-metrics.timer`) running +`/usr/local/bin/zfs-metrics.sh`, which writes pool and dataset capacity to +node_exporter's textfile collector directory (`zfs_metrics_dir`). The +`node_exporter` role must be deployed on the host; the script exits silently +otherwise. + +| Metric | Labels | Description | +| --- | --- | --- | +| `zfs_pool_size_bytes` | `pool` | Total pool size | +| `zfs_pool_alloc_bytes` | `pool` | Allocated bytes | +| `zfs_pool_free_bytes` | `pool` | Free bytes | +| `zfs_pool_capacity_percent` | `pool` | Used capacity | +| `zfs_dataset_used_bytes` | `dataset` | Used bytes (incl. snapshots and children) | +| `zfs_dataset_refer_bytes` | `dataset` | Own referenced bytes | +| `zfs_dataset_quota_bytes` | `dataset` | Quota (0 = none) | + +These feed the **ZFS - Pools & Datasets** Grafana dashboard. node_exporter's +built-in `--collector.zfs` exposes ARC, I/O and pool state only, not capacity. + +Tunables (`defaults/main.yml`): `zfs_metrics_enabled`, `zfs_metrics_dir`, +`zfs_metrics_script_path`, `zfs_metrics_interval`. + ## References - https://docs.ansible.com/ansible/latest/collections/community/general/zfs_module.html diff --git a/roles/zfs/defaults/main.yml b/roles/zfs/defaults/main.yml index e69de29..6043dc4 100644 --- a/roles/zfs/defaults/main.yml +++ b/roles/zfs/defaults/main.yml @@ -0,0 +1,18 @@ +--- +# ZFS capacity metrics (Prometheus) +# --------------------------------- +# Exposes pool size/used/free and per-dataset used/quota to Prometheus via +# node_exporter's textfile collector. node_exporter's built-in --collector.zfs +# only exposes ARC, I/O and pool state — not capacity. +zfs_metrics_enabled: true + +# Directory node_exporter's textfile collector reads `*.prom` files from. +# Must match node_exporter_textfile_dir in roles/node_exporter/defaults/main.yml. +zfs_metrics_dir: /var/lib/node_exporter/textfile_collector + +# Where the collector script is installed. +zfs_metrics_script_path: /usr/local/bin/zfs-metrics.sh + +# systemd OnCalendar expression for the collection timer. Hourly is plenty to +# track daily growth; Prometheus keeps 90d of history on andromeda. +zfs_metrics_interval: "hourly" diff --git a/roles/zfs/handlers/main.yml b/roles/zfs/handlers/main.yml new file mode 100644 index 0000000..c28484f --- /dev/null +++ b/roles/zfs/handlers/main.yml @@ -0,0 +1,4 @@ +--- +- name: Reload systemd + ansible.builtin.systemd: + daemon_reload: true diff --git a/roles/zfs/tasks/main.yml b/roles/zfs/tasks/main.yml index 48d24f3..af61bdc 100644 --- a/roles/zfs/tasks/main.yml +++ b/roles/zfs/tasks/main.yml @@ -5,3 +5,7 @@ ansible.builtin.include_tasks: pools.yml - name: "Setup ZFS datasets: filesystems, snapshots, volumes" ansible.builtin.include_tasks: datasets.yml + +- name: Setup ZFS capacity metrics + ansible.builtin.include_tasks: metrics.yml + when: zfs_metrics_enabled | bool diff --git a/roles/zfs/tasks/metrics.yml b/roles/zfs/tasks/metrics.yml new file mode 100644 index 0000000..d2048a4 --- /dev/null +++ b/roles/zfs/tasks/metrics.yml @@ -0,0 +1,43 @@ +--- +# ZFS capacity metrics, exposed to Prometheus through node_exporter's textfile +# collector (see roles/node_exporter). Runs hourly as a root system timer. + +- name: Deploy ZFS metrics script + ansible.builtin.template: + src: zfs-metrics.sh.j2 + dest: "{{ zfs_metrics_script_path }}" + owner: root + group: root + mode: "0755" + +- name: Deploy ZFS metrics systemd service + ansible.builtin.template: + src: zfs-metrics.service.j2 + dest: /etc/systemd/system/zfs-metrics.service + owner: root + group: root + mode: "0644" + notify: Reload systemd + +- name: Deploy ZFS metrics systemd timer + ansible.builtin.template: + src: zfs-metrics.timer.j2 + dest: /etc/systemd/system/zfs-metrics.timer + owner: root + group: root + mode: "0644" + notify: Reload systemd + +- name: Flush handlers before enabling the timer + ansible.builtin.meta: flush_handlers + +- name: Enable and start ZFS metrics timer + ansible.builtin.systemd: + name: zfs-metrics.timer + enabled: true + state: started + +- name: Collect ZFS metrics now + ansible.builtin.systemd: + name: zfs-metrics.service + state: started diff --git a/roles/zfs/templates/zfs-metrics.service.j2 b/roles/zfs/templates/zfs-metrics.service.j2 new file mode 100644 index 0000000..a63f6a1 --- /dev/null +++ b/roles/zfs/templates/zfs-metrics.service.j2 @@ -0,0 +1,7 @@ +[Unit] +Description=Collect ZFS pool and dataset capacity metrics +After=zfs.target zfs-import.target + +[Service] +Type=oneshot +ExecStart={{ zfs_metrics_script_path }} diff --git a/roles/zfs/templates/zfs-metrics.sh.j2 b/roles/zfs/templates/zfs-metrics.sh.j2 new file mode 100644 index 0000000..86ef3a1 --- /dev/null +++ b/roles/zfs/templates/zfs-metrics.sh.j2 @@ -0,0 +1,49 @@ +#!/bin/sh +# Managed by Ansible - DO NOT EDIT MANUALLY +# +# Writes ZFS pool and dataset capacity metrics as Prometheus textfile metrics +# into node_exporter's textfile collector directory. node_exporter's built-in +# --collector.zfs exposes ARC, pool I/O and pool state only, not capacity. +set -eu +set -o pipefail + +metrics_dir="{{ zfs_metrics_dir }}" + +# node_exporter may not be deployed on this host. +[ -d "$metrics_dir" ] || exit 0 + +out="$metrics_dir/zfs.prom" +tmp="$out.tmp" + +# Written atomically (tmp + mv) so the collector never reads a half-written file. +{ + echo "# HELP zfs_pool_size_bytes Total size of the ZFS pool." + echo "# TYPE zfs_pool_size_bytes gauge" + echo "# HELP zfs_pool_alloc_bytes Allocated bytes in the ZFS pool." + echo "# TYPE zfs_pool_alloc_bytes gauge" + echo "# HELP zfs_pool_free_bytes Free bytes in the ZFS pool." + echo "# TYPE zfs_pool_free_bytes gauge" + echo "# HELP zfs_pool_capacity_percent Used capacity of the ZFS pool, in percent." + echo "# TYPE zfs_pool_capacity_percent gauge" + echo "# HELP zfs_dataset_used_bytes Used bytes of a ZFS dataset, including snapshots and children." + echo "# TYPE zfs_dataset_used_bytes gauge" + echo "# HELP zfs_dataset_refer_bytes Bytes referenced by a ZFS dataset itself." + echo "# TYPE zfs_dataset_refer_bytes gauge" + echo "# HELP zfs_dataset_quota_bytes Quota of a ZFS dataset in bytes (0 = no quota)." + echo "# TYPE zfs_dataset_quota_bytes gauge" + + zpool list -Hp -o name,size,alloc,free,capacity | while read -r pool size alloc free capacity; do + echo "zfs_pool_size_bytes{pool=\"$pool\"} $size" + echo "zfs_pool_alloc_bytes{pool=\"$pool\"} $alloc" + echo "zfs_pool_free_bytes{pool=\"$pool\"} $free" + echo "zfs_pool_capacity_percent{pool=\"$pool\"} $capacity" + done + + zfs list -Hp -o name,used,refer,quota -t filesystem | while read -r dataset used refer quota; do + echo "zfs_dataset_used_bytes{dataset=\"$dataset\"} $used" + echo "zfs_dataset_refer_bytes{dataset=\"$dataset\"} $refer" + echo "zfs_dataset_quota_bytes{dataset=\"$dataset\"} $quota" + done +} > "$tmp" + +mv "$tmp" "$out" diff --git a/roles/zfs/templates/zfs-metrics.timer.j2 b/roles/zfs/templates/zfs-metrics.timer.j2 new file mode 100644 index 0000000..4045d30 --- /dev/null +++ b/roles/zfs/templates/zfs-metrics.timer.j2 @@ -0,0 +1,10 @@ +[Unit] +Description=Timer for ZFS capacity metrics collection + +[Timer] +OnCalendar={{ zfs_metrics_interval }} +Persistent=true +Unit=zfs-metrics.service + +[Install] +WantedBy=timers.target