feat(zfs): add capacity metrics and Grafana dashboard

Deploy an hourly systemd timer that writes pool and dataset capacity to
node_exporter's textfile collector, since --collector.zfs exposes only
ARC/I-O/state. Add a provisioned 'ZFS - Pools & Datasets' dashboard with
per-pool totals, per-dataset used/quota and daily growth panels.

roles/grafana/files/zfs-dashboard.json

@@ -0,0 +1,148 @@
+{
+  "title": "ZFS - Pools & Datasets",
+  "uid": "zfs",
+  "tags": ["zfs", "storage", "disks"],
+  "schemaVersion": 39,
+  "version": 1,
+  "refresh": "30m",
+  "time": { "from": "now-90d", "to": "now" },
+  "timezone": "browser",
+  "editable": true,
+  "graphTooltip": 1,
+  "templating": {
+    "list": [
+      {
+        "name": "pool",
+        "label": "Pool",
+        "type": "query",
+        "datasource": { "type": "prometheus", "uid": "prometheus" },
+        "query": { "query": "label_values(zfs_pool_size_bytes, pool)", "refId": "A" },
+        "refresh": 2,
+        "sort": 1,
+        "multi": true,
+        "includeAll": true,
+        "allValue": ".*",
+        "current": { "selected": true, "text": ["All"], "value": ["$__all"] }
+      },
+      {
+        "name": "dataset",
+        "label": "Dataset",
+        "type": "query",
+        "datasource": { "type": "prometheus", "uid": "prometheus" },
+        "query": { "query": "label_values(zfs_dataset_used_bytes, dataset)", "refId": "A" },
+        "refresh": 2,
+        "sort": 1,
+        "multi": true,
+        "includeAll": true,
+        "allValue": ".*",
+        "current": { "selected": true, "text": ["All"], "value": ["$__all"] }
+      }
+    ]
+  },
+  "panels": [
+    {
+      "id": 1,
+      "type": "stat",
+      "title": "Pool Size",
+      "gridPos": { "h": 4, "w": 6, "x": 0, "y": 0 },
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "fieldConfig": { "defaults": { "unit": "bytes", "color": { "mode": "none" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] } }, "overrides": [] },
+      "options": { "reduceOptions": { "values": false, "calcs": ["lastNotNull"], "fields": "" }, "colorMode": "value", "graphMode": "none", "textMode": "auto", "justifyMode": "auto", "orientation": "auto" },
+      "targets": [ { "refId": "A", "expr": "zfs_pool_size_bytes{pool=~\"$pool\"}", "legendFormat": "{{pool}}" } ]
+    },
+    {
+      "id": 2,
+      "type": "stat",
+      "title": "Pool Allocated",
+      "gridPos": { "h": 4, "w": 6, "x": 6, "y": 0 },
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "fieldConfig": { "defaults": { "unit": "bytes", "color": { "mode": "none" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] } }, "overrides": [] },
+      "options": { "reduceOptions": { "values": false, "calcs": ["lastNotNull"], "fields": "" }, "colorMode": "value", "graphMode": "none", "textMode": "auto", "justifyMode": "auto", "orientation": "auto" },
+      "targets": [ { "refId": "A", "expr": "zfs_pool_alloc_bytes{pool=~\"$pool\"}", "legendFormat": "{{pool}}" } ]
+    },
+    {
+      "id": 3,
+      "type": "stat",
+      "title": "Pool Free",
+      "gridPos": { "h": 4, "w": 6, "x": 12, "y": 0 },
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "fieldConfig": { "defaults": { "unit": "bytes", "color": { "mode": "none" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] } }, "overrides": [] },
+      "options": { "reduceOptions": { "values": false, "calcs": ["lastNotNull"], "fields": "" }, "colorMode": "value", "graphMode": "none", "textMode": "auto", "justifyMode": "auto", "orientation": "auto" },
+      "targets": [ { "refId": "A", "expr": "zfs_pool_free_bytes{pool=~\"$pool\"}", "legendFormat": "{{pool}}" } ]
+    },
+    {
+      "id": 4,
+      "type": "stat",
+      "title": "Pool Capacity",
+      "gridPos": { "h": 4, "w": 6, "x": 18, "y": 0 },
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "fieldConfig": { "defaults": { "unit": "percent", "min": 0, "max": 100, "color": { "mode": "thresholds" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "orange", "value": 80 }, { "color": "red", "value": 90 } ] } }, "overrides": [] },
+      "options": { "reduceOptions": { "values": false, "calcs": ["lastNotNull"], "fields": "" }, "colorMode": "background", "graphMode": "area", "textMode": "auto", "justifyMode": "auto", "orientation": "auto" },
+      "targets": [ { "refId": "A", "expr": "zfs_pool_capacity_percent{pool=~\"$pool\"}", "legendFormat": "{{pool}}" } ]
+    },
+    {
+      "id": 5,
+      "type": "timeseries",
+      "title": "Pool usage over time",
+      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 4 },
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "fieldConfig": { "defaults": { "unit": "bytes", "color": { "mode": "palette-classic" }, "custom": { "fillOpacity": 15, "lineWidth": 1, "drawStyle": "line", "showPoints": "never" } }, "overrides": [] },
+      "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } },
+      "targets": [
+        { "refId": "A", "expr": "zfs_pool_size_bytes{pool=~\"$pool\"}", "legendFormat": "{{pool}} size" },
+        { "refId": "B", "expr": "zfs_pool_alloc_bytes{pool=~\"$pool\"}", "legendFormat": "{{pool}} alloc" },
+        { "refId": "C", "expr": "zfs_pool_free_bytes{pool=~\"$pool\"}", "legendFormat": "{{pool}} free" }
+      ]
+    },
+    {
+      "id": 6,
+      "type": "timeseries",
+      "title": "Pool capacity %",
+      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 4 },
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "fieldConfig": { "defaults": { "unit": "percent", "min": 0, "max": 100, "color": { "mode": "palette-classic" }, "custom": { "fillOpacity": 15, "lineWidth": 1, "drawStyle": "line", "showPoints": "never" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "orange", "value": 80 }, { "color": "red", "value": 90 } ] } }, "overrides": [] },
+      "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } },
+      "targets": [ { "refId": "A", "expr": "zfs_pool_capacity_percent{pool=~\"$pool\"}", "legendFormat": "{{pool}}" } ]
+    },
+    {
+      "id": 7,
+      "type": "timeseries",
+      "title": "Dataset used bytes",
+      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 12 },
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "fieldConfig": { "defaults": { "unit": "bytes", "color": { "mode": "palette-classic" }, "custom": { "fillOpacity": 15, "lineWidth": 1, "drawStyle": "line", "showPoints": "never" } }, "overrides": [] },
+      "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } },
+      "targets": [ { "refId": "A", "expr": "zfs_dataset_used_bytes{dataset=~\"$dataset\"}", "legendFormat": "{{dataset}}" } ]
+    },
+    {
+      "id": 8,
+      "type": "bargauge",
+      "title": "Dataset used (current)",
+      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 12 },
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "fieldConfig": { "defaults": { "unit": "bytes", "min": 0, "color": { "mode": "continuous-GrYlRd" } }, "overrides": [] },
+      "options": { "displayMode": "gradient", "orientation": "horizontal", "reduceOptions": { "values": false, "calcs": ["lastNotNull"], "fields": "" }, "showUnfilled": true },
+      "targets": [ { "refId": "A", "expr": "zfs_dataset_used_bytes{dataset=~\"$dataset\"}", "legendFormat": "{{dataset}}" } ]
+    },
+    {
+      "id": 9,
+      "type": "timeseries",
+      "title": "Daily growth - pool allocated (bytes/day)",
+      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 20 },
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "fieldConfig": { "defaults": { "unit": "bytes", "color": { "mode": "palette-classic" }, "custom": { "fillOpacity": 15, "lineWidth": 1, "drawStyle": "line", "showPoints": "never" } }, "overrides": [] },
+      "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } },
+      "targets": [ { "refId": "A", "expr": "deriv(zfs_pool_alloc_bytes{pool=~\"$pool\"}[1d]) * 86400", "legendFormat": "{{pool}}" } ]
+    },
+    {
+      "id": 10,
+      "type": "timeseries",
+      "title": "Daily growth - datasets (bytes/day)",
+      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 20 },
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "fieldConfig": { "defaults": { "unit": "bytes", "color": { "mode": "palette-classic" }, "custom": { "fillOpacity": 15, "lineWidth": 1, "drawStyle": "line", "showPoints": "never" } }, "overrides": [] },
+      "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } },
+      "targets": [ { "refId": "A", "expr": "deriv(zfs_dataset_used_bytes{dataset=~\"$dataset\"}[1d]) * 86400", "legendFormat": "{{dataset}}" } ]
+    }
+  ]
+}

roles/grafana/tasks/main.yml

@@ -260,6 +260,15 @@
     - systemd-dashboard.json
   notify: Restart grafana
 
+- name: Deploy ZFS dashboard
+  ansible.builtin.copy:
+    src: zfs-dashboard.json
+    dest: "{{ grafana_provisioning_path }}/dashboards/zfs.json"
+    owner: "{{ grafana_user }}"
+    group: "{{ grafana_group }}"
+    mode: "0644"
+  notify: Restart grafana
+
 # Service management
 - name: Enable and start Grafana service
   ansible.builtin.systemd:

roles/zfs/README.md

@@ -1,4 +1,4 @@
-# Disks
+# ZFS
 
 Ansible community support for ZFS is limited to create filesystems, volumes and snapshots. There is no support for managing zpools, so here it is.
 
@@ -36,6 +36,30 @@ zfs_datasets:
           quota: 500G
 ```
 
+## Prometheus metrics
+
+The role deploys an hourly systemd timer (`zfs-metrics.timer`) running
+`/usr/local/bin/zfs-metrics.sh`, which writes pool and dataset capacity to
+node_exporter's textfile collector directory (`zfs_metrics_dir`). The
+`node_exporter` role must be deployed on the host; the script exits silently
+otherwise.
+
+| Metric | Labels | Description |
+| --- | --- | --- |
+| `zfs_pool_size_bytes` | `pool` | Total pool size |
+| `zfs_pool_alloc_bytes` | `pool` | Allocated bytes |
+| `zfs_pool_free_bytes` | `pool` | Free bytes |
+| `zfs_pool_capacity_percent` | `pool` | Used capacity |
+| `zfs_dataset_used_bytes` | `dataset` | Used bytes (incl. snapshots and children) |
+| `zfs_dataset_refer_bytes` | `dataset` | Own referenced bytes |
+| `zfs_dataset_quota_bytes` | `dataset` | Quota (0 = none) |
+
+These feed the **ZFS - Pools & Datasets** Grafana dashboard. node_exporter's
+built-in `--collector.zfs` exposes ARC, I/O and pool state only, not capacity.
+
+Tunables (`defaults/main.yml`): `zfs_metrics_enabled`, `zfs_metrics_dir`,
+`zfs_metrics_script_path`, `zfs_metrics_interval`.
+
 ## References
 
 - https://docs.ansible.com/ansible/latest/collections/community/general/zfs_module.html

roles/zfs/defaults/main.yml

@@ -0,0 +1,18 @@
+---
+# ZFS capacity metrics (Prometheus)
+# ---------------------------------
+# Exposes pool size/used/free and per-dataset used/quota to Prometheus via
+# node_exporter's textfile collector. node_exporter's built-in --collector.zfs
+# only exposes ARC, I/O and pool state — not capacity.
+zfs_metrics_enabled: true
+
+# Directory node_exporter's textfile collector reads `*.prom` files from.
+# Must match node_exporter_textfile_dir in roles/node_exporter/defaults/main.yml.
+zfs_metrics_dir: /var/lib/node_exporter/textfile_collector
+
+# Where the collector script is installed.
+zfs_metrics_script_path: /usr/local/bin/zfs-metrics.sh
+
+# systemd OnCalendar expression for the collection timer. Hourly is plenty to
+# track daily growth; Prometheus keeps 90d of history on andromeda.
+zfs_metrics_interval: "hourly"

roles/zfs/handlers/main.yml

@@ -0,0 +1,4 @@
+---
+- name: Reload systemd
+  ansible.builtin.systemd:
+    daemon_reload: true

roles/zfs/tasks/main.yml

@@ -5,3 +5,7 @@
   ansible.builtin.include_tasks: pools.yml
 - name: "Setup ZFS datasets: filesystems, snapshots, volumes"
   ansible.builtin.include_tasks: datasets.yml
+
+- name: Setup ZFS capacity metrics
+  ansible.builtin.include_tasks: metrics.yml
+  when: zfs_metrics_enabled | bool

roles/zfs/tasks/metrics.yml

@@ -0,0 +1,43 @@
+---
+# ZFS capacity metrics, exposed to Prometheus through node_exporter's textfile
+# collector (see roles/node_exporter). Runs hourly as a root system timer.
+
+- name: Deploy ZFS metrics script
+  ansible.builtin.template:
+    src: zfs-metrics.sh.j2
+    dest: "{{ zfs_metrics_script_path }}"
+    owner: root
+    group: root
+    mode: "0755"
+
+- name: Deploy ZFS metrics systemd service
+  ansible.builtin.template:
+    src: zfs-metrics.service.j2
+    dest: /etc/systemd/system/zfs-metrics.service
+    owner: root
+    group: root
+    mode: "0644"
+  notify: Reload systemd
+
+- name: Deploy ZFS metrics systemd timer
+  ansible.builtin.template:
+    src: zfs-metrics.timer.j2
+    dest: /etc/systemd/system/zfs-metrics.timer
+    owner: root
+    group: root
+    mode: "0644"
+  notify: Reload systemd
+
+- name: Flush handlers before enabling the timer
+  ansible.builtin.meta: flush_handlers
+
+- name: Enable and start ZFS metrics timer
+  ansible.builtin.systemd:
+    name: zfs-metrics.timer
+    enabled: true
+    state: started
+
+- name: Collect ZFS metrics now
+  ansible.builtin.systemd:
+    name: zfs-metrics.service
+    state: started

roles/zfs/templates/zfs-metrics.service.j2

@@ -0,0 +1,7 @@
+[Unit]
+Description=Collect ZFS pool and dataset capacity metrics
+After=zfs.target zfs-import.target
+
+[Service]
+Type=oneshot
+ExecStart={{ zfs_metrics_script_path }}

roles/zfs/templates/zfs-metrics.sh.j2

@@ -0,0 +1,49 @@
+#!/bin/sh
+# Managed by Ansible - DO NOT EDIT MANUALLY
+#
+# Writes ZFS pool and dataset capacity metrics as Prometheus textfile metrics
+# into node_exporter's textfile collector directory. node_exporter's built-in
+# --collector.zfs exposes ARC, pool I/O and pool state only, not capacity.
+set -eu
+set -o pipefail
+
+metrics_dir="{{ zfs_metrics_dir }}"
+
+# node_exporter may not be deployed on this host.
+[ -d "$metrics_dir" ] || exit 0
+
+out="$metrics_dir/zfs.prom"
+tmp="$out.tmp"
+
+# Written atomically (tmp + mv) so the collector never reads a half-written file.
+{
+    echo "# HELP zfs_pool_size_bytes Total size of the ZFS pool."
+    echo "# TYPE zfs_pool_size_bytes gauge"
+    echo "# HELP zfs_pool_alloc_bytes Allocated bytes in the ZFS pool."
+    echo "# TYPE zfs_pool_alloc_bytes gauge"
+    echo "# HELP zfs_pool_free_bytes Free bytes in the ZFS pool."
+    echo "# TYPE zfs_pool_free_bytes gauge"
+    echo "# HELP zfs_pool_capacity_percent Used capacity of the ZFS pool, in percent."
+    echo "# TYPE zfs_pool_capacity_percent gauge"
+    echo "# HELP zfs_dataset_used_bytes Used bytes of a ZFS dataset, including snapshots and children."
+    echo "# TYPE zfs_dataset_used_bytes gauge"
+    echo "# HELP zfs_dataset_refer_bytes Bytes referenced by a ZFS dataset itself."
+    echo "# TYPE zfs_dataset_refer_bytes gauge"
+    echo "# HELP zfs_dataset_quota_bytes Quota of a ZFS dataset in bytes (0 = no quota)."
+    echo "# TYPE zfs_dataset_quota_bytes gauge"
+
+    zpool list -Hp -o name,size,alloc,free,capacity | while read -r pool size alloc free capacity; do
+        echo "zfs_pool_size_bytes{pool=\"$pool\"} $size"
+        echo "zfs_pool_alloc_bytes{pool=\"$pool\"} $alloc"
+        echo "zfs_pool_free_bytes{pool=\"$pool\"} $free"
+        echo "zfs_pool_capacity_percent{pool=\"$pool\"} $capacity"
+    done
+
+    zfs list -Hp -o name,used,refer,quota -t filesystem | while read -r dataset used refer quota; do
+        echo "zfs_dataset_used_bytes{dataset=\"$dataset\"} $used"
+        echo "zfs_dataset_refer_bytes{dataset=\"$dataset\"} $refer"
+        echo "zfs_dataset_quota_bytes{dataset=\"$dataset\"} $quota"
+    done
+} > "$tmp"
+
+mv "$tmp" "$out"

roles/zfs/templates/zfs-metrics.timer.j2

@@ -0,0 +1,10 @@
+[Unit]
+Description=Timer for ZFS capacity metrics collection
+
+[Timer]
+OnCalendar={{ zfs_metrics_interval }}
+Persistent=true
+Unit=zfs-metrics.service
+
+[Install]
+WantedBy=timers.target