feat(zfs): add capacity metrics and Grafana dashboard
Deploy an hourly systemd timer that writes pool and dataset capacity to node_exporter's textfile collector, since --collector.zfs exposes only ARC/I-O/state. Add a provisioned 'ZFS - Pools & Datasets' dashboard with per-pool totals, per-dataset used/quota and daily growth panels.
This commit is contained in:
Clément Désiles
@@ -0,0 +1,148 @@
|
||||
{
|
||||
"title": "ZFS - Pools & Datasets",
|
||||
"uid": "zfs",
|
||||
"tags": ["zfs", "storage", "disks"],
|
||||
"schemaVersion": 39,
|
||||
"version": 1,
|
||||
"refresh": "30m",
|
||||
"time": { "from": "now-90d", "to": "now" },
|
||||
"timezone": "browser",
|
||||
"editable": true,
|
||||
"graphTooltip": 1,
|
||||
"templating": {
|
||||
"list": [
|
||||
{
|
||||
"name": "pool",
|
||||
"label": "Pool",
|
||||
"type": "query",
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"query": { "query": "label_values(zfs_pool_size_bytes, pool)", "refId": "A" },
|
||||
"refresh": 2,
|
||||
"sort": 1,
|
||||
"multi": true,
|
||||
"includeAll": true,
|
||||
"allValue": ".*",
|
||||
"current": { "selected": true, "text": ["All"], "value": ["$__all"] }
|
||||
},
|
||||
{
|
||||
"name": "dataset",
|
||||
"label": "Dataset",
|
||||
"type": "query",
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"query": { "query": "label_values(zfs_dataset_used_bytes, dataset)", "refId": "A" },
|
||||
"refresh": 2,
|
||||
"sort": 1,
|
||||
"multi": true,
|
||||
"includeAll": true,
|
||||
"allValue": ".*",
|
||||
"current": { "selected": true, "text": ["All"], "value": ["$__all"] }
|
||||
}
|
||||
]
|
||||
},
|
||||
"panels": [
|
||||
{
|
||||
"id": 1,
|
||||
"type": "stat",
|
||||
"title": "Pool Size",
|
||||
"gridPos": { "h": 4, "w": 6, "x": 0, "y": 0 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": { "defaults": { "unit": "bytes", "color": { "mode": "none" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] } }, "overrides": [] },
|
||||
"options": { "reduceOptions": { "values": false, "calcs": ["lastNotNull"], "fields": "" }, "colorMode": "value", "graphMode": "none", "textMode": "auto", "justifyMode": "auto", "orientation": "auto" },
|
||||
"targets": [ { "refId": "A", "expr": "zfs_pool_size_bytes{pool=~\"$pool\"}", "legendFormat": "{{pool}}" } ]
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"type": "stat",
|
||||
"title": "Pool Allocated",
|
||||
"gridPos": { "h": 4, "w": 6, "x": 6, "y": 0 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": { "defaults": { "unit": "bytes", "color": { "mode": "none" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] } }, "overrides": [] },
|
||||
"options": { "reduceOptions": { "values": false, "calcs": ["lastNotNull"], "fields": "" }, "colorMode": "value", "graphMode": "none", "textMode": "auto", "justifyMode": "auto", "orientation": "auto" },
|
||||
"targets": [ { "refId": "A", "expr": "zfs_pool_alloc_bytes{pool=~\"$pool\"}", "legendFormat": "{{pool}}" } ]
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"type": "stat",
|
||||
"title": "Pool Free",
|
||||
"gridPos": { "h": 4, "w": 6, "x": 12, "y": 0 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": { "defaults": { "unit": "bytes", "color": { "mode": "none" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] } }, "overrides": [] },
|
||||
"options": { "reduceOptions": { "values": false, "calcs": ["lastNotNull"], "fields": "" }, "colorMode": "value", "graphMode": "none", "textMode": "auto", "justifyMode": "auto", "orientation": "auto" },
|
||||
"targets": [ { "refId": "A", "expr": "zfs_pool_free_bytes{pool=~\"$pool\"}", "legendFormat": "{{pool}}" } ]
|
||||
},
|
||||
{
|
||||
"id": 4,
|
||||
"type": "stat",
|
||||
"title": "Pool Capacity",
|
||||
"gridPos": { "h": 4, "w": 6, "x": 18, "y": 0 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": { "defaults": { "unit": "percent", "min": 0, "max": 100, "color": { "mode": "thresholds" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "orange", "value": 80 }, { "color": "red", "value": 90 } ] } }, "overrides": [] },
|
||||
"options": { "reduceOptions": { "values": false, "calcs": ["lastNotNull"], "fields": "" }, "colorMode": "background", "graphMode": "area", "textMode": "auto", "justifyMode": "auto", "orientation": "auto" },
|
||||
"targets": [ { "refId": "A", "expr": "zfs_pool_capacity_percent{pool=~\"$pool\"}", "legendFormat": "{{pool}}" } ]
|
||||
},
|
||||
{
|
||||
"id": 5,
|
||||
"type": "timeseries",
|
||||
"title": "Pool usage over time",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 4 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": { "defaults": { "unit": "bytes", "color": { "mode": "palette-classic" }, "custom": { "fillOpacity": 15, "lineWidth": 1, "drawStyle": "line", "showPoints": "never" } }, "overrides": [] },
|
||||
"options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } },
|
||||
"targets": [
|
||||
{ "refId": "A", "expr": "zfs_pool_size_bytes{pool=~\"$pool\"}", "legendFormat": "{{pool}} size" },
|
||||
{ "refId": "B", "expr": "zfs_pool_alloc_bytes{pool=~\"$pool\"}", "legendFormat": "{{pool}} alloc" },
|
||||
{ "refId": "C", "expr": "zfs_pool_free_bytes{pool=~\"$pool\"}", "legendFormat": "{{pool}} free" }
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 6,
|
||||
"type": "timeseries",
|
||||
"title": "Pool capacity %",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 4 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": { "defaults": { "unit": "percent", "min": 0, "max": 100, "color": { "mode": "palette-classic" }, "custom": { "fillOpacity": 15, "lineWidth": 1, "drawStyle": "line", "showPoints": "never" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "orange", "value": 80 }, { "color": "red", "value": 90 } ] } }, "overrides": [] },
|
||||
"options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } },
|
||||
"targets": [ { "refId": "A", "expr": "zfs_pool_capacity_percent{pool=~\"$pool\"}", "legendFormat": "{{pool}}" } ]
|
||||
},
|
||||
{
|
||||
"id": 7,
|
||||
"type": "timeseries",
|
||||
"title": "Dataset used bytes",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 12 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": { "defaults": { "unit": "bytes", "color": { "mode": "palette-classic" }, "custom": { "fillOpacity": 15, "lineWidth": 1, "drawStyle": "line", "showPoints": "never" } }, "overrides": [] },
|
||||
"options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } },
|
||||
"targets": [ { "refId": "A", "expr": "zfs_dataset_used_bytes{dataset=~\"$dataset\"}", "legendFormat": "{{dataset}}" } ]
|
||||
},
|
||||
{
|
||||
"id": 8,
|
||||
"type": "bargauge",
|
||||
"title": "Dataset used (current)",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 12 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": { "defaults": { "unit": "bytes", "min": 0, "color": { "mode": "continuous-GrYlRd" } }, "overrides": [] },
|
||||
"options": { "displayMode": "gradient", "orientation": "horizontal", "reduceOptions": { "values": false, "calcs": ["lastNotNull"], "fields": "" }, "showUnfilled": true },
|
||||
"targets": [ { "refId": "A", "expr": "zfs_dataset_used_bytes{dataset=~\"$dataset\"}", "legendFormat": "{{dataset}}" } ]
|
||||
},
|
||||
{
|
||||
"id": 9,
|
||||
"type": "timeseries",
|
||||
"title": "Daily growth - pool allocated (bytes/day)",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 20 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": { "defaults": { "unit": "bytes", "color": { "mode": "palette-classic" }, "custom": { "fillOpacity": 15, "lineWidth": 1, "drawStyle": "line", "showPoints": "never" } }, "overrides": [] },
|
||||
"options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } },
|
||||
"targets": [ { "refId": "A", "expr": "deriv(zfs_pool_alloc_bytes{pool=~\"$pool\"}[1d]) * 86400", "legendFormat": "{{pool}}" } ]
|
||||
},
|
||||
{
|
||||
"id": 10,
|
||||
"type": "timeseries",
|
||||
"title": "Daily growth - datasets (bytes/day)",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 20 },
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"fieldConfig": { "defaults": { "unit": "bytes", "color": { "mode": "palette-classic" }, "custom": { "fillOpacity": 15, "lineWidth": 1, "drawStyle": "line", "showPoints": "never" } }, "overrides": [] },
|
||||
"options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } },
|
||||
"targets": [ { "refId": "A", "expr": "deriv(zfs_dataset_used_bytes{dataset=~\"$dataset\"}[1d]) * 86400", "legendFormat": "{{dataset}}" } ]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -260,6 +260,15 @@
|
||||
- systemd-dashboard.json
|
||||
notify: Restart grafana
|
||||
|
||||
- name: Deploy ZFS dashboard
|
||||
ansible.builtin.copy:
|
||||
src: zfs-dashboard.json
|
||||
dest: "{{ grafana_provisioning_path }}/dashboards/zfs.json"
|
||||
owner: "{{ grafana_user }}"
|
||||
group: "{{ grafana_group }}"
|
||||
mode: "0644"
|
||||
notify: Restart grafana
|
||||
|
||||
# Service management
|
||||
- name: Enable and start Grafana service
|
||||
ansible.builtin.systemd:
|
||||
|
||||
+25
-1
@@ -1,4 +1,4 @@
|
||||
# Disks
|
||||
# ZFS
|
||||
|
||||
Ansible community support for ZFS is limited to create filesystems, volumes and snapshots. There is no support for managing zpools, so here it is.
|
||||
|
||||
@@ -36,6 +36,30 @@ zfs_datasets:
|
||||
quota: 500G
|
||||
```
|
||||
|
||||
## Prometheus metrics
|
||||
|
||||
The role deploys an hourly systemd timer (`zfs-metrics.timer`) running
|
||||
`/usr/local/bin/zfs-metrics.sh`, which writes pool and dataset capacity to
|
||||
node_exporter's textfile collector directory (`zfs_metrics_dir`). The
|
||||
`node_exporter` role must be deployed on the host; the script exits silently
|
||||
otherwise.
|
||||
|
||||
| Metric | Labels | Description |
|
||||
| --- | --- | --- |
|
||||
| `zfs_pool_size_bytes` | `pool` | Total pool size |
|
||||
| `zfs_pool_alloc_bytes` | `pool` | Allocated bytes |
|
||||
| `zfs_pool_free_bytes` | `pool` | Free bytes |
|
||||
| `zfs_pool_capacity_percent` | `pool` | Used capacity |
|
||||
| `zfs_dataset_used_bytes` | `dataset` | Used bytes (incl. snapshots and children) |
|
||||
| `zfs_dataset_refer_bytes` | `dataset` | Own referenced bytes |
|
||||
| `zfs_dataset_quota_bytes` | `dataset` | Quota (0 = none) |
|
||||
|
||||
These feed the **ZFS - Pools & Datasets** Grafana dashboard. node_exporter's
|
||||
built-in `--collector.zfs` exposes ARC, I/O and pool state only, not capacity.
|
||||
|
||||
Tunables (`defaults/main.yml`): `zfs_metrics_enabled`, `zfs_metrics_dir`,
|
||||
`zfs_metrics_script_path`, `zfs_metrics_interval`.
|
||||
|
||||
## References
|
||||
|
||||
- https://docs.ansible.com/ansible/latest/collections/community/general/zfs_module.html
|
||||
|
||||
@@ -0,0 +1,18 @@
|
||||
---
|
||||
# ZFS capacity metrics (Prometheus)
|
||||
# ---------------------------------
|
||||
# Exposes pool size/used/free and per-dataset used/quota to Prometheus via
|
||||
# node_exporter's textfile collector. node_exporter's built-in --collector.zfs
|
||||
# only exposes ARC, I/O and pool state — not capacity.
|
||||
zfs_metrics_enabled: true
|
||||
|
||||
# Directory node_exporter's textfile collector reads `*.prom` files from.
|
||||
# Must match node_exporter_textfile_dir in roles/node_exporter/defaults/main.yml.
|
||||
zfs_metrics_dir: /var/lib/node_exporter/textfile_collector
|
||||
|
||||
# Where the collector script is installed.
|
||||
zfs_metrics_script_path: /usr/local/bin/zfs-metrics.sh
|
||||
|
||||
# systemd OnCalendar expression for the collection timer. Hourly is plenty to
|
||||
# track daily growth; Prometheus keeps 90d of history on andromeda.
|
||||
zfs_metrics_interval: "hourly"
|
||||
|
||||
@@ -0,0 +1,4 @@
|
||||
---
|
||||
- name: Reload systemd
|
||||
ansible.builtin.systemd:
|
||||
daemon_reload: true
|
||||
@@ -5,3 +5,7 @@
|
||||
ansible.builtin.include_tasks: pools.yml
|
||||
- name: "Setup ZFS datasets: filesystems, snapshots, volumes"
|
||||
ansible.builtin.include_tasks: datasets.yml
|
||||
|
||||
- name: Setup ZFS capacity metrics
|
||||
ansible.builtin.include_tasks: metrics.yml
|
||||
when: zfs_metrics_enabled | bool
|
||||
|
||||
@@ -0,0 +1,43 @@
|
||||
---
|
||||
# ZFS capacity metrics, exposed to Prometheus through node_exporter's textfile
|
||||
# collector (see roles/node_exporter). Runs hourly as a root system timer.
|
||||
|
||||
- name: Deploy ZFS metrics script
|
||||
ansible.builtin.template:
|
||||
src: zfs-metrics.sh.j2
|
||||
dest: "{{ zfs_metrics_script_path }}"
|
||||
owner: root
|
||||
group: root
|
||||
mode: "0755"
|
||||
|
||||
- name: Deploy ZFS metrics systemd service
|
||||
ansible.builtin.template:
|
||||
src: zfs-metrics.service.j2
|
||||
dest: /etc/systemd/system/zfs-metrics.service
|
||||
owner: root
|
||||
group: root
|
||||
mode: "0644"
|
||||
notify: Reload systemd
|
||||
|
||||
- name: Deploy ZFS metrics systemd timer
|
||||
ansible.builtin.template:
|
||||
src: zfs-metrics.timer.j2
|
||||
dest: /etc/systemd/system/zfs-metrics.timer
|
||||
owner: root
|
||||
group: root
|
||||
mode: "0644"
|
||||
notify: Reload systemd
|
||||
|
||||
- name: Flush handlers before enabling the timer
|
||||
ansible.builtin.meta: flush_handlers
|
||||
|
||||
- name: Enable and start ZFS metrics timer
|
||||
ansible.builtin.systemd:
|
||||
name: zfs-metrics.timer
|
||||
enabled: true
|
||||
state: started
|
||||
|
||||
- name: Collect ZFS metrics now
|
||||
ansible.builtin.systemd:
|
||||
name: zfs-metrics.service
|
||||
state: started
|
||||
@@ -0,0 +1,7 @@
|
||||
[Unit]
|
||||
Description=Collect ZFS pool and dataset capacity metrics
|
||||
After=zfs.target zfs-import.target
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
ExecStart={{ zfs_metrics_script_path }}
|
||||
@@ -0,0 +1,49 @@
|
||||
#!/bin/sh
|
||||
# Managed by Ansible - DO NOT EDIT MANUALLY
|
||||
#
|
||||
# Writes ZFS pool and dataset capacity metrics as Prometheus textfile metrics
|
||||
# into node_exporter's textfile collector directory. node_exporter's built-in
|
||||
# --collector.zfs exposes ARC, pool I/O and pool state only, not capacity.
|
||||
set -eu
|
||||
set -o pipefail
|
||||
|
||||
metrics_dir="{{ zfs_metrics_dir }}"
|
||||
|
||||
# node_exporter may not be deployed on this host.
|
||||
[ -d "$metrics_dir" ] || exit 0
|
||||
|
||||
out="$metrics_dir/zfs.prom"
|
||||
tmp="$out.tmp"
|
||||
|
||||
# Written atomically (tmp + mv) so the collector never reads a half-written file.
|
||||
{
|
||||
echo "# HELP zfs_pool_size_bytes Total size of the ZFS pool."
|
||||
echo "# TYPE zfs_pool_size_bytes gauge"
|
||||
echo "# HELP zfs_pool_alloc_bytes Allocated bytes in the ZFS pool."
|
||||
echo "# TYPE zfs_pool_alloc_bytes gauge"
|
||||
echo "# HELP zfs_pool_free_bytes Free bytes in the ZFS pool."
|
||||
echo "# TYPE zfs_pool_free_bytes gauge"
|
||||
echo "# HELP zfs_pool_capacity_percent Used capacity of the ZFS pool, in percent."
|
||||
echo "# TYPE zfs_pool_capacity_percent gauge"
|
||||
echo "# HELP zfs_dataset_used_bytes Used bytes of a ZFS dataset, including snapshots and children."
|
||||
echo "# TYPE zfs_dataset_used_bytes gauge"
|
||||
echo "# HELP zfs_dataset_refer_bytes Bytes referenced by a ZFS dataset itself."
|
||||
echo "# TYPE zfs_dataset_refer_bytes gauge"
|
||||
echo "# HELP zfs_dataset_quota_bytes Quota of a ZFS dataset in bytes (0 = no quota)."
|
||||
echo "# TYPE zfs_dataset_quota_bytes gauge"
|
||||
|
||||
zpool list -Hp -o name,size,alloc,free,capacity | while read -r pool size alloc free capacity; do
|
||||
echo "zfs_pool_size_bytes{pool=\"$pool\"} $size"
|
||||
echo "zfs_pool_alloc_bytes{pool=\"$pool\"} $alloc"
|
||||
echo "zfs_pool_free_bytes{pool=\"$pool\"} $free"
|
||||
echo "zfs_pool_capacity_percent{pool=\"$pool\"} $capacity"
|
||||
done
|
||||
|
||||
zfs list -Hp -o name,used,refer,quota -t filesystem | while read -r dataset used refer quota; do
|
||||
echo "zfs_dataset_used_bytes{dataset=\"$dataset\"} $used"
|
||||
echo "zfs_dataset_refer_bytes{dataset=\"$dataset\"} $refer"
|
||||
echo "zfs_dataset_quota_bytes{dataset=\"$dataset\"} $quota"
|
||||
done
|
||||
} > "$tmp"
|
||||
|
||||
mv "$tmp" "$out"
|
||||
@@ -0,0 +1,10 @@
|
||||
[Unit]
|
||||
Description=Timer for ZFS capacity metrics collection
|
||||
|
||||
[Timer]
|
||||
OnCalendar={{ zfs_metrics_interval }}
|
||||
Persistent=true
|
||||
Unit=zfs-metrics.service
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
Reference in New Issue
Block a user