feat(monitoring): add smartctl, fail2ban, wireguard exporters and dashboards

- smartctl_exporter (disks), fail2ban exporter, wireguard exporter
- grafana dashboards for each exporter
- prometheus: firewall source rules, honor_labels/params scrape config
- node_exporter: zfs collector
- wireguard: nftables dep, resolvconf wrapper, reload fix
- remove superseded monitoring role

roles/disks/README.md

@@ -0,0 +1,29 @@
+# Disks
+
+Partitions disks, enables periodic SSD TRIM, and optionally exposes S.M.A.R.T.
+disk health to Prometheus.
+
+## Configuration
+
+See [defaults/main.yml](defaults/main.yml).
+
+### S.M.A.R.T. exporter
+
+Gated by `smartctl_exporter_enabled`. When true, downloads and runs the
+`smartctl_exporter` static binary as a root systemd service, polling `smartctl`
+at `smartctl_exporter_interval` (default `1h`).
+
+```yaml
+smartctl_exporter_enabled: true
+```
+
+Pair with Prometheus:
+
+```yaml
+prometheus_scrape_configs:
+  - job_name: "smartctl"
+    static_configs:
+      - targets: ["127.0.0.1:9633"]
+        labels:
+          instance: andromeda
+```

roles/disks/defaults/main.yml

@@ -1,2 +1,10 @@
 ---
 ssd_trim_periodicity: monthly
+
+# Prometheus S.M.A.R.T. exporter (optional, gated by smartctl_exporter_enabled).
+smartctl_exporter_enabled: false
+smartctl_exporter_version: "0.14.0"
+smartctl_exporter_listen_address: "127.0.0.1:9633"
+smartctl_exporter_binary_dir: "/usr/local/bin"
+smartctl_exporter_smartctl_path: "/usr/sbin/smartctl"
+smartctl_exporter_interval: "1h"

roles/disks/handlers/main.yml

@@ -2,3 +2,9 @@
 - name: Systemd daemon reload
   ansible.builtin.systemd:
     daemon_reload: true
+
+- name: Restart smartctl exporter
+  ansible.builtin.systemd:
+    name: smartctl_exporter
+    daemon_reload: true
+    state: restarted

roles/disks/tasks/exporter.yml

@@ -0,0 +1,86 @@
+---
+# Prometheus S.M.A.R.T. exporter (optional, gated by smartctl_exporter_enabled).
+# Downloads the checksum-verified prometheus-community/smartctl_exporter static
+# binary and runs it as a hardened root systemd service (required to read raw
+# S.M.A.R.T. attributes from block devices via smartctl).
+
+- name: Install smartmontools
+  ansible.builtin.package:
+    name: smartmontools
+    state: present
+
+- name: Map CPU architecture to release asset
+  ansible.builtin.set_fact:
+    smartctl_exporter_arch: >-
+      {{ {
+           'x86_64': 'amd64',
+           'aarch64': 'arm64',
+           'armv7l': 'armv7',
+           'armv6l': 'armv6',
+           'i386': '386',
+           'i686': '386'
+         }[ansible_architecture] }}
+
+- name: Set release filenames
+  ansible.builtin.set_fact:
+    smartctl_exporter_archive: "smartctl_exporter-{{ smartctl_exporter_version }}.linux-{{ smartctl_exporter_arch }}.tar.gz"
+    smartctl_exporter_extracted: "smartctl_exporter-{{ smartctl_exporter_version }}.linux-{{ smartctl_exporter_arch }}"
+    smartctl_exporter_release_base: "https://github.com/prometheus-community/smartctl_exporter/releases/download/v{{ smartctl_exporter_version }}"
+
+- name: Download smartctl_exporter release
+  ansible.builtin.get_url:
+    url: "{{ smartctl_exporter_release_base }}/{{ smartctl_exporter_archive }}"
+    dest: "/tmp/{{ smartctl_exporter_archive }}"
+    mode: "0644"
+
+- name: Download release checksums
+  ansible.builtin.get_url:
+    url: "{{ smartctl_exporter_release_base }}/sha256sums.txt"
+    dest: /tmp/smartctl_exporter_sha256sums.txt
+    mode: "0644"
+
+- name: Verify release archive checksum
+  ansible.builtin.command: "sha256sum -c --ignore-missing smartctl_exporter_sha256sums.txt"
+  args:
+    chdir: /tmp
+  changed_when: false
+
+- name: Extract smartctl_exporter release
+  ansible.builtin.unarchive:
+    src: "/tmp/{{ smartctl_exporter_archive }}"
+    dest: /tmp
+    remote_src: true
+
+- name: Install smartctl_exporter binary
+  ansible.builtin.copy:
+    src: "/tmp/{{ smartctl_exporter_extracted }}/smartctl_exporter"
+    dest: "{{ smartctl_exporter_binary_dir }}/smartctl_exporter"
+    remote_src: true
+    owner: root
+    group: root
+    mode: "0755"
+
+- name: Clean up temporary files
+  ansible.builtin.file:
+    path: "{{ item }}"
+    state: absent
+  loop:
+    - "/tmp/{{ smartctl_exporter_archive }}"
+    - "/tmp/{{ smartctl_exporter_extracted }}"
+    - /tmp/smartctl_exporter_sha256sums.txt
+
+- name: Deploy smartctl_exporter systemd unit
+  ansible.builtin.template:
+    src: smartctl_exporter.service.j2
+    dest: /etc/systemd/system/smartctl_exporter.service
+    owner: root
+    group: root
+    mode: "0644"
+  notify: Restart smartctl exporter
+
+- name: Enable and start smartctl_exporter
+  ansible.builtin.systemd:
+    name: smartctl_exporter
+    enabled: true
+    state: started
+    daemon_reload: true

roles/disks/tasks/main.yml

@@ -5,3 +5,7 @@
 
 - name: Enable trim SSD if there is at least one
   ansible.builtin.include_tasks: trim-ssd.yml
+
+- name: Deploy Prometheus S.M.A.R.T. exporter
+  ansible.builtin.include_tasks: exporter.yml
+  when: smartctl_exporter_enabled | bool

roles/disks/templates/smartctl_exporter.service.j2

@@ -0,0 +1,17 @@
+[Unit]
+Description=Smartctl Prometheus Exporter
+After=network.target
+
+[Service]
+# Runs as root with its capabilities intact: smartctl needs raw device ioctls
+# (SG_IO / CAP_SYS_RAWIO) to read S.M.A.R.T. attributes.
+ExecStart={{ smartctl_exporter_binary_dir }}/smartctl_exporter --smartctl.path={{ smartctl_exporter_smartctl_path }} --smartctl.interval={{ smartctl_exporter_interval }} --web.listen-address={{ smartctl_exporter_listen_address }}
+Restart=on-failure
+RestartSec=5
+NoNewPrivileges=yes
+PrivateTmp=yes
+ProtectSystem=full
+ProtectHome=yes
+
+[Install]
+WantedBy=multi-user.target

roles/fail2ban/README.md

@@ -17,6 +17,19 @@ Bans brute-force sources via UFW. Reads logs from the systemd journal (`backend
 
 See [defaults/main.yml](defaults/main.yml).
 
+## Prometheus exporter
+
+Optionally exposes fail2ban metrics (`f2b_*`) for Prometheus via
+[hectorjsmith/fail2ban-prometheus-exporter](https://gitlab.com/hectorjsmith/fail2ban-prometheus-exporter).
+
+```yaml
+fail2ban_exporter_enabled: true
+```
+
+Downloads the checksum-verified static binary to `/usr/local/bin` and runs it
+as a hardened root systemd service (required to read the root-owned socket) on
+`{{ fail2ban_exporter_listen_address }}` (default `127.0.0.1:9191`).
+
 ## Verify
 
 ```bash

roles/fail2ban/defaults/main.yml

@@ -2,3 +2,10 @@
 fail2ban_firewall: ufw
 fail2ban_backend: systemd
 fail2ban_blocktype: deny
+
+# Prometheus exporter (optional, gated by fail2ban_exporter_enabled).
+fail2ban_exporter_enabled: false
+fail2ban_exporter_version: "0.10.3"
+fail2ban_exporter_listen_address: "127.0.0.1:9191"
+fail2ban_exporter_socket: "/var/run/fail2ban/fail2ban.sock"
+fail2ban_exporter_binary_dir: "/usr/local/bin"

roles/fail2ban/handlers/main.yml

@@ -4,3 +4,9 @@
     name: fail2ban
     daemon_reload: true
     state: restarted
+
+- name: Restart fail2ban_exporter
+  ansible.builtin.systemd:
+    name: fail2ban_exporter
+    daemon_reload: true
+    state: restarted

roles/fail2ban/tasks/exporter.yml

@@ -0,0 +1,83 @@
+---
+# Prometheus fail2ban exporter (optional, gated by fail2ban_exporter_enabled).
+# Downloads the checksum-verified hectorjsmith/fail2ban-prometheus-exporter
+# static binary and runs it as a hardened root systemd service (required to
+# read the root-owned fail2ban socket).
+
+- name: Map CPU architecture to release asset
+  ansible.builtin.set_fact:
+    fail2ban_exporter_arch: >-
+      {{ {
+           'x86_64': 'amd64',
+           'aarch64': 'arm64',
+           'armv7l': 'armv7',
+           'armv6l': 'armv6',
+           'i386': '386',
+           'i686': '386'
+         }[ansible_architecture] }}
+
+- name: Set release filenames
+  ansible.builtin.set_fact:
+    fail2ban_exporter_archive: "fail2ban_exporter_{{ fail2ban_exporter_version }}_linux_{{ fail2ban_exporter_arch }}.tar.gz"
+    fail2ban_exporter_checksums: "fail2ban_exporter_{{ fail2ban_exporter_version }}_checksums.txt"
+    fail2ban_exporter_release_base: "https://gitlab.com/hectorjsmith/fail2ban-prometheus-exporter/-/releases/v{{ fail2ban_exporter_version }}/downloads"
+
+- name: Download fail2ban_exporter release
+  ansible.builtin.get_url:
+    url: "{{ fail2ban_exporter_release_base }}/{{ fail2ban_exporter_archive }}"
+    dest: "/tmp/{{ fail2ban_exporter_archive }}"
+    mode: "0644"
+
+- name: Download release checksums
+  ansible.builtin.get_url:
+    url: "{{ fail2ban_exporter_release_base }}/{{ fail2ban_exporter_checksums }}"
+    dest: "/tmp/{{ fail2ban_exporter_checksums }}"
+    mode: "0644"
+
+- name: Verify release archive checksum
+  ansible.builtin.command: "sha256sum -c --ignore-missing {{ fail2ban_exporter_checksums }}"
+  args:
+    chdir: /tmp
+  changed_when: false
+
+- name: Extract fail2ban_exporter binary
+  ansible.builtin.unarchive:
+    src: "/tmp/{{ fail2ban_exporter_archive }}"
+    dest: /tmp
+    remote_src: true
+    extra_opts:
+      - fail2ban_exporter
+
+- name: Install fail2ban_exporter binary
+  ansible.builtin.copy:
+    src: /tmp/fail2ban_exporter
+    dest: "{{ fail2ban_exporter_binary_dir }}/fail2ban_exporter"
+    remote_src: true
+    owner: root
+    group: root
+    mode: "0755"
+
+- name: Clean up temporary files
+  ansible.builtin.file:
+    path: "{{ item }}"
+    state: absent
+  loop:
+    - "/tmp/{{ fail2ban_exporter_archive }}"
+    - "/tmp/{{ fail2ban_exporter_checksums }}"
+    - /tmp/fail2ban_exporter
+
+- name: Deploy fail2ban_exporter systemd unit
+  ansible.builtin.template:
+    src: fail2ban_exporter.service.j2
+    dest: /etc/systemd/system/fail2ban_exporter.service
+    owner: root
+    group: root
+    mode: "0644"
+  notify: Restart fail2ban_exporter
+
+- name: Enable and start fail2ban_exporter
+  ansible.builtin.systemd:
+    name: fail2ban_exporter
+    enabled: true
+    state: started
+    daemon_reload: true

roles/fail2ban/tasks/main.yml

@@ -70,3 +70,7 @@
     name: fail2ban
     state: started
     enabled: true
+
+- name: Deploy Prometheus fail2ban exporter
+  ansible.builtin.include_tasks: exporter.yml
+  when: fail2ban_exporter_enabled | bool

roles/fail2ban/templates/fail2ban_exporter.service.j2

@@ -0,0 +1,16 @@
+[Unit]
+Description=Fail2Ban Prometheus Exporter
+After=network.target fail2ban.service
+
+[Service]
+ExecStart={{ fail2ban_exporter_binary_dir }}/fail2ban_exporter --collector.f2b.socket={{ fail2ban_exporter_socket }} --web.listen-address={{ fail2ban_exporter_listen_address }}
+Restart=on-failure
+RestartSec=5
+NoNewPrivileges=yes
+PrivateTmp=yes
+ProtectSystem=full
+ProtectHome=yes
+CapabilityBoundingSet=
+
+[Install]
+WantedBy=multi-user.target

roles/grafana/files/fail2ban-dashboard.json

@@ -0,0 +1,880 @@
+{
+  "__inputs": [
+    {
+      "name": "DS_PROMETHEUS",
+      "label": "Prometheus",
+      "description": "",
+      "type": "datasource",
+      "pluginId": "prometheus",
+      "pluginName": "Prometheus"
+    }
+  ],
+  "__elements": {},
+  "__requires": [
+    {
+      "type": "grafana",
+      "id": "grafana",
+      "name": "Grafana",
+      "version": "9.1.8"
+    },
+    {
+      "type": "datasource",
+      "id": "prometheus",
+      "name": "Prometheus",
+      "version": "1.0.0"
+    },
+    {
+      "type": "panel",
+      "id": "table",
+      "name": "Table",
+      "version": ""
+    },
+    {
+      "type": "panel",
+      "id": "timeseries",
+      "name": "Time series",
+      "version": ""
+    }
+  ],
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": {
+          "type": "datasource",
+          "uid": "grafana"
+        },
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "target": {
+          "limit": 100,
+          "matchAny": false,
+          "tags": [],
+          "type": "dashboard"
+        },
+        "type": "dashboard"
+      }
+    ]
+  },
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 2,
+  "id": null,
+  "links": [],
+  "liveNow": false,
+  "panels": [
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "description": "",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "custom": {
+            "align": "auto",
+            "displayMode": "auto",
+            "inspect": false
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              }
+            ]
+          }
+        },
+        "overrides": [
+          {
+            "matcher": {
+              "id": "byRegexp",
+              "options": ".*Time"
+            },
+            "properties": [
+              {
+                "id": "unit",
+                "value": "s"
+              }
+            ]
+          }
+        ]
+      },
+      "gridPos": {
+        "h": 6,
+        "w": 24,
+        "x": 0,
+        "y": 0
+      },
+      "id": 206,
+      "options": {
+        "footer": {
+          "fields": "",
+          "reducer": [
+            "sum"
+          ],
+          "show": false
+        },
+        "showHeader": true
+      },
+      "pluginVersion": "9.1.8",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "editorMode": "code",
+          "exemplar": false,
+          "expr": "f2b_config_jail_max_retries{instance=~\"$instance\"}",
+          "format": "table",
+          "instant": true,
+          "interval": "",
+          "legendFormat": "{{jail}}",
+          "refId": "A"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "editorMode": "code",
+          "exemplar": false,
+          "expr": "f2b_config_jail_ban_time{instance=~\"$instance\"}",
+          "format": "table",
+          "hide": false,
+          "instant": true,
+          "interval": "",
+          "legendFormat": "{{jail}}",
+          "refId": "B"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "editorMode": "code",
+          "exemplar": false,
+          "expr": "f2b_config_jail_find_time{instance=~\"$instance\"}",
+          "format": "table",
+          "hide": false,
+          "instant": true,
+          "interval": "",
+          "legendFormat": "{{jail}}",
+          "refId": "C"
+        }
+      ],
+      "title": "F2B Config",
+      "transformations": [
+        {
+          "id": "merge",
+          "options": {}
+        },
+        {
+          "id": "groupBy",
+          "options": {
+            "fields": {
+              "Value #A": {
+                "aggregations": [
+                  "lastNotNull"
+                ],
+                "operation": "aggregate"
+              },
+              "Value #B": {
+                "aggregations": [
+                  "lastNotNull"
+                ],
+                "operation": "aggregate"
+              },
+              "Value #C": {
+                "aggregations": [
+                  "lastNotNull"
+                ],
+                "operation": "aggregate"
+              },
+              "instance": {
+                "aggregations": [],
+                "operation": "groupby"
+              },
+              "jail": {
+                "aggregations": [],
+                "operation": "groupby"
+              }
+            }
+          }
+        },
+        {
+          "id": "organize",
+          "options": {
+            "excludeByName": {},
+            "indexByName": {},
+            "renameByName": {
+              "Value #A (lastNotNull)": "Max Retries",
+              "Value #B (lastNotNull)": "Ban Time",
+              "Value #C (lastNotNull)": "Find Time",
+              "jail": "Jail"
+            }
+          }
+        }
+      ],
+      "transparent": true,
+      "type": "table"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "description": "",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "never",
+            "spanNulls": true,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "min": 0,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              }
+            ]
+          },
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 6
+      },
+      "id": 190,
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "displayMode": "table",
+          "placement": "right",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "pluginVersion": "8.2.1",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "editorMode": "code",
+          "exemplar": true,
+          "expr": "f2b_jail_failed_total{instance=~\"$instance\"}",
+          "hide": false,
+          "interval": "",
+          "legendFormat": "{{jail}} ({{instance}})",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "Fail2Ban Failures (Total)",
+      "transparent": true,
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "description": "",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "never",
+            "spanNulls": true,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "min": 0,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              }
+            ]
+          },
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 6
+      },
+      "id": 191,
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "displayMode": "table",
+          "placement": "right",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "pluginVersion": "8.2.1",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "editorMode": "code",
+          "exemplar": true,
+          "expr": "f2b_jail_banned_total{instance=~\"$instance\"}",
+          "interval": "",
+          "legendFormat": "{{jail}} ({{instance}})",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "Fail2Ban Bans (Total)",
+      "transparent": true,
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "description": "",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "never",
+            "spanNulls": true,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "min": 0,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              }
+            ]
+          },
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 14
+      },
+      "id": 208,
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "displayMode": "table",
+          "placement": "right",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "pluginVersion": "8.2.1",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "editorMode": "code",
+          "exemplar": true,
+          "expr": "f2b_jail_failed_current{instance=~\"$instance\"}",
+          "interval": "",
+          "legendFormat": "{{jail}} ({{instance}})",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "Fail2Ban Failures (Current)",
+      "transparent": true,
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "description": "",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "never",
+            "spanNulls": true,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "min": 0,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              }
+            ]
+          },
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 14
+      },
+      "id": 209,
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "displayMode": "table",
+          "placement": "right",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "pluginVersion": "8.2.1",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "editorMode": "code",
+          "exemplar": true,
+          "expr": "f2b_jail_banned_current{instance=~\"$instance\"}",
+          "interval": "",
+          "legendFormat": "{{jail}} ({{instance}})",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "Fail2Ban Bans (Current)",
+      "transparent": true,
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "description": "",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "never",
+            "spanNulls": true,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "max": 1,
+          "min": 0,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              }
+            ]
+          },
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 5,
+        "w": 12,
+        "x": 0,
+        "y": 22
+      },
+      "id": 203,
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "displayMode": "table",
+          "placement": "right",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "pluginVersion": "8.2.1",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "editorMode": "code",
+          "exemplar": true,
+          "expr": "f2b_up{instance=~\"$instance\"}",
+          "interval": "",
+          "legendFormat": "Up ({{instance}})",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "Fail2Ban Up",
+      "transparent": true,
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "description": "",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "never",
+            "spanNulls": true,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "max": 1,
+          "min": 0,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              }
+            ]
+          },
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 5,
+        "w": 12,
+        "x": 12,
+        "y": 22
+      },
+      "id": 204,
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "displayMode": "table",
+          "placement": "right",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "pluginVersion": "8.2.1",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "editorMode": "code",
+          "exemplar": true,
+          "expr": "f2b_errors{instance=~\"$instance\"}",
+          "interval": "",
+          "legendFormat": "{{type}} ({{instance}})",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "Fail2Ban Exporter Errors",
+      "transparent": true,
+      "type": "timeseries"
+    }
+  ],
+  "refresh": "30s",
+  "schemaVersion": 37,
+  "style": "dark",
+  "tags": [
+    "security",
+    "networking"
+  ],
+  "templating": {
+    "list": [
+      {
+        "current": {
+          "selected": false,
+          "text": "Prometheus",
+          "value": "Prometheus"
+        },
+        "hide": 0,
+        "includeAll": false,
+        "label": "Data Source",
+        "multi": false,
+        "name": "DataSource",
+        "options": [],
+        "query": "prometheus",
+        "queryValue": "",
+        "refresh": 1,
+        "regex": "",
+        "skipUrlSync": false,
+        "type": "datasource"
+      },
+      {
+        "current": {},
+        "datasource": {
+          "type": "prometheus",
+          "uid": "prometheus"
+        },
+        "definition": "label_values(up{job=\"fail2ban\"}, instance)",
+        "description": "Select which instance(s) to show",
+        "hide": 0,
+        "includeAll": false,
+        "label": "Instance",
+        "multi": false,
+        "name": "instance",
+        "options": [],
+        "query": {
+          "query": "label_values(up{job=\"fail2ban\"}, instance)",
+          "refId": "StandardVariableQuery"
+        },
+        "refresh": 1,
+        "regex": "",
+        "skipUrlSync": false,
+        "sort": 0,
+        "type": "query"
+      }
+    ]
+  },
+  "time": {
+    "from": "now-6h",
+    "to": "now"
+  },
+  "timepicker": {},
+  "timezone": "",
+  "title": "fail2ban",
+  "uid": "cTkH9AT7z",
+  "version": 3,
+  "weekStart": ""
+}

roles/grafana/files/node-exporter-dashboard.json

@@ -1,5 +1,5 @@
 {
-  "title": "Node Exporter - Host Overview",
+  "title": "Host Overview",
   "uid": "node-exporter-overview",
   "tags": ["node-exporter", "linux", "system"],
   "schemaVersion": 39,
@@ -107,7 +107,8 @@
       "targets": [
         { "refId": "A", "expr": "node_memory_MemTotal_bytes{instance=\"$instance\"} - node_memory_MemAvailable_bytes{instance=\"$instance\"}", "legendFormat": "Used" },
         { "refId": "B", "expr": "node_memory_Cached_bytes{instance=\"$instance\"} + node_memory_Buffers_bytes{instance=\"$instance\"}", "legendFormat": "Cached+Buffers" },
-        { "refId": "C", "expr": "node_memory_MemAvailable_bytes{instance=\"$instance\"}", "legendFormat": "Available" }
+        { "refId": "C", "expr": "node_memory_MemAvailable_bytes{instance=\"$instance\"}", "legendFormat": "Available" },
+        { "refId": "D", "expr": "node_zfs_arc_size{instance=\"$instance\"}", "legendFormat": "ZFS ARC" }
       ]
     },
     {

roles/grafana/files/smartctl-dashboard.json

@@ -0,0 +1,151 @@
+{
+  "title": "Disks - S.M.A.R.T.",
+  "uid": "smartctl",
+  "tags": ["disks", "smart", "storage"],
+  "schemaVersion": 39,
+  "version": 1,
+  "refresh": "1h",
+  "time": { "from": "now-24h", "to": "now" },
+  "timezone": "browser",
+  "editable": true,
+  "graphTooltip": 1,
+  "templating": {
+    "list": [
+      {
+        "name": "instance",
+        "label": "Instance",
+        "type": "query",
+        "datasource": { "type": "prometheus", "uid": "prometheus" },
+        "query": { "query": "label_values(smartctl_device, instance)", "refId": "A" },
+        "refresh": 2,
+        "sort": 1,
+        "multi": false,
+        "includeAll": false,
+        "current": { "selected": false, "text": "", "value": "" }
+      },
+      {
+        "name": "device",
+        "label": "Disk",
+        "type": "query",
+        "datasource": { "type": "prometheus", "uid": "prometheus" },
+        "query": { "query": "label_values(smartctl_device, device)", "refId": "A" },
+        "refresh": 2,
+        "sort": 1,
+        "multi": false,
+        "includeAll": false,
+        "current": { "selected": false, "text": "", "value": "" }
+      }
+    ]
+  },
+  "panels": [
+    {
+      "id": 1,
+      "type": "stat",
+      "title": "SMART Health",
+      "gridPos": { "h": 4, "w": 6, "x": 0, "y": 0 },
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "fieldConfig": { "defaults": { "unit": "none", "color": { "mode": "thresholds" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "red", "value": null }, { "color": "green", "value": 1 } ] }, "mappings": [ { "type": "value", "options": { "0": { "text": "FAILED", "color": "red" }, "1": { "text": "OK", "color": "green" } } } ] }, "overrides": [] },
+      "options": { "reduceOptions": { "values": false, "calcs": ["lastNotNull"], "fields": "" }, "colorMode": "background", "graphMode": "none", "textMode": "auto", "justifyMode": "auto", "orientation": "auto" },
+      "targets": [ { "refId": "A", "expr": "smartctl_device_smart_status{instance=~\"$instance\", device=~\"$device\"}" } ]
+    },
+    {
+      "id": 2,
+      "type": "stat",
+      "title": "Temperature",
+      "gridPos": { "h": 4, "w": 6, "x": 6, "y": 0 },
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "fieldConfig": { "defaults": { "unit": "celsius", "color": { "mode": "thresholds" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "orange", "value": 45 }, { "color": "red", "value": 55 } ] } }, "overrides": [] },
+      "options": { "reduceOptions": { "values": false, "calcs": ["lastNotNull"], "fields": "" }, "colorMode": "background", "graphMode": "none", "textMode": "auto", "justifyMode": "auto", "orientation": "auto" },
+      "targets": [ { "refId": "A", "expr": "smartctl_device_temperature{instance=~\"$instance\", device=~\"$device\", temperature_type=\"current\"}" } ]
+    },
+    {
+      "id": 3,
+      "type": "stat",
+      "title": "Power-On Hours",
+      "gridPos": { "h": 4, "w": 6, "x": 12, "y": 0 },
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "fieldConfig": { "defaults": { "unit": "h", "color": { "mode": "thresholds" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "orange", "value": 43800 }, { "color": "red", "value": 73000 } ] } }, "overrides": [] },
+      "options": { "reduceOptions": { "values": false, "calcs": ["lastNotNull"], "fields": "" }, "colorMode": "background", "graphMode": "none", "textMode": "auto", "justifyMode": "auto", "orientation": "auto" },
+      "targets": [ { "refId": "A", "expr": "smartctl_device_power_on_seconds{instance=~\"$instance\", device=~\"$device\"} / 3600" } ]
+    },
+    {
+      "id": 4,
+      "type": "stat",
+      "title": "Power Cycles",
+      "gridPos": { "h": 4, "w": 6, "x": 18, "y": 0 },
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "fieldConfig": { "defaults": { "unit": "none", "color": { "mode": "thresholds" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] } }, "overrides": [] },
+      "options": { "reduceOptions": { "values": false, "calcs": ["lastNotNull"], "fields": "" }, "colorMode": "background", "graphMode": "none", "textMode": "auto", "justifyMode": "auto", "orientation": "auto" },
+      "targets": [ { "refId": "A", "expr": "smartctl_device_power_cycle_count{instance=~\"$instance\", device=~\"$device\"}" } ]
+    },
+    {
+      "id": 5,
+      "type": "stat",
+      "title": "Capacity",
+      "gridPos": { "h": 4, "w": 6, "x": 0, "y": 4 },
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "fieldConfig": { "defaults": { "unit": "bytes", "color": { "mode": "thresholds" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] } }, "overrides": [] },
+      "options": { "reduceOptions": { "values": false, "calcs": ["lastNotNull"], "fields": "" }, "colorMode": "background", "graphMode": "none", "textMode": "auto", "justifyMode": "auto", "orientation": "auto" },
+      "targets": [ { "refId": "A", "expr": "smartctl_device_capacity_bytes{instance=~\"$instance\", device=~\"$device\"}" } ]
+    },
+    {
+      "id": 6,
+      "type": "stat",
+      "title": "Reallocated Sectors",
+      "gridPos": { "h": 4, "w": 6, "x": 6, "y": 4 },
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "fieldConfig": { "defaults": { "unit": "none", "color": { "mode": "thresholds" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "orange", "value": 1 }, { "color": "red", "value": 10 } ] } }, "overrides": [] },
+      "options": { "reduceOptions": { "values": false, "calcs": ["lastNotNull"], "fields": "" }, "colorMode": "background", "graphMode": "none", "textMode": "auto", "justifyMode": "auto", "orientation": "auto" },
+      "targets": [ { "refId": "A", "expr": "smartctl_device_attribute{instance=~\"$instance\", device=~\"$device\", attribute_name=\"Reallocated_Sector_Ct\", attribute_value_type=\"raw\"}" } ]
+    },
+    {
+      "id": 7,
+      "type": "stat",
+      "title": "Pending Sectors",
+      "gridPos": { "h": 4, "w": 6, "x": 12, "y": 4 },
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "fieldConfig": { "defaults": { "unit": "none", "color": { "mode": "thresholds" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "orange", "value": 1 }, { "color": "red", "value": 10 } ] } }, "overrides": [] },
+      "options": { "reduceOptions": { "values": false, "calcs": ["lastNotNull"], "fields": "" }, "colorMode": "background", "graphMode": "none", "textMode": "auto", "justifyMode": "auto", "orientation": "auto" },
+      "targets": [ { "refId": "A", "expr": "smartctl_device_attribute{instance=~\"$instance\", device=~\"$device\", attribute_name=\"Current_Pending_Sector\", attribute_value_type=\"raw\"}" } ]
+    },
+    {
+      "id": 8,
+      "type": "stat",
+      "title": "Offline Uncorrectable",
+      "gridPos": { "h": 4, "w": 6, "x": 18, "y": 4 },
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "fieldConfig": { "defaults": { "unit": "none", "color": { "mode": "thresholds" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "orange", "value": 1 }, { "color": "red", "value": 10 } ] } }, "overrides": [] },
+      "options": { "reduceOptions": { "values": false, "calcs": ["lastNotNull"], "fields": "" }, "colorMode": "background", "graphMode": "none", "textMode": "auto", "justifyMode": "auto", "orientation": "auto" },
+      "targets": [ { "refId": "A", "expr": "smartctl_device_attribute{instance=~\"$instance\", device=~\"$device\", attribute_name=\"Offline_Uncorrectable\", attribute_value_type=\"raw\"}" } ]
+    },
+    {
+      "id": 9,
+      "type": "timeseries",
+      "title": "Temperature",
+      "gridPos": { "h": 8, "w": 24, "x": 0, "y": 8 },
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "fieldConfig": { "defaults": { "unit": "celsius", "color": { "mode": "palette-classic" }, "custom": { "fillOpacity": 20, "lineWidth": 1, "drawStyle": "line", "showPoints": "never" } }, "overrides": [] },
+      "targets": [ { "refId": "A", "expr": "smartctl_device_temperature{instance=~\"$instance\", device=~\"$device\", temperature_type=\"current\"}", "legendFormat": "{{device}}" } ]
+    },
+    {
+      "id": 10,
+      "type": "table",
+      "title": "SMART Attributes (raw)",
+      "gridPos": { "h": 8, "w": 24, "x": 0, "y": 16 },
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { "align": "auto", "displayMode": "auto", "inspect": false }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] } }, "overrides": [] },
+      "options": { "footer": { "fields": "", "reducer": ["sum"], "show": false }, "showHeader": true },
+      "targets": [ { "refId": "A", "expr": "smartctl_device_attribute{instance=~\"$instance\", device=~\"$device\", attribute_value_type=\"raw\"}", "format": "table", "instant": true, "legendFormat": "{{attribute_name}}" } ],
+      "transformations": [
+        {
+          "id": "organize",
+          "options": {
+            "excludeByName": { "attribute_flags_short": true, "attribute_flags_long": true, "attribute_id": true, "Time": true },
+            "indexByName": {},
+            "renameByName": { "attribute_name": "Attribute", "Value": "Raw value" }
+          }
+        }
+      ]
+    }
+  ]
+}

roles/grafana/files/wireguard-dashboard.json

@@ -0,0 +1,100 @@
+{
+  "title": "WireGuard",
+  "uid": "wireguard-overview",
+  "tags": ["wireguard", "vpn", "networking"],
+  "schemaVersion": 39,
+  "version": 1,
+  "refresh": "1m",
+  "time": { "from": "now-6h", "to": "now" },
+  "timezone": "browser",
+  "editable": true,
+  "graphTooltip": 1,
+  "templating": {
+    "list": [
+      {
+        "name": "instance",
+        "label": "Host",
+        "type": "query",
+        "datasource": { "type": "prometheus", "uid": "prometheus" },
+        "query": { "query": "label_values(up{job=\"wireguard\"}, instance)", "refId": "A" },
+        "refresh": 2,
+        "multi": false,
+        "includeAll": false,
+        "sort": 1,
+        "current": { "selected": false, "text": "", "value": "" }
+      }
+    ]
+  },
+  "panels": [
+    {
+      "id": 1,
+      "type": "stat",
+      "title": "Up",
+      "gridPos": { "h": 4, "w": 6, "x": 0, "y": 0 },
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "fieldConfig": { "defaults": { "unit": "none", "color": { "mode": "thresholds" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "red", "value": null }, { "color": "green", "value": 1 } ] }, "mappings": [ { "type": "value", "options": { "0": { "text": "Down", "color": "red" }, "1": { "text": "Up", "color": "green" } } } ] }, "overrides": [] },
+      "options": { "reduceOptions": { "values": false, "calcs": ["lastNotNull"], "fields": "" }, "colorMode": "background", "graphMode": "none", "textMode": "auto", "justifyMode": "auto", "orientation": "auto" },
+      "targets": [ { "refId": "A", "expr": "up{job=\"wireguard\", instance=\"$instance\"}" } ]
+    },
+    {
+      "id": 2,
+      "type": "stat",
+      "title": "Connected Peers (3m)",
+      "gridPos": { "h": 4, "w": 6, "x": 6, "y": 0 },
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "fieldConfig": { "defaults": { "unit": "short", "color": { "mode": "thresholds" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "blue", "value": null } ] }, "mappings": [] }, "overrides": [] },
+      "options": { "reduceOptions": { "values": false, "calcs": ["lastNotNull"], "fields": "" }, "colorMode": "background", "graphMode": "none", "textMode": "auto", "justifyMode": "auto", "orientation": "auto" },
+      "targets": [ { "refId": "A", "expr": "count(wireguard_latest_handshake_seconds{job=\"wireguard\", instance=\"$instance\"} > (time() - 180))" } ]
+    },
+    {
+      "id": 3,
+      "type": "stat",
+      "title": "Total Peers",
+      "gridPos": { "h": 4, "w": 6, "x": 12, "y": 0 },
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "fieldConfig": { "defaults": { "unit": "short", "color": { "mode": "thresholds" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "blue", "value": null } ] }, "mappings": [] }, "overrides": [] },
+      "options": { "reduceOptions": { "values": false, "calcs": ["lastNotNull"], "fields": "" }, "colorMode": "background", "graphMode": "none", "textMode": "auto", "justifyMode": "auto", "orientation": "auto" },
+      "targets": [ { "refId": "A", "expr": "count(wireguard_latest_handshake_seconds{job=\"wireguard\", instance=\"$instance\"})" } ]
+    },
+    {
+      "id": 4,
+      "type": "stat",
+      "title": "Interfaces",
+      "gridPos": { "h": 4, "w": 6, "x": 18, "y": 0 },
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "fieldConfig": { "defaults": { "unit": "short", "color": { "mode": "thresholds" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "blue", "value": null } ] }, "mappings": [] }, "overrides": [] },
+      "options": { "reduceOptions": { "values": false, "calcs": ["lastNotNull"], "fields": "" }, "colorMode": "background", "graphMode": "none", "textMode": "auto", "justifyMode": "auto", "orientation": "auto" },
+      "targets": [ { "refId": "A", "expr": "count(count by (interface) (wireguard_latest_handshake_seconds{job=\"wireguard\", instance=\"$instance\"}))" } ]
+    },
+    {
+      "id": 5,
+      "type": "timeseries",
+      "title": "Throughput",
+      "gridPos": { "h": 8, "w": 24, "x": 0, "y": 4 },
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "fieldConfig": { "defaults": { "unit": "Bps", "color": { "mode": "palette-classic" }, "custom": { "fillOpacity": 20, "lineWidth": 1, "drawStyle": "line", "showPoints": "never" } }, "overrides": [] },
+      "targets": [
+        { "refId": "A", "expr": "sum by (instance) (rate(wireguard_received_bytes_total{job=\"wireguard\", instance=\"$instance\"}[$__rate_interval]))", "legendFormat": "received ({{instance}})" },
+        { "refId": "B", "expr": "sum by (instance) (rate(wireguard_sent_bytes_total{job=\"wireguard\", instance=\"$instance\"}[$__rate_interval]))", "legendFormat": "sent ({{instance}})" }
+      ]
+    },
+    {
+      "id": 6,
+      "type": "timeseries",
+      "title": "Received per peer",
+      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 12 },
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "fieldConfig": { "defaults": { "unit": "Bps", "color": { "mode": "palette-classic" }, "custom": { "fillOpacity": 20, "lineWidth": 1, "drawStyle": "line", "showPoints": "never" } }, "overrides": [] },
+      "targets": [ { "refId": "A", "expr": "rate(wireguard_received_bytes_total{job=\"wireguard\", instance=\"$instance\"}[$__rate_interval])", "legendFormat": "{{allowed_ips}} ({{interface}})" } ]
+    },
+    {
+      "id": 7,
+      "type": "timeseries",
+      "title": "Sent per peer",
+      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 12 },
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "fieldConfig": { "defaults": { "unit": "Bps", "color": { "mode": "palette-classic" }, "custom": { "fillOpacity": 20, "lineWidth": 1, "drawStyle": "line", "showPoints": "never" } }, "overrides": [] },
+      "targets": [ { "refId": "A", "expr": "rate(wireguard_sent_bytes_total{job=\"wireguard\", instance=\"$instance\"}[$__rate_interval])", "legendFormat": "{{allowed_ips}} ({{interface}})" } ]
+    }
+  ]
+}

roles/grafana/tasks/main.yml

@@ -209,6 +209,24 @@
     mode: "0644"
   notify: Restart grafana
 
+- name: Deploy S.M.A.R.T. dashboard
+  ansible.builtin.copy:
+    src: smartctl-dashboard.json
+    dest: "{{ grafana_provisioning_path }}/dashboards/smartctl.json"
+    owner: "{{ grafana_user }}"
+    group: "{{ grafana_group }}"
+    mode: "0644"
+  notify: Restart grafana
+
+- name: Deploy fail2ban dashboard
+  ansible.builtin.copy:
+    src: fail2ban-dashboard.json
+    dest: "{{ grafana_provisioning_path }}/dashboards/fail2ban.json"
+    owner: "{{ grafana_user }}"
+    group: "{{ grafana_group }}"
+    mode: "0644"
+  notify: Restart grafana
+
 - name: Remove legacy combined services dashboard
   ansible.builtin.file:
     path: "{{ grafana_provisioning_path }}/dashboards/services.json"
@@ -227,6 +245,7 @@
     - postgres-dashboard.json
     - valkey-dashboard.json
     - unbound-dashboard.json
+    - wireguard-dashboard.json
   notify: Restart grafana
 
 # Service management

roles/monitoring/tasks/main.yml

@@ -1,5 +0,0 @@
----
-- name: Install oryx
-  ansible.builtin.command: paru -S --noconfirm oryx
-  when: ansible_facts['os_family'] == 'Archlinux'
-  changed_when: false

roles/node_exporter/templates/env.conf.j2

@@ -1,2 +1,2 @@
 # Managed by Ansible - DO NOT EDIT MANUALLY
-{{ node_exporter_args_var }}="--web.listen-address={{ node_exporter_listen_address }}"
+{{ node_exporter_args_var }}="--web.listen-address={{ node_exporter_listen_address }} --collector.zfs"

roles/prometheus/defaults/main.yml

@@ -12,8 +12,14 @@ prometheus_retention_size: 0
 
 # Web interface settings
 prometheus_web_listen_address: "127.0.0.1:9090"
+prometheus_port: 9090
 prometheus_web_external_url: ""
 
+# UFW sources allowed to reach the web UI/federation endpoint. Empty means
+# localhost-only; set on VPN-server hosts so a remote Prometheus can federate
+# over the tunnel.
+prometheus_firewall_allowed_sources: []
+
 # Storage settings (uses prometheus_data_dir from vars/)
 prometheus_storage_tsdb_path: "{{ prometheus_data_dir }}"
 

roles/prometheus/tasks/main.yml

@@ -60,6 +60,21 @@
       enabled: true
       state: started
 
+- name: Setup firewall rules for Prometheus
+  community.general.ufw:
+      rule: allow
+      src: "{{ item }}"
+      port: "{{ prometheus_port }}"
+      proto: tcp
+      direction: in
+      comment: "Prometheus"
+  loop: "{{ prometheus_firewall_allowed_sources }}"
+  retries: 5
+  delay: 2
+  register: ufw_result
+  until: ufw_result is succeeded
+  when: prometheus_firewall_allowed_sources | length > 0
+
 - name: Deploy Nginx vhost configuration
   ansible.builtin.template:
       src: nginx-vhost.conf.j2

roles/prometheus/templates/prometheus.yml.j2

@@ -36,6 +36,18 @@ scrape_configs:
 {% if scrape_config.metrics_path is defined %}
     metrics_path: {{ scrape_config.metrics_path }}
 {% endif %}
+{% if scrape_config.honor_labels is defined %}
+    honor_labels: {{ scrape_config.honor_labels | bool | lower }}
+{% endif %}
+{% if scrape_config.params is defined %}
+    params:
+{% for param_key, param_values in scrape_config.params.items() %}
+      {{ param_key }}:
+{% for param_value in param_values %}
+        - {{ param_value }}
+{% endfor %}
+{% endfor %}
+{% endif %}
 {% if scrape_config.static_configs is defined %}
     static_configs:
 {% for static_config in scrape_config.static_configs %}

roles/wireguard/defaults/main.yml

@@ -19,3 +19,12 @@ wireguard_tunnels: []
 # OS-dependent package name
 wireguard_package_name: >-
   {{ (ansible_facts['os_family'] == 'Archlinux') | ternary('wireguard-tools', 'wireguard') }}
+
+# Prometheus exporter (optional, gated by wireguard_exporter_enabled).
+# Exposes `wg show all dump` as Prometheus metrics (bytes per peer, last
+# handshake). Arch Linux only: prometheus-wireguard-exporter is packaged in
+# [extra]; no Debian/Ubuntu equivalent exists. Bind stays loopback-only:
+# the host's local Prometheus scrapes it and is federated by the central one.
+wireguard_exporter_enabled: false
+wireguard_exporter_listen_address: "127.0.0.1"
+wireguard_exporter_port: 9586

roles/wireguard/handlers/main.yml

@@ -10,8 +10,13 @@
 - name: Reload wireguard tunnel
   ansible.builtin.shell: |
     set -o pipefail
-    wg-quick strip "{{ item.interface }}" | wg syncconf "{{ item.interface }}"
+    wg syncconf "{{ item.interface }}" <(wg-quick strip "{{ item.interface }}")
   args:
     executable: /bin/bash
   loop: "{{ wireguard_tunnels }}"
   changed_when: true
+
+- name: Restart wireguard exporter
+  ansible.builtin.systemd:
+    name: "{{ wireguard_exporter_service }}"
+    state: restarted

roles/wireguard/tasks/exporter.yml

@@ -0,0 +1,40 @@
+---
+# Prometheus WireGuard exporter (optional, gated by wireguard_exporter_enabled).
+# Runs `wg show all dump` and exposes per-peer byte counters and last-handshake
+# timestamps. Arch Linux only: prometheus-wireguard-exporter is packaged in
+# [extra]; there is no Debian/Ubuntu package.
+
+- name: Assert exporter is supported on this OS
+  ansible.builtin.assert:
+    that: ansible_facts['os_family'] == 'Archlinux'
+    fail_msg: |
+      wireguard_exporter is Arch Linux only (prometheus-wireguard-exporter is
+      not packaged for Debian/Ubuntu). Set wireguard_exporter_enabled: false.
+    success_msg: "Exporter OS support validated"
+
+- name: Load OS-specific exporter variables
+  ansible.builtin.include_vars: "{{ item }}"
+  with_first_found:
+    - "{{ ansible_facts['os_family'] }}.yml"
+    - "archlinux.yml"
+
+- name: Install wireguard exporter
+  ansible.builtin.package:
+    name: "{{ wireguard_exporter_package }}"
+    state: present
+
+- name: Deploy exporter environment file
+  ansible.builtin.template:
+    src: exporter-env.conf.j2
+    dest: "{{ wireguard_exporter_args_file }}"
+    owner: root
+    group: root
+    mode: "0644"
+  notify: Restart wireguard exporter
+
+- name: Enable and start wireguard exporter
+  ansible.builtin.systemd:
+    name: "{{ wireguard_exporter_service }}"
+    enabled: true
+    state: started
+    daemon_reload: true

roles/wireguard/tasks/main.yml

@@ -14,10 +14,20 @@
     name: "{{ wireguard_package_name }}"
     state: present
 
+- name: Install nftables (Arch server_mode PostUp dependency)
+  ansible.builtin.package:
+    name: nftables
+    state: present
+  when:
+    - ansible_facts['os_family'] == 'Archlinux'
+    - wireguard_tunnels | selectattr('server_mode', 'defined') | selectattr('server_mode', 'equalto', true) | list | length > 0
+
 # Use systemd-resolved for DNS management (modern approach on all distributions)
-# Install systemd-resolvconf to provide resolvconf compatibility wrapper
-# "systemd-resolved" is prefered over "openresolv"
-- name: Install systemd-resolvconf (Debian only, built into systemd on Arch)
+# wg-quick calls `resolvconf` (not resolvectl) to apply DNS, so the
+# systemd-resolvconf compat wrapper must be installed wherever wg-quick is
+# used to push DNS to the local host. On Arch, tunnels that need this should
+# set `dns:` and install openresolv/systemd-resolvconf explicitly.
+- name: Install systemd-resolvconf (provides resolvconf wrapper for wg-quick)
   ansible.builtin.package:
     name: systemd-resolvconf
     state: present
@@ -42,3 +52,7 @@
   loop: "{{ wireguard_tunnels }}"
   loop_control:
     loop_var: _tunnel
+
+- name: Deploy Prometheus wireguard exporter
+  ansible.builtin.include_tasks: exporter.yml
+  when: wireguard_exporter_enabled | bool

roles/wireguard/templates/exporter-env.conf.j2

@@ -0,0 +1,2 @@
+# Managed by Ansible - DO NOT EDIT MANUALLY
+{{ wireguard_exporter_args_var }}="-l {{ wireguard_exporter_listen_address }} -p {{ wireguard_exporter_port }}"

roles/wireguard/vars/archlinux.yml

@@ -0,0 +1,6 @@
+---
+# Prometheus exporter
+wireguard_exporter_package: prometheus-wireguard-exporter
+wireguard_exporter_service: prometheus-wireguard-exporter
+wireguard_exporter_args_file: /etc/conf.d/prometheus-wireguard-exporter
+wireguard_exporter_args_var: WIREGUARD_EXPORTER_ARGS