feat(monitoring): add smartctl, fail2ban, wireguard exporters and dashboards

- smartctl_exporter (disks), fail2ban exporter, wireguard exporter
- grafana dashboards for each exporter
- prometheus: firewall source rules, honor_labels/params scrape config
- node_exporter: zfs collector
- wireguard: nftables dep, resolvconf wrapper, reload fix
- remove superseded monitoring role

roles/disks/README.md

@@ -0,0 +1,29 @@
+# Disks
+
+Partitions disks, enables periodic SSD TRIM, and optionally exposes S.M.A.R.T.
+disk health to Prometheus.
+
+## Configuration
+
+See [defaults/main.yml](defaults/main.yml).
+
+### S.M.A.R.T. exporter
+
+Gated by `smartctl_exporter_enabled`. When true, downloads and runs the
+`smartctl_exporter` static binary as a root systemd service, polling `smartctl`
+at `smartctl_exporter_interval` (default `1h`).
+
+```yaml
+smartctl_exporter_enabled: true
+```
+
+Pair with Prometheus:
+
+```yaml
+prometheus_scrape_configs:
+  - job_name: "smartctl"
+    static_configs:
+      - targets: ["127.0.0.1:9633"]
+        labels:
+          instance: andromeda
+```

roles/disks/defaults/main.yml

@@ -1,2 +1,10 @@
 ---
 ssd_trim_periodicity: monthly
+
+# Prometheus S.M.A.R.T. exporter (optional, gated by smartctl_exporter_enabled).
+smartctl_exporter_enabled: false
+smartctl_exporter_version: "0.14.0"
+smartctl_exporter_listen_address: "127.0.0.1:9633"
+smartctl_exporter_binary_dir: "/usr/local/bin"
+smartctl_exporter_smartctl_path: "/usr/sbin/smartctl"
+smartctl_exporter_interval: "1h"

roles/disks/handlers/main.yml

@@ -2,3 +2,9 @@
 - name: Systemd daemon reload
   ansible.builtin.systemd:
     daemon_reload: true
+
+- name: Restart smartctl exporter
+  ansible.builtin.systemd:
+    name: smartctl_exporter
+    daemon_reload: true
+    state: restarted

roles/disks/tasks/exporter.yml

@@ -0,0 +1,86 @@
+---
+# Prometheus S.M.A.R.T. exporter (optional, gated by smartctl_exporter_enabled).
+# Downloads the checksum-verified prometheus-community/smartctl_exporter static
+# binary and runs it as a hardened root systemd service (required to read raw
+# S.M.A.R.T. attributes from block devices via smartctl).
+
+- name: Install smartmontools
+  ansible.builtin.package:
+    name: smartmontools
+    state: present
+
+- name: Map CPU architecture to release asset
+  ansible.builtin.set_fact:
+    smartctl_exporter_arch: >-
+      {{ {
+           'x86_64': 'amd64',
+           'aarch64': 'arm64',
+           'armv7l': 'armv7',
+           'armv6l': 'armv6',
+           'i386': '386',
+           'i686': '386'
+         }[ansible_architecture] }}
+
+- name: Set release filenames
+  ansible.builtin.set_fact:
+    smartctl_exporter_archive: "smartctl_exporter-{{ smartctl_exporter_version }}.linux-{{ smartctl_exporter_arch }}.tar.gz"
+    smartctl_exporter_extracted: "smartctl_exporter-{{ smartctl_exporter_version }}.linux-{{ smartctl_exporter_arch }}"
+    smartctl_exporter_release_base: "https://github.com/prometheus-community/smartctl_exporter/releases/download/v{{ smartctl_exporter_version }}"
+
+- name: Download smartctl_exporter release
+  ansible.builtin.get_url:
+    url: "{{ smartctl_exporter_release_base }}/{{ smartctl_exporter_archive }}"
+    dest: "/tmp/{{ smartctl_exporter_archive }}"
+    mode: "0644"
+
+- name: Download release checksums
+  ansible.builtin.get_url:
+    url: "{{ smartctl_exporter_release_base }}/sha256sums.txt"
+    dest: /tmp/smartctl_exporter_sha256sums.txt
+    mode: "0644"
+
+- name: Verify release archive checksum
+  ansible.builtin.command: "sha256sum -c --ignore-missing smartctl_exporter_sha256sums.txt"
+  args:
+    chdir: /tmp
+  changed_when: false
+
+- name: Extract smartctl_exporter release
+  ansible.builtin.unarchive:
+    src: "/tmp/{{ smartctl_exporter_archive }}"
+    dest: /tmp
+    remote_src: true
+
+- name: Install smartctl_exporter binary
+  ansible.builtin.copy:
+    src: "/tmp/{{ smartctl_exporter_extracted }}/smartctl_exporter"
+    dest: "{{ smartctl_exporter_binary_dir }}/smartctl_exporter"
+    remote_src: true
+    owner: root
+    group: root
+    mode: "0755"
+
+- name: Clean up temporary files
+  ansible.builtin.file:
+    path: "{{ item }}"
+    state: absent
+  loop:
+    - "/tmp/{{ smartctl_exporter_archive }}"
+    - "/tmp/{{ smartctl_exporter_extracted }}"
+    - /tmp/smartctl_exporter_sha256sums.txt
+
+- name: Deploy smartctl_exporter systemd unit
+  ansible.builtin.template:
+    src: smartctl_exporter.service.j2
+    dest: /etc/systemd/system/smartctl_exporter.service
+    owner: root
+    group: root
+    mode: "0644"
+  notify: Restart smartctl exporter
+
+- name: Enable and start smartctl_exporter
+  ansible.builtin.systemd:
+    name: smartctl_exporter
+    enabled: true
+    state: started
+    daemon_reload: true

roles/disks/tasks/main.yml

@@ -5,3 +5,7 @@
 
 - name: Enable trim SSD if there is at least one
   ansible.builtin.include_tasks: trim-ssd.yml
+
+- name: Deploy Prometheus S.M.A.R.T. exporter
+  ansible.builtin.include_tasks: exporter.yml
+  when: smartctl_exporter_enabled | bool

roles/disks/templates/smartctl_exporter.service.j2

@@ -0,0 +1,17 @@
+[Unit]
+Description=Smartctl Prometheus Exporter
+After=network.target
+
+[Service]
+# Runs as root with its capabilities intact: smartctl needs raw device ioctls
+# (SG_IO / CAP_SYS_RAWIO) to read S.M.A.R.T. attributes.
+ExecStart={{ smartctl_exporter_binary_dir }}/smartctl_exporter --smartctl.path={{ smartctl_exporter_smartctl_path }} --smartctl.interval={{ smartctl_exporter_interval }} --web.listen-address={{ smartctl_exporter_listen_address }}
+Restart=on-failure
+RestartSec=5
+NoNewPrivileges=yes
+PrivateTmp=yes
+ProtectSystem=full
+ProtectHome=yes
+
+[Install]
+WantedBy=multi-user.target