feat(unbound): add prometheus exporter, dashboard, fix socket buffer sysctl

examples/inventory/host_vars.example.yml

@@ -45,6 +45,9 @@ unbound_firewall_allowed_sources:
   - { src: "192.168.1.0/24", comment: "DNS from LAN" }
   - { src: "192.168.20.0/27", comment: "DNS from VPN" }
 
+# Unbound Prometheus exporter (Arch Linux only)
+# unbound_exporter_enabled: true
+
 # NTP servers configuration
 # -------------------------
 ntp_pools:

roles/grafana/files/unbound-dashboard.json

@@ -0,0 +1,178 @@
+{
+  "title": "Unbound",
+  "uid": "unbound-overview",
+  "tags": ["unbound", "dns", "resolver"],
+  "schemaVersion": 39,
+  "version": 1,
+  "refresh": "1m",
+  "time": { "from": "now-6h", "to": "now" },
+  "timezone": "browser",
+  "editable": true,
+  "graphTooltip": 1,
+  "templating": {
+    "list": [
+      {
+        "name": "instance",
+        "label": "Host",
+        "type": "query",
+        "datasource": { "type": "prometheus", "uid": "prometheus" },
+        "query": { "query": "label_values(up{job=\"unbound\"}, instance)", "refId": "A" },
+        "refresh": 2,
+        "multi": false,
+        "includeAll": false,
+        "sort": 1,
+        "current": { "selected": false, "text": "", "value": "" }
+      }
+    ]
+  },
+  "panels": [
+    {
+      "id": 1,
+      "type": "stat",
+      "title": "Up",
+      "gridPos": { "h": 4, "w": 4, "x": 0, "y": 0 },
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "fieldConfig": { "defaults": { "unit": "none", "color": { "mode": "thresholds" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "red", "value": null }, { "color": "green", "value": 1 } ] }, "mappings": [ { "type": "value", "options": { "0": { "text": "Down", "color": "red" }, "1": { "text": "Up", "color": "green" } } } ] }, "overrides": [] },
+      "options": { "reduceOptions": { "values": false, "calcs": ["lastNotNull"], "fields": "" }, "colorMode": "background", "graphMode": "none", "textMode": "auto", "justifyMode": "auto", "orientation": "auto" },
+      "targets": [ { "refId": "A", "expr": "unbound_up{instance=\"$instance\"}" } ]
+    },
+    {
+      "id": 2,
+      "type": "stat",
+      "title": "Uptime",
+      "gridPos": { "h": 4, "w": 4, "x": 4, "y": 0 },
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "fieldConfig": { "defaults": { "unit": "s", "color": { "mode": "none" } }, "overrides": [] },
+      "options": { "reduceOptions": { "values": false, "calcs": ["lastNotNull"], "fields": "" }, "colorMode": "value", "graphMode": "none", "textMode": "auto", "justifyMode": "auto", "orientation": "auto" },
+      "targets": [ { "refId": "A", "expr": "unbound_time_up_seconds_total{instance=\"$instance\"}" } ]
+    },
+    {
+      "id": 3,
+      "type": "timeseries",
+      "title": "Queries/sec",
+      "gridPos": { "h": 4, "w": 8, "x": 8, "y": 0 },
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "fieldConfig": { "defaults": { "unit": "ops", "color": { "mode": "palette-classic" }, "custom": { "fillOpacity": 20, "lineWidth": 1, "drawStyle": "line", "showPoints": "never" } }, "overrides": [] },
+      "targets": [ { "refId": "A", "expr": "sum by (instance) (rate(unbound_queries_total{instance=\"$instance\"}[$__rate_interval]))", "legendFormat": "queries" } ]
+    },
+    {
+      "id": 4,
+      "type": "timeseries",
+      "title": "Cache Hit Rate",
+      "gridPos": { "h": 4, "w": 8, "x": 16, "y": 0 },
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "fieldConfig": { "defaults": { "unit": "percentunit", "min": 0, "max": 1, "color": { "mode": "palette-classic" }, "custom": { "fillOpacity": 20, "lineWidth": 1, "drawStyle": "line", "showPoints": "never" } }, "overrides": [] },
+      "targets": [ { "refId": "A", "expr": "sum by (instance) (rate(unbound_cache_hits_total{instance=\"$instance\"}[$__rate_interval])) / (sum by (instance) (rate(unbound_cache_hits_total{instance=\"$instance\"}[$__rate_interval])) + sum by (instance) (rate(unbound_cache_misses_total{instance=\"$instance\"}[$__rate_interval])))", "legendFormat": "hit rate" } ]
+    },
+    {
+      "id": 5,
+      "type": "timeseries",
+      "title": "Response Time",
+      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 4 },
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "fieldConfig": { "defaults": { "unit": "s", "min": 0, "color": { "mode": "palette-classic" }, "custom": { "fillOpacity": 20, "lineWidth": 1, "drawStyle": "line", "showPoints": "never" } }, "overrides": [] },
+      "targets": [
+        { "refId": "A", "expr": "histogram_quantile(0.95, sum by (le) (rate(unbound_response_time_seconds_bucket{instance=\"$instance\"}[$__rate_interval])))", "legendFormat": "p95" },
+        { "refId": "B", "expr": "histogram_quantile(0.50, sum by (le) (rate(unbound_response_time_seconds_bucket{instance=\"$instance\"}[$__rate_interval])))", "legendFormat": "p50" }
+      ]
+    },
+    {
+      "id": 6,
+      "type": "timeseries",
+      "title": "Recursion Time",
+      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 4 },
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "fieldConfig": { "defaults": { "unit": "s", "min": 0, "color": { "mode": "palette-classic" }, "custom": { "fillOpacity": 20, "lineWidth": 1, "drawStyle": "line", "showPoints": "never" } }, "overrides": [] },
+      "targets": [
+        { "refId": "A", "expr": "unbound_recursion_time_seconds_avg{instance=\"$instance\"}", "legendFormat": "avg" },
+        { "refId": "B", "expr": "unbound_recursion_time_seconds_median{instance=\"$instance\"}", "legendFormat": "median" }
+      ]
+    },
+    {
+      "id": 7,
+      "type": "timeseries",
+      "title": "Answers by Rcode",
+      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 12 },
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "fieldConfig": { "defaults": { "unit": "ops", "color": { "mode": "palette-classic" }, "custom": { "fillOpacity": 20, "lineWidth": 1, "drawStyle": "line", "showPoints": "never" } }, "overrides": [] },
+      "targets": [ { "refId": "A", "expr": "sum by (rcode) (rate(unbound_answer_rcodes_total{instance=\"$instance\"}[$__rate_interval]))", "legendFormat": "{{rcode}}" } ]
+    },
+    {
+      "id": 8,
+      "type": "timeseries",
+      "title": "Query Types",
+      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 12 },
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "fieldConfig": { "defaults": { "unit": "ops", "color": { "mode": "palette-classic" }, "custom": { "fillOpacity": 20, "lineWidth": 1, "drawStyle": "line", "showPoints": "never" } }, "overrides": [] },
+      "targets": [ { "refId": "A", "expr": "sum by (type) (rate(unbound_query_types_total{instance=\"$instance\"}[$__rate_interval]))", "legendFormat": "{{type}}" } ]
+    },
+    {
+      "id": 9,
+      "type": "timeseries",
+      "title": "Request List (backlog)",
+      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 20 },
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "fieldConfig": { "defaults": { "unit": "short", "min": 0, "color": { "mode": "palette-classic" }, "custom": { "fillOpacity": 20, "lineWidth": 1, "drawStyle": "line", "showPoints": "never" } }, "overrides": [] },
+      "targets": [ { "refId": "A", "expr": "sum by (instance) (unbound_request_list_current_user{instance=\"$instance\"})", "legendFormat": "current user requests" } ]
+    },
+    {
+      "id": 10,
+      "type": "timeseries",
+      "title": "Dropped Queries",
+      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 20 },
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "fieldConfig": { "defaults": { "unit": "ops", "color": { "mode": "palette-classic" }, "custom": { "fillOpacity": 20, "lineWidth": 1, "drawStyle": "line", "showPoints": "never" } }, "overrides": [] },
+      "targets": [
+        { "refId": "A", "expr": "sum by (instance) (rate(unbound_request_list_exceeded_total{instance=\"$instance\"}[$__rate_interval]))", "legendFormat": "exceeded" },
+        { "refId": "B", "expr": "sum by (instance) (rate(unbound_request_list_overwritten_total{instance=\"$instance\"}[$__rate_interval]))", "legendFormat": "overwritten" }
+      ]
+    },
+    {
+      "id": 11,
+      "type": "timeseries",
+      "title": "Transport",
+      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 28 },
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "fieldConfig": { "defaults": { "unit": "ops", "color": { "mode": "palette-classic" }, "custom": { "fillOpacity": 20, "lineWidth": 1, "drawStyle": "line", "showPoints": "never" } }, "overrides": [] },
+      "targets": [
+        { "refId": "A", "expr": "rate(unbound_query_tcp_total{instance=\"$instance\"}[$__rate_interval])", "legendFormat": "TCP" },
+        { "refId": "B", "expr": "rate(unbound_query_tls_total{instance=\"$instance\"}[$__rate_interval])", "legendFormat": "TLS (DoT)" },
+        { "refId": "C", "expr": "rate(unbound_query_https_total{instance=\"$instance\"}[$__rate_interval])", "legendFormat": "HTTPS (DoH)" }
+      ]
+    },
+    {
+      "id": 12,
+      "type": "timeseries",
+      "title": "DNSSEC Validation",
+      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 28 },
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "fieldConfig": { "defaults": { "unit": "ops", "color": { "mode": "palette-classic" }, "custom": { "fillOpacity": 20, "lineWidth": 1, "drawStyle": "line", "showPoints": "never" } }, "overrides": [] },
+      "targets": [
+        { "refId": "A", "expr": "rate(unbound_answers_secure_total{instance=\"$instance\"}[$__rate_interval])", "legendFormat": "secure" },
+        { "refId": "B", "expr": "rate(unbound_answers_bogus{instance=\"$instance\"}[$__rate_interval])", "legendFormat": "bogus" }
+      ]
+    },
+    {
+      "id": 13,
+      "type": "timeseries",
+      "title": "Cache Entries",
+      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 36 },
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "fieldConfig": { "defaults": { "unit": "short", "color": { "mode": "palette-classic" }, "custom": { "fillOpacity": 20, "lineWidth": 1, "drawStyle": "line", "showPoints": "never" } }, "overrides": [] },
+      "targets": [
+        { "refId": "A", "expr": "unbound_msg_cache_count{instance=\"$instance\"}", "legendFormat": "msg" },
+        { "refId": "B", "expr": "unbound_rrset_cache_count{instance=\"$instance\"}", "legendFormat": "rrset" },
+        { "refId": "C", "expr": "unbound_infra_cache_count{instance=\"$instance\"}", "legendFormat": "infra" }
+      ]
+    },
+    {
+      "id": 14,
+      "type": "timeseries",
+      "title": "Recursive vs Cache Replies",
+      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 36 },
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "fieldConfig": { "defaults": { "unit": "ops", "color": { "mode": "palette-classic" }, "custom": { "fillOpacity": 20, "lineWidth": 1, "drawStyle": "line", "showPoints": "never" } }, "overrides": [] },
+      "targets": [ { "refId": "A", "expr": "sum by (instance) (rate(unbound_recursive_replies_total{instance=\"$instance\"}[$__rate_interval]))", "legendFormat": "recursive replies" } ]
+    }
+  ]
+}

roles/grafana/tasks/main.yml

@@ -215,7 +215,7 @@
     state: absent
   notify: Restart grafana
 
-- name: Deploy services dashboards (nginx, postgres, valkey)
+- name: Deploy services dashboards (nginx, postgres, valkey, unbound)
   ansible.builtin.copy:
     src: "{{ item }}"
     dest: "{{ grafana_provisioning_path }}/dashboards/{{ item }}"
@@ -226,6 +226,7 @@
     - nginx-dashboard.json
     - postgres-dashboard.json
     - valkey-dashboard.json
+    - unbound-dashboard.json
   notify: Restart grafana
 
 # Service management

roles/unbound/README.md

@@ -1,3 +1,28 @@
+# Metrics (Prometheus)
+
+Set `unbound_exporter_enabled: true` in inventory/host_vars. This enables
+`remote-control` (no TLS, loopback only) and deploys `prometheus-unbound-exporter`
+(Arch Linux only), which scrapes `unbound-control stats_noreset` on
+`127.0.0.1:9167`.
+
+Verify:
+
+```sh
+curl -s 127.0.0.1:9167/metrics | grep '^unbound_up'
+```
+
+Add a scrape job in `prometheus_scrape_configs`:
+
+```yaml
+- job_name: 'unbound'
+  static_configs:
+    - targets: ['localhost:9167']
+```
+
+Key QoS metrics: cache hit ratio (`unbound_cache_hit_rate` or
+`num.cachehits`/`num.cachemiss`), recursion latency (`num.query.time.avg/median`),
+requestlist backlog, and `num.answer.rcode.SERVFAIL` failure rate.
+
 # Testing
 
 ## DNS leaks

roles/unbound/defaults/main.yml

@@ -25,3 +25,9 @@ unbound_firewall_allowed_sources:
 unbound_custom_lan_records:
   "example.lan":
     v4: 192.168.1.2
+
+# Prometheus exporter (optional, via prometheus-unbound-exporter).
+# Scrapes unbound-control stats_noreset over the local control interface.
+# Arch Linux only — the exporter is not packaged in Debian/Ubuntu.
+unbound_exporter_enabled: false
+unbound_exporter_listen_address: "127.0.0.1:9167"

roles/unbound/handlers/main.yml

@@ -14,3 +14,8 @@
 - name: Reload AppArmor profile
   ansible.builtin.command: apparmor_parser -r {{ unbound_apparmor_profile_path }}
   when: ansible_facts.apparmor.status == "enabled"
+
+- name: Restart unbound exporter
+  ansible.builtin.systemd:
+    name: "{{ unbound_exporter_service }}"
+    state: restarted

roles/unbound/tasks/exporter.yml

@@ -0,0 +1,35 @@
+---
+# Prometheus unbound exporter (optional, gated by unbound_exporter_enabled).
+# Scrapes unbound-control stats_noreset over the local control interface.
+
+- name: Assert exporter is supported (Arch Linux only)
+  ansible.builtin.assert:
+    that: ansible_facts['os_family'] == 'Archlinux'
+    fail_msg: |
+      The unbound exporter (prometheus-unbound-exporter) is only packaged for
+      Arch Linux. It is not available in Debian/Ubuntu repositories.
+    success_msg: "Unbound exporter supported"
+
+- name: Load OS-specific exporter variables
+  ansible.builtin.include_vars: archlinux.yml
+
+- name: Install prometheus-unbound-exporter
+  ansible.builtin.package:
+    name: "{{ unbound_exporter_package }}"
+    state: present
+
+- name: Deploy exporter environment file
+  ansible.builtin.template:
+    src: exporter-env.conf.j2
+    dest: "{{ unbound_exporter_args_file }}"
+    owner: root
+    group: root
+    mode: "0644"
+  notify: Restart unbound exporter
+
+- name: Enable and start prometheus-unbound-exporter
+  ansible.builtin.systemd:
+    name: "{{ unbound_exporter_service }}"
+    enabled: true
+    state: started
+    daemon_reload: true

roles/unbound/tasks/main.yml

@@ -137,13 +137,20 @@
     group: root
     mode: "0644"
 
+# Match so-rcvbuf/so-sndbuf (4m) in unbound.conf. rmem_max was 1 MiB and
+# wmem_max was left at its ~208 KiB default, silently capping unbound's
+# requested 4 MiB buffers (the kernel halves SO_RCVBUF/SO_SNDBUF requests
+# against these sysctls before doubling them).
 - name: Configure sysctl for unbound socket buffers
   ansible.posix.sysctl:
-    name: net.core.rmem_max
+    name: "{{ item.name }}"
-    value: "1048576"
+    value: "{{ item.value }}"
     state: present
     sysctl_set: true
     reload: true
+  loop:
+    - { name: net.core.rmem_max, value: "4194304" }
+    - { name: net.core.wmem_max, value: "4194304" }
 
 - name: Install unbound config
   ansible.builtin.template:
@@ -184,3 +191,7 @@
   delay: 2
   register: ufw_result
   until: ufw_result is succeeded
+
+- name: Deploy Prometheus unbound exporter
+  ansible.builtin.include_tasks: exporter.yml
+  when: unbound_exporter_enabled | bool

roles/unbound/templates/exporter-env.conf.j2

@@ -0,0 +1,4 @@
+# Managed by Ansible - DO NOT EDIT MANUALLY
+# Empty cert flags (-unbound.ca= etc.) make the exporter speak plain-text to
+# unbound's control interface (control-use-cert: no).
+{{ unbound_exporter_args_var }}="-web.listen-address={{ unbound_exporter_listen_address }} -unbound.ca= -unbound.cert= -unbound.key= -unbound.host tcp://127.0.0.1:8953"

roles/unbound/templates/unbound.conf.j2

@@ -130,5 +130,18 @@ server:
     # Enable DNSSEC
     auto-trust-anchor-file: "{{ unbound_anchor_root_key }}"
 
+    # Statistics for monitoring (unbound-control stats / prometheus exporter)
+    statistics-interval: 0
+    statistics-cumulative: yes
+    extended-statistics: yes
+
+# Remote control for unbound-control and the prometheus exporter.
+# control-use-cert: no — unbound only accepts plain control connections from
+# loopback in this mode, so no TLS certs need to be generated.
+remote-control:
+    control-enable: yes
+    control-interface: 127.0.0.1
+    control-use-cert: no
+
 include: "{{ unbound_custom_lan_config_path }}"
 include: "{{ unbound_custom_vpn_config_path }}"

roles/unbound/vars/archlinux.yml

@@ -0,0 +1,6 @@
+---
+# Prometheus exporter
+unbound_exporter_package: prometheus-unbound-exporter
+unbound_exporter_service: prometheus-unbound-exporter
+unbound_exporter_args_file: /etc/conf.d/prometheus-unbound-exporter
+unbound_exporter_args_var: UNBOUND_EXPORTER_ARGS