diff --git a/examples/inventory/host_vars.example.yml b/examples/inventory/host_vars.example.yml index e12fa22..70daf52 100644 --- a/examples/inventory/host_vars.example.yml +++ b/examples/inventory/host_vars.example.yml @@ -45,6 +45,9 @@ unbound_firewall_allowed_sources: - { src: "192.168.1.0/24", comment: "DNS from LAN" } - { src: "192.168.20.0/27", comment: "DNS from VPN" } +# Unbound Prometheus exporter (Arch Linux only) +# unbound_exporter_enabled: true + # NTP servers configuration # ------------------------- ntp_pools: diff --git a/roles/grafana/files/unbound-dashboard.json b/roles/grafana/files/unbound-dashboard.json new file mode 100644 index 0000000..c702d87 --- /dev/null +++ b/roles/grafana/files/unbound-dashboard.json @@ -0,0 +1,178 @@ +{ + "title": "Unbound", + "uid": "unbound-overview", + "tags": ["unbound", "dns", "resolver"], + "schemaVersion": 39, + "version": 1, + "refresh": "1m", + "time": { "from": "now-6h", "to": "now" }, + "timezone": "browser", + "editable": true, + "graphTooltip": 1, + "templating": { + "list": [ + { + "name": "instance", + "label": "Host", + "type": "query", + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "query": { "query": "label_values(up{job=\"unbound\"}, instance)", "refId": "A" }, + "refresh": 2, + "multi": false, + "includeAll": false, + "sort": 1, + "current": { "selected": false, "text": "", "value": "" } + } + ] + }, + "panels": [ + { + "id": 1, + "type": "stat", + "title": "Up", + "gridPos": { "h": 4, "w": 4, "x": 0, "y": 0 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { "defaults": { "unit": "none", "color": { "mode": "thresholds" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "red", "value": null }, { "color": "green", "value": 1 } ] }, "mappings": [ { "type": "value", "options": { "0": { "text": "Down", "color": "red" }, "1": { "text": "Up", "color": "green" } } } ] }, "overrides": [] }, + "options": { "reduceOptions": { "values": false, "calcs": ["lastNotNull"], "fields": "" }, "colorMode": "background", "graphMode": "none", "textMode": "auto", "justifyMode": "auto", "orientation": "auto" }, + "targets": [ { "refId": "A", "expr": "unbound_up{instance=\"$instance\"}" } ] + }, + { + "id": 2, + "type": "stat", + "title": "Uptime", + "gridPos": { "h": 4, "w": 4, "x": 4, "y": 0 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { "defaults": { "unit": "s", "color": { "mode": "none" } }, "overrides": [] }, + "options": { "reduceOptions": { "values": false, "calcs": ["lastNotNull"], "fields": "" }, "colorMode": "value", "graphMode": "none", "textMode": "auto", "justifyMode": "auto", "orientation": "auto" }, + "targets": [ { "refId": "A", "expr": "unbound_time_up_seconds_total{instance=\"$instance\"}" } ] + }, + { + "id": 3, + "type": "timeseries", + "title": "Queries/sec", + "gridPos": { "h": 4, "w": 8, "x": 8, "y": 0 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { "defaults": { "unit": "ops", "color": { "mode": "palette-classic" }, "custom": { "fillOpacity": 20, "lineWidth": 1, "drawStyle": "line", "showPoints": "never" } }, "overrides": [] }, + "targets": [ { "refId": "A", "expr": "sum by (instance) (rate(unbound_queries_total{instance=\"$instance\"}[$__rate_interval]))", "legendFormat": "queries" } ] + }, + { + "id": 4, + "type": "timeseries", + "title": "Cache Hit Rate", + "gridPos": { "h": 4, "w": 8, "x": 16, "y": 0 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { "defaults": { "unit": "percentunit", "min": 0, "max": 1, "color": { "mode": "palette-classic" }, "custom": { "fillOpacity": 20, "lineWidth": 1, "drawStyle": "line", "showPoints": "never" } }, "overrides": [] }, + "targets": [ { "refId": "A", "expr": "sum by (instance) (rate(unbound_cache_hits_total{instance=\"$instance\"}[$__rate_interval])) / (sum by (instance) (rate(unbound_cache_hits_total{instance=\"$instance\"}[$__rate_interval])) + sum by (instance) (rate(unbound_cache_misses_total{instance=\"$instance\"}[$__rate_interval])))", "legendFormat": "hit rate" } ] + }, + { + "id": 5, + "type": "timeseries", + "title": "Response Time", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 4 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { "defaults": { "unit": "s", "min": 0, "color": { "mode": "palette-classic" }, "custom": { "fillOpacity": 20, "lineWidth": 1, "drawStyle": "line", "showPoints": "never" } }, "overrides": [] }, + "targets": [ + { "refId": "A", "expr": "histogram_quantile(0.95, sum by (le) (rate(unbound_response_time_seconds_bucket{instance=\"$instance\"}[$__rate_interval])))", "legendFormat": "p95" }, + { "refId": "B", "expr": "histogram_quantile(0.50, sum by (le) (rate(unbound_response_time_seconds_bucket{instance=\"$instance\"}[$__rate_interval])))", "legendFormat": "p50" } + ] + }, + { + "id": 6, + "type": "timeseries", + "title": "Recursion Time", + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 4 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { "defaults": { "unit": "s", "min": 0, "color": { "mode": "palette-classic" }, "custom": { "fillOpacity": 20, "lineWidth": 1, "drawStyle": "line", "showPoints": "never" } }, "overrides": [] }, + "targets": [ + { "refId": "A", "expr": "unbound_recursion_time_seconds_avg{instance=\"$instance\"}", "legendFormat": "avg" }, + { "refId": "B", "expr": "unbound_recursion_time_seconds_median{instance=\"$instance\"}", "legendFormat": "median" } + ] + }, + { + "id": 7, + "type": "timeseries", + "title": "Answers by Rcode", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 12 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { "defaults": { "unit": "ops", "color": { "mode": "palette-classic" }, "custom": { "fillOpacity": 20, "lineWidth": 1, "drawStyle": "line", "showPoints": "never" } }, "overrides": [] }, + "targets": [ { "refId": "A", "expr": "sum by (rcode) (rate(unbound_answer_rcodes_total{instance=\"$instance\"}[$__rate_interval]))", "legendFormat": "{{rcode}}" } ] + }, + { + "id": 8, + "type": "timeseries", + "title": "Query Types", + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 12 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { "defaults": { "unit": "ops", "color": { "mode": "palette-classic" }, "custom": { "fillOpacity": 20, "lineWidth": 1, "drawStyle": "line", "showPoints": "never" } }, "overrides": [] }, + "targets": [ { "refId": "A", "expr": "sum by (type) (rate(unbound_query_types_total{instance=\"$instance\"}[$__rate_interval]))", "legendFormat": "{{type}}" } ] + }, + { + "id": 9, + "type": "timeseries", + "title": "Request List (backlog)", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 20 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { "defaults": { "unit": "short", "min": 0, "color": { "mode": "palette-classic" }, "custom": { "fillOpacity": 20, "lineWidth": 1, "drawStyle": "line", "showPoints": "never" } }, "overrides": [] }, + "targets": [ { "refId": "A", "expr": "sum by (instance) (unbound_request_list_current_user{instance=\"$instance\"})", "legendFormat": "current user requests" } ] + }, + { + "id": 10, + "type": "timeseries", + "title": "Dropped Queries", + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 20 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { "defaults": { "unit": "ops", "color": { "mode": "palette-classic" }, "custom": { "fillOpacity": 20, "lineWidth": 1, "drawStyle": "line", "showPoints": "never" } }, "overrides": [] }, + "targets": [ + { "refId": "A", "expr": "sum by (instance) (rate(unbound_request_list_exceeded_total{instance=\"$instance\"}[$__rate_interval]))", "legendFormat": "exceeded" }, + { "refId": "B", "expr": "sum by (instance) (rate(unbound_request_list_overwritten_total{instance=\"$instance\"}[$__rate_interval]))", "legendFormat": "overwritten" } + ] + }, + { + "id": 11, + "type": "timeseries", + "title": "Transport", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 28 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { "defaults": { "unit": "ops", "color": { "mode": "palette-classic" }, "custom": { "fillOpacity": 20, "lineWidth": 1, "drawStyle": "line", "showPoints": "never" } }, "overrides": [] }, + "targets": [ + { "refId": "A", "expr": "rate(unbound_query_tcp_total{instance=\"$instance\"}[$__rate_interval])", "legendFormat": "TCP" }, + { "refId": "B", "expr": "rate(unbound_query_tls_total{instance=\"$instance\"}[$__rate_interval])", "legendFormat": "TLS (DoT)" }, + { "refId": "C", "expr": "rate(unbound_query_https_total{instance=\"$instance\"}[$__rate_interval])", "legendFormat": "HTTPS (DoH)" } + ] + }, + { + "id": 12, + "type": "timeseries", + "title": "DNSSEC Validation", + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 28 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { "defaults": { "unit": "ops", "color": { "mode": "palette-classic" }, "custom": { "fillOpacity": 20, "lineWidth": 1, "drawStyle": "line", "showPoints": "never" } }, "overrides": [] }, + "targets": [ + { "refId": "A", "expr": "rate(unbound_answers_secure_total{instance=\"$instance\"}[$__rate_interval])", "legendFormat": "secure" }, + { "refId": "B", "expr": "rate(unbound_answers_bogus{instance=\"$instance\"}[$__rate_interval])", "legendFormat": "bogus" } + ] + }, + { + "id": 13, + "type": "timeseries", + "title": "Cache Entries", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 36 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { "defaults": { "unit": "short", "color": { "mode": "palette-classic" }, "custom": { "fillOpacity": 20, "lineWidth": 1, "drawStyle": "line", "showPoints": "never" } }, "overrides": [] }, + "targets": [ + { "refId": "A", "expr": "unbound_msg_cache_count{instance=\"$instance\"}", "legendFormat": "msg" }, + { "refId": "B", "expr": "unbound_rrset_cache_count{instance=\"$instance\"}", "legendFormat": "rrset" }, + { "refId": "C", "expr": "unbound_infra_cache_count{instance=\"$instance\"}", "legendFormat": "infra" } + ] + }, + { + "id": 14, + "type": "timeseries", + "title": "Recursive vs Cache Replies", + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 36 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { "defaults": { "unit": "ops", "color": { "mode": "palette-classic" }, "custom": { "fillOpacity": 20, "lineWidth": 1, "drawStyle": "line", "showPoints": "never" } }, "overrides": [] }, + "targets": [ { "refId": "A", "expr": "sum by (instance) (rate(unbound_recursive_replies_total{instance=\"$instance\"}[$__rate_interval]))", "legendFormat": "recursive replies" } ] + } + ] +} diff --git a/roles/grafana/tasks/main.yml b/roles/grafana/tasks/main.yml index f9c41bd..51605f8 100644 --- a/roles/grafana/tasks/main.yml +++ b/roles/grafana/tasks/main.yml @@ -215,7 +215,7 @@ state: absent notify: Restart grafana -- name: Deploy services dashboards (nginx, postgres, valkey) +- name: Deploy services dashboards (nginx, postgres, valkey, unbound) ansible.builtin.copy: src: "{{ item }}" dest: "{{ grafana_provisioning_path }}/dashboards/{{ item }}" @@ -226,6 +226,7 @@ - nginx-dashboard.json - postgres-dashboard.json - valkey-dashboard.json + - unbound-dashboard.json notify: Restart grafana # Service management diff --git a/roles/unbound/README.md b/roles/unbound/README.md index 314c01d..477661f 100644 --- a/roles/unbound/README.md +++ b/roles/unbound/README.md @@ -1,3 +1,28 @@ +# Metrics (Prometheus) + +Set `unbound_exporter_enabled: true` in inventory/host_vars. This enables +`remote-control` (no TLS, loopback only) and deploys `prometheus-unbound-exporter` +(Arch Linux only), which scrapes `unbound-control stats_noreset` on +`127.0.0.1:9167`. + +Verify: + +```sh +curl -s 127.0.0.1:9167/metrics | grep '^unbound_up' +``` + +Add a scrape job in `prometheus_scrape_configs`: + +```yaml +- job_name: 'unbound' + static_configs: + - targets: ['localhost:9167'] +``` + +Key QoS metrics: cache hit ratio (`unbound_cache_hit_rate` or +`num.cachehits`/`num.cachemiss`), recursion latency (`num.query.time.avg/median`), +requestlist backlog, and `num.answer.rcode.SERVFAIL` failure rate. + # Testing ## DNS leaks diff --git a/roles/unbound/defaults/main.yml b/roles/unbound/defaults/main.yml index 7050804..5e55cb0 100644 --- a/roles/unbound/defaults/main.yml +++ b/roles/unbound/defaults/main.yml @@ -25,3 +25,9 @@ unbound_firewall_allowed_sources: unbound_custom_lan_records: "example.lan": v4: 192.168.1.2 + +# Prometheus exporter (optional, via prometheus-unbound-exporter). +# Scrapes unbound-control stats_noreset over the local control interface. +# Arch Linux only — the exporter is not packaged in Debian/Ubuntu. +unbound_exporter_enabled: false +unbound_exporter_listen_address: "127.0.0.1:9167" diff --git a/roles/unbound/handlers/main.yml b/roles/unbound/handlers/main.yml index a31deb3..e85b62e 100644 --- a/roles/unbound/handlers/main.yml +++ b/roles/unbound/handlers/main.yml @@ -14,3 +14,8 @@ - name: Reload AppArmor profile ansible.builtin.command: apparmor_parser -r {{ unbound_apparmor_profile_path }} when: ansible_facts.apparmor.status == "enabled" + +- name: Restart unbound exporter + ansible.builtin.systemd: + name: "{{ unbound_exporter_service }}" + state: restarted diff --git a/roles/unbound/tasks/exporter.yml b/roles/unbound/tasks/exporter.yml new file mode 100644 index 0000000..0fff1f3 --- /dev/null +++ b/roles/unbound/tasks/exporter.yml @@ -0,0 +1,35 @@ +--- +# Prometheus unbound exporter (optional, gated by unbound_exporter_enabled). +# Scrapes unbound-control stats_noreset over the local control interface. + +- name: Assert exporter is supported (Arch Linux only) + ansible.builtin.assert: + that: ansible_facts['os_family'] == 'Archlinux' + fail_msg: | + The unbound exporter (prometheus-unbound-exporter) is only packaged for + Arch Linux. It is not available in Debian/Ubuntu repositories. + success_msg: "Unbound exporter supported" + +- name: Load OS-specific exporter variables + ansible.builtin.include_vars: archlinux.yml + +- name: Install prometheus-unbound-exporter + ansible.builtin.package: + name: "{{ unbound_exporter_package }}" + state: present + +- name: Deploy exporter environment file + ansible.builtin.template: + src: exporter-env.conf.j2 + dest: "{{ unbound_exporter_args_file }}" + owner: root + group: root + mode: "0644" + notify: Restart unbound exporter + +- name: Enable and start prometheus-unbound-exporter + ansible.builtin.systemd: + name: "{{ unbound_exporter_service }}" + enabled: true + state: started + daemon_reload: true diff --git a/roles/unbound/tasks/main.yml b/roles/unbound/tasks/main.yml index 0301179..e4ad2c3 100644 --- a/roles/unbound/tasks/main.yml +++ b/roles/unbound/tasks/main.yml @@ -137,13 +137,20 @@ group: root mode: "0644" +# Match so-rcvbuf/so-sndbuf (4m) in unbound.conf. rmem_max was 1 MiB and +# wmem_max was left at its ~208 KiB default, silently capping unbound's +# requested 4 MiB buffers (the kernel halves SO_RCVBUF/SO_SNDBUF requests +# against these sysctls before doubling them). - name: Configure sysctl for unbound socket buffers ansible.posix.sysctl: - name: net.core.rmem_max - value: "1048576" + name: "{{ item.name }}" + value: "{{ item.value }}" state: present sysctl_set: true reload: true + loop: + - { name: net.core.rmem_max, value: "4194304" } + - { name: net.core.wmem_max, value: "4194304" } - name: Install unbound config ansible.builtin.template: @@ -184,3 +191,7 @@ delay: 2 register: ufw_result until: ufw_result is succeeded + +- name: Deploy Prometheus unbound exporter + ansible.builtin.include_tasks: exporter.yml + when: unbound_exporter_enabled | bool diff --git a/roles/unbound/templates/exporter-env.conf.j2 b/roles/unbound/templates/exporter-env.conf.j2 new file mode 100644 index 0000000..5cadcb7 --- /dev/null +++ b/roles/unbound/templates/exporter-env.conf.j2 @@ -0,0 +1,4 @@ +# Managed by Ansible - DO NOT EDIT MANUALLY +# Empty cert flags (-unbound.ca= etc.) make the exporter speak plain-text to +# unbound's control interface (control-use-cert: no). +{{ unbound_exporter_args_var }}="-web.listen-address={{ unbound_exporter_listen_address }} -unbound.ca= -unbound.cert= -unbound.key= -unbound.host tcp://127.0.0.1:8953" diff --git a/roles/unbound/templates/unbound.conf.j2 b/roles/unbound/templates/unbound.conf.j2 index ed573c6..64ecabe 100644 --- a/roles/unbound/templates/unbound.conf.j2 +++ b/roles/unbound/templates/unbound.conf.j2 @@ -130,5 +130,18 @@ server: # Enable DNSSEC auto-trust-anchor-file: "{{ unbound_anchor_root_key }}" + # Statistics for monitoring (unbound-control stats / prometheus exporter) + statistics-interval: 0 + statistics-cumulative: yes + extended-statistics: yes + +# Remote control for unbound-control and the prometheus exporter. +# control-use-cert: no — unbound only accepts plain control connections from +# loopback in this mode, so no TLS certs need to be generated. +remote-control: + control-enable: yes + control-interface: 127.0.0.1 + control-use-cert: no + include: "{{ unbound_custom_lan_config_path }}" include: "{{ unbound_custom_vpn_config_path }}" diff --git a/roles/unbound/vars/archlinux.yml b/roles/unbound/vars/archlinux.yml new file mode 100644 index 0000000..b12f3fa --- /dev/null +++ b/roles/unbound/vars/archlinux.yml @@ -0,0 +1,6 @@ +--- +# Prometheus exporter +unbound_exporter_package: prometheus-unbound-exporter +unbound_exporter_service: prometheus-unbound-exporter +unbound_exporter_args_file: /etc/conf.d/prometheus-unbound-exporter +unbound_exporter_args_var: UNBOUND_EXPORTER_ARGS