feat(backup): add restic backup role with encrypted incremental snapshots

Restic-backed backups using a many-to-many repository/job model.

- repository = destination: restic repository URL verbatim (s3, local,
  removable device), backend credentials via env passthrough, retention
  policy, and an optional per-repository ntfy notify channel
- job = source set: paths, sqlite and postgres dumps (consistent .backup /
  pg_dump instead of torn live-file copies), pushed to one or more repos
- systemd timer per job, per-repo + per-job flock serialisation, restic
  check with read_data_subset for rotating integrity verification

Service roles declare what to back up (ntfy_backup_*, uptime_kuma_backup_*);
the inventory declares where.

roles/backup/tasks/job.yml

@@ -0,0 +1,36 @@
+---
+# Included per job with `job` set to the job dict.
+
+- name: Deploy job definition for {{ job.name }}
+  ansible.builtin.template:
+    src: job.conf.j2
+    dest: "{{ backup_config_dir }}/jobs/{{ job.name }}.conf"
+    owner: root
+    group: root
+    mode: "0600"
+
+- name: Create timer override directory for {{ job.name }}
+  ansible.builtin.file:
+    path: "/etc/systemd/system/backup@{{ job.name }}.timer.d"
+    state: directory
+    owner: root
+    group: root
+    mode: "0755"
+  when: job.schedule is defined
+
+- name: Deploy schedule override for {{ job.name }}
+  ansible.builtin.template:
+    src: timer-schedule-override.conf.j2
+    dest: "/etc/systemd/system/backup@{{ job.name }}.timer.d/schedule.conf"
+    owner: root
+    group: root
+    mode: "0644"
+  when: job.schedule is defined
+  notify: Reload systemd
+
+- name: Remove stale schedule override for {{ job.name }}
+  ansible.builtin.file:
+    path: "/etc/systemd/system/backup@{{ job.name }}.timer.d"
+    state: absent
+  when: job.schedule is not defined
+  notify: Reload systemd

roles/backup/tasks/main.yml

@@ -0,0 +1,147 @@
+---
+- name: Validate backup configuration
+  ansible.builtin.import_tasks: validate.yml
+
+- name: Load OS-specific variables
+  ansible.builtin.include_vars: "{{ item }}"
+  with_first_found:
+    - "{{ ansible_facts['os_family'] }}.yml"
+    - debian.yml
+
+- name: Install backup packages
+  ansible.builtin.package:
+    name: "{{ backup_packages }}"
+    state: present
+
+- name: Install sqlite tooling
+  ansible.builtin.package:
+    name: "{{ backup_sqlite_packages }}"
+    state: present
+  when: backup_jobs | selectattr('sqlite', 'defined') | list | length > 0
+
+- name: Install PostgreSQL client tooling
+  ansible.builtin.package:
+    name: "{{ backup_postgres_packages }}"
+    state: present
+  when: backup_jobs | selectattr('postgres', 'defined') | list | length > 0
+
+- name: Create backup configuration directories
+  ansible.builtin.file:
+    path: "{{ item }}"
+    state: directory
+    owner: root
+    group: root
+    mode: "0700"
+  loop:
+    - "{{ backup_config_dir }}"
+    - "{{ backup_config_dir }}/repos"
+    - "{{ backup_config_dir }}/jobs"
+    - "{{ backup_dumps_dir }}"
+    - "{{ backup_cache_dir }}"
+
+- name: Deploy backup script
+  ansible.builtin.template:
+    src: backup.sh.j2
+    dest: "{{ backup_script_path }}"
+    owner: root
+    group: root
+    mode: "0700"
+  no_log: true
+
+- name: Configure repositories
+  ansible.builtin.include_tasks: repository.yml
+  loop: "{{ backup_repositories }}"
+  loop_control:
+    loop_var: repo
+    label: "{{ repo.name }}"
+
+- name: Configure jobs
+  ansible.builtin.include_tasks: job.yml
+  loop: "{{ backup_jobs }}"
+  loop_control:
+    loop_var: job
+    label: "{{ job.name }}"
+
+- name: Find existing repository configurations
+  ansible.builtin.find:
+    paths: "{{ backup_config_dir }}/repos"
+    patterns: "*.env"
+  register: backup_existing_repos
+
+- name: Remove orphaned repository configurations
+  ansible.builtin.file:
+    path: "{{ item.path }}"
+    state: absent
+  loop: "{{ backup_existing_repos.files }}"
+  loop_control:
+    label: "{{ item.path | basename }}"
+  when: (item.path | basename | splitext | first)
+    not in (backup_repositories | map(attribute='name') | list)
+
+- name: Find existing job configurations
+  ansible.builtin.find:
+    paths: "{{ backup_config_dir }}/jobs"
+    patterns: "*.conf"
+  register: backup_existing_jobs
+
+- name: Remove orphaned job configurations
+  ansible.builtin.file:
+    path: "{{ item.path }}"
+    state: absent
+  loop: "{{ backup_existing_jobs.files }}"
+  loop_control:
+    label: "{{ item.path | basename }}"
+  when: (item.path | basename | splitext | first)
+    not in (backup_jobs | map(attribute='name') | list)
+
+- name: Deploy systemd service template
+  ansible.builtin.template:
+    src: backup@.service.j2
+    dest: /etc/systemd/system/backup@.service
+    owner: root
+    group: root
+    mode: "0644"
+  notify: Reload systemd
+
+- name: Deploy systemd timer template
+  ansible.builtin.template:
+    src: backup@.timer.j2
+    dest: /etc/systemd/system/backup@.timer
+    owner: root
+    group: root
+    mode: "0644"
+  notify: Reload systemd
+
+- name: Flush handlers before enabling timers
+  ansible.builtin.meta: flush_handlers
+
+- name: Enable and start backup timers
+  ansible.builtin.systemd:
+    name: "backup@{{ item.name }}.timer"
+    enabled: true
+    state: started
+  loop: "{{ backup_jobs }}"
+  loop_control:
+    label: "{{ item.name }}"
+
+# Failure alerts are the only thing standing between a broken timer and a
+# backup you believe exists. ntfy runs with auth-default-access deny-all, so a
+# token without write access to the topic would fail silently at 03:00. Prove
+# each configured channel works now instead.
+- name: Send a test notification
+  ansible.builtin.uri:
+    url: "{{ item.notify.server | default('https://ntfy.sh') }}/{{ item.notify.topic }}"
+    method: POST
+    body: "Backup role deployed on {{ inventory_hostname }}. Repository '{{ item.name }}' alerts are configured and reachable."
+    headers: >-
+      {{
+        {'Title': inventory_hostname ~ ': Backup notifications configured',
+         'Tags': 'floppy_disk',
+         'Priority': 'low'}
+        | combine({'Authorization': 'Bearer ' ~ item.notify.token} if item.notify.token is defined else {})
+      }}
+    status_code: 200
+  loop: "{{ backup_repositories | selectattr('notify', 'defined') | list }}"
+  loop_control:
+    label: "{{ item.name }}"
+  changed_when: false

roles/backup/tasks/repository.yml

@@ -0,0 +1,53 @@
+---
+# Included per repository with `repo` set to the repository dict.
+
+- name: Deploy repository environment for {{ repo.name }}
+  ansible.builtin.template:
+    src: repo.env.j2
+    dest: "{{ backup_config_dir }}/repos/{{ repo.name }}.env"
+    owner: root
+    group: root
+    mode: "0600"
+  no_log: true
+
+- name: Create local repository directory for {{ repo.name }}
+  ansible.builtin.file:
+    path: "{{ repo.repository }}"
+    state: directory
+    owner: root
+    group: root
+    mode: "0700"
+  when: backup_repo_is_local | bool
+
+- name: Create device mountpoint for {{ repo.name }}
+  ansible.builtin.file:
+    path: "{{ repo.mountpoint }}"
+    state: directory
+    owner: root
+    group: root
+    mode: "0700"
+  when: backup_repo_is_device | bool
+
+# Device repositories are initialised lazily by the backup script the first
+# time the disk is actually attached. Everything else is initialised here so
+# that bad credentials or an unreachable endpoint fail at deploy time rather
+# than at 03:00 on a Sunday.
+- name: Check initialisation state of repository {{ repo.name }}
+  ansible.builtin.command:
+    cmd: restic cat config
+  environment: "{{ backup_repo_environment }}"
+  register: backup_repo_state
+  failed_when: false
+  changed_when: false
+  no_log: true
+  when: not backup_repo_is_device | bool
+
+- name: Initialise repository {{ repo.name }}
+  ansible.builtin.command:
+    cmd: restic init
+  environment: "{{ backup_repo_environment }}"
+  when:
+    - not backup_repo_is_device | bool
+    - backup_repo_state.rc | default(1) != 0
+  changed_when: true
+  no_log: true

roles/backup/tasks/validate.yml

@@ -0,0 +1,146 @@
+---
+- name: Validate repository names are unique
+  ansible.builtin.assert:
+    that:
+      - backup_repositories | map(attribute='name') | list | length
+        == backup_repositories | map(attribute='name') | unique | list | length
+    fail_msg: "backup_repositories contains duplicate names"
+    success_msg: "Repository names are unique"
+
+- name: Validate repository common keys
+  ansible.builtin.assert:
+    that:
+      - item.name is defined
+      - item.name | length > 0
+      - item.repository is defined
+      - item.repository | length > 0
+      - item.password is defined
+      - item.password | length >= 12
+      - item.retention is defined
+      - item.retention | length > 0
+    fail_msg: |
+      Repository '{{ item.name | default('<unnamed>') }}' is invalid.
+      Required: name, repository (a restic repository URL, verbatim),
+      password (min 12 chars), retention (at least one keep_* key).
+      A repository with an empty retention policy would make `restic forget`
+      fail at 03:00 rather than here.
+      See roles/backup/defaults/main.yml for configuration instructions.
+    quiet: true
+  loop: "{{ backup_repositories }}"
+  loop_control:
+    label: "{{ item.name | default('<unnamed>') }}"
+  no_log: true
+
+- name: Validate retention keys are recognised
+  ansible.builtin.assert:
+    that:
+      - item.retention.keys() | list | difference(backup_retention_keys) | length == 0
+    fail_msg: >-
+      Repository '{{ item.name }}' has unknown retention key(s):
+      {{ item.retention.keys() | list | difference(backup_retention_keys) | join(', ') }}.
+      Allowed: {{ backup_retention_keys | join(', ') }}.
+      An unrecognised key would be silently dropped and the repository would
+      grow without bound.
+    quiet: true
+  loop: "{{ backup_repositories }}"
+  loop_control:
+    label: "{{ item.name }}"
+
+- name: Validate repository environment is a mapping of strings
+  ansible.builtin.assert:
+    that:
+      - item.env.values() | reject('string') | list | length == 0
+    fail_msg: >-
+      Repository '{{ item.name }}' has a non-string env value.
+      Environment values are exported verbatim and must be strings.
+    quiet: true
+  loop: "{{ backup_repositories | selectattr('env', 'defined') | list }}"
+  loop_control:
+    label: "{{ item.name }}"
+
+- name: Validate device repositories
+  ansible.builtin.assert:
+    that:
+      - item.uuid is defined
+      - item.uuid | length > 0
+      - item.mountpoint is defined
+      - item.mountpoint is match('^/')
+      - item.repository is match('^' ~ item.mountpoint)
+    fail_msg: |
+      Device repository '{{ item.name }}' (declares mountpoint) requires uuid,
+      mountpoint, and a repository path below the mountpoint.
+      Identify the disk by filesystem UUID (blkid), never by /dev/sdX - kernel
+      device names are not stable across boots and would eventually point the
+      backup at the wrong disk.
+    quiet: true
+  loop: "{{ backup_repositories | selectattr('mountpoint', 'defined') | list }}"
+  loop_control:
+    label: "{{ item.name }}"
+
+- name: Validate no orphaned device keys
+  ansible.builtin.assert:
+    that:
+      - item.mountpoint is defined
+    fail_msg: >-
+      Repository '{{ item.name }}' declares 'uuid' but no 'mountpoint'.
+      A removable device is recognised by its mountpoint; without one the
+      repository is treated as a plain path and the disk is never mounted.
+    quiet: true
+  loop: "{{ backup_repositories | selectattr('uuid', 'defined') | list }}"
+  loop_control:
+    label: "{{ item.name }}"
+
+- name: Validate job names are unique
+  ansible.builtin.assert:
+    that:
+      - backup_jobs | map(attribute='name') | list | length
+        == backup_jobs | map(attribute='name') | unique | list | length
+    fail_msg: "backup_jobs contains duplicate names"
+    success_msg: "Job names are unique"
+
+- name: Validate job definitions
+  ansible.builtin.assert:
+    that:
+      - item.name is defined
+      - item.name | length > 0
+      - item.name is match('^[A-Za-z0-9_.-]+$')
+      - item.repositories is defined
+      - item.repositories | length > 0
+      - (item.paths | default([]) | length > 0)
+        or (item.sqlite | default([]) | length > 0)
+        or (item.postgres | default([]) | length > 0)
+        or (item.command | default('') | length > 0)
+    fail_msg: |
+      Job '{{ item.name | default('<unnamed>') }}' is invalid.
+      Required: name (systemd instance-safe characters only), a non-empty
+      repositories list, and at least one source among paths, sqlite,
+      postgres, command.
+    quiet: true
+  loop: "{{ backup_jobs }}"
+  loop_control:
+    label: "{{ item.name | default('<unnamed>') }}"
+
+- name: Validate job repository references resolve
+  ansible.builtin.assert:
+    that:
+      - item.1 in (backup_repositories | map(attribute='name') | list)
+    fail_msg: |
+      Job '{{ item.0.name }}' references unknown repository '{{ item.1 }}'.
+      Known repositories: {{ backup_repositories | map(attribute='name') | join(', ') | default('none', true) }}
+    quiet: true
+  loop: "{{ backup_jobs | subelements('repositories') }}"
+  loop_control:
+    label: "{{ item.0.name }} -> {{ item.1 }}"
+
+- name: Validate repository notification configuration
+  ansible.builtin.assert:
+    that:
+      - item.notify.topic is defined
+      - item.notify.topic | length > 0
+    fail_msg: >-
+      Repository '{{ item.name }}' declares `notify` but no `topic`.
+      A notification channel needs at least a topic to send to.
+    quiet: true
+  loop: "{{ backup_repositories | selectattr('notify', 'defined') | list }}"
+  loop_control:
+    label: "{{ item.name }}"