From ceca06ff2eaf1ee22a8ec9d9c94dafe6f22b8481 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20D=C3=A9siles?= <1536672+cdesiles@users.noreply.github.com> Date: Tue, 1 Sep 2026 22:20:27 +0200 Subject: [PATCH] feat(backup): add restic backup role with encrypted incremental snapshots Restic-backed backups using a many-to-many repository/job model. - repository = destination: restic repository URL verbatim (s3, local, removable device), backend credentials via env passthrough, retention policy, and an optional per-repository ntfy notify channel - job = source set: paths, sqlite and postgres dumps (consistent .backup / pg_dump instead of torn live-file copies), pushed to one or more repos - systemd timer per job, per-repo + per-job flock serialisation, restic check with read_data_subset for rotating integrity verification Service roles declare what to back up (ntfy_backup_*, uptime_kuma_backup_*); the inventory declares where. --- examples/inventory/host_vars.example.yml | 68 +++ roles/backup/README.md | 273 +++++++++++ roles/backup/defaults/main.yml | 154 ++++++ roles/backup/handlers/main.yml | 4 + roles/backup/meta/main.yml | 2 + roles/backup/tasks/job.yml | 36 ++ roles/backup/tasks/main.yml | 147 ++++++ roles/backup/tasks/repository.yml | 53 +++ roles/backup/tasks/validate.yml | 146 ++++++ roles/backup/templates/backup.sh.j2 | 449 ++++++++++++++++++ roles/backup/templates/backup@.service.j2 | 21 + roles/backup/templates/backup@.timer.j2 | 13 + roles/backup/templates/job.conf.j2 | 45 ++ roles/backup/templates/repo.env.j2 | 64 +++ .../templates/timer-schedule-override.conf.j2 | 6 + roles/backup/vars/archlinux.yml | 15 + roles/backup/vars/debian.yml | 15 + roles/backup/vars/main.yml | 51 ++ roles/ntfy/defaults/main.yml | 16 + roles/uptime_kuma/defaults/main.yml | 13 + 20 files changed, 1591 insertions(+) create mode 100644 roles/backup/README.md create mode 100644 roles/backup/defaults/main.yml create mode 100644 roles/backup/handlers/main.yml create mode 100644 roles/backup/meta/main.yml create mode 100644 roles/backup/tasks/job.yml create mode 100644 roles/backup/tasks/main.yml create mode 100644 roles/backup/tasks/repository.yml create mode 100644 roles/backup/tasks/validate.yml create mode 100644 roles/backup/templates/backup.sh.j2 create mode 100644 roles/backup/templates/backup@.service.j2 create mode 100644 roles/backup/templates/backup@.timer.j2 create mode 100644 roles/backup/templates/job.conf.j2 create mode 100644 roles/backup/templates/repo.env.j2 create mode 100644 roles/backup/templates/timer-schedule-override.conf.j2 create mode 100644 roles/backup/vars/archlinux.yml create mode 100644 roles/backup/vars/debian.yml create mode 100644 roles/backup/vars/main.yml diff --git a/examples/inventory/host_vars.example.yml b/examples/inventory/host_vars.example.yml index 70659c2..e12fa22 100644 --- a/examples/inventory/host_vars.example.yml +++ b/examples/inventory/host_vars.example.yml @@ -177,3 +177,71 @@ valkey_acl_users: # -------------------- immich_postgres_password: "{{ vault_immich_postgres_password }}" immich_valkey_password: "{{ vault_immich_valkey_password }}" + +# Backup configuration +# -------------------- +# restic: encrypted, deduplicated, incremental. See roles/backup/README.md. + +# Destinations. `repository` is RESTIC_REPOSITORY verbatim. Retention lives +# here: a large local disk can afford more history than metered object storage. +# Each destination can carry its own `notify` channel for fine-tuned alerts. +backup_repositories: + - name: offsite + # Two hosts must never share a repository - keep the paths distinct. + repository: "s3:https://s3.fr1.next.ink/lisez-next/{{ inventory_hostname }}" + password: "{{ vault_backup_offsite_password }}" + env: + AWS_ACCESS_KEY_ID: "{{ vault_backup_s3_access_key }}" + AWS_SECRET_ACCESS_KEY: "{{ vault_backup_s3_secret_key }}" + AWS_DEFAULT_REGION: fr1 + retention: + keep_weekly: 4 + # Re-hash 5% of the pack files on each run, so the whole repository gets + # verified over time without a full read every week. + read_data_subset: "5%" + notify: + server: https://ntfy.example.org + topic: sys-alerts + token: "{{ vault_backup_ntfy_token }}" + + # Removable disk. Skipped without error when it is not plugged in - alert at + # low priority rather than urgent, so a disk in a drawer is a reminder, not + # an alarm. + # - name: usb + # repository: /mnt/backup-usb/restic + # uuid: 1234-5678-90ab # blkid; never /dev/sdX, it renumbers + # mountpoint: /mnt/backup-usb + # fstype: ext4 + # password: "{{ vault_backup_usb_password }}" + # retention: + # keep_weekly: 12 + # notify: + # topic: backups-reminders + + # Always-present filesystem, e.g. a ZFS dataset. + # - name: nas + # repository: /mnt/andromeda/backups/restic + # password: "{{ vault_backup_nas_password }}" + # retention: + # keep_daily: 7 + # keep_weekly: 4 + +# Sources. Dumps are produced once and pushed to every listed repository. +backup_jobs: + # WHAT gets backed up is declared by the service role (ntfy_backup_paths / + # ntfy_backup_exclude / ntfy_backup_sqlite in roles/ntfy/defaults/main.yml); + # only WHERE (repositories, retention, schedule) is decided here. The ntfy + # role knows its storage layout - cache.db is ephemeral and skipped, user.db + # is dumped with sqlite3 .backup rather than copied (torn -wal risk). + - name: ntfy + repositories: [offsite] + paths: "{{ ntfy_backup_paths }}" + exclude: "{{ ntfy_backup_exclude }}" + sqlite: "{{ ntfy_backup_sqlite }}" + + # Databases only, no files. pg_dumpall --globals-only is always included. + # - name: databases + # repositories: [offsite, nas] + # postgres: all + # command: | + # valkey-cli --rdb "$DUMPS/valkey.rdb" diff --git a/roles/backup/README.md b/roles/backup/README.md new file mode 100644 index 0000000..60ef427 --- /dev/null +++ b/roles/backup/README.md @@ -0,0 +1,273 @@ +# backup + +Encrypted, deduplicated, incremental backups with [restic](https://restic.readthedocs.io/). + +Variables: [defaults/main.yml](defaults/main.yml) + +## Model + +Two independent lists, related many-to-many. + +- A **repository** is a destination: a restic repository URL, credentials, retention policy. +- A **job** is a source set: paths and databases, plus the repositories it pushes to. + +One job feeds several destinations from a single set of dumps. That is what makes 3-2-1 expressible without dumping the same database twice. + +`repository` is `RESTIC_REPOSITORY` verbatim, so any backend restic supports works: + +| repository URL | destination | +| --- | --- | +| `s3:https://host/bucket/prefix` | object storage | +| `/local/path` | filesystem directory | +| `sftp:user@host:/path` | remote server over SSH | +| `rest:https://host:8000/` | rest-server | + +Credentials for a backend are passed through the `env` mapping, exported as written. A repository that also declares `mountpoint` is a removable device, mounted on demand and unmounted after the run. + +### What goes where + +The **what** of a backup (which files, which databases, how to dump them consistently) is owned by the service role, not this one. The ntfy role knows its storage layout — that `user.db` is durable but `cache.db` is ephemeral, and that `user.db` must be dumped with `sqlite3 .backup` rather than copied — so it publishes a manifest in its own defaults: + +```yaml +# roles/ntfy/defaults/main.yml +ntfy_backup_paths: ["{{ ntfy_data_dir | dirname }}"] +ntfy_backup_exclude: ["{{ ntfy_cache_dir }}/cache.db*", "{{ ntfy_data_dir }}/user.db*"] +ntfy_backup_sqlite: ["{{ ntfy_data_dir }}/user.db"] +``` + +The **where** (repositories, retention, schedule) is operator policy, decided in the inventory: + +```yaml +backup_jobs: + - name: ntfy + repositories: [offsite] + paths: "{{ ntfy_backup_paths }}" + exclude: "{{ ntfy_backup_exclude }}" + sqlite: "{{ ntfy_backup_sqlite }}" +``` + +This keeps the storage knowledge in one place (the service role), so a change to ntfy's layout cannot silently drift from its backup. The inventory stays a single, auditable list of jobs and policy. There is no role-to-role dependency — only a naming convention: a service that wants backing up declares `_backup_paths` / `_exclude` / `_sqlite` / `_postgres` in its defaults, and the inventory references them. Role defaults load at play-compile time, so a targeted `--tags backup` still sees them. + +## Why restic and not borg + +borg 1.x has no S3 backend. The S3 connector lives in [borgstore](https://github.com/borgbackup/borgstore), used only by the Borg 2 beta, whose own README states there are no data migration tools while the format is unstable. Reaching S3 with borg means keeping a local repository and mirroring it with rclone — two stages, double the disk, and a `sync` that can propagate a local corruption to the only offsite copy. + +restic reaches all three destinations natively. All operations here stay behind `repo_backup`, `repo_forget` and `repo_check` in `templates/backup.sh.j2`, so adding a second engine later is a contained change. + +SFTP and rest-server are supported by the `repository` URL with no role changes — they are simply not used in this inventory yet. + +## Configuration + +```yaml +# inventory/host_vars//vars.yml +backup_repositories: + - name: offsite + repository: "s3:https://s3.fr1.next.ink/lisez-next/{{ inventory_hostname }}" + password: "{{ vault_backup_offsite_password }}" + env: + AWS_ACCESS_KEY_ID: "{{ vault_backup_s3_access_key }}" + AWS_SECRET_ACCESS_KEY: "{{ vault_backup_s3_secret_key }}" + AWS_DEFAULT_REGION: fr1 + retention: + keep_weekly: 4 + read_data_subset: "5%" + notify: + server: https://ntfy.jokester.fr + topic: backups + token: "{{ vault_backup_ntfy_token }}" + + # Removable disk, skipped without error when not plugged in. + # - name: usb + # repository: /mnt/backup-usb/restic + # uuid: 1234-5678-90ab + # mountpoint: /mnt/backup-usb + # fstype: ext4 + # password: "{{ vault_backup_usb_password }}" + # retention: + # keep_weekly: 12 + # notify: + # topic: backups-reminders # low-priority reminder, not an alarm + +backup_jobs: + - name: ntfy + repositories: [offsite] + paths: "{{ ntfy_backup_paths }}" + exclude: "{{ ntfy_backup_exclude }}" + sqlite: "{{ ntfy_backup_sqlite }}" +``` + +Generate the repository password and **store it outside this repository and outside the machine being backed up**: + +```bash +openssl rand -base64 32 +ansible-vault encrypt_string '' --name 'vault_backup_offsite_password' +``` + +Losing it means losing every snapshot. There is no recovery path, and a password kept only in a vault on the host being backed up protects nothing after that host dies. + +Two hosts must never share a repository — keep their `repository` paths distinct. A restic repository is not designed for concurrent writers from different machines. + +## Sources + +| key | behaviour | +| --- | --- | +| `paths` / `exclude` | files, straight into the snapshot | +| `sqlite: [path…]` | dumped with `sqlite3 ".backup"` | +| `postgres: [db…]` or `all` | `pg_dump -Fc` per database, plus `pg_dumpall --globals-only` | +| `command:` | escape hatch, `$DUMPS` points at the staging dir | + +A job needs at least one source. `paths` alone is fine, as is `postgres` alone. + +### Databases are dumped, not copied + +Copying a database file while the service is writing produces a torn copy. SQLite keeps recent writes in a `-wal` sidecar, PostgreSQL in its own WAL. The result restores cleanly and is silently inconsistent — the worst possible failure mode, because you only discover it when you need it. + +So the live file is excluded from `paths` and a consistent dump is written into the staging directory instead: + +``` +//sqlite/opt/podman/ntfy/data/user.db # mirrors the source path +//postgres/.dump +//postgres/globals.sql +``` + +The staging directory is wiped before and after every run, so plaintext dumps never linger. + +`globals.sql` carries roles, grants and tablespaces. Without it a restored dump has no owners to grant privileges to. + +## Operations + +```bash +# Run a job now and watch it +sudo systemctl start backup@ntfy.service +sudo journalctl -u backup@ntfy.service -f + +# Timers +systemctl list-timers 'backup@*' + +# Any interactive restic command: source the repository environment first +sudo -i +set -a; . /etc/backup/repos/offsite.env; set +a + +restic snapshots +restic snapshots --tag ntfy +restic stats +restic check --read-data +``` + +## Restore + +```bash +sudo -i +set -a; . /etc/backup/repos/offsite.env; set +a + +restic snapshots --tag ntfy +restic restore --target /tmp/restore # inspect first +restic restore latest --tag ntfy --target / # restore in place +``` + +`--target /` writes absolute paths back where they came from. Restore to a scratch directory first unless you are certain. + +Browsing without extracting: + +```bash +restic mount /mnt/restic # FUSE, read-only +``` + +### From a device repository + +The disk must be attached and mounted first — the automatic mount only happens during a backup run: + +```bash +mount /dev/disk/by-uuid/ /mnt/backup-usb +set -a; . /etc/backup/repos/usb.env; set +a +restic snapshots +``` + +### On a machine that never ran this role + +Nothing host-specific is needed. Install restic and point it at the repository: + +```bash +export RESTIC_REPOSITORY='s3:https://s3.fr1.next.ink/lisez-next/omega' +export RESTIC_PASSWORD='' +export AWS_ACCESS_KEY_ID='...' +export AWS_SECRET_ACCESS_KEY='...' +export AWS_DEFAULT_REGION='fr1' +restic snapshots +``` + +### Restoring ntfy + +```bash +sudo systemctl --user -M jokester@ stop ntfy.service + +sudo -i +set -a; . /etc/backup/repos/offsite.env; set +a +restic restore latest --tag ntfy --target /tmp/ntfy-restore + +# Config and attachments +rsync -a /tmp/ntfy-restore/opt/podman/ntfy/ /opt/podman/ntfy/ + +# The consistent user.db copy lives under the staging path, not in place +install -o jokester -g jokester -m 0644 \ + /tmp/ntfy-restore/var/lib/backup/dumps/ntfy/sqlite/opt/podman/ntfy/data/user.db \ + /opt/podman/ntfy/data/user.db +rm -f /opt/podman/ntfy/data/user.db-wal /opt/podman/ntfy/data/user.db-shm + +sudo systemctl --user -M jokester@ start ntfy.service +``` + +Deleting the stale `-wal`/`-shm` sidecars matters: a fresh `user.db` next to an old WAL is a corrupt pair. + +### Restoring PostgreSQL + +```bash +sudo -u postgres psql -f /postgres/globals.sql +sudo -u postgres createdb immich +sudo -u postgres pg_restore -d immich /postgres/immich.dump +``` + +## Design notes + +### One job, several destinations + +Dumps are produced once and pushed to each repository in turn. A failure on one destination is recorded and the remaining ones are still attempted; the unit exits non-zero if any failed, and the notification lists which. + +### A missing device is a skip, not a failure + +If `/dev/disk/by-uuid/` is absent the destination is skipped and the run still succeeds. Alerting every week about a disk that lives in a drawer trains you to ignore the alerts that matter. + +If *every* destination of a job is skipped, that is reported — silence there would be indistinguishable from a successful backup. + +### Locking + +- **Per repository** (`/run/backup/repo-.lock`): two jobs sharing a destination queue; jobs on different destinations run in parallel. +- **Per job** (`/run/backup/job-.lock`, non-blocking): a second run of the same job exits immediately rather than fighting over the staging directory. systemd already prevents this for timer-driven runs; the lock covers manual invocation. + +### Retention + +Retention lives on the repository, because a large local disk should keep more history than metered object storage. `restic forget` is scoped with `--tag `, so a weekly job cannot prune the snapshots of a daily one sharing the repository. `--prune` runs in the same invocation, otherwise unreferenced data is never reclaimed. + +Every snapshot is a logically complete view; "incremental" describes only how it is stored. `keep_weekly: 4` gives four complete, independently restorable backups. + +### Integrity checks + +`restic check` runs after every backup: structure, indexes and metadata. Setting `read_data_subset` (e.g. `"5%"`) additionally re-hashes that fraction of the pack files, so the whole repository gets verified over time without paying for a full read on every run. + +### Excludes that are easy to get wrong + +The role always excludes restic's own cache, and the staging directories of *other* jobs — otherwise a job backing up `/var` would sweep up both. + +### Failure notification + +Each repository can carry a `notify` channel: `{ topic, server, token, notify_success }`. A repository with `notify.topic` alerts on that destination's own result — an offsite failure is `urgent`, a missing USB disk is a low-priority `warning`, and a destination without `notify` is silent. Success alerts are opt-in per channel (`notify_success`), off by default because routine chatter trains you to ignore the failure alerts. + +A job-level failure (a database dump, before any destination runs) alerts the first repository's channel, so order a job's repositories with the primary channel first. + +Every configured channel gets a test notification during the play. ntfy runs with `auth-default-access: deny-all`, so a token lacking write access to the topic would otherwise fail silently at 03:00 — this surfaces it at deploy time instead. + +### What this does not do + +- **No automated restore test.** An untested backup is a hypothesis. Restore one by hand, on a schedule. +- **No append-only protection.** The host holds credentials that can delete its own snapshots, so an attacker with root can destroy the backups too. Mitigate with object versioning or an object-lock policy on the bucket, which is outside this role. +- **No metrics.** Failures are visible through `systemctl --failed`, the journal, and ntfy. diff --git a/roles/backup/defaults/main.yml b/roles/backup/defaults/main.yml new file mode 100644 index 0000000..07b6ec6 --- /dev/null +++ b/roles/backup/defaults/main.yml @@ -0,0 +1,154 @@ +--- +# --------------------------------------------------------------------------- +# Layout +# --------------------------------------------------------------------------- +backup_config_dir: /etc/backup +backup_script_path: /usr/local/bin/backup + +# Scratch space for database dumps. Wiped before and after every run so +# plaintext dumps never linger on disk. +backup_dumps_dir: /var/lib/backup/dumps + +# restic keeps a local metadata cache. Pinned to a predictable path so it can +# be excluded from snapshots rather than silently backing itself up. +backup_cache_dir: /var/cache/restic + +# --------------------------------------------------------------------------- +# Schedule (systemd OnCalendar syntax) - per-job `schedule` overrides this +# --------------------------------------------------------------------------- +backup_schedule: "Sun *-*-* 03:00:00" +backup_randomized_delay: 30m + +# Hard cap on a single job run, all destinations included. +backup_timeout: 4h + +# --------------------------------------------------------------------------- +# Repositories (destinations) +# --------------------------------------------------------------------------- +# A repository is where bytes land: a restic repository URL, its passphrase, +# and a retention policy. Jobs reference repositories by name, many-to-many. +# +# The configuration maps 1:1 onto restic's own concepts. `repository` is +# RESTIC_REPOSITORY verbatim, `env` is exported into the environment, `password` +# is RESTIC_PASSWORD. There is no translation layer to drift from what restic +# actually runs. +# +# Common keys: +# name (required) identifier referenced by jobs +# repository (required) RESTIC_REPOSITORY, verbatim. Any backend restic +# supports works: +# s3:https://s3.fr1.next.ink/lisez-next/omega +# sftp:backup@host:/srv/restic-repo +# rest:https://host:8000/ +# /local/path (absolute = local) +# password (required) repository encryption passphrase, min 12 chars. +# This is the ONLY secret protecting the data. Losing +# it means losing every snapshot - there is no recovery. +# env (optional) mapping of extra environment variables, exported as +# written. Backend credentials go here: +# AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY for s3, +# B2_ACCOUNT_ID / B2_ACCOUNT_KEY for Backblaze, ... +# Values must be strings. One repository must not rely +# on ambient credentials (AWS_PROFILE, default chain) +# when another sets explicit keys - env vars are +# per-repository and cleared between runs. +# retention (required) at least one keep_* key. Maps 1:1 onto restic +# `forget`: keep_last, keep_hourly, keep_daily, +# keep_weekly, keep_monthly, keep_yearly. +# read_data_subset (optional) e.g. "5%" or "1/12". Makes `restic check` +# also re-hash that fraction of the pack files, so the +# whole repository gets verified over time without +# paying for a full read on every run. +# options (optional) extra restic flags, e.g. ["-o", "s3.list-objects-v1=true"] +# +# notify (optional) ntfy notification channel for THIS repository. Each +# destination can alert differently: offsite failure is +# urgent, a missing USB disk is a low-priority reminder. +# Keys: +# topic (required) ntfy topic +# server (optional) default https://ntfy.sh +# token (optional) bearer token for restricted topics +# notify_success (optional, default false) also alert +# on success - off by default because +# routine chatter trains you to ignore +# the failure alerts +# Absent = this repository does not send notifications. +# A job-level failure (database dump) alerts the first +# repository's channel. +# +# Removable-device repositories add: +# uuid (required) filesystem UUID, resolved via /dev/disk/by-uuid. +# Never use /dev/sdX - it renumbers between boots. +# mountpoint (required) mounted on demand, unmounted after the run +# fstype (optional) passed to mount(8) when set +# +# A device repository is any repository that declares `mountpoint`. Its +# `repository` path must live below the mountpoint. A device that is not +# plugged in is skipped, not failed - alerting every week about a disk that +# lives in a drawer trains you to ignore alerts. +# +# Example: +# backup_repositories: +# - name: offsite +# repository: "s3:https://s3.fr1.next.ink/lisez-next/{{ inventory_hostname }}" +# password: "{{ vault_backup_offsite_password }}" +# env: +# AWS_ACCESS_KEY_ID: "{{ vault_backup_s3_access_key }}" +# AWS_SECRET_ACCESS_KEY: "{{ vault_backup_s3_secret_key }}" +# AWS_DEFAULT_REGION: fr1 +# retention: +# keep_weekly: 4 +# read_data_subset: "5%" +# notify: +# server: https://ntfy.jokester.fr +# topic: sys-updates +# token: "{{ sys_autoupdate_ntfy_token }}" +backup_repositories: [] + +# --------------------------------------------------------------------------- +# Jobs (sources) +# --------------------------------------------------------------------------- +# Each job gets its own systemd timer and its own restic tag. Retention is +# applied per tag, so jobs never delete each other's snapshots. +# +# Keys: +# name (required) identifier, used for the unit name and restic tag +# repositories (required) list of repository names to push to. Dumps are +# produced once and pushed to each in turn; one +# failing destination does not abort the others. +# schedule (optional) OnCalendar override for this job +# paths (optional) files and directories to snapshot +# exclude (optional) restic --exclude patterns +# sqlite (optional) SQLite files. Dumped with the online backup API +# into the staging dir - see below. +# postgres (optional) list of database names, or the string "all". +# Dumped with pg_dump -Fc, plus pg_dumpall +# --globals-only for roles and grants. +# command (optional) escape hatch. Shell snippet run with $DUMPS +# pointing at this job's staging dir. +# +# A job needs at least one of paths / sqlite / postgres / command. +# +# Why databases are dumped instead of copied: a live SQLite file keeps recent +# writes in a -wal sidecar and PostgreSQL keeps them in its own WAL. Copying +# the file gives a torn snapshot that restores cleanly and is silently +# inconsistent. Dumps land under the staging dir mirroring their source path: +# sqlite /opt/podman/ntfy/data/user.db +# -> //sqlite/opt/podman/ntfy/data/user.db +# postgres -> //postgres/.dump and postgres/globals.sql +# +# Example: +# backup_jobs: +# - name: ntfy +# repositories: [offsite] +# paths: +# - "{{ podman_projects_dir | default('/opt/podman') }}/ntfy" +# exclude: +# # Ephemeral message cache: churns every run for no restore value. +# # attachments/ lives under cache/, hence the narrow glob. +# - "{{ podman_projects_dir | default('/opt/podman') }}/ntfy/cache/cache.db*" +# # Excluded from the live tree because it is dumped consistently below. +# - "{{ podman_projects_dir | default('/opt/podman') }}/ntfy/data/user.db*" +# sqlite: +# - "{{ podman_projects_dir | default('/opt/podman') }}/ntfy/data/user.db" +backup_jobs: [] diff --git a/roles/backup/handlers/main.yml b/roles/backup/handlers/main.yml new file mode 100644 index 0000000..c28484f --- /dev/null +++ b/roles/backup/handlers/main.yml @@ -0,0 +1,4 @@ +--- +- name: Reload systemd + ansible.builtin.systemd: + daemon_reload: true diff --git a/roles/backup/meta/main.yml b/roles/backup/meta/main.yml new file mode 100644 index 0000000..23d65c7 --- /dev/null +++ b/roles/backup/meta/main.yml @@ -0,0 +1,2 @@ +--- +dependencies: [] diff --git a/roles/backup/tasks/job.yml b/roles/backup/tasks/job.yml new file mode 100644 index 0000000..ed8f261 --- /dev/null +++ b/roles/backup/tasks/job.yml @@ -0,0 +1,36 @@ +--- +# Included per job with `job` set to the job dict. + +- name: Deploy job definition for {{ job.name }} + ansible.builtin.template: + src: job.conf.j2 + dest: "{{ backup_config_dir }}/jobs/{{ job.name }}.conf" + owner: root + group: root + mode: "0600" + +- name: Create timer override directory for {{ job.name }} + ansible.builtin.file: + path: "/etc/systemd/system/backup@{{ job.name }}.timer.d" + state: directory + owner: root + group: root + mode: "0755" + when: job.schedule is defined + +- name: Deploy schedule override for {{ job.name }} + ansible.builtin.template: + src: timer-schedule-override.conf.j2 + dest: "/etc/systemd/system/backup@{{ job.name }}.timer.d/schedule.conf" + owner: root + group: root + mode: "0644" + when: job.schedule is defined + notify: Reload systemd + +- name: Remove stale schedule override for {{ job.name }} + ansible.builtin.file: + path: "/etc/systemd/system/backup@{{ job.name }}.timer.d" + state: absent + when: job.schedule is not defined + notify: Reload systemd diff --git a/roles/backup/tasks/main.yml b/roles/backup/tasks/main.yml new file mode 100644 index 0000000..44a6c88 --- /dev/null +++ b/roles/backup/tasks/main.yml @@ -0,0 +1,147 @@ +--- +- name: Validate backup configuration + ansible.builtin.import_tasks: validate.yml + +- name: Load OS-specific variables + ansible.builtin.include_vars: "{{ item }}" + with_first_found: + - "{{ ansible_facts['os_family'] }}.yml" + - debian.yml + +- name: Install backup packages + ansible.builtin.package: + name: "{{ backup_packages }}" + state: present + +- name: Install sqlite tooling + ansible.builtin.package: + name: "{{ backup_sqlite_packages }}" + state: present + when: backup_jobs | selectattr('sqlite', 'defined') | list | length > 0 + +- name: Install PostgreSQL client tooling + ansible.builtin.package: + name: "{{ backup_postgres_packages }}" + state: present + when: backup_jobs | selectattr('postgres', 'defined') | list | length > 0 + +- name: Create backup configuration directories + ansible.builtin.file: + path: "{{ item }}" + state: directory + owner: root + group: root + mode: "0700" + loop: + - "{{ backup_config_dir }}" + - "{{ backup_config_dir }}/repos" + - "{{ backup_config_dir }}/jobs" + - "{{ backup_dumps_dir }}" + - "{{ backup_cache_dir }}" + +- name: Deploy backup script + ansible.builtin.template: + src: backup.sh.j2 + dest: "{{ backup_script_path }}" + owner: root + group: root + mode: "0700" + no_log: true + +- name: Configure repositories + ansible.builtin.include_tasks: repository.yml + loop: "{{ backup_repositories }}" + loop_control: + loop_var: repo + label: "{{ repo.name }}" + +- name: Configure jobs + ansible.builtin.include_tasks: job.yml + loop: "{{ backup_jobs }}" + loop_control: + loop_var: job + label: "{{ job.name }}" + +- name: Find existing repository configurations + ansible.builtin.find: + paths: "{{ backup_config_dir }}/repos" + patterns: "*.env" + register: backup_existing_repos + +- name: Remove orphaned repository configurations + ansible.builtin.file: + path: "{{ item.path }}" + state: absent + loop: "{{ backup_existing_repos.files }}" + loop_control: + label: "{{ item.path | basename }}" + when: (item.path | basename | splitext | first) + not in (backup_repositories | map(attribute='name') | list) + +- name: Find existing job configurations + ansible.builtin.find: + paths: "{{ backup_config_dir }}/jobs" + patterns: "*.conf" + register: backup_existing_jobs + +- name: Remove orphaned job configurations + ansible.builtin.file: + path: "{{ item.path }}" + state: absent + loop: "{{ backup_existing_jobs.files }}" + loop_control: + label: "{{ item.path | basename }}" + when: (item.path | basename | splitext | first) + not in (backup_jobs | map(attribute='name') | list) + +- name: Deploy systemd service template + ansible.builtin.template: + src: backup@.service.j2 + dest: /etc/systemd/system/backup@.service + owner: root + group: root + mode: "0644" + notify: Reload systemd + +- name: Deploy systemd timer template + ansible.builtin.template: + src: backup@.timer.j2 + dest: /etc/systemd/system/backup@.timer + owner: root + group: root + mode: "0644" + notify: Reload systemd + +- name: Flush handlers before enabling timers + ansible.builtin.meta: flush_handlers + +- name: Enable and start backup timers + ansible.builtin.systemd: + name: "backup@{{ item.name }}.timer" + enabled: true + state: started + loop: "{{ backup_jobs }}" + loop_control: + label: "{{ item.name }}" + +# Failure alerts are the only thing standing between a broken timer and a +# backup you believe exists. ntfy runs with auth-default-access deny-all, so a +# token without write access to the topic would fail silently at 03:00. Prove +# each configured channel works now instead. +- name: Send a test notification + ansible.builtin.uri: + url: "{{ item.notify.server | default('https://ntfy.sh') }}/{{ item.notify.topic }}" + method: POST + body: "Backup role deployed on {{ inventory_hostname }}. Repository '{{ item.name }}' alerts are configured and reachable." + headers: >- + {{ + {'Title': inventory_hostname ~ ': Backup notifications configured', + 'Tags': 'floppy_disk', + 'Priority': 'low'} + | combine({'Authorization': 'Bearer ' ~ item.notify.token} if item.notify.token is defined else {}) + }} + status_code: 200 + loop: "{{ backup_repositories | selectattr('notify', 'defined') | list }}" + loop_control: + label: "{{ item.name }}" + changed_when: false diff --git a/roles/backup/tasks/repository.yml b/roles/backup/tasks/repository.yml new file mode 100644 index 0000000..7f70b09 --- /dev/null +++ b/roles/backup/tasks/repository.yml @@ -0,0 +1,53 @@ +--- +# Included per repository with `repo` set to the repository dict. + +- name: Deploy repository environment for {{ repo.name }} + ansible.builtin.template: + src: repo.env.j2 + dest: "{{ backup_config_dir }}/repos/{{ repo.name }}.env" + owner: root + group: root + mode: "0600" + no_log: true + +- name: Create local repository directory for {{ repo.name }} + ansible.builtin.file: + path: "{{ repo.repository }}" + state: directory + owner: root + group: root + mode: "0700" + when: backup_repo_is_local | bool + +- name: Create device mountpoint for {{ repo.name }} + ansible.builtin.file: + path: "{{ repo.mountpoint }}" + state: directory + owner: root + group: root + mode: "0700" + when: backup_repo_is_device | bool + +# Device repositories are initialised lazily by the backup script the first +# time the disk is actually attached. Everything else is initialised here so +# that bad credentials or an unreachable endpoint fail at deploy time rather +# than at 03:00 on a Sunday. +- name: Check initialisation state of repository {{ repo.name }} + ansible.builtin.command: + cmd: restic cat config + environment: "{{ backup_repo_environment }}" + register: backup_repo_state + failed_when: false + changed_when: false + no_log: true + when: not backup_repo_is_device | bool + +- name: Initialise repository {{ repo.name }} + ansible.builtin.command: + cmd: restic init + environment: "{{ backup_repo_environment }}" + when: + - not backup_repo_is_device | bool + - backup_repo_state.rc | default(1) != 0 + changed_when: true + no_log: true diff --git a/roles/backup/tasks/validate.yml b/roles/backup/tasks/validate.yml new file mode 100644 index 0000000..5ccd053 --- /dev/null +++ b/roles/backup/tasks/validate.yml @@ -0,0 +1,146 @@ +--- +- name: Validate repository names are unique + ansible.builtin.assert: + that: + - backup_repositories | map(attribute='name') | list | length + == backup_repositories | map(attribute='name') | unique | list | length + fail_msg: "backup_repositories contains duplicate names" + success_msg: "Repository names are unique" + +- name: Validate repository common keys + ansible.builtin.assert: + that: + - item.name is defined + - item.name | length > 0 + - item.repository is defined + - item.repository | length > 0 + - item.password is defined + - item.password | length >= 12 + - item.retention is defined + - item.retention | length > 0 + fail_msg: | + Repository '{{ item.name | default('') }}' is invalid. + Required: name, repository (a restic repository URL, verbatim), + password (min 12 chars), retention (at least one keep_* key). + A repository with an empty retention policy would make `restic forget` + fail at 03:00 rather than here. + See roles/backup/defaults/main.yml for configuration instructions. + quiet: true + loop: "{{ backup_repositories }}" + loop_control: + label: "{{ item.name | default('') }}" + no_log: true + +- name: Validate retention keys are recognised + ansible.builtin.assert: + that: + - item.retention.keys() | list | difference(backup_retention_keys) | length == 0 + fail_msg: >- + Repository '{{ item.name }}' has unknown retention key(s): + {{ item.retention.keys() | list | difference(backup_retention_keys) | join(', ') }}. + Allowed: {{ backup_retention_keys | join(', ') }}. + An unrecognised key would be silently dropped and the repository would + grow without bound. + quiet: true + loop: "{{ backup_repositories }}" + loop_control: + label: "{{ item.name }}" + +- name: Validate repository environment is a mapping of strings + ansible.builtin.assert: + that: + - item.env.values() | reject('string') | list | length == 0 + fail_msg: >- + Repository '{{ item.name }}' has a non-string env value. + Environment values are exported verbatim and must be strings. + quiet: true + loop: "{{ backup_repositories | selectattr('env', 'defined') | list }}" + loop_control: + label: "{{ item.name }}" + +- name: Validate device repositories + ansible.builtin.assert: + that: + - item.uuid is defined + - item.uuid | length > 0 + - item.mountpoint is defined + - item.mountpoint is match('^/') + - item.repository is match('^' ~ item.mountpoint) + fail_msg: | + Device repository '{{ item.name }}' (declares mountpoint) requires uuid, + mountpoint, and a repository path below the mountpoint. + Identify the disk by filesystem UUID (blkid), never by /dev/sdX - kernel + device names are not stable across boots and would eventually point the + backup at the wrong disk. + quiet: true + loop: "{{ backup_repositories | selectattr('mountpoint', 'defined') | list }}" + loop_control: + label: "{{ item.name }}" + +- name: Validate no orphaned device keys + ansible.builtin.assert: + that: + - item.mountpoint is defined + fail_msg: >- + Repository '{{ item.name }}' declares 'uuid' but no 'mountpoint'. + A removable device is recognised by its mountpoint; without one the + repository is treated as a plain path and the disk is never mounted. + quiet: true + loop: "{{ backup_repositories | selectattr('uuid', 'defined') | list }}" + loop_control: + label: "{{ item.name }}" + +- name: Validate job names are unique + ansible.builtin.assert: + that: + - backup_jobs | map(attribute='name') | list | length + == backup_jobs | map(attribute='name') | unique | list | length + fail_msg: "backup_jobs contains duplicate names" + success_msg: "Job names are unique" + +- name: Validate job definitions + ansible.builtin.assert: + that: + - item.name is defined + - item.name | length > 0 + - item.name is match('^[A-Za-z0-9_.-]+$') + - item.repositories is defined + - item.repositories | length > 0 + - (item.paths | default([]) | length > 0) + or (item.sqlite | default([]) | length > 0) + or (item.postgres | default([]) | length > 0) + or (item.command | default('') | length > 0) + fail_msg: | + Job '{{ item.name | default('') }}' is invalid. + Required: name (systemd instance-safe characters only), a non-empty + repositories list, and at least one source among paths, sqlite, + postgres, command. + quiet: true + loop: "{{ backup_jobs }}" + loop_control: + label: "{{ item.name | default('') }}" + +- name: Validate job repository references resolve + ansible.builtin.assert: + that: + - item.1 in (backup_repositories | map(attribute='name') | list) + fail_msg: | + Job '{{ item.0.name }}' references unknown repository '{{ item.1 }}'. + Known repositories: {{ backup_repositories | map(attribute='name') | join(', ') | default('none', true) }} + quiet: true + loop: "{{ backup_jobs | subelements('repositories') }}" + loop_control: + label: "{{ item.0.name }} -> {{ item.1 }}" + +- name: Validate repository notification configuration + ansible.builtin.assert: + that: + - item.notify.topic is defined + - item.notify.topic | length > 0 + fail_msg: >- + Repository '{{ item.name }}' declares `notify` but no `topic`. + A notification channel needs at least a topic to send to. + quiet: true + loop: "{{ backup_repositories | selectattr('notify', 'defined') | list }}" + loop_control: + label: "{{ item.name }}" diff --git a/roles/backup/templates/backup.sh.j2 b/roles/backup/templates/backup.sh.j2 new file mode 100644 index 0000000..519544b --- /dev/null +++ b/roles/backup/templates/backup.sh.j2 @@ -0,0 +1,449 @@ +#!/usr/bin/env bash +# +# Managed by Ansible (roles/backup). Do not edit on the host. +# +# Runs one backup job: +# 1. produce consistent database dumps into the job staging dir +# 2. for each repository the job targets: +# prepare (mount a removable device if needed) +# restic backup / forget --prune / check +# release (unmount) +# 3. report - one failing destination does not abort the others +# +# -E propagates the ERR trap into functions, so a failure during the dump +# phase still sends the failure notification. +# +# NOTE ON ERROR HANDLING: run_repository() is invoked as `run_repository X || +# rc=$?`. POSIX shells disable errexit for the whole call tree of a command on +# the left of `||`, so `set -e` does NOT apply inside it. Every fallible +# command below that point therefore carries an explicit `|| return`. Do not +# "simplify" those away. +# +set -Eeuo pipefail + +JOB="${1:-}" +if [[ -z "$JOB" ]]; then + echo "usage: $(basename "$0") " >&2 + exit 2 +fi + +CONFIG_DIR="{{ backup_config_dir }}" +DUMPS_ROOT="{{ backup_dumps_dir }}" +CACHE_DIR="{{ backup_cache_dir }}" +LOCK_DIR="/run/backup" + +HOSTNAME_SHORT="{{ ansible_facts['hostname'] }}" + +# Exit code meaning "destination unavailable, nothing was attempted". +# Distinct from a real failure so an unplugged disk does not raise an alarm. +EXIT_SKIP=75 + +DUMPS="$DUMPS_ROOT/$JOB" + +# Notification channel of the repository currently being processed. Defaults +# here keep the ERR trap safe under `set -u` before any repository is sourced. +NTFY_ENABLED=false +NTFY_SERVER="" +NTFY_TOPIC="" +NTFY_TOKEN="" +NTFY_ON_SUCCESS=false + +log() { + echo "[$(date '+%Y-%m-%d %H:%M:%S')] [$JOB] $*" +} + +notify() { + local title="$1" message="$2" tags="$3" priority="${4:-default}" + + [[ "$NTFY_ENABLED" == "true" ]] || return 0 + [[ -n "$NTFY_TOPIC" ]] || return 0 + + local auth_args=() + if [[ -n "$NTFY_TOKEN" ]]; then + auth_args=(-H "Authorization: Bearer $NTFY_TOKEN") + fi + + curl -s --max-time 15 \ + "${auth_args[@]}" \ + -H "Title: $title" \ + -H "Tags: $tags" \ + -H "Priority: $priority" \ + -d "$message" \ + "$NTFY_SERVER/$NTFY_TOPIC" >/dev/null \ + || log "WARNING: failed to send notification" +} + +# Only the run that created the staging directory may remove it. Without this +# flag, a second invocation bailing out on the job lock would delete the dumps +# of the run that is still using them. +OWNS_DUMPS=false + +cleanup() { + [[ "$OWNS_DUMPS" == "true" ]] || return 0 + # Dumps are plaintext copies of live databases: never leave them behind. + rm -rf -- "$DUMPS" +} + +on_error() { + local line="$1" + log "ERROR: aborted at line $line" + notify \ + "$HOSTNAME_SHORT: Backup FAILED ($JOB)" \ + "Job '$JOB' aborted at line $line before reaching any destination. Check: journalctl -u backup@$JOB.service" \ + "x" \ + "urgent" +} + +trap 'on_error $LINENO' ERR +trap cleanup EXIT + +# --------------------------------------------------------------------------- +# Job configuration +# --------------------------------------------------------------------------- + +JOB_CONF="$CONFIG_DIR/jobs/$JOB.conf" +if [[ ! -f "$JOB_CONF" ]]; then + log "ERROR: no such job, $JOB_CONF is missing" + exit 1 +fi +# shellcheck source=/dev/null +source "$JOB_CONF" + +if [[ -z "${JOB_REPOSITORIES[*]:-}" ]]; then + log "ERROR: job targets no repositories" + exit 1 +fi + +# Job-level failures (a database dump) happen before any repository runs, so +# there is no current destination to source. Alert the first repository's +# channel: order the job's repositories with the primary channel first. +load_job_notify() { + local env_file="$CONFIG_DIR/repos/${JOB_REPOSITORIES[0]}.env" + if [[ -f "$env_file" ]]; then + # shellcheck source=/dev/null + source "$env_file" + fi +} + +# --------------------------------------------------------------------------- +# Dump phase - runs once, feeds every destination +# --------------------------------------------------------------------------- + +dump_sqlite() { + local src dest + for src in ${JOB_SQLITE[@]+"${JOB_SQLITE[@]}"}; do + if [[ ! -f "$src" ]]; then + log "ERROR: sqlite source $src does not exist" + return 1 + fi + # Mirror the source path inside the staging dir so a restore is + # self-describing and two databases with the same basename cannot + # collide. + dest="$DUMPS/sqlite$src" + mkdir -p "$(dirname "$dest")" + log "Dumping sqlite $src" + # .backup uses SQLite's online backup API: a consistent copy with the + # service still running. Copying the file instead would capture a torn + # state, because recent writes live in the -wal sidecar. + sqlite3 "$src" ".backup '$dest'" + done +} + +dump_postgres() { + local -a databases=() + local db + + [[ -n "${JOB_POSTGRES[*]:-}" ]] || return 0 + + mkdir -p "$DUMPS/postgres" + + if [[ "${JOB_POSTGRES[0]}" == "all" ]]; then + log "Enumerating PostgreSQL databases" + mapfile -t databases < <( + runuser -u postgres -- psql -tAc \ + "SELECT datname FROM pg_database WHERE datallowconn AND NOT datistemplate ORDER BY datname" + ) + else + databases=("${JOB_POSTGRES[@]}") + fi + + # Roles, grants and tablespaces live outside any single database. Without + # them a restored dump has no owners to grant privileges to. + log "Dumping PostgreSQL globals" + runuser -u postgres -- pg_dumpall --globals-only > "$DUMPS/postgres/globals.sql" + + for db in ${databases[@]+"${databases[@]}"}; do + log "Dumping PostgreSQL database $db" + # Redirection is performed by root, so the postgres user does not need + # write access to the staging directory. + runuser -u postgres -- pg_dump --format=custom "$db" > "$DUMPS/postgres/$db.dump" + done +} + +stage_dumps() { + rm -rf -- "$DUMPS" + mkdir -p "$DUMPS" + chmod 0700 "$DUMPS" + OWNS_DUMPS=true + + dump_sqlite + dump_postgres + + if [[ -n "${JOB_COMMAND:-}" ]]; then + log "Running job command hook" + # DUMPS is exported so the hook can write its own dumps into it. + DUMPS="$DUMPS" bash -Eeuo pipefail -c "$JOB_COMMAND" + fi +} + +# --------------------------------------------------------------------------- +# Destination handling +# --------------------------------------------------------------------------- + +WE_MOUNTED=false + +dest_prepare() { + WE_MOUNTED=false + + # A repository with a mountpoint is a removable device: mount it if the + # disk is attached. The restic URL itself tells us the rest - an absolute + # path is a local directory, a scheme URL (s3:, sftp:, rest:) needs no + # local preparation at all. + if [[ -n "${REPO_MOUNTPOINT:-}" ]]; then + local dev="/dev/disk/by-uuid/$REPO_DEVICE_UUID" + if [[ ! -e "$dev" ]]; then + log "Device $REPO_DEVICE_UUID is not attached, skipping this destination" + return "$EXIT_SKIP" + fi + if mountpoint -q "$REPO_MOUNTPOINT"; then + log "Device already mounted at $REPO_MOUNTPOINT" + else + mkdir -p "$REPO_MOUNTPOINT" || return 1 + log "Mounting $dev at $REPO_MOUNTPOINT" + local -a mount_args=() + if [[ -n "${REPO_FSTYPE:-}" ]]; then + mount_args+=(-t "$REPO_FSTYPE") + fi + mount "${mount_args[@]+"${mount_args[@]}"}" "$dev" "$REPO_MOUNTPOINT" || return 1 + WE_MOUNTED=true + fi + mkdir -p "$RESTIC_REPOSITORY" || return 1 + return 0 + fi + + if [[ "$RESTIC_REPOSITORY" == /* ]]; then + mkdir -p "$RESTIC_REPOSITORY" || return 1 + fi + return 0 +} + +dest_release() { + [[ "$WE_MOUNTED" == "true" ]] || return 0 + # Flush before unmounting so the disk is safe to physically remove the + # moment the run finishes. + sync + log "Unmounting $REPO_MOUNTPOINT" + umount "$REPO_MOUNTPOINT" || log "WARNING: failed to unmount $REPO_MOUNTPOINT" + WE_MOUNTED=false +} + +# --------------------------------------------------------------------------- +# restic operations +# --------------------------------------------------------------------------- + +repo_backup() { + local -a paths=() excludes=() sibling + + paths=(${JOB_PATHS[@]+"${JOB_PATHS[@]}"}) + + # Include the staging dir only when something was actually dumped into it, + # otherwise restic would record an empty directory on every run. + if find "$DUMPS" -mindepth 1 -print -quit 2>/dev/null | grep -q .; then + paths+=("$DUMPS") + fi + + if [[ -z "${paths[*]:-}" ]]; then + log "ERROR: nothing to back up" + return 1 + fi + + for pattern in ${JOB_EXCLUDES[@]+"${JOB_EXCLUDES[@]}"}; do + excludes+=(--exclude "$pattern") + done + + # restic's own cache must never end up inside a snapshot. + excludes+=(--exclude "$CACHE_DIR") + + # A job whose paths cover /var would otherwise recurse into the staging + # directories of every other job. Exclude the siblings, keep our own. + for sibling in "$DUMPS_ROOT"/*; do + [[ -e "$sibling" ]] || continue + [[ "$sibling" != "$DUMPS" ]] || continue + excludes+=(--exclude "$sibling") + done + + log "Backing up to repository '$REPO_NAME'" + # --group-by tags makes restic pick the previous snapshot of this same job + # as the parent, which is what keeps the scan incremental. + restic backup \ + --tag "$JOB" \ + --group-by tags \ + --exclude-caches \ + "${excludes[@]}" \ + ${REPO_OPTIONS[@]+"${REPO_OPTIONS[@]}"} \ + "${paths[@]}" || return 1 +} + +repo_forget() { + log "Applying retention policy on '$REPO_NAME'" + # --tag scopes the policy to this job. Without it, a job running weekly + # would prune the snapshots of a job running daily into the same repo. + restic forget \ + --tag "$JOB" \ + --group-by tag \ + "${REPO_RETENTION[@]}" \ + --prune \ + ${REPO_OPTIONS[@]+"${REPO_OPTIONS[@]}"} || return 1 +} + +repo_check() { + local -a check_args=() + + if [[ -n "${REPO_READ_DATA_SUBSET:-}" ]]; then + # Re-hashes a fraction of the pack files. Over enough runs the whole + # repository gets verified without paying for a full read every time. + check_args+=(--read-data-subset="$REPO_READ_DATA_SUBSET") + fi + + log "Verifying repository '$REPO_NAME'" + restic check \ + "${check_args[@]+"${check_args[@]}"}" \ + ${REPO_OPTIONS[@]+"${REPO_OPTIONS[@]}"} || return 1 +} + +repo_work() { + if ! restic cat config >/dev/null 2>&1; then + log "Initialising repository '$REPO_NAME'" + restic init || return 1 + fi + + repo_backup || return 1 + repo_forget || return 1 + repo_check || return 1 +} + +run_repository() { + local repo="$1" + local rc=0 + + # Clear structural variables left by the previous repository in the loop, + # so a missing key in one config cannot be silently satisfied by another's + # value. The repository's env vars are cleared by the env file itself. + unset REPO_NAME REPO_DEVICE_UUID REPO_MOUNTPOINT REPO_FSTYPE + unset REPO_OPTIONS REPO_RETENTION REPO_READ_DATA_SUBSET + + local repo_env="$CONFIG_DIR/repos/$repo.env" + if [[ ! -f "$repo_env" ]]; then + log "ERROR: no configuration for repository '$repo'" + return 1 + fi + # shellcheck source=/dev/null + source "$repo_env" || return 1 + + dest_prepare || return $? + + # Serialise access per repository: two jobs sharing a destination queue + # instead of colliding, while jobs on different destinations run freely. + exec 9>"$LOCK_DIR/repo-$repo.lock" || return 1 + if ! flock -w 3600 9; then + log "ERROR: timed out waiting for the lock on '$repo'" + dest_release + exec 9>&- + return 1 + fi + + repo_work || rc=$? + + dest_release + exec 9>&- + return "$rc" +} + +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- + +main() { + local -a succeeded=() skipped=() failed=() + local repo rc n_failed=0 + + export RESTIC_CACHE_DIR="$CACHE_DIR" + mkdir -p "$CACHE_DIR" + chmod 0700 "$CACHE_DIR" + mkdir -p "$LOCK_DIR" + + # Two runs of the same job would fight over the same staging directory, + # each wiping the other's half-written dumps. systemd already refuses to + # start a second instance of the same unit, but nothing stops someone + # running the script by hand while the timer fires. + exec 8>"$LOCK_DIR/job-$JOB.lock" + if ! flock -n 8; then + log "Another run of job '$JOB' is in progress, exiting" + exit 0 + fi + + # Give the ERR trap a channel for dump-phase failures. + load_job_notify + + log "=== Starting backup job '$JOB' ===" + stage_dumps + + # From here on failures are collected rather than fatal: a broken + # destination must not stop the others from receiving the backup. + trap - ERR + + for repo in "${JOB_REPOSITORIES[@]}"; do + rc=0 + run_repository "$repo" || rc=$? + case "$rc" in + 0) + succeeded+=("$repo") + [[ "$NTFY_ON_SUCCESS" == "true" ]] && notify \ + "$HOSTNAME_SHORT: Backup OK ($JOB / $repo)" \ + "Repository '$repo' backed up and verified." \ + "white_check_mark" "low" + ;; + "$EXIT_SKIP") + skipped+=("$repo") + notify \ + "$HOSTNAME_SHORT: Backup skipped ($JOB / $repo)" \ + "Repository '$repo' skipped (device not attached)." \ + "warning" "default" + ;; + *) + failed+=("$repo") + n_failed=$((n_failed + 1)) + log "ERROR: repository '$repo' failed with exit code $rc" + notify \ + "$HOSTNAME_SHORT: Backup FAILED ($JOB / $repo)" \ + "Repository '$repo' failed (exit $rc). Check: journalctl -u backup@$JOB.service" \ + "x" "urgent" + ;; + esac + done + + local summary="ok: ${succeeded[*]:-none}" + summary+=$'\n'"skipped: ${skipped[*]:-none}" + summary+=$'\n'"failed: ${failed[*]:-none}" + log "=== Backup job '$JOB' complete ===" + log "$summary" + + # Notifications are per-repository above; the exit code still tells systemd + # whether any destination failed. + if [[ "$n_failed" -gt 0 ]]; then + return 1 + fi + return 0 +} + +main "$@" diff --git a/roles/backup/templates/backup@.service.j2 b/roles/backup/templates/backup@.service.j2 new file mode 100644 index 0000000..dcef5e7 --- /dev/null +++ b/roles/backup/templates/backup@.service.j2 @@ -0,0 +1,21 @@ +[Unit] +Description=Backup job %i +After=network-online.target +Wants=network-online.target + +[Service] +Type=oneshot +ExecStart={{ backup_script_path }} %i + +# Creates /run/backup for the per-repository lock files, and removes it on +# stop. /run is a tmpfs, so the directory cannot simply be created once. +RuntimeDirectory=backup +RuntimeDirectoryMode=0700 + +# Backups are I/O heavy and never urgent: stay out of the way of live services. +Nice=10 +IOSchedulingClass=idle + +# A hung run holds the repository lock and blocks every other job against the +# same destination, so it must be killed rather than left to hang forever. +TimeoutStartSec={{ backup_timeout }} diff --git a/roles/backup/templates/backup@.timer.j2 b/roles/backup/templates/backup@.timer.j2 new file mode 100644 index 0000000..1c25a7c --- /dev/null +++ b/roles/backup/templates/backup@.timer.j2 @@ -0,0 +1,13 @@ +[Unit] +Description=Backup timer for job %i + +[Timer] +OnCalendar={{ backup_schedule }} +RandomizedDelaySec={{ backup_randomized_delay }} +# Run on next boot if the machine was off when the timer was due. Without this +# a box that is down every Sunday would never back up. +Persistent=true +Unit=backup@%i.service + +[Install] +WantedBy=timers.target diff --git a/roles/backup/templates/job.conf.j2 b/roles/backup/templates/job.conf.j2 new file mode 100644 index 0000000..7d03393 --- /dev/null +++ b/roles/backup/templates/job.conf.j2 @@ -0,0 +1,45 @@ +# Managed by Ansible (roles/backup). Do not edit on the host. +# Job definition sourced by {{ backup_script_path }}. + +JOB_REPOSITORIES=( +{% for name in job.repositories %} + {{ name | quote }} +{% endfor %} +) + +JOB_PATHS=( +{% for path in job.paths | default([]) %} + {{ path | quote }} +{% endfor %} +) + +JOB_EXCLUDES=( +{% for pattern in job.exclude | default([]) %} + {{ pattern | quote }} +{% endfor %} +) + +JOB_SQLITE=( +{% for path in job.sqlite | default([]) %} + {{ path | quote }} +{% endfor %} +) + +JOB_POSTGRES=( +{% if job.postgres is defined %} +{% if job.postgres is string %} + {{ job.postgres | quote }} +{% else %} +{% for database in job.postgres %} + {{ database | quote }} +{% endfor %} +{% endif %} +{% endif %} +) + +{% if job.command is defined and job.command | length > 0 %} +# Runs after the declarative dumps, with $DUMPS pointing at the staging dir. +JOB_COMMAND={{ job.command | trim | quote }} +{% else %} +JOB_COMMAND="" +{% endif %} diff --git a/roles/backup/templates/repo.env.j2 b/roles/backup/templates/repo.env.j2 new file mode 100644 index 0000000..07566b7 --- /dev/null +++ b/roles/backup/templates/repo.env.j2 @@ -0,0 +1,64 @@ +# Managed by Ansible (roles/backup). Do not edit on the host. +# Contains the repository passphrase{% if repo.env is defined and repo.env | length > 0 %} and backend credentials{% endif %}. Mode 0600, root-owned. +# +# Source this to run restic against this repository by hand: +# set -a; . /etc/backup/repos/{{ repo.name }}.env; set +a + +# Clear credential variables a previously-sourced repository may have set, so +# env vars never leak from one repository into another. +{% for key in backup_env_keys %} +unset {{ key }} +{% endfor %} + +REPO_NAME={{ repo.name | quote }} + +export RESTIC_REPOSITORY={{ repo.repository | quote }} + +{% for key, value in (repo.env | default({})).items() %} +export {{ key }}={{ value | string | quote }} +{% endfor %} + +{% if repo.mountpoint is defined %} +REPO_DEVICE_UUID={{ repo.uuid | quote }} +REPO_MOUNTPOINT={{ repo.mountpoint | quote }} +{% if repo.fstype | default('') | length > 0 %} +REPO_FSTYPE={{ repo.fstype | quote }} +{% endif %} +{% endif %} + +export RESTIC_PASSWORD={{ repo.password | quote }} +export RESTIC_CACHE_DIR={{ backup_cache_dir | quote }} + +# Notification channel for this repository. Absent notify config -> disabled. +{% if repo.notify is defined %} +NTFY_ENABLED=true +NTFY_SERVER={{ (repo.notify.server | default('https://ntfy.sh')) | quote }} +NTFY_TOPIC={{ repo.notify.topic | quote }} +NTFY_TOKEN={{ (repo.notify.token | default('')) | quote }} +NTFY_ON_SUCCESS={{ (repo.notify.notify_success | default(false)) | lower }} +{% else %} +NTFY_ENABLED=false +NTFY_SERVER="" +NTFY_TOPIC="" +NTFY_TOKEN="" +NTFY_ON_SUCCESS=false +{% endif %} + +# Retention policy, applied per job tag by `restic forget`. +REPO_RETENTION=( +{% for key, value in repo.retention.items() %} + --{{ key | replace('_', '-') }} {{ value }} +{% endfor %} +) + +{% if repo.read_data_subset | default('') | length > 0 %} +REPO_READ_DATA_SUBSET={{ repo.read_data_subset | quote }} +{% else %} +REPO_READ_DATA_SUBSET="" +{% endif %} + +REPO_OPTIONS=( +{% for option in repo.options | default([]) %} + {{ option | quote }} +{% endfor %} +) diff --git a/roles/backup/templates/timer-schedule-override.conf.j2 b/roles/backup/templates/timer-schedule-override.conf.j2 new file mode 100644 index 0000000..9746a53 --- /dev/null +++ b/roles/backup/templates/timer-schedule-override.conf.j2 @@ -0,0 +1,6 @@ +# Managed by Ansible (roles/backup). Per-job schedule override for {{ job.name }}. +[Timer] +# Reset the inherited value first: OnCalendar is additive, so without an empty +# assignment the job would fire on both the default and the override schedule. +OnCalendar= +OnCalendar={{ job.schedule }} diff --git a/roles/backup/vars/archlinux.yml b/roles/backup/vars/archlinux.yml new file mode 100644 index 0000000..0a4b039 --- /dev/null +++ b/roles/backup/vars/archlinux.yml @@ -0,0 +1,15 @@ +--- +backup_packages: + - restic + - util-linux # mountpoint, runuser, flock + - curl + +# Only installed when a job declares a `sqlite` source. +backup_sqlite_packages: + - sqlite + +# Only installed when a job declares a `postgres` source. On a host that also +# runs the postgres role this is already present; the package is listed so the +# role works on a host that only dumps a remote database. +backup_postgres_packages: + - postgresql diff --git a/roles/backup/vars/debian.yml b/roles/backup/vars/debian.yml new file mode 100644 index 0000000..1b8dae4 --- /dev/null +++ b/roles/backup/vars/debian.yml @@ -0,0 +1,15 @@ +--- +backup_packages: + - restic + - util-linux # mountpoint, runuser, flock + - curl + +# Only installed when a job declares a `sqlite` source. +backup_sqlite_packages: + - sqlite3 + +# Only installed when a job declares a `postgres` source. On a host that also +# runs the postgres role this is already present; the package is listed so the +# role works on a host that only dumps a remote database. +backup_postgres_packages: + - postgresql-client diff --git a/roles/backup/vars/main.yml b/roles/backup/vars/main.yml new file mode 100644 index 0000000..22d4422 --- /dev/null +++ b/roles/backup/vars/main.yml @@ -0,0 +1,51 @@ +--- +# Retention keys accepted in a repository's `retention` mapping. Each maps to +# the restic `forget` flag of the same name with dashes. +backup_retention_keys: + - keep_last + - keep_hourly + - keep_daily + - keep_weekly + - keep_monthly + - keep_yearly + +# Environment variables a repository's `env` mapping is allowed to set, and +# which the generated env file clears between repositories. These are the +# credential/connection variables restic and its backends read; anything a +# previously-processed repository exported must not leak into the next one. +backup_env_keys: + - RESTIC_REPOSITORY + - RESTIC_PASSWORD + - RESTIC_PASSWORD_FILE + - RESTIC_PASSWORD_COMMAND + - RESTIC_CACHE_DIR + - RESTIC_REST_USERNAME + - RESTIC_REST_PASSWORD + - AWS_ACCESS_KEY_ID + - AWS_SECRET_ACCESS_KEY + - AWS_SESSION_TOKEN + - AWS_DEFAULT_REGION + - AWS_PROFILE + - B2_ACCOUNT_ID + - B2_ACCOUNT_KEY + +# Derived values. Lazily evaluated and only valid inside a loop that defines +# `repo`, i.e. tasks/repository.yml. + +# RESTIC_REPOSITORY is written verbatim; the environment for the deploy-time +# init tasks is the same `env` mapping the backup script will use, so the two +# can never drift apart. +backup_repo_environment: >- + {{ + { + 'RESTIC_REPOSITORY': repo.repository, + 'RESTIC_PASSWORD': repo.password, + 'RESTIC_CACHE_DIR': backup_cache_dir, + } + | combine(repo.env | default({})) + }} + +# A device repository is recognised by the presence of a mountpoint, not a +# `type` field. A path-style repository without one is a plain local directory. +backup_repo_is_device: "{{ repo.mountpoint is defined }}" +backup_repo_is_local: "{{ repo.mountpoint is not defined and repo.repository is match('^/') }}" diff --git a/roles/ntfy/defaults/main.yml b/roles/ntfy/defaults/main.yml index 8d834e8..f56e552 100644 --- a/roles/ntfy/defaults/main.yml +++ b/roles/ntfy/defaults/main.yml @@ -6,6 +6,22 @@ ntfy_version: latest ntfy_data_dir: "{{ podman_projects_dir }}/ntfy/data" ntfy_cache_dir: "{{ podman_projects_dir }}/ntfy/cache" +# Backup manifest - consumed by roles/backup via inventory `backup_jobs`. +# The ntfy role owns WHAT gets backed up: it knows its storage layout. The +# operator owns WHERE (repositories, retention, schedule) in the inventory. +# - cache.db is the ephemeral message cache: excluded, churns every run for +# no restore value. attachments/ lives under cache/, hence the narrow glob. +# - user.db holds users, tokens and ACLs: excluded from the live tree and +# dumped with sqlite3 .backup instead, because copying a live SQLite file +# yields a torn snapshot (recent writes sit in the -wal sidecar). +ntfy_backup_paths: + - "{{ ntfy_data_dir | dirname }}" # project root: data/, cache/, server.yml, ntfy.yaml +ntfy_backup_exclude: + - "{{ ntfy_cache_dir }}/cache.db*" + - "{{ ntfy_data_dir }}/user.db*" +ntfy_backup_sqlite: + - "{{ ntfy_data_dir }}/user.db" + # Authentication configuration (REQUIRED - must be set explicitly) # Ntfy admin user for managing topics and access control ntfy_admin_user: admin diff --git a/roles/uptime_kuma/defaults/main.yml b/roles/uptime_kuma/defaults/main.yml index 739c4bb..1ec5381 100644 --- a/roles/uptime_kuma/defaults/main.yml +++ b/roles/uptime_kuma/defaults/main.yml @@ -6,6 +6,19 @@ uptime_kuma_version: "2" # NFS is NOT supported - will cause SQLite corruption uptime_kuma_data_dir: "{{ podman_projects_dir }}/uptime-kuma/data" +# Backup manifest - consumed by roles/backup via inventory `backup_jobs`. +# The uptime_kuma role owns WHAT gets backed up: kuma.db holds every monitor, +# notification channel and status page, and is a live SQLite file - so it is +# excluded from the live tree and dumped with sqlite3 .backup instead (copying +# a live SQLite file yields a torn snapshot, recent writes sit in the -wal +# sidecar). The rest of the data dir is archived as-is. +uptime_kuma_backup_paths: + - "{{ uptime_kuma_data_dir }}" +uptime_kuma_backup_exclude: + - "{{ uptime_kuma_data_dir }}/kuma.db*" +uptime_kuma_backup_sqlite: + - "{{ uptime_kuma_data_dir }}/kuma.db" + # Network configuration uptime_kuma_port: 3300