feat(backup): per-job schedules, stale-lock recovery, service manifests

- Add per-job `schedule` (systemd OnCalendar) overrides and document the
  global `backup_schedule` / `backup_randomized_delay` knobs.
- Run `restic unlock` before backup/forget/check so a run killed mid-flight
  no longer wedges every later run on a stale exclusive lock.
- Publish backup manifests for fdroid, immich and unifi, so the inventory
  decides only where (repositories, retention, schedule).
- Document that `read_data_subset` re-reads on every job targeting the
  repository, so a large repository wants a small fraction.

examples/inventory/host_vars.example.yml

@@ -184,6 +184,11 @@ immich_valkey_password: "{{ vault_immich_valkey_password }}"
 # Backup configuration
 # --------------------
 # restic: encrypted, deduplicated, incremental. See roles/backup/README.md.
+#
+# Schedule is a systemd OnCalendar expression. This global default applies to
+# every job; a job's own `schedule` overrides it.
+# backup_schedule: "Sun *-*-* 03:00:00"   # weekly
+# backup_randomized_delay: 30m
 
 # Destinations. `repository` is RESTIC_REPOSITORY verbatim. Retention lives
 # here: a large local disk can afford more history than metered object storage.
@@ -241,6 +246,9 @@ backup_jobs:
     paths: "{{ ntfy_backup_paths }}"
     exclude: "{{ ntfy_backup_exclude }}"
     sqlite: "{{ ntfy_backup_sqlite }}"
+    # Daily. Weekly jobs omit this and inherit backup_schedule. Retention is
+    # per-tag, so jobs sharing a repository never prune each other's snapshots.
+    schedule: "*-*-* 03:00:00"
 
   # Databases only, no files. pg_dumpall --globals-only is always included.
   # - name: databases

roles/backup/README.md

@@ -85,8 +85,8 @@ backup_repositories:
   #   password: "{{ vault_backup_usb_password }}"
   #   retention:
   #     keep_weekly: 12
-  #   notify:
+#   notify:
-  #     topic: backups-reminders   # low-priority reminder, not an alarm
+#     topic: backups-reminders   # low-priority reminder, not an alarm
 
 backup_jobs:
   - name: ntfy
@@ -94,6 +94,7 @@ backup_jobs:
     paths: "{{ ntfy_backup_paths }}"
     exclude: "{{ ntfy_backup_exclude }}"
     sqlite: "{{ ntfy_backup_sqlite }}"
+    schedule: "*-*-* 03:00:00"   # OnCalendar; omit to use the global backup_schedule
 ```
 
 Generate the repository password and **store it outside this repository and outside the machine being backed up**:

roles/backup/defaults/main.yml

@@ -65,7 +65,10 @@ backup_timeout: 4h
 #   read_data_subset  (optional) e.g. "5%" or "1/12". Makes `restic check`
 #                         also re-hash that fraction of the pack files, so the
 #                         whole repository gets verified over time without
-#                         paying for a full read on every run.
+#                         paying for a full read on every run. The check runs on
+#                         every job that targets the repository, so a large
+#                         repository with frequent jobs wants a small fraction
+#                         (1%), or each run pays the read again.
 #   options    (optional) extra restic flags, e.g. ["-o", "s3.list-objects-v1=true"]
 #
 #   notify     (optional) ntfy notification channel for THIS repository. Each

roles/backup/templates/backup.sh.j2

@@ -327,6 +327,15 @@ repo_work() {
         restic init || return 1
     fi
 
+    # A run killed mid-flight (systemd timeout, OOM, reboot) leaves an exclusive
+    # lock behind. `forget --prune` and `check` both need that lock, so every
+    # later run backs up successfully and then fails, until someone clears it by
+    # hand. The per-repository flock above makes us the only local writer, and a
+    # repository is never shared between hosts (see README), so any lock left
+    # here belongs to a dead process. `unlock` only removes locks it considers
+    # stale, so a genuinely live one is left alone.
+    restic unlock || return 1
+
     repo_backup || return 1
     repo_forget || return 1
     repo_check || return 1

roles/fdroid/defaults/main.yml

@@ -11,6 +11,10 @@ fdroid_port: 8070
 # Data directory (keystore, config, metadata, repo APKs)
 fdroid_data_dir: "{{ podman_projects_dir }}/fdroid/data"
 
+# Backup manifest - signing keystore (irreplaceable), config, metadata, published APKs.
+fdroid_backup_paths:
+  - "{{ fdroid_data_dir }}"
+
 # Repository metadata
 fdroid_repo_url: "https://apk.mysite.fr/repo"
 fdroid_repo_name: "F-Droid Repository"

roles/immich/defaults/main.yml

@@ -5,6 +5,18 @@ immich_version: release
 # Storage location (@see https://docs.immich.app/install/environment-variables/)
 immich_upload_location: "{{ podman_projects_dir }}/immich/data/upload"
 
+# Backup manifest - original photos/videos (the irreplaceable data). The
+# PostgreSQL side is covered by the `postgres: all` job in the inventory.
+immich_backup_paths:
+  - "{{ immich_upload_location }}"
+
+# Regenerable derivatives. Immich rebuilds these on demand (Generate
+# Thumbnails / Transcode Videos jobs), and they are many small files that
+# dominate S3 request count - exclude them for restore value.
+immich_backup_exclude:
+  - "{{ immich_upload_location }}/thumbs"
+  - "{{ immich_upload_location }}/encoded-video"
+
 # External libraries (read-only host paths exposed to the server container)
 # Use the in-container `mount_path` when registering the library in the Immich UI.
 # Example:

roles/unifi/defaults/main.yml

@@ -8,6 +8,10 @@ unifi_image: docker.io/jacobalberty/unifi
 # Storage location for persistent data (MongoDB, config, backups)
 unifi_data_dir: "{{ podman_projects_dir }}/unifi/data"
 
+# Backup manifest - MongoDB (device/AP config), settings and auto-backups.
+unifi_backup_paths:
+  - "{{ unifi_data_dir }}"
+
 # Network binding — REQUIRED, must be set in inventory
 # IP address of the LAN interface for AP communication (inform, STUN, discovery).
 # APs must be able to reach this address.