feat(sys_autoupdate): notify on pending kernel reboot
Detect drift between the running kernel and the newest installed one (Arch: /usr/lib/modules vs uname -r; Debian: /var/run/reboot-required) and push a weekly ntfy reminder until the host is rebooted. Also bundles the in-flight ignore-packages feature and hostname refactor in the autoupdate script.
This commit is contained in:
Clément Désiles
@@ -27,6 +27,26 @@ sys_autoupdate_ntfy_topic: your-notification-topic
|
||||
|
||||
OS-specific commands are defined in `vars/archlinux.yml` and `vars/debian.yml`, loaded automatically via `ansible_facts['os_family']`.
|
||||
|
||||
## Excluding packages from auto-update
|
||||
|
||||
Set `sys_autoupdate_ignore_packages` to a list of package names to hold back. Useful when running out-of-tree kernel modules (ZFS DKMS, Nvidia, VirtualBox) that must stay in lockstep with the kernel.
|
||||
|
||||
```yaml
|
||||
# Arch with ZFS DKMS
|
||||
sys_autoupdate_ignore_packages:
|
||||
- linux
|
||||
- linux-headers
|
||||
- zfs-dkms
|
||||
- zfs-utils
|
||||
```
|
||||
|
||||
Translation per OS:
|
||||
|
||||
- **Arch:** passed as `pacman --ignore=pkg1,pkg2` on each run. Held packages are also filtered out of the pending-update detection.
|
||||
- **Debian/Ubuntu:** reconciled via `apt-mark hold` at role-apply time. Holds added by this role are not automatically released when removed from the list — run `sudo apt-mark unhold <package>` to release them manually.
|
||||
|
||||
**Warning:** when holding the kernel, also hold matching headers and any DKMS modules together. Holding the kernel while letting `zfs-dkms` advance can break the DKMS build on next boot.
|
||||
|
||||
## Podman image updates
|
||||
|
||||
When `sys_autoupdate_podman_enabled: true` (default), the script scans `podman_projects_dir` for `docker-compose.yml` files, pulls images via `podman-compose pull`, and recreates containers with `podman-compose up -d` for projects with updated images. Dangling images are pruned after each run.
|
||||
@@ -41,3 +61,4 @@ The script runs as root (for package management) and uses `sudo -u {{ ansible_us
|
||||
| `x` | Update or pod restart failed |
|
||||
| `warning` | Distro news requires manual review (Arch) |
|
||||
| `whale` | Podman images updated |
|
||||
| `arrows_counterclockwise` | Running kernel older than installed — reboot pending (notified weekly, `sys_autoupdate_reboot_notify_weekday`) |
|
||||
|
||||
@@ -17,8 +17,34 @@ sys_autoupdate_randomized_delay: 60m
|
||||
sys_autoupdate_check_news: true
|
||||
sys_autoupdate_news_hours: 24
|
||||
|
||||
# Kernel reboot check cadence: notify drift (running vs installed kernel) only
|
||||
# on this day of the week (1=Monday .. 7=Sunday) to avoid daily nagging.
|
||||
sys_autoupdate_reboot_notify_weekday: 1
|
||||
|
||||
sys_autoupdate_allow_downgrade: false
|
||||
|
||||
# Packages excluded from auto-update.
|
||||
# Arch: passed as `pacman --ignore=pkg1,pkg2`
|
||||
# Debian/Ubuntu: reconciled via `apt-mark hold` / `apt-mark unhold` on each run
|
||||
#
|
||||
# Common use case: hold kernel back when running out-of-tree modules (ZFS DKMS,
|
||||
# Nvidia, VirtualBox, etc.). When holding the kernel, also hold matching headers
|
||||
# and any DKMS modules to keep them in lockstep.
|
||||
#
|
||||
# Example (Arch with ZFS DKMS):
|
||||
# sys_autoupdate_ignore_packages:
|
||||
# - linux
|
||||
# - linux-headers
|
||||
# - zfs-dkms
|
||||
# - zfs-utils
|
||||
#
|
||||
# Example (Debian/Ubuntu with ZFS DKMS):
|
||||
# sys_autoupdate_ignore_packages:
|
||||
# - linux-image-generic
|
||||
# - linux-headers-generic
|
||||
# - zfs-dkms
|
||||
sys_autoupdate_ignore_packages: []
|
||||
|
||||
# Podman image auto-update (rootless, runs as ansible_user)
|
||||
# Pulls latest images via podman-compose and recreates containers if changed
|
||||
sys_autoupdate_podman_enabled: true
|
||||
|
||||
@@ -19,6 +19,26 @@
|
||||
name: "{{ sys_autoupdate_packages }}"
|
||||
state: present
|
||||
|
||||
- name: Reconcile apt package holds (Debian/Ubuntu)
|
||||
when: ansible_facts['os_family'] == 'Debian'
|
||||
block:
|
||||
- name: Get currently held apt packages
|
||||
ansible.builtin.command: apt-mark showhold
|
||||
register: sys_autoupdate_current_holds
|
||||
changed_when: false
|
||||
|
||||
- name: Hold packages listed in sys_autoupdate_ignore_packages
|
||||
ansible.builtin.dpkg_selections:
|
||||
name: "{{ item }}"
|
||||
selection: hold
|
||||
loop: "{{ sys_autoupdate_ignore_packages }}"
|
||||
when: item not in sys_autoupdate_current_holds.stdout_lines
|
||||
|
||||
# NOTE: holds set manually (outside this role) or in previous runs are not
|
||||
# automatically released when removed from sys_autoupdate_ignore_packages.
|
||||
# This is intentional: avoids accidentally unholding operator-pinned packages.
|
||||
# To release a hold, run: sudo apt-mark unhold <package>
|
||||
|
||||
- name: Deploy autoupdate script
|
||||
ansible.builtin.template:
|
||||
src: sys-autoupdate.sh.j2
|
||||
|
||||
@@ -13,6 +13,7 @@ PODMAN_PROJECTS_DIR="{{ sys_autoupdate_podman_projects_dir }}"
|
||||
PODMAN_USER="{{ ansible_user }}"
|
||||
PODMAN_PRUNE_UNTIL="{{ sys_autoupdate_podman_prune_until }}"
|
||||
|
||||
HOSTNAME="{{ ansible_facts['hostname'] }}"
|
||||
DISTRO_NAME="{{ sys_autoupdate_distro_name }}"
|
||||
OS_KEY="{{ sys_autoupdate_os_key }}"
|
||||
ALLOW_DOWNGRADE="{{ sys_autoupdate_allow_downgrade | lower }}"
|
||||
@@ -20,6 +21,8 @@ HAS_NEWS_CHECK="{{ sys_autoupdate_has_news_check | lower }}"
|
||||
CHECK_NEWS="{{ sys_autoupdate_check_news | lower }}"
|
||||
NEWS_HOURS="{{ sys_autoupdate_news_hours }}"
|
||||
NEWS_URL="{{ sys_autoupdate_news_url }}"
|
||||
IGNORE_PACKAGES="{{ sys_autoupdate_ignore_packages | join(',') }}"
|
||||
REBOOT_NOTIFY_WEEKDAY="{{ sys_autoupdate_reboot_notify_weekday }}"
|
||||
|
||||
{% raw %}
|
||||
log() {
|
||||
@@ -53,10 +56,20 @@ check_pending_updates() {
|
||||
archlinux)
|
||||
# checkupdates (pacman-contrib) exits 2 when no updates are pending;
|
||||
# swallow non-zero so set -e doesn't abort the script.
|
||||
checkupdates 2>/dev/null || true
|
||||
# Honor ignored packages by filtering them out of the pending list.
|
||||
local raw
|
||||
raw=$(checkupdates 2>/dev/null || true)
|
||||
if [[ -n "$IGNORE_PACKAGES" && -n "$raw" ]]; then
|
||||
local ignore_regex
|
||||
ignore_regex="^($(echo "$IGNORE_PACKAGES" | tr ',' '|')) "
|
||||
echo "$raw" | grep -Ev "$ignore_regex" || true
|
||||
else
|
||||
echo "$raw"
|
||||
fi
|
||||
;;
|
||||
debian)
|
||||
apt-get update -qq >/dev/null 2>&1
|
||||
# Held packages are automatically excluded from dist-upgrade simulation.
|
||||
apt-get -s dist-upgrade 2>/dev/null | grep '^Inst ' || true
|
||||
;;
|
||||
*)
|
||||
@@ -69,13 +82,18 @@ check_pending_updates() {
|
||||
apply_upgrade() {
|
||||
case "$OS_KEY" in
|
||||
archlinux)
|
||||
local -a pacman_args=(-Syu --noconfirm)
|
||||
if [[ "$ALLOW_DOWNGRADE" == "true" ]]; then
|
||||
pacman -Syu --noconfirm --allow-downgrade
|
||||
else
|
||||
pacman -Syu --noconfirm
|
||||
pacman_args+=(--allow-downgrade)
|
||||
fi
|
||||
if [[ -n "$IGNORE_PACKAGES" ]]; then
|
||||
pacman_args+=(--ignore="$IGNORE_PACKAGES")
|
||||
fi
|
||||
pacman "${pacman_args[@]}"
|
||||
;;
|
||||
debian)
|
||||
# Holds are reconciled by Ansible (apt-mark hold/unhold) before each
|
||||
# run. dist-upgrade automatically skips held packages.
|
||||
local -a apt_args=(dist-upgrade -y)
|
||||
if [[ "$ALLOW_DOWNGRADE" == "true" ]]; then
|
||||
apt_args+=(--allow-downgrades)
|
||||
@@ -137,7 +155,7 @@ check_distro_news() {
|
||||
log "Recent $DISTRO_NAME news detected (${#recent_titles[@]} items within ${hours}h):"
|
||||
log "$news_list"
|
||||
send_notification \
|
||||
"$(hostname): Manual Intervention Required" \
|
||||
"$HOSTNAME: Manual Intervention Required" \
|
||||
"Recent $DISTRO_NAME news (last ${hours}h). Review before updating.\n\n${news_list}\n\n$NEWS_URL" \
|
||||
"warning" \
|
||||
"high"
|
||||
@@ -165,14 +183,14 @@ update_system() {
|
||||
if apply_upgrade 2>&1; then
|
||||
log "System updates completed successfully ($update_count packages)"
|
||||
send_notification \
|
||||
"$(hostname): System Update Success" \
|
||||
"$HOSTNAME: System Update Success" \
|
||||
"$update_count packages updated" \
|
||||
"white_check_mark" \
|
||||
"default"
|
||||
else
|
||||
log "ERROR: System update failed"
|
||||
send_notification \
|
||||
"$(hostname): System Update FAILED" \
|
||||
"$HOSTNAME: System Update FAILED" \
|
||||
"Update failed. Check: journalctl -u sys-autoupdate.service" \
|
||||
"x" \
|
||||
"urgent"
|
||||
@@ -180,6 +198,45 @@ update_system() {
|
||||
fi
|
||||
}
|
||||
|
||||
# Detect drift between the running kernel and the newest installed one.
|
||||
# Fires a notification when a reboot is pending, so a long-lived host isn't
|
||||
# silently left on a stale kernel.
|
||||
check_kernel_reboot() {
|
||||
# Only notify once a week (1=Mon .. 7=Sun) to avoid daily nagging.
|
||||
if [[ "$(date +%u)" != "$REBOOT_NOTIFY_WEEKDAY" ]]; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
local detail=""
|
||||
|
||||
case "$OS_KEY" in
|
||||
archlinux)
|
||||
local running latest
|
||||
running=$(uname -r)
|
||||
latest=$(ls /usr/lib/modules/ 2>/dev/null | sort -V | tail -1 || echo "")
|
||||
if [[ -n "$latest" && "$running" != "$latest" ]]; then
|
||||
detail="Running kernel $running is older than installed $latest."
|
||||
fi
|
||||
;;
|
||||
debian)
|
||||
if [[ -f /var/run/reboot-required ]]; then
|
||||
detail=$(cat /var/run/reboot-required)
|
||||
fi
|
||||
;;
|
||||
esac
|
||||
|
||||
if [[ -n "$detail" ]]; then
|
||||
log "Kernel drift detected: $detail"
|
||||
send_notification \
|
||||
"$HOSTNAME: Reboot Required" \
|
||||
"${detail}\n\nA reboot is needed to load the new kernel." \
|
||||
"arrows_counterclockwise" \
|
||||
"high"
|
||||
else
|
||||
log "Running kernel is up to date (no reboot pending)"
|
||||
fi
|
||||
}
|
||||
|
||||
# Run a command as the rootless Podman user with a proper login environment.
|
||||
# `runuser -l` invokes a login shell, so XDG_RUNTIME_DIR, DBUS_SESSION_BUS_ADDRESS,
|
||||
# PATH, and HOME are set correctly for rootless Podman state under
|
||||
@@ -223,7 +280,7 @@ update_podman_images() {
|
||||
local pull_tail
|
||||
pull_tail=$(echo "$pull_output" | tail -n 5)
|
||||
send_notification \
|
||||
"$(hostname): Podman Pull Failed ($project_name)" \
|
||||
"$HOSTNAME: Podman Pull Failed ($project_name)" \
|
||||
"Failed to pull images.\n\nLast output:\n$pull_tail\n\nFull log: journalctl -u sys-autoupdate.service" \
|
||||
"x" \
|
||||
"high"
|
||||
@@ -260,7 +317,7 @@ update_podman_images() {
|
||||
up_tail=$(echo "$up_output" | tail -n 5)
|
||||
|
||||
send_notification \
|
||||
"$(hostname): Podman Restart FAILED ($project_name)" \
|
||||
"$HOSTNAME: Podman Restart FAILED ($project_name)" \
|
||||
"up -d exit=$up_status, $running_count container(s) running.\n\nLast output:\n$up_tail\n\nFull log: journalctl -u sys-autoupdate.service" \
|
||||
"x" \
|
||||
"urgent"
|
||||
@@ -283,7 +340,7 @@ update_podman_images() {
|
||||
service_list=$(printf '%s, ' "${updated_services[@]}")
|
||||
service_list="${service_list%, }"
|
||||
send_notification \
|
||||
"$(hostname): Podman Images Updated" \
|
||||
"$HOSTNAME: Podman Images Updated" \
|
||||
"${#updated_services[@]} project(s) updated: $service_list" \
|
||||
"whale" \
|
||||
"default"
|
||||
@@ -309,6 +366,8 @@ main() {
|
||||
update_podman_images
|
||||
fi
|
||||
|
||||
check_kernel_reboot
|
||||
|
||||
log "=== Auto-update run complete ==="
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user