From 0b6efc9ad84e32b3dcda2927e7aec2de5bf211ef Mon Sep 17 00:00:00 2001 From: Fabio Scotto di Santolo Date: Fri, 25 Sep 2026 08:02:44 +0200 Subject: [PATCH] Harden Atlas USB backup cleanup and document restorecon workflow --- AGENTS.md | 17 +++++-- README.md | 37 ++++++++++++-- ansible/inventory/host_vars/atlas.yml | 4 ++ ansible/inventory/host_vars/prometheus.yml | 1 + ansible/roles/profile_atlas/defaults/main.yml | 3 ++ ansible/roles/profile_atlas/tasks/main.yml | 3 ++ .../roles/profile_atlas/tasks/restorecon.yml | 27 ++++++++++ .../roles/profile_atlas/tasks/usb_backup.yml | 10 ++++ .../templates/atlas-usb-backup.service.j2 | 1 + .../templates/atlas-usb-backup.sh.j2 | 31 +++++------ .../atlas-usb-snapshot-cleanup.sh.j2 | 51 +++++++++++++++++++ 11 files changed, 161 insertions(+), 24 deletions(-) create mode 100644 ansible/roles/profile_atlas/tasks/restorecon.yml create mode 100644 ansible/roles/profile_atlas/templates/atlas-usb-snapshot-cleanup.sh.j2 diff --git a/AGENTS.md b/AGENTS.md index 412d2be..faaac61 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -67,6 +67,8 @@ Ansible-driven personal infrastructure repo for Fedora and Void desktops, Fedora `ansible-playbook ansible/site.yml --limit atlas --tags borg_logging --check --diff` - Atlas manual offline USB backup and 45Drives Alerts reminder: `ansible-playbook ansible/site.yml --limit atlas --tags usb_backup,usb_reminder --check --diff` + - Atlas explicit post-restore SELinux relabeling: + `ansible-playbook ansible/site.yml --limit atlas --tags restorecon --check -e '{"atlas_restorecon_paths":["/zpool/archive"]}'` - Prometheus/Aegis WireGuard gateway: `ansible-playbook ansible/site.yml --limit prometheus,aegis --tags wireguard --check --diff` - DuckDNS config only: `ansible-playbook ansible/site.yml --limit prometheus --tags duckdns --check --diff` @@ -202,8 +204,11 @@ scheduled retention prune and monthly scrub remain runtime checks. scheduled for the first Saturday of each month at 10:00 Europe/Rome via the existing 45Drives notifier. A manual test produced an Alerts notification, not an email. The first USB attempt failed on a `security.selinux` xattr and was interrupted; the xattr filter is deployed and the temporary - recursive snapshot, open LUKS mapper, and failed service state were cleaned up. No complete backup or - tested USB restore exists; do not mark this item complete yet. + recursive snapshot and open LUKS mapper were cleaned up. A later run reported checksum verification + and published the USB version, but failed while removing host-namespace ZFS snapshot mounts. Those + exact mounts and snapshots were cleaned up. An `ExecStopPost` helper now removes only the named + temporary snapshot after the backup process exits; a disposable-snapshot test passed. A new full + successful service run and an independent USB restore remain unverified; do not mark this item complete yet. - [ ] Test restores independently from a ZFS snapshot, Borg, and the offline USB backup before relying on any backup path. - [ ] Add monitoring and alerting for pool health, scrub/resilver, SMART data, temperatures, free space, @@ -230,9 +235,11 @@ scheduled retention prune and monthly scrub remain runtime checks. container paths, and the required Vault database secret. ### Priority 4 - Optional workflows -- [ ] Optionally design iCloud photo ingestion and an Aegis persistent NFS mount as a separate workflow - after the storage and backup layers are validated; do not make either a dependency of the Atlas - baseline. +- [ ] After data protection is validated, move iCloudPD photo ingestion from Aegis to Atlas as a + temporary service until Uranus is ready. Plan to store photos in `/zpool/archive/Pictures` and + persistent application/MFA state outside `Archive`; validate permissions, SELinux, backups and + recovery before cutover. Keep the current Aegis service and Photobook NFS export unchanged until + the Atlas workflow is tested, then retire them explicitly if no longer needed. ## Cerberus Management Node (Deferred) `cerberus` is postponed until the office in the new house is physically set up. It is not an inventory diff --git a/README.md b/README.md index 3bc2afc..3ff1f72 100644 --- a/README.md +++ b/README.md @@ -390,8 +390,17 @@ copies every dataset to a versioned `atlas/snapshots//` directory usi verifies the result with a checksum-based dry run, atomically updates `atlas/latest`, unmounts and closes LUKS. A failed run never replaces `latest` or removes an earlier complete version. Borg and the USB backup may run concurrently from separate snapshots; both reading the same pool can reduce throughput. -The USB copy preserves ACLs and extended attributes except `security.selinux`, which the target -SELinux policy must recreate during a restore; do not restore data into service paths without relabeling. +The USB copy preserves ACLs but not generic extended attributes; `security.selinux` is also intentionally +excluded because the target SELinux policy must recreate labels during a restore. Do not restore data into +service paths without relabeling. After restoring an explicit dataset path, apply its destination policy with: + +```bash +ansible-playbook ansible/site.yml --limit atlas --tags restorecon \ + -e '{"atlas_restorecon_paths":["/zpool/archive"]}' +``` + +The task accepts only paths below the Atlas pool mount root, runs `restorecon -RFv` only for the paths +provided at invocation, and is otherwise a no-op. It must not be used on the whole pool during routine runs. Old USB versions are not pruned automatically, to avoid deleting the only offline copy without an explicitly chosen retention policy; capacity checks include an estimated transfer size and a 10 GiB free-space reserve. The disk must be physically disconnected after a successful backup @@ -404,6 +413,13 @@ ANSIBLE_LOCAL_TEMP=/tmp/ansible-local \ ansible-playbook ansible/site.yml --limit atlas --tags usb_backup,usb_reminder --check --diff ``` +To validate a planned, explicit post-restore relabel operation without changing labels, run: + +```bash +ansible-playbook ansible/site.yml --limit atlas --tags restorecon --check \ + -e '{"atlas_restorecon_paths":["/zpool/archive"]}' +``` + Before the first **manual** service start, safely unmount the currently mounted `/mnt/zpool-backup`; never run it on an arbitrary mounted disk. Future starts can begin with the mapper closed: `sudo systemctl start atlas-usb-backup.service` prompts for the @@ -417,8 +433,8 @@ A manual test confirmed a notification in 45Drives Alerts, **not** an email. The reports notification submission, not email delivery; the role does not depend on SMTP/OAuth settings. The reminder never starts the backup. Check its schedule with `systemctl list-timers atlas-usb-reminder.timer` and the result in 45Drives Alerts. -The timer was verified active with its first scheduled run at 2026-10-03 10:00 CEST. The USB backup -service was verified inactive after deployment; no successful backup or email delivery is claimed. +The timer was verified active with its first scheduled run at 2026-10-03 10:00 CEST. No email +delivery is claimed. The first manual USB attempt on 2026-09-23 did not complete: rsync was denied while removing `security.selinux` on the USB filesystem, then the interrupted service left its recursive `atlas-usb-20260923T185748Z-2469168` snapshot and the `zpool-backup` LUKS mapper open. The @@ -426,12 +442,25 @@ rsync xattr filter was deployed afterward. The incomplete USB directory was abse the exact failed snapshot was removed, the verified and unmounted mapper closed, and the service failed state cleared. A final check found no remnant snapshot, mount, mapper, or staging directory. The failed attempt is not a valid backup, and no USB restore has been tested. +On 2026-09-24 a later run reported a checksum-verified, published USB version and closed the LUKS +mapper, but the service failed while destroying its temporary ZFS snapshot: OpenZFS still had +on-demand `.zfs/snapshot` mounts open in the host namespace. Those exact temporary snapshots were +unmounted normally and removed; no force or rollback was used. The backup service now records its +snapshot name and runs a narrowly scoped `ExecStopPost` cleanup after the private backup process +exits. The cleanup helper was tested with a disposable recursive snapshot and an active snapshot +mount, but a complete new backup run and independent USB restore remain unverified. A temporary Nextcloud deployment on Atlas is also planned before Uranus: it requires separately declared persistent application, database, and cache storage, Vault-backed credentials, NPM-only publishing through Aegis, and defined backup, upgrade, and eventual migration procedures. Do not deploy it before the data-protection checklist is complete. +The desired future iCloud photo-ingestion host is Atlas, not Aegis. After data-protection validation, +plan an explicit iCloudPD migration with photos under `/zpool/archive/Pictures` and application/MFA +state outside `Archive`, then test permissions, SELinux, backups and recovery before cutting over. +The current Aegis iCloudPD service and Atlas Photobook NFS export remain configured until that +separate migration is approved and validated; the eventual Atlas service is temporary until Uranus. + Prometheus backup pulls, USB backup, monitoring, and disaster-recovery tests remain follow-up work. The prioritized operational backlog is kept in `AGENTS.md`. diff --git a/ansible/inventory/host_vars/atlas.yml b/ansible/inventory/host_vars/atlas.yml index 58375d5..0a5701d 100644 --- a/ansible/inventory/host_vars/atlas.yml +++ b/ansible/inventory/host_vars/atlas.yml @@ -92,6 +92,9 @@ atlas_manage_usb_reminder: true atlas_usb_reminder_calendar: "Sat *-*-01..07 10:00:00 Europe/Rome" atlas_manage_sharing: true atlas_manage_media_stack: false +# Planned after data-protection validation: move iCloudPD photo ingestion from +# Aegis to Atlas, with photos under /zpool/archive/Pictures and persistent +# application/MFA state outside Archive. Do not deploy or cut over yet. # WireGuard is retired on Atlas. These rootless services are a temporary home # until Uranus replaces them. @@ -117,6 +120,7 @@ host_packages: - policycoreutils - policycoreutils-python-utils - python3-libselinux + - setroubleshoot-server - samba - samba-client - samba-common-tools diff --git a/ansible/inventory/host_vars/prometheus.yml b/ansible/inventory/host_vars/prometheus.yml index a81b8bc..7ca16d9 100644 --- a/ansible/inventory/host_vars/prometheus.yml +++ b/ansible/inventory/host_vars/prometheus.yml @@ -33,6 +33,7 @@ host_packages: - cockpit-navigator - cockpit-podman - cockpit-storaged + - setroubleshoot-server host_enabled_services: - cockpit.socket diff --git a/ansible/roles/profile_atlas/defaults/main.yml b/ansible/roles/profile_atlas/defaults/main.yml index 287ff50..91833c5 100644 --- a/ansible/roles/profile_atlas/defaults/main.yml +++ b/ansible/roles/profile_atlas/defaults/main.yml @@ -108,6 +108,9 @@ atlas_manage_usb_reminder: false atlas_usb_reminder_calendar: "" atlas_usb_reminder_notifier: /opt/45drives/houston/houston-notify +# Explicit post-restore relabeling only; never relabel datasets during ordinary runs. +atlas_restorecon_paths: [] + atlas_archive_mountpoint: "{{ atlas_mount_root }}/{{ atlas_zfs_dataset_archive }}" atlas_services_mountpoint: "{{ atlas_mount_root }}/{{ atlas_zfs_dataset_services }}" atlas_app_data_mountpoint: "{{ atlas_mount_root }}/{{ atlas_zfs_dataset_app_data }}" diff --git a/ansible/roles/profile_atlas/tasks/main.yml b/ansible/roles/profile_atlas/tasks/main.yml index 86f6277..f294504 100644 --- a/ansible/roles/profile_atlas/tasks/main.yml +++ b/ansible/roles/profile_atlas/tasks/main.yml @@ -23,6 +23,9 @@ - name: Import Atlas offline USB backup tasks ansible.builtin.import_tasks: usb_backup.yml +- name: Import Atlas post-restore SELinux relabeling tasks + ansible.builtin.import_tasks: restorecon.yml + - name: Import Atlas file sharing tasks ansible.builtin.import_tasks: sharing.yml diff --git a/ansible/roles/profile_atlas/tasks/restorecon.yml b/ansible/roles/profile_atlas/tasks/restorecon.yml new file mode 100644 index 0000000..cd0c02a --- /dev/null +++ b/ansible/roles/profile_atlas/tasks/restorecon.yml @@ -0,0 +1,27 @@ +--- +- name: Validate requested Atlas post-restore relabel paths + tags: [atlas, restorecon, recovery] + ansible.builtin.assert: + that: + - item is string + - item.startswith(atlas_mount_root ~ '/') + - item != atlas_mount_root + fail_msg: >- + Post-restore relabeling accepts only explicit paths below the Atlas pool + mount root. Do not relabel the whole pool during routine provisioning. + loop: "{{ atlas_restorecon_paths }}" + when: atlas_restorecon_paths | length > 0 + +- name: Restore SELinux labels on explicitly restored Atlas paths + tags: [atlas, restorecon, recovery] + ansible.builtin.command: + argv: + - restorecon + - -RFv + - "{{ item }}" + register: atlas_restorecon_result + changed_when: atlas_restorecon_result.stdout | length > 0 + loop: "{{ atlas_restorecon_paths }}" + when: + - atlas_restorecon_paths | length > 0 + - not ansible_check_mode diff --git a/ansible/roles/profile_atlas/tasks/usb_backup.yml b/ansible/roles/profile_atlas/tasks/usb_backup.yml index 69686b0..94b3dee 100644 --- a/ansible/roles/profile_atlas/tasks/usb_backup.yml +++ b/ansible/roles/profile_atlas/tasks/usb_backup.yml @@ -36,6 +36,16 @@ mode: "0750" when: atlas_manage_usb_backup | bool +- name: Install the Atlas USB snapshot cleanup helper + tags: [atlas, storage, backup, usb_backup] + ansible.builtin.template: + src: atlas-usb-snapshot-cleanup.sh.j2 + dest: /usr/local/sbin/atlas-usb-snapshot-cleanup + owner: root + group: root + mode: "0750" + when: atlas_manage_usb_backup | bool + - name: Install the manual Atlas offline USB backup service tags: [atlas, storage, backup, usb_backup] ansible.builtin.template: diff --git a/ansible/roles/profile_atlas/templates/atlas-usb-backup.service.j2 b/ansible/roles/profile_atlas/templates/atlas-usb-backup.service.j2 index 5b1e78a..337e770 100644 --- a/ansible/roles/profile_atlas/templates/atlas-usb-backup.service.j2 +++ b/ansible/roles/profile_atlas/templates/atlas-usb-backup.service.j2 @@ -7,6 +7,7 @@ ConditionFileIsExecutable=/usr/local/sbin/atlas-usb-backup [Service] Type=oneshot ExecStart=/usr/local/sbin/atlas-usb-backup +ExecStopPost=+/usr/local/sbin/atlas-usb-snapshot-cleanup User=root Group=root UMask=0077 diff --git a/ansible/roles/profile_atlas/templates/atlas-usb-backup.sh.j2 b/ansible/roles/profile_atlas/templates/atlas-usb-backup.sh.j2 index 1dbdd41..e8c216d 100644 --- a/ansible/roles/profile_atlas/templates/atlas-usb-backup.sh.j2 +++ b/ansible/roles/profile_atlas/templates/atlas-usb-backup.sh.j2 @@ -14,12 +14,12 @@ readonly min_free_bytes={{ atlas_usb_backup_min_free_bytes | int }} readonly mapper="/dev/mapper/${mapper_name}" readonly outer="/dev/disk/by-uuid/${luks_uuid}" readonly runtime_dir=/run/atlas-usb-backup +readonly snapshot_marker="${runtime_dir}/snapshot-name" readonly source_dir="${runtime_dir}/source" readonly usb_mount="${runtime_dir}/target" readonly backup_root="${usb_mount}/atlas" snapshot_name="" -snapshot_created=false mapper_opened_by_script=false usb_mounted=false published=false @@ -45,10 +45,14 @@ cleanup() { {% raw %} for ((index = ${#mounted_targets[@]} - 1; index >= 0; index--)); do {% endraw %} - if mountpoint -q "${mounted_targets[$index]}"; then - umount "${mounted_targets[$index]}" || source_mount_failed=true + if mountpoint -q "${mounted_targets[$index]}" && ! umount -R "${mounted_targets[$index]}"; then + printf 'Source snapshot mount cleanup failed: %s\n' "${mounted_targets[$index]}" >&2 + source_mount_failed=true fi - if ! mountpoint -q "${mounted_targets[$index]}"; then + if mountpoint -q "${mounted_targets[$index]}"; then + printf 'Source snapshot mount is still active: %s\n' "${mounted_targets[$index]}" >&2 + source_mount_failed=true + else rmdir -- "${mounted_targets[$index]}" 2>/dev/null || true fi done @@ -59,11 +63,6 @@ cleanup() { printf 'Source bind mount cleanup failed; keeping the snapshot for recovery\n' >&2 fi - if [[ "$snapshot_created" == true && "$source_mount_failed" == false ]]; then - flock 9 - zfs destroy -r "${pool}@${snapshot_name}" || cleanup_status=2 - flock -u 9 - fi if [[ "$usb_mounted" == true || "$mapper_opened_by_script" == true ]] && ! mountpoint -q "$usb_mount" && ! findmnt -rn -S "$mapper" >/dev/null; then @@ -148,8 +147,8 @@ mkdir -m 0700 "$source_dir" flock 9 timestamp="$(date -u +%Y%m%dT%H%M%SZ)" snapshot_name="${snapshot_prefix}-${timestamp}-$$" +printf '%s\n' "$snapshot_name" >"$snapshot_marker" zfs snapshot -r "${pool}@${snapshot_name}" -snapshot_created=true flock -u 9 printf 'Created recursive USB source snapshot %s@%s\n' "$pool" "$snapshot_name" @@ -189,10 +188,12 @@ mkdir -m 0700 "$candidate_partial" partial="$candidate_partial" printf 'Copying the consistent pool tree to USB backup %s\n' "$backup_name" -# Preserve ACLs and other xattrs, but let SELinux relabel restored data on the -# destination host instead of trying to write source security.selinux labels to USB. -rsync_args=(-aHAXS --numeric-ids '--filter=-x security.selinux' "--info=progress2,stats2") -estimate_args=(-aHAXS --numeric-ids '--filter=-x security.selinux' --dry-run --stats) +# Preserve POSIX ACLs, ownership, modes, timestamps, hard links, and sparse +# files. Do not preserve generic xattrs: Rocky 9's rsync 3.2.7 fails when +# combining xattrs with --link-dest, while SELinux labels were intentionally +# excluded because restores must relabel for their destination host. +rsync_args=(-aHAS --numeric-ids "--info=progress2,stats2") +estimate_args=(-aHAS --numeric-ids --dry-run --stats) if [[ -n "$previous" ]]; then rsync_args+=("--link-dest=$previous") estimate_args+=("--link-dest=$previous") @@ -217,7 +218,7 @@ rsync "${rsync_args[@]}" "${source_dir}/" "${partial}/" printf 'Verifying USB backup %s with a checksum-based dry run\n' "$backup_name" verification="${runtime_dir}/verification.out" -rsync -aHAXS --numeric-ids --filter='-x security.selinux' \ +rsync -aHAS --numeric-ids \ --checksum --dry-run --delete --itemize-changes \ "${source_dir}/" "${partial}/" >"$verification" if [[ -s "$verification" ]]; then diff --git a/ansible/roles/profile_atlas/templates/atlas-usb-snapshot-cleanup.sh.j2 b/ansible/roles/profile_atlas/templates/atlas-usb-snapshot-cleanup.sh.j2 new file mode 100644 index 0000000..2f50bbc --- /dev/null +++ b/ansible/roles/profile_atlas/templates/atlas-usb-snapshot-cleanup.sh.j2 @@ -0,0 +1,51 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +export PATH=/usr/sbin:/usr/bin:/sbin:/bin + +readonly pool={{ atlas_zfs_pool | quote }} +readonly mount_root={{ atlas_mount_root | quote }} +readonly snapshot_prefix={{ atlas_usb_backup_snapshot_prefix | quote }} +readonly marker=/run/atlas-usb-backup/snapshot-name + +[[ -e "$marker" ]] || exit 0 +[[ -f "$marker" && ! -L "$marker" ]] || { + printf 'Unsafe Atlas USB snapshot marker; leaving snapshots unchanged\n' >&2 + exit 2 +} +IFS= read -r snapshot_name <"$marker" +[[ "$snapshot_name" =~ ^${snapshot_prefix}-[0-9]{8}T[0-9]{6}Z-[0-9]+$ ]] || { + printf 'Invalid Atlas USB snapshot marker; leaving snapshots unchanged\n' >&2 + exit 2 +} + +exec 9>/run/lock/atlas-zfs-snapshot.lock +flock 9 +if zfs list -H -t snapshot -o name "${pool}@${snapshot_name}" >/dev/null 2>&1; then + # ZFS can leave its on-demand .zfs/snapshot mounts in the host namespace + # even after the backup's private bind mounts and process have exited. + snapshot_mounts=() + snapshot_sources=() + while IFS=$'\t' read -r dataset dataset_mountpoint; do + [[ "$dataset_mountpoint" == "$mount_root" || "$dataset_mountpoint" == "$mount_root/"* ]] || continue + snapshot_mounts+=("${dataset_mountpoint}/.zfs/snapshot/${snapshot_name}") + snapshot_sources+=("${dataset}@${snapshot_name}") + done < <(zfs list -H -o name,mountpoint -s name -r "$pool") + +{% raw %} + for ((index = ${#snapshot_mounts[@]} - 1; index >= 0; index--)); do +{% endraw %} + mounted_source="$(findmnt -rn -M "${snapshot_mounts[$index]}" -o SOURCE || true)" + [[ -n "$mounted_source" ]] || continue + [[ "$mounted_source" == "${snapshot_sources[$index]}" ]] || { + printf 'Unexpected source on Atlas USB snapshot mount: %s\n' \ + "${snapshot_mounts[$index]}" >&2 + exit 2 + } + umount "${snapshot_mounts[$index]}" + done + + zfs destroy -r "${pool}@${snapshot_name}" + printf 'Removed recursive Atlas USB source snapshot %s@%s after backup exit\n' \ + "$pool" "$snapshot_name" +fi