From 361ee77d7222fcad3e8f14858449d07ba5c139be Mon Sep 17 00:00:00 2001 From: Fabio Scotto di Santolo Date: Thu, 24 Sep 2026 08:55:52 +0200 Subject: [PATCH] Add manual Atlas USB backup and reminder services --- AGENTS.md | 11 +- README.md | 49 ++++ ansible/inventory/host_vars/atlas.yml | 7 + ansible/roles/profile_atlas/defaults/main.yml | 11 + ansible/roles/profile_atlas/tasks/main.yml | 3 + .../roles/profile_atlas/tasks/usb_backup.yml | 135 ++++++++++ .../templates/atlas-usb-backup.service.j2 | 29 +++ .../templates/atlas-usb-backup.sh.j2 | 241 ++++++++++++++++++ .../templates/atlas-usb-reminder.py.j2 | 31 +++ .../templates/atlas-usb-reminder.service.j2 | 22 ++ .../templates/atlas-usb-reminder.timer.j2 | 10 + 11 files changed, 548 insertions(+), 1 deletion(-) create mode 100644 ansible/roles/profile_atlas/tasks/usb_backup.yml create mode 100644 ansible/roles/profile_atlas/templates/atlas-usb-backup.service.j2 create mode 100644 ansible/roles/profile_atlas/templates/atlas-usb-backup.sh.j2 create mode 100644 ansible/roles/profile_atlas/templates/atlas-usb-reminder.py.j2 create mode 100644 ansible/roles/profile_atlas/templates/atlas-usb-reminder.service.j2 create mode 100644 ansible/roles/profile_atlas/templates/atlas-usb-reminder.timer.j2 diff --git a/AGENTS.md b/AGENTS.md index 99761d9..412d2be 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -65,6 +65,8 @@ Ansible-driven personal infrastructure repo for Fedora and Void desktops, Fedora `ansible-playbook ansible/site.yml --limit atlas --tags packages,borg --check --diff` - Atlas Borg progress logging only: `ansible-playbook ansible/site.yml --limit atlas --tags borg_logging --check --diff` + - Atlas manual offline USB backup and 45Drives Alerts reminder: + `ansible-playbook ansible/site.yml --limit atlas --tags usb_backup,usb_reminder --check --diff` - Prometheus/Aegis WireGuard gateway: `ansible-playbook ansible/site.yml --limit prometheus,aegis --tags wireguard --check --diff` - DuckDNS config only: `ansible-playbook ansible/site.yml --limit prometheus --tags duckdns --check --diff` @@ -194,7 +196,14 @@ scheduled retention prune and monthly scrub remain runtime checks. - [ ] Run and evaluate Borg against the populated pool: duration, repository capacity, deduplication, and a subsequent incremental archive must be observed before relying on the offline USB test. - [ ] Add the UUID-bound offline USB backup with versioned rsync, locking, capacity checks, verification, - safe unmounting and a tested restore procedure; never trigger it for an arbitrary USB disk. + safe unmounting and a tested restore procedure; never trigger it for an arbitrary USB disk. The + LUKS/ext4 identities were read-only verified; the manual service and 45Drives Alerts reminder timer were + deployed on Atlas. Interactive LUKS unlock is part of the manual service; only the reminder is + scheduled for the first Saturday of each month at 10:00 Europe/Rome via the existing 45Drives + notifier. A manual test produced an Alerts notification, not an email. The first USB attempt failed + on a `security.selinux` xattr and was interrupted; the xattr filter is deployed and the temporary + recursive snapshot, open LUKS mapper, and failed service state were cleaned up. No complete backup or + tested USB restore exists; do not mark this item complete yet. - [ ] Test restores independently from a ZFS snapshot, Borg, and the offline USB backup before relying on any backup path. - [ ] Add monitoring and alerting for pool health, scrub/resilver, SMART data, temperatures, free space, diff --git a/README.md b/README.md index 32e07c3..3bc2afc 100644 --- a/README.md +++ b/README.md @@ -378,6 +378,55 @@ Atlas runtime activation is complete: the initial backup and repository check su to a temporary directory was validated against the live `Archive` tree, the recovery-key export was copied to offline storage, and the temporary snapshot and bind mounts were cleaned up. +The offline USB backup is deployed as a manual-only service (`atlas_manage_usb_backup: true`): +Ansible never formats, unlocks, mounts, backs up to, or schedules the disk. Atlas' existing USB disk was verified +read-only on 2026-09-23 as LUKS UUID `577b3c43-ea37-4611-81a9-39d555cdfbd4`, containing ext4 UUID +`758e2d2e-a427-4797-aad9-39c3a9f17c7e` through mapper `zpool-backup`. It was mounted at +`/mnt/zpool-backup` at inspection time. The service deliberately requires the verified mapper to be +**not mounted** before starting. When necessary, `systemd-ask-password` requests the LUKS passphrase +through the `systemctl start` password agent; it is piped directly to `cryptsetup` without saving it, +passing it as a command argument, or caching it. The service then mounts the disk privately, takes a recursive ZFS snapshot, +copies every dataset to a versioned `atlas/snapshots//` directory using `rsync --link-dest`, +verifies the result with a checksum-based dry run, atomically updates `atlas/latest`, unmounts and closes +LUKS. A failed run never replaces `latest` or removes an earlier complete version. Borg and the USB +backup may run concurrently from separate snapshots; both reading the same pool can reduce throughput. +The USB copy preserves ACLs and extended attributes except `security.selinux`, which the target +SELinux policy must recreate during a restore; do not restore data into service paths without relabeling. +Old USB versions are not pruned automatically, to avoid deleting the only offline +copy without an explicitly chosen retention policy; capacity checks include an estimated transfer size +and a 10 GiB free-space reserve. The disk must be physically disconnected after a successful backup +to make the copy offline. + +To check the USB backup and reminder configuration without starting a backup, run: + +```bash +ANSIBLE_LOCAL_TEMP=/tmp/ansible-local \ +ansible-playbook ansible/site.yml --limit atlas --tags usb_backup,usb_reminder --check --diff +``` + +Before the first **manual** service start, safely unmount the currently mounted +`/mnt/zpool-backup`; never run it on an arbitrary mounted disk. Future starts +can begin with the mapper closed: `sudo systemctl start atlas-usb-backup.service` prompts for the +passphrase interactively and then performs the backup. Neither the LUKS password nor a key file belongs +in Ansible. Inspect the run with +`sudo journalctl -fu atlas-usb-backup.service`. There is intentionally no timer. Independently test a +read-only mount and restore from `atlas/latest` into an empty temporary directory before marking the +USB recovery path complete. Only `atlas-usb-reminder.timer` is enabled, for the first Saturday of each +month at 10:00 Europe/Rome. Its warning notification uses the existing 45Drives Houston notifier. +A manual test confirmed a notification in 45Drives Alerts, **not** an email. The reminder service log +reports notification submission, not email delivery; the role does not depend on SMTP/OAuth settings. +The reminder never starts the backup. Check its schedule with +`systemctl list-timers atlas-usb-reminder.timer` and the result in 45Drives Alerts. +The timer was verified active with its first scheduled run at 2026-10-03 10:00 CEST. The USB backup +service was verified inactive after deployment; no successful backup or email delivery is claimed. +The first manual USB attempt on 2026-09-23 did not complete: rsync was denied while removing +`security.selinux` on the USB filesystem, then the interrupted service left its recursive +`atlas-usb-20260923T185748Z-2469168` snapshot and the `zpool-backup` LUKS mapper open. The +rsync xattr filter was deployed afterward. The incomplete USB directory was absent on inspection; +the exact failed snapshot was removed, the verified and unmounted mapper closed, and the service +failed state cleared. A final check found no remnant snapshot, mount, mapper, or staging directory. +The failed attempt is not a valid backup, and no USB restore has been tested. + A temporary Nextcloud deployment on Atlas is also planned before Uranus: it requires separately declared persistent application, database, and cache storage, Vault-backed credentials, NPM-only publishing through Aegis, and defined backup, upgrade, and eventual migration procedures. Do not deploy diff --git a/ansible/inventory/host_vars/atlas.yml b/ansible/inventory/host_vars/atlas.yml index 2f7dc8a..58375d5 100644 --- a/ansible/inventory/host_vars/atlas.yml +++ b/ansible/inventory/host_vars/atlas.yml @@ -83,6 +83,13 @@ atlas_borg_randomized_delay: 30m atlas_borg_keep_daily: 30 atlas_borg_keep_weekly: 8 atlas_borg_keep_monthly: 12 +atlas_manage_usb_backup: true +# Read-only lsblk verification on Atlas, 2026-09-23. Never store the LUKS password here. +atlas_usb_backup_luks_uuid: 577b3c43-ea37-4611-81a9-39d555cdfbd4 +atlas_usb_backup_fs_uuid: 758e2d2e-a427-4797-aad9-39c3a9f17c7e +atlas_usb_backup_mapper_name: zpool-backup +atlas_manage_usb_reminder: true +atlas_usb_reminder_calendar: "Sat *-*-01..07 10:00:00 Europe/Rome" atlas_manage_sharing: true atlas_manage_media_stack: false diff --git a/ansible/roles/profile_atlas/defaults/main.yml b/ansible/roles/profile_atlas/defaults/main.yml index 56408d3..287ff50 100644 --- a/ansible/roles/profile_atlas/defaults/main.yml +++ b/ansible/roles/profile_atlas/defaults/main.yml @@ -97,6 +97,17 @@ atlas_borg_cache_dir: /var/cache/atlas-borg atlas_borg_lock_path: /var/lib/atlas-borg/backup.lock atlas_borg_recovery_export_path: "{{ playbook_dir }}/../secrets/recovery/atlas-borg-repokey.export" +# Manual-only offline backup. No USB device is formatted or mounted by Ansible. +atlas_manage_usb_backup: false +atlas_usb_backup_luks_uuid: "" +atlas_usb_backup_fs_uuid: "" +atlas_usb_backup_mapper_name: atlas-usb-backup +atlas_usb_backup_min_free_bytes: 10737418240 +atlas_usb_backup_snapshot_prefix: atlas-usb +atlas_manage_usb_reminder: false +atlas_usb_reminder_calendar: "" +atlas_usb_reminder_notifier: /opt/45drives/houston/houston-notify + atlas_archive_mountpoint: "{{ atlas_mount_root }}/{{ atlas_zfs_dataset_archive }}" atlas_services_mountpoint: "{{ atlas_mount_root }}/{{ atlas_zfs_dataset_services }}" atlas_app_data_mountpoint: "{{ atlas_mount_root }}/{{ atlas_zfs_dataset_app_data }}" diff --git a/ansible/roles/profile_atlas/tasks/main.yml b/ansible/roles/profile_atlas/tasks/main.yml index d17119b..86f6277 100644 --- a/ansible/roles/profile_atlas/tasks/main.yml +++ b/ansible/roles/profile_atlas/tasks/main.yml @@ -20,6 +20,9 @@ - name: Import Atlas Borg backup tasks ansible.builtin.import_tasks: borg_backup.yml +- name: Import Atlas offline USB backup tasks + ansible.builtin.import_tasks: usb_backup.yml + - name: Import Atlas file sharing tasks ansible.builtin.import_tasks: sharing.yml diff --git a/ansible/roles/profile_atlas/tasks/usb_backup.yml b/ansible/roles/profile_atlas/tasks/usb_backup.yml new file mode 100644 index 0000000..69686b0 --- /dev/null +++ b/ansible/roles/profile_atlas/tasks/usb_backup.yml @@ -0,0 +1,135 @@ +--- +- name: Validate Atlas offline USB backup configuration + tags: [atlas, storage, backup, usb_backup] + ansible.builtin.assert: + that: + - atlas_manage_storage | bool + - atlas_zfs_pool != 'CHANGEME_ZFS_POOL' + - atlas_mount_root.startswith('/') + - atlas_usb_backup_luks_uuid is match('^[0-9a-fA-F]{8}(-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12}$') + - atlas_usb_backup_fs_uuid is match('^[0-9a-fA-F]{8}(-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12}$') + - atlas_usb_backup_luks_uuid != atlas_usb_backup_fs_uuid + - atlas_usb_backup_mapper_name is match('^[a-z][a-z0-9_-]*$') + - atlas_usb_backup_min_free_bytes | int > 0 + - atlas_usb_backup_snapshot_prefix is match('^[a-z0-9][a-z0-9_-]*$') + - atlas_usb_backup_snapshot_prefix != atlas_borg_snapshot_prefix + - atlas_usb_backup_snapshot_prefix != atlas_zfs_snapshot_prefix + fail_msg: >- + The manual Atlas USB backup needs verified LUKS and ext4 UUIDs, a safe + mapper name, positive free-space reserve, and a unique snapshot prefix. + when: atlas_manage_usb_backup | bool + +- name: Install rsync for the Atlas offline USB backup + tags: [atlas, storage, backup, usb_backup] + ansible.builtin.dnf: + name: rsync + state: present + when: atlas_manage_usb_backup | bool + +- name: Install the manual Atlas offline USB backup helper + tags: [atlas, storage, backup, usb_backup] + ansible.builtin.template: + src: atlas-usb-backup.sh.j2 + dest: /usr/local/sbin/atlas-usb-backup + owner: root + group: root + mode: "0750" + when: atlas_manage_usb_backup | bool + +- name: Install the manual Atlas offline USB backup service + tags: [atlas, storage, backup, usb_backup] + ansible.builtin.template: + src: atlas-usb-backup.service.j2 + dest: /etc/systemd/system/atlas-usb-backup.service + owner: root + group: root + mode: "0644" + when: atlas_manage_usb_backup | bool + +- name: Reload systemd for the Atlas offline USB backup service + tags: [atlas, storage, backup, usb_backup] + ansible.builtin.systemd: + daemon_reload: true + when: + - atlas_manage_usb_backup | bool + - not ansible_check_mode + +- name: Validate the 45Drives Atlas USB reminder configuration + tags: [atlas, backup, usb_reminder] + ansible.builtin.assert: + that: + - atlas_manage_usb_backup | bool + - atlas_usb_reminder_calendar | length > 0 + - atlas_usb_reminder_notifier.startswith('/opt/45drives/houston/') + fail_msg: >- + Enable the manual USB backup and declare a systemd calendar before + enabling its 45Drives Alerts reminder. + when: atlas_manage_usb_reminder | bool + +- name: Validate the Atlas USB reminder calendar + tags: [atlas, backup, usb_reminder] + ansible.builtin.command: + argv: + - systemd-analyze + - calendar + - "{{ atlas_usb_reminder_calendar }}" + changed_when: false + check_mode: false + when: atlas_manage_usb_reminder | bool + +- name: Inspect the existing 45Drives notifier + tags: [atlas, backup, usb_reminder] + ansible.builtin.stat: + path: "{{ atlas_usb_reminder_notifier }}" + register: atlas_usb_reminder_notifier_file + when: atlas_manage_usb_reminder | bool + +- name: Require the configured 45Drives notifier for USB reminders + tags: [atlas, backup, usb_reminder] + ansible.builtin.assert: + that: + - atlas_usb_reminder_notifier_file.stat.executable | default(false) + fail_msg: >- + The existing 45Drives Houston notifier must be executable. + when: atlas_manage_usb_reminder | bool + +- name: Install the 45Drives Atlas USB reminder helper + tags: [atlas, backup, usb_reminder] + ansible.builtin.template: + src: atlas-usb-reminder.py.j2 + dest: /usr/local/libexec/atlas-usb-reminder + owner: root + group: root + mode: "0750" + when: atlas_manage_usb_reminder | bool + +- name: Install the 45Drives Atlas USB reminder service + tags: [atlas, backup, usb_reminder] + ansible.builtin.template: + src: atlas-usb-reminder.service.j2 + dest: /etc/systemd/system/atlas-usb-reminder.service + owner: root + group: root + mode: "0644" + when: atlas_manage_usb_reminder | bool + +- name: Install the 45Drives Atlas USB reminder timer + tags: [atlas, backup, usb_reminder] + ansible.builtin.template: + src: atlas-usb-reminder.timer.j2 + dest: /etc/systemd/system/atlas-usb-reminder.timer + owner: root + group: root + mode: "0644" + when: atlas_manage_usb_reminder | bool + +- name: Enable only the Atlas USB notification reminder timer + tags: [atlas, backup, usb_reminder] + ansible.builtin.systemd: + name: atlas-usb-reminder.timer + enabled: true + state: started + daemon_reload: true + when: + - atlas_manage_usb_reminder | bool + - not ansible_check_mode diff --git a/ansible/roles/profile_atlas/templates/atlas-usb-backup.service.j2 b/ansible/roles/profile_atlas/templates/atlas-usb-backup.service.j2 new file mode 100644 index 0000000..5b1e78a --- /dev/null +++ b/ansible/roles/profile_atlas/templates/atlas-usb-backup.service.j2 @@ -0,0 +1,29 @@ +[Unit] +Description=Run a manual, UUID-bound offline USB backup of Atlas ZFS datasets +Requires=zfs.target +After=zfs.target +ConditionFileIsExecutable=/usr/local/sbin/atlas-usb-backup + +[Service] +Type=oneshot +ExecStart=/usr/local/sbin/atlas-usb-backup +User=root +Group=root +UMask=0077 +TimeoutStartSec=infinity +RuntimeDirectory=atlas-usb-backup +RuntimeDirectoryMode=0700 +Nice=15 +IOSchedulingClass=best-effort +IOSchedulingPriority=7 +PrivateMounts=true +PrivateTmp=true +ProtectHome=true +ProtectSystem=strict +ReadWritePaths=/run/atlas-usb-backup /run/lock +ProtectKernelTunables=true +ProtectKernelModules=true +ProtectControlGroups=true +RestrictAddressFamilies=AF_UNIX +RestrictRealtime=true +LockPersonality=true diff --git a/ansible/roles/profile_atlas/templates/atlas-usb-backup.sh.j2 b/ansible/roles/profile_atlas/templates/atlas-usb-backup.sh.j2 new file mode 100644 index 0000000..1dbdd41 --- /dev/null +++ b/ansible/roles/profile_atlas/templates/atlas-usb-backup.sh.j2 @@ -0,0 +1,241 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +export LC_ALL=C.utf8 +export PATH=/usr/sbin:/usr/bin:/sbin:/bin + +readonly pool={{ atlas_zfs_pool | quote }} +readonly mount_root={{ atlas_mount_root | quote }} +readonly luks_uuid={{ atlas_usb_backup_luks_uuid | quote }} +readonly fs_uuid={{ atlas_usb_backup_fs_uuid | quote }} +readonly mapper_name={{ atlas_usb_backup_mapper_name | quote }} +readonly snapshot_prefix={{ atlas_usb_backup_snapshot_prefix | quote }} +readonly min_free_bytes={{ atlas_usb_backup_min_free_bytes | int }} +readonly mapper="/dev/mapper/${mapper_name}" +readonly outer="/dev/disk/by-uuid/${luks_uuid}" +readonly runtime_dir=/run/atlas-usb-backup +readonly source_dir="${runtime_dir}/source" +readonly usb_mount="${runtime_dir}/target" +readonly backup_root="${usb_mount}/atlas" + +snapshot_name="" +snapshot_created=false +mapper_opened_by_script=false +usb_mounted=false +published=false +partial="" +mounted_targets=() + +# shellcheck disable=SC2329 +cleanup() { + local status=$? + local cleanup_status=0 + local index + local source_mount_failed=false + trap - EXIT HUP INT TERM + set +e + + if [[ -n "$partial" && "$published" == false && "$usb_mounted" == true ]]; then + rm -rf -- "$partial" || cleanup_status=2 + fi + if [[ "$usb_mounted" == true ]]; then + umount "$usb_mount" || cleanup_status=2 + fi + +{% raw %} + for ((index = ${#mounted_targets[@]} - 1; index >= 0; index--)); do +{% endraw %} + if mountpoint -q "${mounted_targets[$index]}"; then + umount "${mounted_targets[$index]}" || source_mount_failed=true + fi + if ! mountpoint -q "${mounted_targets[$index]}"; then + rmdir -- "${mounted_targets[$index]}" 2>/dev/null || true + fi + done + if [[ "$source_mount_failed" == false ]]; then + rmdir -- "$source_dir" 2>/dev/null || true + else + cleanup_status=2 + printf 'Source bind mount cleanup failed; keeping the snapshot for recovery\n' >&2 + fi + + if [[ "$snapshot_created" == true && "$source_mount_failed" == false ]]; then + flock 9 + zfs destroy -r "${pool}@${snapshot_name}" || cleanup_status=2 + flock -u 9 + fi + if [[ "$usb_mounted" == true || "$mapper_opened_by_script" == true ]] && + ! mountpoint -q "$usb_mount" && + ! findmnt -rn -S "$mapper" >/dev/null; then + cryptsetup close "$mapper_name" || cleanup_status=2 + fi + rmdir -- "$usb_mount" 2>/dev/null || true + + if ((status == 0 && cleanup_status != 0)); then + status=$cleanup_status + fi + exit "$status" +} + +trap cleanup EXIT +trap 'exit 143' HUP INT TERM + +exec 8>/run/lock/atlas-usb-backup.lock +flock -n 8 || { printf 'Atlas USB backup is already running\n' >&2; exit 75; } +exec 9>/run/lock/atlas-zfs-snapshot.lock + +zpool list -H -o name "$pool" >/dev/null +[[ -b "$outer" ]] || { printf 'Configured LUKS UUID is not connected\n' >&2; exit 66; } +[[ "$(blkid -s TYPE -o value "$outer")" == crypto_LUKS ]] || { + printf 'Configured outer UUID is not a LUKS container\n' >&2 + exit 65 +} +[[ "$(blkid -s UUID -o value "$outer")" == "$luks_uuid" ]] || exit 65 +if ! cryptsetup status "$mapper_name" >/dev/null; then + printf 'Requesting the LUKS passphrase for the configured USB disk\n' + systemd-ask-password -n --no-tty --timeout=300 \ + --id="atlas-usb-backup:${luks_uuid}" \ + 'Atlas offline USB backup LUKS passphrase:' | + cryptsetup open --type luks2 --key-file - "$outer" "$mapper_name" + mapper_opened_by_script=true +fi +backing_device="$(cryptsetup status "$mapper_name" | awk '$1 == "device:" { print $2 }')" +[[ -n "$backing_device" && "$(readlink -f "$backing_device")" == "$(readlink -f "$outer")" ]] || { + printf 'The unlocked mapper does not belong to the configured LUKS UUID\n' >&2 + exit 65 +} +[[ "$(blkid -s TYPE -o value "$mapper")" == ext4 ]] || { + printf 'The unlocked USB filesystem is not ext4\n' >&2 + exit 65 +} +[[ "$(blkid -s UUID -o value "$mapper")" == "$fs_uuid" ]] || { + printf 'The unlocked USB filesystem UUID does not match\n' >&2 + exit 65 +} +if findmnt -rn -S "$mapper" >/dev/null; then + printf 'The USB filesystem is already mounted elsewhere\n' >&2 + exit 65 +fi +[[ ! -e "$source_dir" && ! -e "$usb_mount" ]] || { + printf 'USB backup staging directories already exist; inspect them manually\n' >&2 + exit 65 +} + +mkdir -m 0700 "$usb_mount" +mount -t ext4 -o nodev,nosuid,noexec "$mapper" "$usb_mount" +usb_mounted=true +[[ "$(readlink -f "$(findmnt -nro SOURCE --target "$usb_mount")")" == "$(readlink -f "$mapper")" ]] || { + printf 'Mounted USB source does not match the verified mapper\n' >&2 + exit 65 +} + +for path in "$backup_root" "$backup_root/snapshots"; do + [[ ! -L "$path" ]] || { printf 'Unsafe symlink in USB backup destination\n' >&2; exit 65; } + mkdir -p -- "$path" + [[ -d "$path" ]] || exit 65 + chown root:root -- "$path" + chmod 0700 -- "$path" +done + +free_bytes="$(df -B1 --output=avail "$usb_mount" | tail -n 1 | tr -d ' ')" +if ((free_bytes < min_free_bytes)); then + printf 'USB free space (%s bytes) is below the required reserve (%s bytes)\n' \ + "$free_bytes" "$min_free_bytes" >&2 + exit 73 +fi + +mkdir -m 0700 "$source_dir" +flock 9 +timestamp="$(date -u +%Y%m%dT%H%M%SZ)" +snapshot_name="${snapshot_prefix}-${timestamp}-$$" +zfs snapshot -r "${pool}@${snapshot_name}" +snapshot_created=true +flock -u 9 +printf 'Created recursive USB source snapshot %s@%s\n' "$pool" "$snapshot_name" + +while IFS=$'\t' read -r dataset dataset_mountpoint mounted; do + if [[ "$mounted" != yes ]]; then + printf 'Dataset %s is not mounted; refusing an incomplete backup\n' "$dataset" >&2 + exit 65 + fi + if [[ "$dataset_mountpoint" != "$mount_root" && "$dataset_mountpoint" != "$mount_root/"* ]]; then + printf 'Dataset %s has unexpected mountpoint %s\n' "$dataset" "$dataset_mountpoint" >&2 + exit 65 + fi + dataset_suffix="${dataset#"$pool"}" + source_path="${dataset_mountpoint}/.zfs/snapshot/${snapshot_name}" + target_path="${source_dir}${dataset_suffix}" + mkdir -p "$target_path" + mount --bind "$source_path" "$target_path" + mounted_targets+=("$target_path") + mount -o remount,bind,ro "$target_path" +done < <(zfs list -H -o name,mountpoint,mounted -s name -r "$pool") + +previous="" +if [[ -e "$backup_root/latest" || -L "$backup_root/latest" ]]; then + [[ -L "$backup_root/latest" ]] || { printf 'latest is not a symlink\n' >&2; exit 65; } + previous="$(readlink -e "$backup_root/latest")" + [[ -n "$previous" && "$previous" == "$backup_root/snapshots/"* && -d "$previous" ]] || { + printf 'latest does not point to a complete snapshot on the USB disk\n' >&2 + exit 65 + } +fi + +backup_name="${timestamp}-$$" +candidate_partial="${backup_root}/snapshots/.incomplete-${backup_name}" +complete="${backup_root}/snapshots/${backup_name}" +[[ ! -e "$candidate_partial" && ! -L "$candidate_partial" && ! -e "$complete" && ! -L "$complete" ]] || exit 65 +mkdir -m 0700 "$candidate_partial" +partial="$candidate_partial" + +printf 'Copying the consistent pool tree to USB backup %s\n' "$backup_name" +# Preserve ACLs and other xattrs, but let SELinux relabel restored data on the +# destination host instead of trying to write source security.selinux labels to USB. +rsync_args=(-aHAXS --numeric-ids '--filter=-x security.selinux' "--info=progress2,stats2") +estimate_args=(-aHAXS --numeric-ids '--filter=-x security.selinux' --dry-run --stats) +if [[ -n "$previous" ]]; then + rsync_args+=("--link-dest=$previous") + estimate_args+=("--link-dest=$previous") +fi + +# The rsync dry run estimates changed file bytes after link-dest deduplication. +# Metadata and filesystem allocation still require the separate free-space reserve. +estimate="$(rsync "${estimate_args[@]}" "${source_dir}/" "${partial}/")" +transfer_bytes="$(printf '%s\n' "$estimate" | awk -F: \ + '/^Total transferred file size:/ { gsub(/[^0-9]/, "", $2); print $2 }')" +[[ "$transfer_bytes" =~ ^[0-9]+$ ]] || { + printf 'Could not determine the USB transfer size\n' >&2 + exit 74 +} +if ((free_bytes - transfer_bytes < min_free_bytes)); then + printf 'Insufficient USB space: %s bytes free, %s estimated transfer, %s reserved\n' \ + "$free_bytes" "$transfer_bytes" "$min_free_bytes" >&2 + exit 73 +fi + +rsync "${rsync_args[@]}" "${source_dir}/" "${partial}/" + +printf 'Verifying USB backup %s with a checksum-based dry run\n' "$backup_name" +verification="${runtime_dir}/verification.out" +rsync -aHAXS --numeric-ids --filter='-x security.selinux' \ + --checksum --dry-run --delete --itemize-changes \ + "${source_dir}/" "${partial}/" >"$verification" +if [[ -s "$verification" ]]; then + printf 'USB verification found mismatches; refusing to publish the backup\n' >&2 + exit 74 +fi + +free_bytes="$(df -B1 --output=avail "$usb_mount" | tail -n 1 | tr -d ' ')" +if ((free_bytes < min_free_bytes)); then + printf 'USB backup completed below the free-space reserve; refusing to publish it\n' >&2 + exit 73 +fi + +mv -- "$partial" "$complete" +partial="" +ln -s "snapshots/${backup_name}" "${backup_root}/.latest-${backup_name}" +mv -Tf -- "${backup_root}/.latest-${backup_name}" "${backup_root}/latest" +published=true +sync -f "$complete" +sync -f "$backup_root" +printf 'USB backup %s verified and published; unmounting and closing LUKS\n' "$backup_name" diff --git a/ansible/roles/profile_atlas/templates/atlas-usb-reminder.py.j2 b/ansible/roles/profile_atlas/templates/atlas-usb-reminder.py.j2 new file mode 100644 index 0000000..fdb8335 --- /dev/null +++ b/ansible/roles/profile_atlas/templates/atlas-usb-reminder.py.j2 @@ -0,0 +1,31 @@ +#!/usr/bin/python3 +"""Submit a manual-backup reminder through Atlas' existing Houston notifier.""" + +import json +import subprocess +from datetime import datetime, timezone + + +now = datetime.now(timezone.utc) +message = { + "timestamp": now.isoformat(timespec="seconds"), + "unixtime": int(now.timestamp()), + "event": "atlas_usb_backup_reminder", + "severity": "warning", + "subject": "Promemoria backup USB offline Atlas", + "email_message": ( + "Collega il disco USB di backup ad Atlas ed esegui manualmente il backup offline.\n" + "Il promemoria non avvia il backup. Controlla che il disco non sia\n" + "montato; poi esegui:\n\n" + " sudo systemctl start atlas-usb-backup.service\n\n" + "Verifica l'esito con:\n" + " sudo journalctl -u atlas-usb-backup.service -n 100 --no-pager\n\n" + "Dopo la riuscita, scollega fisicamente il disco." + ), +} + +subprocess.run( + [{{ atlas_usb_reminder_notifier | to_json }}, json.dumps(message)], + check=True, +) +print("Atlas USB backup reminder submitted to 45Drives Alerts; email delivery is not verified.", flush=True) diff --git a/ansible/roles/profile_atlas/templates/atlas-usb-reminder.service.j2 b/ansible/roles/profile_atlas/templates/atlas-usb-reminder.service.j2 new file mode 100644 index 0000000..221fafa --- /dev/null +++ b/ansible/roles/profile_atlas/templates/atlas-usb-reminder.service.j2 @@ -0,0 +1,22 @@ +[Unit] +Description=45Drives Alerts reminder to run the manual Atlas offline USB backup +Requires=houston-dbus.service +After=houston-dbus.service +ConditionFileIsExecutable=/usr/local/libexec/atlas-usb-reminder + +[Service] +Type=oneshot +ExecStart=/usr/local/libexec/atlas-usb-reminder +User=root +Group=root +UMask=0077 +NoNewPrivileges=true +PrivateTmp=true +ProtectHome=true +ProtectSystem=strict +ProtectKernelTunables=true +ProtectKernelModules=true +ProtectControlGroups=true +RestrictAddressFamilies=AF_UNIX +RestrictRealtime=true +LockPersonality=true diff --git a/ansible/roles/profile_atlas/templates/atlas-usb-reminder.timer.j2 b/ansible/roles/profile_atlas/templates/atlas-usb-reminder.timer.j2 new file mode 100644 index 0000000..8051de2 --- /dev/null +++ b/ansible/roles/profile_atlas/templates/atlas-usb-reminder.timer.j2 @@ -0,0 +1,10 @@ +[Unit] +Description=Remind the administrator to run the manual Atlas offline USB backup + +[Timer] +OnCalendar={{ atlas_usb_reminder_calendar }} +Persistent=true +Unit=atlas-usb-reminder.service + +[Install] +WantedBy=timers.target