Harden Atlas USB backup cleanup and document restorecon workflow

ansible/roles/profile_atlas/defaults/main.yml

@@ -108,6 +108,9 @@ atlas_manage_usb_reminder: false
 atlas_usb_reminder_calendar: ""
 atlas_usb_reminder_notifier: /opt/45drives/houston/houston-notify
 
+# Explicit post-restore relabeling only; never relabel datasets during ordinary runs.
+atlas_restorecon_paths: []
+
 atlas_archive_mountpoint: "{{ atlas_mount_root }}/{{ atlas_zfs_dataset_archive }}"
 atlas_services_mountpoint: "{{ atlas_mount_root }}/{{ atlas_zfs_dataset_services }}"
 atlas_app_data_mountpoint: "{{ atlas_mount_root }}/{{ atlas_zfs_dataset_app_data }}"

ansible/roles/profile_atlas/tasks/main.yml

@@ -23,6 +23,9 @@
 - name: Import Atlas offline USB backup tasks
   ansible.builtin.import_tasks: usb_backup.yml
 
+- name: Import Atlas post-restore SELinux relabeling tasks
+  ansible.builtin.import_tasks: restorecon.yml
+
 - name: Import Atlas file sharing tasks
   ansible.builtin.import_tasks: sharing.yml
 

ansible/roles/profile_atlas/tasks/restorecon.yml

@@ -0,0 +1,27 @@
+---
+- name: Validate requested Atlas post-restore relabel paths
+  tags: [atlas, restorecon, recovery]
+  ansible.builtin.assert:
+    that:
+      - item is string
+      - item.startswith(atlas_mount_root ~ '/')
+      - item != atlas_mount_root
+    fail_msg: >-
+      Post-restore relabeling accepts only explicit paths below the Atlas pool
+      mount root. Do not relabel the whole pool during routine provisioning.
+  loop: "{{ atlas_restorecon_paths }}"
+  when: atlas_restorecon_paths | length > 0
+
+- name: Restore SELinux labels on explicitly restored Atlas paths
+  tags: [atlas, restorecon, recovery]
+  ansible.builtin.command:
+    argv:
+      - restorecon
+      - -RFv
+      - "{{ item }}"
+  register: atlas_restorecon_result
+  changed_when: atlas_restorecon_result.stdout | length > 0
+  loop: "{{ atlas_restorecon_paths }}"
+  when:
+    - atlas_restorecon_paths | length > 0
+    - not ansible_check_mode

ansible/roles/profile_atlas/tasks/usb_backup.yml

@@ -36,6 +36,16 @@
     mode: "0750"
   when: atlas_manage_usb_backup | bool
 
+- name: Install the Atlas USB snapshot cleanup helper
+  tags: [atlas, storage, backup, usb_backup]
+  ansible.builtin.template:
+    src: atlas-usb-snapshot-cleanup.sh.j2
+    dest: /usr/local/sbin/atlas-usb-snapshot-cleanup
+    owner: root
+    group: root
+    mode: "0750"
+  when: atlas_manage_usb_backup | bool
+
 - name: Install the manual Atlas offline USB backup service
   tags: [atlas, storage, backup, usb_backup]
   ansible.builtin.template:

ansible/roles/profile_atlas/templates/atlas-usb-backup.service.j2

@@ -7,6 +7,7 @@ ConditionFileIsExecutable=/usr/local/sbin/atlas-usb-backup
 [Service]
 Type=oneshot
 ExecStart=/usr/local/sbin/atlas-usb-backup
+ExecStopPost=+/usr/local/sbin/atlas-usb-snapshot-cleanup
 User=root
 Group=root
 UMask=0077

ansible/roles/profile_atlas/templates/atlas-usb-backup.sh.j2

@@ -14,12 +14,12 @@ readonly min_free_bytes={{ atlas_usb_backup_min_free_bytes | int }}
 readonly mapper="/dev/mapper/${mapper_name}"
 readonly outer="/dev/disk/by-uuid/${luks_uuid}"
 readonly runtime_dir=/run/atlas-usb-backup
+readonly snapshot_marker="${runtime_dir}/snapshot-name"
 readonly source_dir="${runtime_dir}/source"
 readonly usb_mount="${runtime_dir}/target"
 readonly backup_root="${usb_mount}/atlas"
 
 snapshot_name=""
-snapshot_created=false
 mapper_opened_by_script=false
 usb_mounted=false
 published=false
@@ -45,10 +45,14 @@ cleanup() {
 {% raw %}
   for ((index = ${#mounted_targets[@]} - 1; index >= 0; index--)); do
 {% endraw %}
-    if mountpoint -q "${mounted_targets[$index]}"; then
-      umount "${mounted_targets[$index]}" || source_mount_failed=true
+    if mountpoint -q "${mounted_targets[$index]}" && ! umount -R "${mounted_targets[$index]}"; then
+      printf 'Source snapshot mount cleanup failed: %s\n' "${mounted_targets[$index]}" >&2
+      source_mount_failed=true
     fi
-    if ! mountpoint -q "${mounted_targets[$index]}"; then
+    if mountpoint -q "${mounted_targets[$index]}"; then
+      printf 'Source snapshot mount is still active: %s\n' "${mounted_targets[$index]}" >&2
+      source_mount_failed=true
+    else
       rmdir -- "${mounted_targets[$index]}" 2>/dev/null || true
     fi
   done
@@ -59,11 +63,6 @@ cleanup() {
     printf 'Source bind mount cleanup failed; keeping the snapshot for recovery\n' >&2
   fi
 
-  if [[ "$snapshot_created" == true && "$source_mount_failed" == false ]]; then
-    flock 9
-    zfs destroy -r "${pool}@${snapshot_name}" || cleanup_status=2
-    flock -u 9
-  fi
   if [[ "$usb_mounted" == true || "$mapper_opened_by_script" == true ]] &&
       ! mountpoint -q "$usb_mount" &&
       ! findmnt -rn -S "$mapper" >/dev/null; then
@@ -148,8 +147,8 @@ mkdir -m 0700 "$source_dir"
 flock 9
 timestamp="$(date -u +%Y%m%dT%H%M%SZ)"
 snapshot_name="${snapshot_prefix}-${timestamp}-$$"
+printf '%s\n' "$snapshot_name" >"$snapshot_marker"
 zfs snapshot -r "${pool}@${snapshot_name}"
-snapshot_created=true
 flock -u 9
 printf 'Created recursive USB source snapshot %s@%s\n' "$pool" "$snapshot_name"
 
@@ -189,10 +188,12 @@ mkdir -m 0700 "$candidate_partial"
 partial="$candidate_partial"
 
 printf 'Copying the consistent pool tree to USB backup %s\n' "$backup_name"
-# Preserve ACLs and other xattrs, but let SELinux relabel restored data on the
-# destination host instead of trying to write source security.selinux labels to USB.
-rsync_args=(-aHAXS --numeric-ids '--filter=-x security.selinux' "--info=progress2,stats2")
-estimate_args=(-aHAXS --numeric-ids '--filter=-x security.selinux' --dry-run --stats)
+# Preserve POSIX ACLs, ownership, modes, timestamps, hard links, and sparse
+# files.  Do not preserve generic xattrs: Rocky 9's rsync 3.2.7 fails when
+# combining xattrs with --link-dest, while SELinux labels were intentionally
+# excluded because restores must relabel for their destination host.
+rsync_args=(-aHAS --numeric-ids "--info=progress2,stats2")
+estimate_args=(-aHAS --numeric-ids --dry-run --stats)
 if [[ -n "$previous" ]]; then
   rsync_args+=("--link-dest=$previous")
   estimate_args+=("--link-dest=$previous")
@@ -217,7 +218,7 @@ rsync "${rsync_args[@]}" "${source_dir}/" "${partial}/"
 
 printf 'Verifying USB backup %s with a checksum-based dry run\n' "$backup_name"
 verification="${runtime_dir}/verification.out"
-rsync -aHAXS --numeric-ids --filter='-x security.selinux' \
+rsync -aHAS --numeric-ids \
   --checksum --dry-run --delete --itemize-changes \
   "${source_dir}/" "${partial}/" >"$verification"
 if [[ -s "$verification" ]]; then

ansible/roles/profile_atlas/templates/atlas-usb-snapshot-cleanup.sh.j2

@@ -0,0 +1,51 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+export PATH=/usr/sbin:/usr/bin:/sbin:/bin
+
+readonly pool={{ atlas_zfs_pool | quote }}
+readonly mount_root={{ atlas_mount_root | quote }}
+readonly snapshot_prefix={{ atlas_usb_backup_snapshot_prefix | quote }}
+readonly marker=/run/atlas-usb-backup/snapshot-name
+
+[[ -e "$marker" ]] || exit 0
+[[ -f "$marker" && ! -L "$marker" ]] || {
+  printf 'Unsafe Atlas USB snapshot marker; leaving snapshots unchanged\n' >&2
+  exit 2
+}
+IFS= read -r snapshot_name <"$marker"
+[[ "$snapshot_name" =~ ^${snapshot_prefix}-[0-9]{8}T[0-9]{6}Z-[0-9]+$ ]] || {
+  printf 'Invalid Atlas USB snapshot marker; leaving snapshots unchanged\n' >&2
+  exit 2
+}
+
+exec 9>/run/lock/atlas-zfs-snapshot.lock
+flock 9
+if zfs list -H -t snapshot -o name "${pool}@${snapshot_name}" >/dev/null 2>&1; then
+  # ZFS can leave its on-demand .zfs/snapshot mounts in the host namespace
+  # even after the backup's private bind mounts and process have exited.
+  snapshot_mounts=()
+  snapshot_sources=()
+  while IFS=$'\t' read -r dataset dataset_mountpoint; do
+    [[ "$dataset_mountpoint" == "$mount_root" || "$dataset_mountpoint" == "$mount_root/"* ]] || continue
+    snapshot_mounts+=("${dataset_mountpoint}/.zfs/snapshot/${snapshot_name}")
+    snapshot_sources+=("${dataset}@${snapshot_name}")
+  done < <(zfs list -H -o name,mountpoint -s name -r "$pool")
+
+{% raw %}
+  for ((index = ${#snapshot_mounts[@]} - 1; index >= 0; index--)); do
+{% endraw %}
+    mounted_source="$(findmnt -rn -M "${snapshot_mounts[$index]}" -o SOURCE || true)"
+    [[ -n "$mounted_source" ]] || continue
+    [[ "$mounted_source" == "${snapshot_sources[$index]}" ]] || {
+      printf 'Unexpected source on Atlas USB snapshot mount: %s\n' \
+        "${snapshot_mounts[$index]}" >&2
+      exit 2
+    }
+    umount "${snapshot_mounts[$index]}"
+  done
+
+  zfs destroy -r "${pool}@${snapshot_name}"
+  printf 'Removed recursive Atlas USB source snapshot %s@%s after backup exit\n' \
+    "$pool" "$snapshot_name"
+fi