Document Atlas NAS backups and monitoring

ansible/roles/profile_atlas/templates/atlas-borg-backup.service.j2

@@ -14,6 +14,7 @@ ConditionPathExists={{ atlas_borg_known_hosts_path }}
 [Service]
 Type=oneshot
 ExecStart=/usr/local/sbin/atlas-borg-backup
+ExecStopPost=+/usr/local/sbin/atlas-borg-snapshot-cleanup
 User=root
 Group=root
 UMask=0077

ansible/roles/profile_atlas/templates/atlas-borg-backup.sh.j2

@@ -17,6 +17,7 @@ readonly archive_prefix={{ atlas_borg_archive_prefix | quote }}
 readonly snapshot_prefix={{ atlas_borg_snapshot_prefix | quote }}
 readonly compression={{ atlas_borg_compression | quote }}
 readonly stage=/run/atlas-borg/source
+readonly snapshot_marker=/run/atlas-borg/snapshot-name
 readonly borg_user={{ atlas_borg_username | quote }}
 readonly borg_group={{ atlas_borg_group | quote }}
 readonly borg_home={{ atlas_borg_home | quote }}
@@ -24,7 +25,6 @@ readonly borg_lock={{ atlas_borg_lock_path | quote }}
 readonly progress_filter=/usr/local/libexec/atlas-borg-progress
 
 snapshot_name=""
-snapshot_created=false
 mounted_targets=()
 
 # Invoked through the EXIT trap below.
@@ -33,22 +33,31 @@ cleanup() {
   local status=$?
   local cleanup_status=0
   local index
+  local source_mount_failed=false
   trap - EXIT HUP INT TERM
   set +e
 
 {% raw %}
   for ((index = ${#mounted_targets[@]} - 1; index >= 0; index--)); do
 {% endraw %}
+    if mountpoint -q "${mounted_targets[$index]}" && ! umount -R "${mounted_targets[$index]}"; then
+      printf 'Source snapshot mount cleanup failed: %s\n' "${mounted_targets[$index]}" >&2
+      source_mount_failed=true
+    fi
     if mountpoint -q "${mounted_targets[$index]}"; then
-      umount "${mounted_targets[$index]}" || cleanup_status=2
+      printf 'Source snapshot mount is still active: %s\n' "${mounted_targets[$index]}" >&2
+      source_mount_failed=true
+    else
+      rmdir -- "${mounted_targets[$index]}" 2>/dev/null || true
     fi
   done
-  rm -rf "$stage" || cleanup_status=2
-
-  if [[ "$snapshot_created" == true ]]; then
-    flock 9
-    zfs destroy -r "${pool}@${snapshot_name}" || cleanup_status=2
-    flock -u 9
+  if [[ "$source_mount_failed" == false ]]; then
+    if [[ -d "$stage" ]]; then
+      rmdir -- "$stage" 2>/dev/null || cleanup_status=2
+    fi
+  else
+    cleanup_status=2
+    printf 'Source bind mount cleanup failed; keeping the snapshot for recovery\n' >&2
   fi
 
   if ((status == 0 && cleanup_status != 0)); then
@@ -97,8 +106,8 @@ timestamp="$(date -u +%Y%m%dT%H%M%SZ)"
 readonly timestamp
 snapshot_name="${snapshot_prefix}-${timestamp}"
 readonly snapshot_name
+printf '%s\n' "$snapshot_name" >"$snapshot_marker"
 zfs snapshot -r "${pool}@${snapshot_name}"
-snapshot_created=true
 flock -u 9
 printf 'Created recursive Borg source snapshot %s@%s\n' "$pool" "$snapshot_name"
 
@@ -117,10 +126,27 @@ while IFS=$'\t' read -r dataset dataset_mountpoint mounted; do
   target_path="${stage}${dataset_suffix}"
   mkdir -p "$target_path"
   mount --bind "$source_path" "$target_path"
-  mount -o remount,bind,ro "$target_path"
   mounted_targets+=("$target_path")
+  mount -o remount,bind,ro "$target_path"
 done < <(zfs list -H -o name,mountpoint,mounted -s name -r "$pool")
 
+estimated_source_bytes=0
+while IFS=$'\t' read -r source_snapshot logical_bytes; do
+  if [[ "$source_snapshot" == *"@${snapshot_name}" ]]; then
+    [[ "$logical_bytes" =~ ^[0-9]+$ ]] || {
+      printf 'Invalid logical size for Borg source snapshot %s\n' "$source_snapshot" >&2
+      exit 74
+    }
+    estimated_source_bytes=$((estimated_source_bytes + logical_bytes))
+  fi
+done < <(zfs list -H -p -t snapshot -o name,logicalreferenced -r "$pool")
+((estimated_source_bytes > 0)) || {
+  printf 'Could not estimate the Borg source snapshot size\n' >&2
+  exit 74
+}
+printf 'Estimated Borg source logical size: %s bytes (ZFS; progress percentage is approximate)\n' \
+  "$estimated_source_bytes"
+
 archive="${archive_prefix}-${timestamp}"
 readonly archive
 borg_status=0
@@ -136,7 +162,7 @@ set +e
     --compression "$compression" \
     "${repository}::${archive}" \
     source 2>&1
-) | /usr/bin/python3 -u "$progress_filter"
+) | /usr/bin/python3 -u "$progress_filter" --estimated-total-bytes "$estimated_source_bytes"
 create_pipeline_status=("${PIPESTATUS[@]}")
 set -e
 create_status=${create_pipeline_status[0]}

ansible/roles/profile_atlas/templates/atlas-borg-snapshot-cleanup.sh.j2

@@ -0,0 +1,51 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+export PATH=/usr/sbin:/usr/bin:/sbin:/bin
+
+readonly pool={{ atlas_zfs_pool | quote }}
+readonly mount_root={{ atlas_mount_root | quote }}
+readonly snapshot_prefix={{ atlas_borg_snapshot_prefix | quote }}
+readonly marker=/run/atlas-borg/snapshot-name
+
+[[ -e "$marker" ]] || exit 0
+[[ -f "$marker" && ! -L "$marker" ]] || {
+  printf 'Unsafe Atlas Borg snapshot marker; leaving snapshots unchanged\n' >&2
+  exit 2
+}
+IFS= read -r snapshot_name <"$marker"
+[[ "$snapshot_name" =~ ^${snapshot_prefix}-[0-9]{8}T[0-9]{6}Z$ ]] || {
+  printf 'Invalid Atlas Borg snapshot marker; leaving snapshots unchanged\n' >&2
+  exit 2
+}
+
+exec 9>/run/lock/atlas-zfs-snapshot.lock
+flock 9
+if zfs list -H -t snapshot -o name "${pool}@${snapshot_name}" >/dev/null 2>&1; then
+  # The private bind mounts are gone, but ZFS may leave its on-demand
+  # .zfs/snapshot mounts in the host namespace until explicitly unmounted.
+  snapshot_mounts=()
+  snapshot_sources=()
+  while IFS=$'\t' read -r dataset dataset_mountpoint; do
+    [[ "$dataset_mountpoint" == "$mount_root" || "$dataset_mountpoint" == "$mount_root/"* ]] || continue
+    snapshot_mounts+=("${dataset_mountpoint}/.zfs/snapshot/${snapshot_name}")
+    snapshot_sources+=("${dataset}@${snapshot_name}")
+  done < <(zfs list -H -o name,mountpoint -s name -r "$pool")
+
+{% raw %}
+  for ((index = ${#snapshot_mounts[@]} - 1; index >= 0; index--)); do
+{% endraw %}
+    mounted_source="$(findmnt -rn -M "${snapshot_mounts[$index]}" -o SOURCE || true)"
+    [[ -n "$mounted_source" ]] || continue
+    [[ "$mounted_source" == "${snapshot_sources[$index]}" ]] || {
+      printf 'Unexpected source on Atlas Borg snapshot mount: %s\n' \
+        "${snapshot_mounts[$index]}" >&2
+      exit 2
+    }
+    umount "${snapshot_mounts[$index]}"
+  done
+
+  zfs destroy -r "${pool}@${snapshot_name}"
+  printf 'Removed recursive Atlas Borg source snapshot %s@%s after backup exit\n' \
+    "$pool" "$snapshot_name"
+fi

ansible/roles/profile_atlas/templates/atlas-health-monitor.json.j2

@@ -0,0 +1,20 @@
+{
+  "pool": {{ atlas_zfs_pool | to_json }},
+  "backup_dataset": {{ (atlas_zfs_pool ~ '/' ~ atlas_zfs_dataset_backup) | to_json }},
+  "notifier": {{ atlas_monitor_notifier | to_json }},
+  "smart_devices": {{ atlas_monitor_smart_devices | to_json }},
+  "timers": {{ atlas_monitor_timers | to_json }},
+  "failure_units": {{ atlas_monitor_failure_units | to_json }},
+  "remote_capacity": {{ atlas_monitor_remote_capacity | to_json }},
+  "pool_warning_percent": {{ atlas_monitor_pool_warning_percent | int }},
+  "pool_critical_percent": {{ atlas_monitor_pool_critical_percent | int }},
+  "root_warning_percent": {{ atlas_monitor_root_warning_percent | int }},
+  "root_critical_percent": {{ atlas_monitor_root_critical_percent | int }},
+  "snapshot_warning_percent": {{ atlas_monitor_snapshot_warning_percent | int }},
+  "snapshot_critical_percent": {{ atlas_monitor_snapshot_critical_percent | int }},
+  "snapshot_growth_warning_gib_day": {{ atlas_monitor_snapshot_growth_warning_gib_day | int }},
+  "backup_growth_warning_gib_day": {{ atlas_monitor_backup_growth_warning_gib_day | int }},
+  "cpu_warning_c": {{ atlas_monitor_cpu_warning_c | int }},
+  "cpu_critical_c": {{ atlas_monitor_cpu_critical_c | int }},
+  "borg_max_runtime_days": {{ atlas_monitor_borg_max_runtime_days | int }}
+}

ansible/roles/profile_atlas/templates/atlas-health-monitor.service.j2

@@ -0,0 +1,25 @@
+[Unit]
+Description=Check Atlas pool, disks, capacity, temperatures and maintenance jobs
+Wants=houston-dbus.service network-online.target
+After=zfs.target houston-dbus.service network-online.target
+ConditionFileIsExecutable=/usr/local/libexec/atlas-health-monitor
+
+[Service]
+Type=oneshot
+ExecStart=/usr/local/libexec/atlas-health-monitor
+User=root
+Group=root
+UMask=0077
+StateDirectory=atlas-health-monitor
+StateDirectoryMode=0700
+NoNewPrivileges=true
+PrivateTmp=true
+ProtectHome=true
+ProtectSystem=strict
+ReadWritePaths=/var/lib/atlas-health-monitor
+ProtectKernelTunables=true
+ProtectKernelModules=true
+ProtectControlGroups=true
+RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6
+RestrictRealtime=true
+LockPersonality=true

ansible/roles/profile_atlas/templates/atlas-health-monitor.timer.j2

@@ -0,0 +1,11 @@
+[Unit]
+Description=Schedule Atlas health checks
+
+[Timer]
+OnCalendar={{ atlas_monitor_calendar }}
+Persistent=true
+RandomizedDelaySec=5min
+Unit=atlas-health-monitor.service
+
+[Install]
+WantedBy=timers.target

ansible/roles/profile_atlas/templates/atlas-monitor-failure.conf.j2

@@ -0,0 +1,2 @@
+[Unit]
+OnFailure=atlas-monitor-failure@%n.service

ansible/roles/profile_atlas/templates/atlas-monitor-failure@.service.j2

@@ -0,0 +1,19 @@
+[Unit]
+Description=Submit a 45Drives Alert for failed Atlas job %I
+Requires=houston-dbus.service
+After=houston-dbus.service
+ConditionFileIsExecutable=/usr/local/libexec/atlas-health-monitor
+
+[Service]
+Type=oneshot
+ExecStart=/usr/local/libexec/atlas-health-monitor --job-failed %I
+User=root
+Group=root
+UMask=0077
+NoNewPrivileges=true
+PrivateTmp=true
+ProtectHome=true
+ProtectSystem=strict
+RestrictAddressFamilies=AF_UNIX
+RestrictRealtime=true
+LockPersonality=true