Add progress logging to Atlas Borg backups

ansible/roles/profile_atlas/files/atlas-borg-progress.py

@@ -0,0 +1,43 @@
+#!/usr/bin/env python3
+"""Turn Borg's JSON progress stream into bounded, readable journal entries."""
+
+import json
+import sys
+import time
+
+
+def size(value):
+    if not isinstance(value, (int, float)):
+        return "unknown"
+    return f"{value / (1024 ** 3):.2f} GiB"
+
+
+last_progress = 0.0
+for line in sys.stdin:
+    try:
+        event = json.loads(line)
+    except json.JSONDecodeError:
+        print(line.rstrip(), flush=True)
+        continue
+
+    kind = event.get("type")
+    if kind == "archive_progress":
+        now = time.monotonic()
+        if now - last_progress < 60 and not event.get("finished"):
+            continue
+        path = event.get("path") or ""
+        parts = path.split("/")
+        dataset = parts[1] if len(parts) > 1 and parts[0] == "source" else "unknown"
+        print(
+            "Borg create progress: "
+            f"dataset={dataset} files={event.get('nfiles', 'unknown')} "
+            f"original={size(event.get('original_size'))} "
+            f"compressed={size(event.get('compressed_size'))} "
+            f"deduplicated={size(event.get('deduplicated_size'))}",
+            flush=True,
+        )
+        last_progress = now
+    elif kind == "log_message":
+        print(f"Borg {event.get('levelname', 'INFO')}: {event.get('message', '')}", flush=True)
+    elif kind == "progress_message" and event.get("message"):
+        print(f"Borg: {event['message']}", flush=True)

ansible/roles/profile_atlas/tasks/borg_backup.yml

@@ -224,7 +224,7 @@
   when: atlas_manage_borg_backup | bool
 
 - name: Install the Atlas Borg backup helper
-  tags: [atlas, storage, backup, borg]
+  tags: [atlas, storage, backup, borg, borg_logging]
   ansible.builtin.template:
     src: atlas-borg-backup.sh.j2
     dest: /usr/local/sbin/atlas-borg-backup
@@ -244,7 +244,7 @@
   when: atlas_manage_borg_backup | bool
 
 - name: Create the local libexec directory for the Atlas Borg SSH wrapper
-  tags: [atlas, storage, backup, borg]
+  tags: [atlas, storage, backup, borg, borg_logging]
   ansible.builtin.file:
     path: "{{ atlas_borg_ssh_wrapper_path | dirname }}"
     state: directory
@@ -253,6 +253,16 @@
     mode: "0755"
   when: atlas_manage_borg_backup | bool
 
+- name: Install the Atlas Borg progress formatter
+  tags: [atlas, storage, backup, borg, borg_logging]
+  ansible.builtin.copy:
+    src: atlas-borg-progress.py
+    dest: /usr/local/libexec/atlas-borg-progress
+    owner: root
+    group: root
+    mode: "0755"
+  when: atlas_manage_borg_backup | bool
+
 - name: Install the capability-dropping Atlas Borg SSH wrapper
   tags: [atlas, storage, backup, borg]
   ansible.builtin.template:

ansible/roles/profile_atlas/templates/atlas-borg-backup.sh.j2

@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 set -Eeuo pipefail
 
-export LC_ALL=C
+export LC_ALL=C.utf8
 export PATH=/usr/sbin:/usr/bin:/sbin:/bin
 export BORG_CACHE_DIR={{ atlas_borg_cache_dir | quote }}
 export BORG_CONFIG_DIR={{ atlas_borg_config_dir | quote }}
@@ -21,6 +21,7 @@ readonly borg_user={{ atlas_borg_username | quote }}
 readonly borg_group={{ atlas_borg_group | quote }}
 readonly borg_home={{ atlas_borg_home | quote }}
 readonly borg_lock={{ atlas_borg_lock_path | quote }}
+readonly progress_filter=/usr/local/libexec/atlas-borg-progress
 
 snapshot_name=""
 snapshot_created=false
@@ -124,24 +125,31 @@ archive="${archive_prefix}-${timestamp}"
 readonly archive
 borg_status=0
 
+printf 'Starting Borg archive %s from snapshot %s@%s\n' "$archive" "$pool" "$snapshot_name"
 set +e
 (
   cd /run/atlas-borg
-  run_as_borg borg --remote-path "$remote_path" --lock-wait 600 create \
+  run_as_borg borg --remote-path "$remote_path" --lock-wait 600 --log-json --progress create \
     --show-rc \
     --stats \
     --checkpoint-interval 900 \
     --compression "$compression" \
     "${repository}::${archive}" \
-    source
-)
-create_status=$?
+    source 2>&1
+) | /usr/bin/python3 -u "$progress_filter"
+create_pipeline_status=("${PIPESTATUS[@]}")
 set -e
+create_status=${create_pipeline_status[0]}
+if ((create_pipeline_status[1] != 0)); then
+  printf 'Borg progress logging failed with status %s\n' "${create_pipeline_status[1]}" >&2
+  exit 2
+fi
 if ((create_status >= 2)); then
   exit "$create_status"
 fi
 borg_status=$create_status
 
+printf 'Borg archive %s created; applying retention\n' "$archive"
 set +e
 run_as_borg borg --remote-path "$remote_path" --lock-wait 600 prune \
   --show-rc \
@@ -160,6 +168,7 @@ if ((prune_status > borg_status)); then
   borg_status=$prune_status
 fi
 
+printf 'Borg retention complete; compacting repository\n'
 set +e
 run_as_borg borg --remote-path "$remote_path" --lock-wait 600 compact \
   --show-rc \
@@ -173,4 +182,5 @@ if ((compact_status > borg_status)); then
   borg_status=$compact_status
 fi
 
+printf 'Borg backup %s completed with status %s\n' "$archive" "$borg_status"
 exit "$borg_status"