diff --git a/AGENTS.md b/AGENTS.md index 5db0715..340306a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -63,6 +63,8 @@ Ansible-driven personal infrastructure repo for Fedora and Void desktops, Fedora `ansible-playbook ansible/site.yml --limit atlas --tags snapshots,scrub --check --diff` - Atlas encrypted Borg backup: `ansible-playbook ansible/site.yml --limit atlas --tags packages,borg --check --diff` + - Atlas Borg progress logging only: + `ansible-playbook ansible/site.yml --limit atlas --tags borg_logging --check --diff` - Prometheus/Aegis WireGuard gateway: `ansible-playbook ansible/site.yml --limit prometheus,aegis --tags wireguard --check --diff` - DuckDNS config only: `ansible-playbook ansible/site.yml --limit prometheus --tags duckdns --check --diff` diff --git a/README.md b/README.md index e940387..a4f6ead 100644 --- a/README.md +++ b/README.md @@ -328,6 +328,12 @@ the Borg client as `borg` with temporary read-search capability and no ZFS, sudo privileges. Borg retains 30 daily, 8 weekly, and 12 monthly archives, then compacts the standard read-write repository. A full metadata and repository check runs as `borg` on the fifteenth day of each month at 06:00. Both operations use a common lock, journal logging, and bounded systemd retries. +New backup runs also log the create phase and a compact progress line at most once per minute: dataset, +files processed, and original/compressed/deduplicated bytes. Progress lines omit individual filenames +and a percentage, since Borg does not know the total in advance; warnings may still name affected files. +Follow the current run with +`sudo journalctl -fu atlas-borg-backup.service` on Atlas; changes to the helper do not alter a run +already in progress. Initial activation remains explicit: diff --git a/ansible/roles/profile_atlas/files/atlas-borg-progress.py b/ansible/roles/profile_atlas/files/atlas-borg-progress.py new file mode 100644 index 0000000..6506880 --- /dev/null +++ b/ansible/roles/profile_atlas/files/atlas-borg-progress.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python3 +"""Turn Borg's JSON progress stream into bounded, readable journal entries.""" + +import json +import sys +import time + + +def size(value): + if not isinstance(value, (int, float)): + return "unknown" + return f"{value / (1024 ** 3):.2f} GiB" + + +last_progress = 0.0 +for line in sys.stdin: + try: + event = json.loads(line) + except json.JSONDecodeError: + print(line.rstrip(), flush=True) + continue + + kind = event.get("type") + if kind == "archive_progress": + now = time.monotonic() + if now - last_progress < 60 and not event.get("finished"): + continue + path = event.get("path") or "" + parts = path.split("/") + dataset = parts[1] if len(parts) > 1 and parts[0] == "source" else "unknown" + print( + "Borg create progress: " + f"dataset={dataset} files={event.get('nfiles', 'unknown')} " + f"original={size(event.get('original_size'))} " + f"compressed={size(event.get('compressed_size'))} " + f"deduplicated={size(event.get('deduplicated_size'))}", + flush=True, + ) + last_progress = now + elif kind == "log_message": + print(f"Borg {event.get('levelname', 'INFO')}: {event.get('message', '')}", flush=True) + elif kind == "progress_message" and event.get("message"): + print(f"Borg: {event['message']}", flush=True) diff --git a/ansible/roles/profile_atlas/tasks/borg_backup.yml b/ansible/roles/profile_atlas/tasks/borg_backup.yml index a246a7d..9cbe89f 100644 --- a/ansible/roles/profile_atlas/tasks/borg_backup.yml +++ b/ansible/roles/profile_atlas/tasks/borg_backup.yml @@ -224,7 +224,7 @@ when: atlas_manage_borg_backup | bool - name: Install the Atlas Borg backup helper - tags: [atlas, storage, backup, borg] + tags: [atlas, storage, backup, borg, borg_logging] ansible.builtin.template: src: atlas-borg-backup.sh.j2 dest: /usr/local/sbin/atlas-borg-backup @@ -244,7 +244,7 @@ when: atlas_manage_borg_backup | bool - name: Create the local libexec directory for the Atlas Borg SSH wrapper - tags: [atlas, storage, backup, borg] + tags: [atlas, storage, backup, borg, borg_logging] ansible.builtin.file: path: "{{ atlas_borg_ssh_wrapper_path | dirname }}" state: directory @@ -253,6 +253,16 @@ mode: "0755" when: atlas_manage_borg_backup | bool +- name: Install the Atlas Borg progress formatter + tags: [atlas, storage, backup, borg, borg_logging] + ansible.builtin.copy: + src: atlas-borg-progress.py + dest: /usr/local/libexec/atlas-borg-progress + owner: root + group: root + mode: "0755" + when: atlas_manage_borg_backup | bool + - name: Install the capability-dropping Atlas Borg SSH wrapper tags: [atlas, storage, backup, borg] ansible.builtin.template: diff --git a/ansible/roles/profile_atlas/templates/atlas-borg-backup.sh.j2 b/ansible/roles/profile_atlas/templates/atlas-borg-backup.sh.j2 index 5d1beb0..6468af7 100644 --- a/ansible/roles/profile_atlas/templates/atlas-borg-backup.sh.j2 +++ b/ansible/roles/profile_atlas/templates/atlas-borg-backup.sh.j2 @@ -1,7 +1,7 @@ #!/usr/bin/env bash set -Eeuo pipefail -export LC_ALL=C +export LC_ALL=C.utf8 export PATH=/usr/sbin:/usr/bin:/sbin:/bin export BORG_CACHE_DIR={{ atlas_borg_cache_dir | quote }} export BORG_CONFIG_DIR={{ atlas_borg_config_dir | quote }} @@ -21,6 +21,7 @@ readonly borg_user={{ atlas_borg_username | quote }} readonly borg_group={{ atlas_borg_group | quote }} readonly borg_home={{ atlas_borg_home | quote }} readonly borg_lock={{ atlas_borg_lock_path | quote }} +readonly progress_filter=/usr/local/libexec/atlas-borg-progress snapshot_name="" snapshot_created=false @@ -124,24 +125,31 @@ archive="${archive_prefix}-${timestamp}" readonly archive borg_status=0 +printf 'Starting Borg archive %s from snapshot %s@%s\n' "$archive" "$pool" "$snapshot_name" set +e ( cd /run/atlas-borg - run_as_borg borg --remote-path "$remote_path" --lock-wait 600 create \ + run_as_borg borg --remote-path "$remote_path" --lock-wait 600 --log-json --progress create \ --show-rc \ --stats \ --checkpoint-interval 900 \ --compression "$compression" \ "${repository}::${archive}" \ - source -) -create_status=$? + source 2>&1 +) | /usr/bin/python3 -u "$progress_filter" +create_pipeline_status=("${PIPESTATUS[@]}") set -e +create_status=${create_pipeline_status[0]} +if ((create_pipeline_status[1] != 0)); then + printf 'Borg progress logging failed with status %s\n' "${create_pipeline_status[1]}" >&2 + exit 2 +fi if ((create_status >= 2)); then exit "$create_status" fi borg_status=$create_status +printf 'Borg archive %s created; applying retention\n' "$archive" set +e run_as_borg borg --remote-path "$remote_path" --lock-wait 600 prune \ --show-rc \ @@ -160,6 +168,7 @@ if ((prune_status > borg_status)); then borg_status=$prune_status fi +printf 'Borg retention complete; compacting repository\n' set +e run_as_borg borg --remote-path "$remote_path" --lock-wait 600 compact \ --show-rc \ @@ -173,4 +182,5 @@ if ((compact_status > borg_status)); then borg_status=$compact_status fi +printf 'Borg backup %s completed with status %s\n' "$archive" "$borg_status" exit "$borg_status"