mirror of
https://github.com/fscotto/infra.git
synced 2026-09-27 11:02:47 +00:00
213 lines
6.9 KiB
Django/Jinja
213 lines
6.9 KiB
Django/Jinja
#!/usr/bin/env bash
|
|
set -Eeuo pipefail
|
|
|
|
export LC_ALL=C.utf8
|
|
export PATH=/usr/sbin:/usr/bin:/sbin:/bin
|
|
export BORG_CACHE_DIR={{ atlas_borg_cache_dir | quote }}
|
|
export BORG_CONFIG_DIR={{ atlas_borg_config_dir | quote }}
|
|
export BORG_PASSCOMMAND={{ ('cat ' ~ atlas_borg_passphrase_path) | quote }}
|
|
export BORG_RSH={{ atlas_borg_ssh_wrapper_path | quote }}
|
|
|
|
readonly pool={{ atlas_zfs_pool | quote }}
|
|
readonly mount_root={{ atlas_mount_root | quote }}
|
|
readonly repository={{ ('ssh://' ~ atlas_borg_repository_user ~ '@' ~ atlas_borg_repository_host
|
|
~ ':' ~ (atlas_borg_repository_port | string) ~ '/' ~ atlas_borg_repository_path) | quote }}
|
|
readonly remote_path={{ atlas_borg_remote_path | quote }}
|
|
readonly archive_prefix={{ atlas_borg_archive_prefix | quote }}
|
|
readonly snapshot_prefix={{ atlas_borg_snapshot_prefix | quote }}
|
|
readonly compression={{ atlas_borg_compression | quote }}
|
|
readonly stage=/run/atlas-borg/source
|
|
readonly snapshot_marker=/run/atlas-borg/snapshot-name
|
|
readonly borg_user={{ atlas_borg_username | quote }}
|
|
readonly borg_group={{ atlas_borg_group | quote }}
|
|
readonly borg_home={{ atlas_borg_home | quote }}
|
|
readonly borg_lock={{ atlas_borg_lock_path | quote }}
|
|
readonly progress_filter=/usr/local/libexec/atlas-borg-progress
|
|
|
|
snapshot_name=""
|
|
mounted_targets=()
|
|
|
|
# Invoked through the EXIT trap below.
|
|
# shellcheck disable=SC2329
|
|
cleanup() {
|
|
local status=$?
|
|
local cleanup_status=0
|
|
local index
|
|
local source_mount_failed=false
|
|
trap - EXIT HUP INT TERM
|
|
set +e
|
|
|
|
{% raw %}
|
|
for ((index = ${#mounted_targets[@]} - 1; index >= 0; index--)); do
|
|
{% endraw %}
|
|
if mountpoint -q "${mounted_targets[$index]}" && ! umount -R "${mounted_targets[$index]}"; then
|
|
printf 'Source snapshot mount cleanup failed: %s\n' "${mounted_targets[$index]}" >&2
|
|
source_mount_failed=true
|
|
fi
|
|
if mountpoint -q "${mounted_targets[$index]}"; then
|
|
printf 'Source snapshot mount is still active: %s\n' "${mounted_targets[$index]}" >&2
|
|
source_mount_failed=true
|
|
else
|
|
rmdir -- "${mounted_targets[$index]}" 2>/dev/null || true
|
|
fi
|
|
done
|
|
if [[ "$source_mount_failed" == false ]]; then
|
|
if [[ -d "$stage" ]]; then
|
|
rmdir -- "$stage" 2>/dev/null || cleanup_status=2
|
|
fi
|
|
else
|
|
cleanup_status=2
|
|
printf 'Source bind mount cleanup failed; keeping the snapshot for recovery\n' >&2
|
|
fi
|
|
|
|
if ((status == 0 && cleanup_status != 0)); then
|
|
status=$cleanup_status
|
|
fi
|
|
exit "$status"
|
|
}
|
|
|
|
trap cleanup EXIT
|
|
trap 'exit 143' HUP INT TERM
|
|
|
|
run_as_borg() {
|
|
setpriv \
|
|
--reuid "$borg_user" \
|
|
--regid "$borg_group" \
|
|
--clear-groups \
|
|
--inh-caps=-all,+dac_read_search \
|
|
--ambient-caps=-all,+dac_read_search \
|
|
--bounding-set=-all,+dac_read_search \
|
|
-- env HOME="$borg_home" USER="$borg_user" LOGNAME="$borg_user" "$@"
|
|
}
|
|
|
|
exec 8>"$borg_lock"
|
|
flock 8
|
|
exec 9>/run/lock/atlas-zfs-snapshot.lock
|
|
|
|
zpool list -H -o name "$pool" >/dev/null
|
|
rm -rf "$stage"
|
|
mkdir -p "$stage"
|
|
chown root:"$borg_group" /run/atlas-borg "$stage"
|
|
chmod 0750 /run/atlas-borg "$stage"
|
|
|
|
flock 9
|
|
while IFS= read -r stale_snapshot; do
|
|
stale_suffix="${stale_snapshot#"${pool}@${snapshot_prefix}-"}"
|
|
if [[ "$stale_suffix" =~ ^[0-9]{8}T[0-9]{6}Z$ ]]; then
|
|
zfs destroy -r "$stale_snapshot"
|
|
printf 'Removed stale Borg source snapshot %s\n' "$stale_snapshot"
|
|
fi
|
|
done < <(
|
|
zfs list -H -t snapshot -o name -r "$pool" |
|
|
grep -E "^${pool}@${snapshot_prefix}-[0-9]{8}T[0-9]{6}Z$" || true
|
|
)
|
|
|
|
timestamp="$(date -u +%Y%m%dT%H%M%SZ)"
|
|
readonly timestamp
|
|
snapshot_name="${snapshot_prefix}-${timestamp}"
|
|
readonly snapshot_name
|
|
printf '%s\n' "$snapshot_name" >"$snapshot_marker"
|
|
zfs snapshot -r "${pool}@${snapshot_name}"
|
|
flock -u 9
|
|
printf 'Created recursive Borg source snapshot %s@%s\n' "$pool" "$snapshot_name"
|
|
|
|
while IFS=$'\t' read -r dataset dataset_mountpoint mounted; do
|
|
if [[ "$mounted" != yes ]]; then
|
|
printf 'Dataset %s is not mounted; refusing an incomplete backup\n' "$dataset" >&2
|
|
exit 65
|
|
fi
|
|
if [[ "$dataset_mountpoint" != "$mount_root" && "$dataset_mountpoint" != "$mount_root/"* ]]; then
|
|
printf 'Dataset %s has unexpected mountpoint %s\n' "$dataset" "$dataset_mountpoint" >&2
|
|
exit 65
|
|
fi
|
|
|
|
dataset_suffix="${dataset#"$pool"}"
|
|
source_path="${dataset_mountpoint}/.zfs/snapshot/${snapshot_name}"
|
|
target_path="${stage}${dataset_suffix}"
|
|
mkdir -p "$target_path"
|
|
mount --bind "$source_path" "$target_path"
|
|
mounted_targets+=("$target_path")
|
|
mount -o remount,bind,ro "$target_path"
|
|
done < <(zfs list -H -o name,mountpoint,mounted -s name -r "$pool")
|
|
|
|
estimated_source_bytes=0
|
|
while IFS=$'\t' read -r source_snapshot logical_bytes; do
|
|
if [[ "$source_snapshot" == *"@${snapshot_name}" ]]; then
|
|
[[ "$logical_bytes" =~ ^[0-9]+$ ]] || {
|
|
printf 'Invalid logical size for Borg source snapshot %s\n' "$source_snapshot" >&2
|
|
exit 74
|
|
}
|
|
estimated_source_bytes=$((estimated_source_bytes + logical_bytes))
|
|
fi
|
|
done < <(zfs list -H -p -t snapshot -o name,logicalreferenced -r "$pool")
|
|
((estimated_source_bytes > 0)) || {
|
|
printf 'Could not estimate the Borg source snapshot size\n' >&2
|
|
exit 74
|
|
}
|
|
printf 'Estimated Borg source logical size: %s bytes (ZFS; progress percentage is approximate)\n' \
|
|
"$estimated_source_bytes"
|
|
|
|
archive="${archive_prefix}-${timestamp}"
|
|
readonly archive
|
|
borg_status=0
|
|
|
|
printf 'Starting Borg archive %s from snapshot %s@%s\n' "$archive" "$pool" "$snapshot_name"
|
|
set +e
|
|
(
|
|
cd /run/atlas-borg
|
|
run_as_borg borg --remote-path "$remote_path" --lock-wait 600 --log-json --progress create \
|
|
--show-rc \
|
|
--stats \
|
|
--checkpoint-interval 900 \
|
|
--compression "$compression" \
|
|
"${repository}::${archive}" \
|
|
source 2>&1
|
|
) | /usr/bin/python3 -u "$progress_filter" --estimated-total-bytes "$estimated_source_bytes"
|
|
create_pipeline_status=("${PIPESTATUS[@]}")
|
|
set -e
|
|
create_status=${create_pipeline_status[0]}
|
|
if ((create_pipeline_status[1] != 0)); then
|
|
printf 'Borg progress logging failed with status %s\n' "${create_pipeline_status[1]}" >&2
|
|
exit 2
|
|
fi
|
|
if ((create_status >= 2)); then
|
|
exit "$create_status"
|
|
fi
|
|
borg_status=$create_status
|
|
|
|
printf 'Borg archive %s created; applying retention\n' "$archive"
|
|
set +e
|
|
run_as_borg borg --remote-path "$remote_path" --lock-wait 600 prune \
|
|
--show-rc \
|
|
--list \
|
|
--glob-archives "${archive_prefix}-*" \
|
|
--keep-daily {{ atlas_borg_keep_daily | int }} \
|
|
--keep-weekly {{ atlas_borg_keep_weekly | int }} \
|
|
--keep-monthly {{ atlas_borg_keep_monthly | int }} \
|
|
"$repository"
|
|
prune_status=$?
|
|
set -e
|
|
if ((prune_status >= 2)); then
|
|
exit "$prune_status"
|
|
fi
|
|
if ((prune_status > borg_status)); then
|
|
borg_status=$prune_status
|
|
fi
|
|
|
|
printf 'Borg retention complete; compacting repository\n'
|
|
set +e
|
|
run_as_borg borg --remote-path "$remote_path" --lock-wait 600 compact \
|
|
--show-rc \
|
|
"$repository"
|
|
compact_status=$?
|
|
set -e
|
|
if ((compact_status >= 2)); then
|
|
exit "$compact_status"
|
|
fi
|
|
if ((compact_status > borg_status)); then
|
|
borg_status=$compact_status
|
|
fi
|
|
|
|
printf 'Borg backup %s completed with status %s\n' "$archive" "$borg_status"
|
|
exit "$borg_status"
|