Add encrypted Borg backups for Atlas

ansible/roles/profile_atlas/defaults/main.yml

@@ -67,6 +67,36 @@ atlas_zfs_snapshot_policies: []
 atlas_manage_zfs_scrub: false
 atlas_zfs_scrub_calendar: ""
 
+atlas_manage_borg_backup: false
+atlas_borg_username: borg
+atlas_borg_group: borg
+atlas_borg_home: /var/lib/atlas-borg
+atlas_borg_repository_host: CHANGEME_BORG_HOST
+atlas_borg_repository_user: CHANGEME_BORG_USER
+atlas_borg_repository_port: 23
+atlas_borg_repository_path: ./borg-data
+atlas_borg_remote_path: borg-1.4
+atlas_borg_host_key: ""
+atlas_borg_ssh_private_key_path: /etc/atlas-borg/id_ed25519
+atlas_borg_known_hosts_path: /etc/atlas-borg/known_hosts
+atlas_borg_passphrase_path: /etc/atlas-borg/passphrase
+atlas_borg_ssh_wrapper_path: /usr/local/libexec/atlas-borg-ssh
+atlas_borg_passphrase: "{{ vault_atlas_borg_passphrase | default('') }}"
+atlas_borg_encryption_mode: repokey
+atlas_borg_archive_prefix: atlas
+atlas_borg_snapshot_prefix: atlas-borg
+atlas_borg_compression: auto,zstd,3
+atlas_borg_backup_calendar: ""
+atlas_borg_check_calendar: ""
+atlas_borg_randomized_delay: 30m
+atlas_borg_keep_daily: 30
+atlas_borg_keep_weekly: 8
+atlas_borg_keep_monthly: 12
+atlas_borg_config_dir: /var/lib/atlas-borg
+atlas_borg_cache_dir: /var/cache/atlas-borg
+atlas_borg_lock_path: /var/lib/atlas-borg/backup.lock
+atlas_borg_recovery_export_path: "{{ playbook_dir }}/../secrets/recovery/atlas-borg-repokey.export"
+
 atlas_archive_mountpoint: "{{ atlas_mount_root }}/{{ atlas_zfs_dataset_archive }}"
 atlas_services_mountpoint: "{{ atlas_mount_root }}/{{ atlas_zfs_dataset_services }}"
 atlas_app_data_mountpoint: "{{ atlas_mount_root }}/{{ atlas_zfs_dataset_app_data }}"

ansible/roles/profile_atlas/handlers/main.yml

@@ -28,6 +28,18 @@
     name: smb
     state: restarted
 
+- name: Restart Atlas Borg timers
+  ansible.builtin.systemd:
+    name: "{{ item }}"
+    state: restarted
+    daemon_reload: true
+  loop:
+    - atlas-borg-backup.timer
+    - atlas-borg-check.timer
+  when:
+    - atlas_manage_borg_backup | bool
+    - not ansible_check_mode
+
 - name: Restart Atlas media Quadlets
   ansible.builtin.systemd:
     name: "{{ item }}"

ansible/roles/profile_atlas/tasks/borg_backup.yml

@@ -0,0 +1,518 @@
+---
+- name: Validate Atlas Borg backup configuration
+  tags: [atlas, storage, backup, borg, borg_key]
+  ansible.builtin.assert:
+    that:
+      - atlas_manage_storage | bool
+      - atlas_zfs_pool != 'CHANGEME_ZFS_POOL'
+      - atlas_mount_root.startswith('/')
+      - atlas_borg_username is match('^[a-z_][a-z0-9_-]*$')
+      - atlas_borg_group is match('^[a-z_][a-z0-9_-]*$')
+      - atlas_borg_username not in ['root', atlas_admin_username]
+      - atlas_borg_group != 'wheel'
+      - atlas_borg_home.startswith('/var/lib/')
+      - atlas_borg_repository_host is match('^[A-Za-z0-9.-]+$')
+      - atlas_borg_repository_user is match('^[A-Za-z0-9_-]+$')
+      - atlas_borg_repository_port | int > 0
+      - atlas_borg_repository_port | int < 65536
+      - atlas_borg_repository_path is match('^\./[A-Za-z0-9][A-Za-z0-9._/-]*$')
+      - "'/../' not in ('/' ~ atlas_borg_repository_path ~ '/')"
+      - atlas_borg_remote_path is match('^borg-[0-9]+\.[0-9]+$')
+      - atlas_borg_host_key.startswith(
+          '[' ~ atlas_borg_repository_host ~ ']:' ~ (atlas_borg_repository_port | string) ~ ' ssh-ed25519 '
+        )
+      - atlas_borg_ssh_private_key_path.startswith('/etc/atlas-borg/')
+      - atlas_borg_known_hosts_path.startswith('/etc/atlas-borg/')
+      - atlas_borg_passphrase_path.startswith('/etc/atlas-borg/')
+      - atlas_borg_ssh_wrapper_path.startswith('/usr/local/libexec/')
+      - atlas_borg_encryption_mode == 'repokey'
+      - atlas_borg_archive_prefix is match('^[a-z0-9][a-z0-9_-]*$')
+      - atlas_borg_snapshot_prefix is match('^[a-z0-9][a-z0-9_-]*$')
+      - atlas_borg_keep_daily | int > 0
+      - atlas_borg_keep_weekly | int > 0
+      - atlas_borg_keep_monthly | int > 0
+    fail_msg: >-
+      Atlas Borg needs a safe relative repository path, a pinned ED25519 host
+      key, positive retention counts, and valid dedicated SSH settings.
+  when: atlas_manage_borg_backup | bool
+
+- name: Create the Atlas Borg system group
+  tags: [atlas, storage, backup, borg, borg_key]
+  ansible.builtin.group:
+    name: "{{ atlas_borg_group }}"
+    system: true
+    state: present
+  when: atlas_manage_borg_backup | bool
+
+- name: Create the least-privilege Atlas Borg account
+  tags: [atlas, storage, backup, borg, borg_key]
+  ansible.builtin.user:
+    name: "{{ atlas_borg_username }}"
+    group: "{{ atlas_borg_group }}"
+    groups: []
+    append: false
+    comment: Atlas Borg backup service
+    home: "{{ atlas_borg_home }}"
+    create_home: false
+    shell: /sbin/nologin
+    password_lock: true
+    system: true
+    state: present
+  when: atlas_manage_borg_backup | bool
+
+- name: Read Atlas Borg account group membership
+  tags: [atlas, storage, backup, borg, borg_key]
+  ansible.builtin.command:
+    argv:
+      - id
+      - -nG
+      - "{{ atlas_borg_username }}"
+  register: atlas_borg_account_groups
+  changed_when: false
+  check_mode: false
+  when:
+    - atlas_manage_borg_backup | bool
+    - not ansible_check_mode
+
+- name: Require the Atlas Borg account to have no supplementary groups
+  tags: [atlas, storage, backup, borg, borg_key]
+  ansible.builtin.assert:
+    that:
+      - atlas_borg_account_groups.stdout.split() == [atlas_borg_group]
+    fail_msg: >-
+      The Atlas Borg service account must belong only to its private primary
+      group and must never receive wheel or other supplementary membership.
+  when:
+    - atlas_manage_borg_backup | bool
+    - not ansible_check_mode
+
+- name: Validate Atlas Borg systemd calendars
+  tags: [atlas, storage, backup, borg]
+  ansible.builtin.command:
+    argv:
+      - systemd-analyze
+      - calendar
+      - "{{ item }}"
+  loop:
+    - "{{ atlas_borg_backup_calendar }}"
+    - "{{ atlas_borg_check_calendar }}"
+  changed_when: false
+  check_mode: false
+  when: atlas_manage_borg_backup | bool
+
+- name: Create Atlas Borg configuration directory
+  tags: [atlas, storage, backup, borg, borg_key]
+  ansible.builtin.file:
+    path: /etc/atlas-borg
+    state: directory
+    owner: root
+    group: "{{ atlas_borg_group }}"
+    mode: "0750"
+  when: atlas_manage_borg_backup | bool
+
+- name: Generate the dedicated Atlas Borg SSH identity
+  tags: [atlas, storage, backup, borg, borg_key]
+  ansible.builtin.command:
+    argv:
+      - ssh-keygen
+      - -q
+      - -t
+      - ed25519
+      - -N
+      - ""
+      - -C
+      - atlas-borg@atlas
+      - -f
+      - "{{ atlas_borg_ssh_private_key_path }}"
+    creates: "{{ atlas_borg_ssh_private_key_path }}"
+  when: atlas_manage_borg_backup | bool
+
+- name: Protect the Atlas Borg private SSH identity
+  tags: [atlas, storage, backup, borg, borg_key]
+  ansible.builtin.file:
+    path: "{{ atlas_borg_ssh_private_key_path }}"
+    owner: "{{ atlas_borg_username }}"
+    group: "{{ atlas_borg_group }}"
+    mode: "0600"
+  when:
+    - atlas_manage_borg_backup | bool
+    - not ansible_check_mode
+
+- name: Set permissions on the Atlas Borg public SSH identity
+  tags: [atlas, storage, backup, borg, borg_key]
+  ansible.builtin.file:
+    path: "{{ atlas_borg_ssh_private_key_path }}.pub"
+    owner: "{{ atlas_borg_username }}"
+    group: "{{ atlas_borg_group }}"
+    mode: "0644"
+  when:
+    - atlas_manage_borg_backup | bool
+    - not ansible_check_mode
+
+- name: Read the dedicated Atlas Borg public SSH identity
+  tags: [atlas, storage, backup, borg, borg_key]
+  ansible.builtin.slurp:
+    src: "{{ atlas_borg_ssh_private_key_path }}.pub"
+  register: atlas_borg_public_key
+  when:
+    - atlas_manage_borg_backup | bool
+    - not ansible_check_mode
+
+- name: Report the public SSH identity to install in the Hetzner sub-account
+  tags: [atlas, storage, backup, borg, borg_key]
+  ansible.builtin.debug:
+    msg: "{{ atlas_borg_public_key.content | b64decode | trim }}"
+  when:
+    - atlas_manage_borg_backup | bool
+    - not ansible_check_mode
+
+- name: Pin the Hetzner Storage Box SSH host key
+  tags: [atlas, storage, backup, borg]
+  ansible.builtin.copy:
+    content: "{{ atlas_borg_host_key }}\n"
+    dest: "{{ atlas_borg_known_hosts_path }}"
+    owner: "{{ atlas_borg_username }}"
+    group: "{{ atlas_borg_group }}"
+    mode: "0600"
+  when: atlas_manage_borg_backup | bool
+
+- name: Create Atlas Borg state directories
+  tags: [atlas, storage, backup, borg]
+  ansible.builtin.file:
+    path: "{{ item }}"
+    state: directory
+    owner: "{{ atlas_borg_username }}"
+    group: "{{ atlas_borg_group }}"
+    mode: "0700"
+  loop:
+    - "{{ atlas_borg_config_dir }}"
+    - "{{ atlas_borg_cache_dir }}"
+  when: atlas_manage_borg_backup | bool
+
+- name: Create the shared Atlas Borg operation lock
+  tags: [atlas, storage, backup, borg]
+  ansible.builtin.copy:
+    content: ""
+    dest: "{{ atlas_borg_lock_path }}"
+    owner: "{{ atlas_borg_username }}"
+    group: "{{ atlas_borg_group }}"
+    mode: "0600"
+    force: false
+  when: atlas_manage_borg_backup | bool
+
+- name: Require the Atlas Borg encryption passphrase from Vault
+  tags: [atlas, storage, backup, borg]
+  ansible.builtin.assert:
+    that:
+      - atlas_borg_passphrase | length >= 20
+    fail_msg: >-
+      Define vault_atlas_borg_passphrase with a strong unique value in the
+      encrypted Vault before activating the Borg repository.
+  no_log: true
+  when: atlas_manage_borg_backup | bool
+
+- name: Install the Atlas Borg passphrase
+  tags: [atlas, storage, backup, borg]
+  ansible.builtin.copy:
+    content: "{{ atlas_borg_passphrase }}\n"
+    dest: "{{ atlas_borg_passphrase_path }}"
+    owner: "{{ atlas_borg_username }}"
+    group: "{{ atlas_borg_group }}"
+    mode: "0600"
+  diff: false
+  no_log: true
+  when: atlas_manage_borg_backup | bool
+
+- name: Install the Atlas Borg backup helper
+  tags: [atlas, storage, backup, borg]
+  ansible.builtin.template:
+    src: atlas-borg-backup.sh.j2
+    dest: /usr/local/sbin/atlas-borg-backup
+    owner: root
+    group: root
+    mode: "0750"
+  when: atlas_manage_borg_backup | bool
+
+- name: Install the Atlas Borg check helper
+  tags: [atlas, storage, backup, borg]
+  ansible.builtin.template:
+    src: atlas-borg-check.sh.j2
+    dest: /usr/local/sbin/atlas-borg-check
+    owner: root
+    group: "{{ atlas_borg_group }}"
+    mode: "0750"
+  when: atlas_manage_borg_backup | bool
+
+- name: Create the local libexec directory for the Atlas Borg SSH wrapper
+  tags: [atlas, storage, backup, borg]
+  ansible.builtin.file:
+    path: "{{ atlas_borg_ssh_wrapper_path | dirname }}"
+    state: directory
+    owner: root
+    group: root
+    mode: "0755"
+  when: atlas_manage_borg_backup | bool
+
+- name: Install the capability-dropping Atlas Borg SSH wrapper
+  tags: [atlas, storage, backup, borg]
+  ansible.builtin.template:
+    src: atlas-borg-ssh.sh.j2
+    dest: "{{ atlas_borg_ssh_wrapper_path }}"
+    owner: root
+    group: root
+    mode: "0755"
+  when: atlas_manage_borg_backup | bool
+
+- name: Install Atlas Borg systemd units
+  tags: [atlas, storage, backup, borg]
+  ansible.builtin.template:
+    src: "{{ item }}.j2"
+    dest: "/etc/systemd/system/{{ item }}"
+    owner: root
+    group: root
+    mode: "0644"
+  loop:
+    - atlas-borg-backup.service
+    - atlas-borg-backup.timer
+    - atlas-borg-check.service
+    - atlas-borg-check.timer
+  notify: Restart Atlas Borg timers
+  when: atlas_manage_borg_backup | bool
+
+- name: Verify dedicated SSH access to the Hetzner Storage Box
+  tags: [atlas, storage, backup, borg]
+  ansible.builtin.command:
+    argv:
+      - ssh
+      - -T
+      - -i
+      - "{{ atlas_borg_ssh_private_key_path }}"
+      - -p
+      - "{{ atlas_borg_repository_port | string }}"
+      - -o
+      - BatchMode=yes
+      - -o
+      - IdentitiesOnly=yes
+      - -o
+      - StrictHostKeyChecking=yes
+      - -o
+      - "UserKnownHostsFile={{ atlas_borg_known_hosts_path }}"
+      - "{{ atlas_borg_repository_user }}@{{ atlas_borg_repository_host }}"
+      - pwd
+  register: atlas_borg_ssh_probe
+  become: true
+  become_user: "{{ atlas_borg_username }}"
+  changed_when: false
+  failed_when: false
+  when:
+    - atlas_manage_borg_backup | bool
+    - not ansible_check_mode
+
+- name: Require the dedicated public key on the Hetzner sub-account
+  tags: [atlas, storage, backup, borg]
+  ansible.builtin.assert:
+    that:
+      - atlas_borg_ssh_probe.rc == 0
+    fail_msg: >-
+      Install the reported Atlas Borg public key in the Hetzner sub-account
+      before rerunning the Borg tasks. Password authentication is never used.
+  when:
+    - atlas_manage_borg_backup | bool
+    - not ansible_check_mode
+
+- name: Probe the remote Atlas Borg repository path
+  tags: [atlas, storage, backup, borg]
+  ansible.builtin.command:
+    argv:
+      - ssh
+      - -T
+      - -i
+      - "{{ atlas_borg_ssh_private_key_path }}"
+      - -p
+      - "{{ atlas_borg_repository_port | string }}"
+      - -o
+      - BatchMode=yes
+      - -o
+      - IdentitiesOnly=yes
+      - -o
+      - StrictHostKeyChecking=yes
+      - -o
+      - "UserKnownHostsFile={{ atlas_borg_known_hosts_path }}"
+      - "{{ atlas_borg_repository_user }}@{{ atlas_borg_repository_host }}"
+      - stat
+      - "{{ atlas_borg_repository_path }}"
+  register: atlas_borg_repository_path_probe
+  become: true
+  become_user: "{{ atlas_borg_username }}"
+  changed_when: false
+  failed_when: false
+  when:
+    - atlas_manage_borg_backup | bool
+    - not ansible_check_mode
+
+- name: Probe the Atlas Borg repository
+  tags: [atlas, storage, backup, borg]
+  ansible.builtin.command:
+    argv:
+      - /usr/bin/borg
+      - --remote-path
+      - "{{ atlas_borg_remote_path }}"
+      - info
+      - >-
+        ssh://{{ atlas_borg_repository_user }}@{{ atlas_borg_repository_host }}:
+        {{- atlas_borg_repository_port }}/{{ atlas_borg_repository_path }}
+  environment:
+    BORG_CACHE_DIR: "{{ atlas_borg_cache_dir }}"
+    BORG_CONFIG_DIR: "{{ atlas_borg_config_dir }}"
+    BORG_PASSCOMMAND: "cat {{ atlas_borg_passphrase_path }}"
+    BORG_RSH: "{{ atlas_borg_ssh_wrapper_path }}"
+  register: atlas_borg_repository_probe
+  become: true
+  become_user: "{{ atlas_borg_username }}"
+  changed_when: false
+  failed_when: false
+  when:
+    - atlas_manage_borg_backup | bool
+    - not ansible_check_mode
+    - atlas_borg_repository_path_probe.rc == 0
+
+- name: Reject an existing path that is not the configured Borg repository
+  tags: [atlas, storage, backup, borg]
+  ansible.builtin.assert:
+    that:
+      - atlas_borg_repository_probe.rc == 0
+    fail_msg: >-
+      The remote repository path already exists but Borg could not open it.
+      Refusing to initialize over existing data; verify the path, passphrase,
+      and repository state manually.
+  when:
+    - atlas_manage_borg_backup | bool
+    - not ansible_check_mode
+    - atlas_borg_repository_path_probe.rc == 0
+
+- name: Initialize the encrypted Atlas Borg repository
+  tags: [atlas, storage, backup, borg]
+  ansible.builtin.command:
+    argv:
+      - /usr/bin/borg
+      - --remote-path
+      - "{{ atlas_borg_remote_path }}"
+      - init
+      - --encryption
+      - "{{ atlas_borg_encryption_mode }}"
+      - >-
+        ssh://{{ atlas_borg_repository_user }}@{{ atlas_borg_repository_host }}:
+        {{- atlas_borg_repository_port }}/{{ atlas_borg_repository_path }}
+  environment:
+    BORG_CACHE_DIR: "{{ atlas_borg_cache_dir }}"
+    BORG_CONFIG_DIR: "{{ atlas_borg_config_dir }}"
+    BORG_PASSCOMMAND: "cat {{ atlas_borg_passphrase_path }}"
+    BORG_RSH: "{{ atlas_borg_ssh_wrapper_path }}"
+  become: true
+  become_user: "{{ atlas_borg_username }}"
+  when:
+    - atlas_manage_borg_backup | bool
+    - not ansible_check_mode
+    - atlas_borg_repository_path_probe.rc != 0
+
+- name: Verify the encrypted Atlas Borg repository
+  tags: [atlas, storage, backup, borg]
+  ansible.builtin.command:
+    argv:
+      - /usr/bin/borg
+      - --remote-path
+      - "{{ atlas_borg_remote_path }}"
+      - info
+      - >-
+        ssh://{{ atlas_borg_repository_user }}@{{ atlas_borg_repository_host }}:
+        {{- atlas_borg_repository_port }}/{{ atlas_borg_repository_path }}
+  environment:
+    BORG_CACHE_DIR: "{{ atlas_borg_cache_dir }}"
+    BORG_CONFIG_DIR: "{{ atlas_borg_config_dir }}"
+    BORG_PASSCOMMAND: "cat {{ atlas_borg_passphrase_path }}"
+    BORG_RSH: "{{ atlas_borg_ssh_wrapper_path }}"
+  become: true
+  become_user: "{{ atlas_borg_username }}"
+  changed_when: false
+  when:
+    - atlas_manage_borg_backup | bool
+    - not ansible_check_mode
+
+- name: Check for the local Atlas Borg recovery-key export
+  tags: [atlas, storage, backup, borg]
+  ansible.builtin.stat:
+    path: "{{ atlas_borg_recovery_export_path }}"
+  register: atlas_borg_recovery_export
+  delegate_to: localhost
+  become: false
+  when:
+    - atlas_manage_borg_backup | bool
+    - not ansible_check_mode
+
+- name: Export the Atlas Borg recovery key for offline preservation
+  tags: [atlas, storage, backup, borg]
+  when:
+    - atlas_manage_borg_backup | bool
+    - not ansible_check_mode
+    - not atlas_borg_recovery_export.stat.exists
+  no_log: true
+  block:
+    - name: Create the local recovery-material directory
+      ansible.builtin.file:
+        path: "{{ atlas_borg_recovery_export_path | dirname }}"
+        state: directory
+        mode: "0700"
+      delegate_to: localhost
+      become: false
+
+    - name: Export the encrypted Borg repository key on Atlas
+      ansible.builtin.command:
+        argv:
+          - /usr/bin/borg
+          - --remote-path
+          - "{{ atlas_borg_remote_path }}"
+          - key
+          - export
+          - >-
+            ssh://{{ atlas_borg_repository_user }}@{{ atlas_borg_repository_host }}:
+            {{- atlas_borg_repository_port }}/{{ atlas_borg_repository_path }}
+          - "{{ atlas_borg_config_dir }}/atlas-borg-repokey.export"
+      environment:
+        BORG_CACHE_DIR: "{{ atlas_borg_cache_dir }}"
+        BORG_CONFIG_DIR: "{{ atlas_borg_config_dir }}"
+        BORG_PASSCOMMAND: "cat {{ atlas_borg_passphrase_path }}"
+        BORG_RSH: "{{ atlas_borg_ssh_wrapper_path }}"
+      become: true
+      become_user: "{{ atlas_borg_username }}"
+
+    - name: Fetch the encrypted Borg recovery key from Atlas
+      ansible.builtin.fetch:
+        src: "{{ atlas_borg_config_dir }}/atlas-borg-repokey.export"
+        dest: "{{ atlas_borg_recovery_export_path }}"
+        flat: true
+
+    - name: Protect the local Borg recovery-key export
+      ansible.builtin.file:
+        path: "{{ atlas_borg_recovery_export_path }}"
+        mode: "0600"
+      delegate_to: localhost
+      become: false
+  always:
+    - name: Remove the temporary recovery-key export from Atlas
+      ansible.builtin.file:
+        path: "{{ atlas_borg_config_dir }}/atlas-borg-repokey.export"
+        state: absent
+
+- name: Enable Atlas Borg backup and check timers
+  tags: [atlas, storage, backup, borg]
+  ansible.builtin.systemd:
+    name: "{{ item }}"
+    enabled: true
+    state: started
+    daemon_reload: true
+  loop:
+    - atlas-borg-backup.timer
+    - atlas-borg-check.timer
+  when:
+    - atlas_manage_borg_backup | bool
+    - not ansible_check_mode

ansible/roles/profile_atlas/tasks/main.yml

@@ -17,6 +17,9 @@
 - name: Import Atlas ZFS maintenance tasks
   ansible.builtin.import_tasks: zfs_maintenance.yml
 
+- name: Import Atlas Borg backup tasks
+  ansible.builtin.import_tasks: borg_backup.yml
+
 - name: Import Atlas file sharing tasks
   ansible.builtin.import_tasks: sharing.yml
 

ansible/roles/profile_atlas/templates/atlas-borg-backup.service.j2

@@ -0,0 +1,40 @@
+[Unit]
+Description=Back up Atlas ZFS datasets to the encrypted Borg repository
+Documentation=man:borg-create(1) man:borg-prune(1) man:borg-compact(1)
+Requires=zfs.target
+Wants=network-online.target
+After=zfs.target network-online.target
+StartLimitIntervalSec=6h
+StartLimitBurst=3
+ConditionFileIsExecutable=/usr/local/sbin/atlas-borg-backup
+ConditionPathExists={{ atlas_borg_passphrase_path }}
+ConditionPathExists={{ atlas_borg_ssh_private_key_path }}
+ConditionPathExists={{ atlas_borg_known_hosts_path }}
+
+[Service]
+Type=oneshot
+ExecStart=/usr/local/sbin/atlas-borg-backup
+User=root
+Group=root
+UMask=0077
+SuccessExitStatus=1
+Restart=on-failure
+RestartSec=30m
+TimeoutStartSec=infinity
+RuntimeDirectory=atlas-borg
+RuntimeDirectoryMode=0750
+Nice=15
+IOSchedulingClass=best-effort
+IOSchedulingPriority=7
+NoNewPrivileges=true
+PrivateMounts=true
+PrivateTmp=true
+ProtectHome=true
+ProtectSystem=strict
+ReadWritePaths={{ atlas_borg_cache_dir }} {{ atlas_borg_config_dir }} /run/atlas-borg /run/lock
+ProtectKernelTunables=true
+ProtectKernelModules=true
+ProtectControlGroups=true
+RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6
+RestrictRealtime=true
+LockPersonality=true

ansible/roles/profile_atlas/templates/atlas-borg-backup.sh.j2

@@ -0,0 +1,176 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+export LC_ALL=C
+export PATH=/usr/sbin:/usr/bin:/sbin:/bin
+export BORG_CACHE_DIR={{ atlas_borg_cache_dir | quote }}
+export BORG_CONFIG_DIR={{ atlas_borg_config_dir | quote }}
+export BORG_PASSCOMMAND={{ ('cat ' ~ atlas_borg_passphrase_path) | quote }}
+export BORG_RSH={{ atlas_borg_ssh_wrapper_path | quote }}
+
+readonly pool={{ atlas_zfs_pool | quote }}
+readonly mount_root={{ atlas_mount_root | quote }}
+readonly repository={{ ('ssh://' ~ atlas_borg_repository_user ~ '@' ~ atlas_borg_repository_host
+  ~ ':' ~ (atlas_borg_repository_port | string) ~ '/' ~ atlas_borg_repository_path) | quote }}
+readonly remote_path={{ atlas_borg_remote_path | quote }}
+readonly archive_prefix={{ atlas_borg_archive_prefix | quote }}
+readonly snapshot_prefix={{ atlas_borg_snapshot_prefix | quote }}
+readonly compression={{ atlas_borg_compression | quote }}
+readonly stage=/run/atlas-borg/source
+readonly borg_user={{ atlas_borg_username | quote }}
+readonly borg_group={{ atlas_borg_group | quote }}
+readonly borg_home={{ atlas_borg_home | quote }}
+readonly borg_lock={{ atlas_borg_lock_path | quote }}
+
+snapshot_name=""
+snapshot_created=false
+mounted_targets=()
+
+# Invoked through the EXIT trap below.
+# shellcheck disable=SC2329
+cleanup() {
+  local status=$?
+  local cleanup_status=0
+  local index
+  trap - EXIT HUP INT TERM
+  set +e
+
+{% raw %}
+  for ((index = ${#mounted_targets[@]} - 1; index >= 0; index--)); do
+{% endraw %}
+    if mountpoint -q "${mounted_targets[$index]}"; then
+      umount "${mounted_targets[$index]}" || cleanup_status=2
+    fi
+  done
+  rm -rf "$stage" || cleanup_status=2
+
+  if [[ "$snapshot_created" == true ]]; then
+    flock 9
+    zfs destroy -r "${pool}@${snapshot_name}" || cleanup_status=2
+    flock -u 9
+  fi
+
+  if ((status == 0 && cleanup_status != 0)); then
+    status=$cleanup_status
+  fi
+  exit "$status"
+}
+
+trap cleanup EXIT
+trap 'exit 143' HUP INT TERM
+
+run_as_borg() {
+  setpriv \
+    --reuid "$borg_user" \
+    --regid "$borg_group" \
+    --clear-groups \
+    --inh-caps=-all,+dac_read_search \
+    --ambient-caps=-all,+dac_read_search \
+    --bounding-set=-all,+dac_read_search \
+    -- env HOME="$borg_home" USER="$borg_user" LOGNAME="$borg_user" "$@"
+}
+
+exec 8>"$borg_lock"
+flock 8
+exec 9>/run/lock/atlas-zfs-snapshot.lock
+
+zpool list -H -o name "$pool" >/dev/null
+rm -rf "$stage"
+mkdir -p "$stage"
+chown root:"$borg_group" /run/atlas-borg "$stage"
+chmod 0750 /run/atlas-borg "$stage"
+
+flock 9
+while IFS= read -r stale_snapshot; do
+  stale_suffix="${stale_snapshot#"${pool}@${snapshot_prefix}-"}"
+  if [[ "$stale_suffix" =~ ^[0-9]{8}T[0-9]{6}Z$ ]]; then
+    zfs destroy -r "$stale_snapshot"
+    printf 'Removed stale Borg source snapshot %s\n' "$stale_snapshot"
+  fi
+done < <(
+  zfs list -H -t snapshot -o name -r "$pool" |
+    grep -E "^${pool}@${snapshot_prefix}-[0-9]{8}T[0-9]{6}Z$" || true
+)
+
+timestamp="$(date -u +%Y%m%dT%H%M%SZ)"
+readonly timestamp
+snapshot_name="${snapshot_prefix}-${timestamp}"
+readonly snapshot_name
+zfs snapshot -r "${pool}@${snapshot_name}"
+snapshot_created=true
+flock -u 9
+printf 'Created recursive Borg source snapshot %s@%s\n' "$pool" "$snapshot_name"
+
+while IFS=$'\t' read -r dataset dataset_mountpoint mounted; do
+  if [[ "$mounted" != yes ]]; then
+    printf 'Dataset %s is not mounted; refusing an incomplete backup\n' "$dataset" >&2
+    exit 65
+  fi
+  if [[ "$dataset_mountpoint" != "$mount_root" && "$dataset_mountpoint" != "$mount_root/"* ]]; then
+    printf 'Dataset %s has unexpected mountpoint %s\n' "$dataset" "$dataset_mountpoint" >&2
+    exit 65
+  fi
+
+  dataset_suffix="${dataset#"$pool"}"
+  source_path="${dataset_mountpoint}/.zfs/snapshot/${snapshot_name}"
+  target_path="${stage}${dataset_suffix}"
+  mkdir -p "$target_path"
+  mount --bind "$source_path" "$target_path"
+  mount -o remount,bind,ro "$target_path"
+  mounted_targets+=("$target_path")
+done < <(zfs list -H -o name,mountpoint,mounted -s name -r "$pool")
+
+archive="${archive_prefix}-${timestamp}"
+readonly archive
+borg_status=0
+
+set +e
+(
+  cd /run/atlas-borg
+  run_as_borg borg --remote-path "$remote_path" --lock-wait 600 create \
+    --show-rc \
+    --stats \
+    --checkpoint-interval 900 \
+    --compression "$compression" \
+    "${repository}::${archive}" \
+    source
+)
+create_status=$?
+set -e
+if ((create_status >= 2)); then
+  exit "$create_status"
+fi
+borg_status=$create_status
+
+set +e
+run_as_borg borg --remote-path "$remote_path" --lock-wait 600 prune \
+  --show-rc \
+  --list \
+  --glob-archives "${archive_prefix}-*" \
+  --keep-daily {{ atlas_borg_keep_daily | int }} \
+  --keep-weekly {{ atlas_borg_keep_weekly | int }} \
+  --keep-monthly {{ atlas_borg_keep_monthly | int }} \
+  "$repository"
+prune_status=$?
+set -e
+if ((prune_status >= 2)); then
+  exit "$prune_status"
+fi
+if ((prune_status > borg_status)); then
+  borg_status=$prune_status
+fi
+
+set +e
+run_as_borg borg --remote-path "$remote_path" --lock-wait 600 compact \
+  --show-rc \
+  "$repository"
+compact_status=$?
+set -e
+if ((compact_status >= 2)); then
+  exit "$compact_status"
+fi
+if ((compact_status > borg_status)); then
+  borg_status=$compact_status
+fi
+
+exit "$borg_status"

ansible/roles/profile_atlas/templates/atlas-borg-backup.timer.j2

@@ -0,0 +1,12 @@
+[Unit]
+Description=Schedule the encrypted Atlas Borg backup
+
+[Timer]
+OnCalendar={{ atlas_borg_backup_calendar }}
+Persistent=true
+RandomizedDelaySec={{ atlas_borg_randomized_delay }}
+AccuracySec=1min
+Unit=atlas-borg-backup.service
+
+[Install]
+WantedBy=timers.target

ansible/roles/profile_atlas/templates/atlas-borg-check.service.j2

@@ -0,0 +1,32 @@
+[Unit]
+Description=Check the encrypted Atlas Borg repository
+Documentation=man:borg-check(1)
+Wants=network-online.target
+After=network-online.target atlas-borg-backup.service
+ConditionFileIsExecutable=/usr/local/sbin/atlas-borg-check
+ConditionPathExists={{ atlas_borg_passphrase_path }}
+ConditionPathExists={{ atlas_borg_ssh_private_key_path }}
+ConditionPathExists={{ atlas_borg_known_hosts_path }}
+
+[Service]
+Type=oneshot
+ExecStart=/usr/local/sbin/atlas-borg-check
+User={{ atlas_borg_username }}
+Group={{ atlas_borg_group }}
+UMask=0077
+SuccessExitStatus=1
+TimeoutStartSec=infinity
+Nice=15
+IOSchedulingClass=best-effort
+IOSchedulingPriority=7
+NoNewPrivileges=true
+PrivateTmp=true
+ProtectHome=true
+ProtectSystem=strict
+ReadWritePaths={{ atlas_borg_cache_dir }} {{ atlas_borg_config_dir }}
+ProtectKernelTunables=true
+ProtectKernelModules=true
+ProtectControlGroups=true
+RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6
+RestrictRealtime=true
+LockPersonality=true

ansible/roles/profile_atlas/templates/atlas-borg-check.sh.j2

@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+export LC_ALL=C
+export PATH=/usr/sbin:/usr/bin:/sbin:/bin
+export BORG_CACHE_DIR={{ atlas_borg_cache_dir | quote }}
+export BORG_CONFIG_DIR={{ atlas_borg_config_dir | quote }}
+export BORG_PASSCOMMAND={{ ('cat ' ~ atlas_borg_passphrase_path) | quote }}
+export BORG_RSH={{ atlas_borg_ssh_wrapper_path | quote }}
+
+readonly repository={{ ('ssh://' ~ atlas_borg_repository_user ~ '@' ~ atlas_borg_repository_host
+  ~ ':' ~ (atlas_borg_repository_port | string) ~ '/' ~ atlas_borg_repository_path) | quote }}
+readonly remote_path={{ atlas_borg_remote_path | quote }}
+readonly archive_prefix={{ atlas_borg_archive_prefix | quote }}
+readonly borg_lock={{ atlas_borg_lock_path | quote }}
+
+exec 8>"$borg_lock"
+flock 8
+
+exec borg --remote-path "$remote_path" --lock-wait 600 check \
+  --show-rc \
+  --glob-archives "${archive_prefix}-*" \
+  "$repository"

ansible/roles/profile_atlas/templates/atlas-borg-check.timer.j2

@@ -0,0 +1,12 @@
+[Unit]
+Description=Schedule checks of the encrypted Atlas Borg repository
+
+[Timer]
+OnCalendar={{ atlas_borg_check_calendar }}
+Persistent=true
+RandomizedDelaySec={{ atlas_borg_randomized_delay }}
+AccuracySec=1min
+Unit=atlas-borg-check.service
+
+[Install]
+WantedBy=timers.target

ansible/roles/profile_atlas/templates/atlas-borg-ssh.sh.j2

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# Borg receives CAP_DAC_READ_SEARCH only for local snapshot traversal. Drop it
+# before starting the network transport so SSH runs as the plain service user.
+exec setpriv \
+  --inh-caps=-all \
+  --ambient-caps=-all \
+  -- /usr/bin/ssh \
+  -i {{ atlas_borg_ssh_private_key_path | quote }} \
+  -p {{ atlas_borg_repository_port | int }} \
+  -o BatchMode=yes \
+  -o IdentitiesOnly=yes \
+  -o StrictHostKeyChecking=yes \
+  -o UserKnownHostsFile={{ atlas_borg_known_hosts_path | quote }} \
+  -o ConnectTimeout=30 \
+  -o ServerAliveInterval=60 \
+  -o ServerAliveCountMax=3 \
+  "$@"