Manage Atlas ZFS snapshots and scrubs

ansible/roles/profile_atlas/defaults/main.yml

@@ -61,6 +61,12 @@ atlas_zfs_backup_reservation: 500G
 atlas_zfs_dataset_photobook: media/photobook
 atlas_mount_root: /CHANGEME_ATLAS_MOUNT_ROOT
 
+atlas_manage_zfs_snapshots: false
+atlas_zfs_snapshot_prefix: atlas-auto
+atlas_zfs_snapshot_policies: []
+atlas_manage_zfs_scrub: false
+atlas_zfs_scrub_calendar: ""
+
 atlas_archive_mountpoint: "{{ atlas_mount_root }}/{{ atlas_zfs_dataset_archive }}"
 atlas_services_mountpoint: "{{ atlas_mount_root }}/{{ atlas_zfs_dataset_services }}"
 atlas_app_data_mountpoint: "{{ atlas_mount_root }}/{{ atlas_zfs_dataset_app_data }}"

ansible/roles/profile_atlas/tasks/main.yml

@@ -14,6 +14,9 @@
 - name: Import Atlas storage tasks
   ansible.builtin.import_tasks: storage.yml
 
+- name: Import Atlas ZFS maintenance tasks
+  ansible.builtin.import_tasks: zfs_maintenance.yml
+
 - name: Import Atlas file sharing tasks
   ansible.builtin.import_tasks: sharing.yml
 

ansible/roles/profile_atlas/tasks/zfs_maintenance.yml

@@ -0,0 +1,175 @@
+---
+- name: Validate Atlas ZFS snapshot policy
+  tags: [atlas, storage, snapshots]
+  ansible.builtin.assert:
+    that:
+      - atlas_manage_storage | bool
+      - atlas_zfs_pool != 'CHANGEME_ZFS_POOL'
+      - atlas_zfs_snapshot_prefix is match('^[a-z0-9][a-z0-9_-]*$')
+      - atlas_zfs_snapshot_policies | length > 0
+      - >-
+        (atlas_zfs_snapshot_policies | map(attribute='name') | unique | list | length)
+        == (atlas_zfs_snapshot_policies | length)
+    fail_msg: >-
+      Enable Atlas storage and declare a non-empty snapshot policy with a safe
+      prefix and unique policy names before managing automatic snapshots.
+  when: atlas_manage_zfs_snapshots | bool
+
+- name: Validate Atlas ZFS snapshot policy entries
+  tags: [atlas, storage, snapshots]
+  ansible.builtin.assert:
+    that:
+      - item.name is match('^[a-z][a-z0-9_-]*$')
+      - item.keep | int > 0
+      - item.calendar | length > 0
+    fail_msg: >-
+      Every Atlas snapshot policy needs a safe name, a positive retention
+      count, and a systemd calendar expression.
+  loop: "{{ atlas_zfs_snapshot_policies }}"
+  loop_control:
+    label: "{{ item.name | default('unnamed') }}"
+  when: atlas_manage_zfs_snapshots | bool
+
+- name: Validate Atlas ZFS snapshot calendars
+  tags: [atlas, storage, snapshots]
+  ansible.builtin.command:
+    argv:
+      - systemd-analyze
+      - calendar
+      - "{{ item.calendar }}"
+  loop: "{{ atlas_zfs_snapshot_policies }}"
+  loop_control:
+    label: "{{ item.name }}: {{ item.calendar }}"
+  changed_when: false
+  check_mode: false
+  when: atlas_manage_zfs_snapshots | bool
+
+- name: Install Atlas ZFS snapshot and retention helper
+  tags: [atlas, storage, snapshots]
+  ansible.builtin.template:
+    src: atlas-zfs-snapshot.sh.j2
+    dest: /usr/local/sbin/atlas-zfs-snapshot
+    owner: root
+    group: root
+    mode: "0750"
+  when: atlas_manage_zfs_snapshots | bool
+
+- name: Install Atlas ZFS snapshot systemd service
+  tags: [atlas, storage, snapshots]
+  ansible.builtin.template:
+    src: atlas-zfs-snapshot@.service.j2
+    dest: /etc/systemd/system/atlas-zfs-snapshot@.service
+    owner: root
+    group: root
+    mode: "0644"
+  when: atlas_manage_zfs_snapshots | bool
+
+- name: Install Atlas ZFS snapshot systemd timers
+  tags: [atlas, storage, snapshots]
+  ansible.builtin.template:
+    src: atlas-zfs-snapshot.timer.j2
+    dest: "/etc/systemd/system/atlas-zfs-snapshot-{{ item.name }}.timer"
+    owner: root
+    group: root
+    mode: "0644"
+  loop: "{{ atlas_zfs_snapshot_policies }}"
+  loop_control:
+    label: "{{ item.name }}"
+  when: atlas_manage_zfs_snapshots | bool
+
+- name: Enable Atlas ZFS snapshot systemd timers
+  tags: [atlas, storage, snapshots]
+  ansible.builtin.systemd:
+    name: "atlas-zfs-snapshot-{{ item.name }}.timer"
+    enabled: true
+    state: started
+    daemon_reload: true
+  loop: "{{ atlas_zfs_snapshot_policies }}"
+  loop_control:
+    label: "{{ item.name }}"
+  when:
+    - atlas_manage_zfs_snapshots | bool
+    - not ansible_check_mode
+
+- name: Validate Atlas ZFS scrub policy
+  tags: [atlas, storage, scrub]
+  ansible.builtin.assert:
+    that:
+      - atlas_manage_storage | bool
+      - atlas_zfs_pool != 'CHANGEME_ZFS_POOL'
+      - atlas_zfs_scrub_calendar | length > 0
+    fail_msg: >-
+      Enable Atlas storage and declare a systemd calendar expression before
+      managing periodic ZFS scrubs.
+  when: atlas_manage_zfs_scrub | bool
+
+- name: Validate Atlas ZFS scrub calendar
+  tags: [atlas, storage, scrub]
+  ansible.builtin.command:
+    argv:
+      - systemd-analyze
+      - calendar
+      - "{{ atlas_zfs_scrub_calendar }}"
+  changed_when: false
+  check_mode: false
+  when: atlas_manage_zfs_scrub | bool
+
+- name: Require OpenZFS scrub systemd units
+  tags: [atlas, storage, scrub]
+  ansible.builtin.command:
+    argv:
+      - systemctl
+      - cat
+      - "{{ item }}"
+  loop:
+    - "zfs-scrub@{{ atlas_zfs_pool }}.service"
+    - "zfs-scrub-monthly@{{ atlas_zfs_pool }}.timer"
+    - "zfs-scrub-weekly@{{ atlas_zfs_pool }}.timer"
+  loop_control:
+    label: "{{ item }}"
+  changed_when: false
+  check_mode: false
+  when: atlas_manage_zfs_scrub | bool
+
+- name: Create Atlas ZFS scrub timer override directory
+  tags: [atlas, storage, scrub]
+  ansible.builtin.file:
+    path: "/etc/systemd/system/zfs-scrub-monthly@{{ atlas_zfs_pool }}.timer.d"
+    state: directory
+    owner: root
+    group: root
+    mode: "0755"
+  when: atlas_manage_zfs_scrub | bool
+
+- name: Configure Atlas ZFS monthly scrub schedule
+  tags: [atlas, storage, scrub]
+  ansible.builtin.template:
+    src: atlas-zfs-scrub-timer.conf.j2
+    dest: >-
+      /etc/systemd/system/zfs-scrub-monthly@{{ atlas_zfs_pool }}.timer.d/override.conf
+    owner: root
+    group: root
+    mode: "0644"
+  when: atlas_manage_zfs_scrub | bool
+
+- name: Disable the conflicting weekly OpenZFS scrub timer
+  tags: [atlas, storage, scrub]
+  ansible.builtin.systemd:
+    name: "zfs-scrub-weekly@{{ atlas_zfs_pool }}.timer"
+    enabled: false
+    state: stopped
+    daemon_reload: true
+  when:
+    - atlas_manage_zfs_scrub | bool
+    - not ansible_check_mode
+
+- name: Enable the Atlas monthly OpenZFS scrub timer
+  tags: [atlas, storage, scrub]
+  ansible.builtin.systemd:
+    name: "zfs-scrub-monthly@{{ atlas_zfs_pool }}.timer"
+    enabled: true
+    state: started
+    daemon_reload: true
+  when:
+    - atlas_manage_zfs_scrub | bool
+    - not ansible_check_mode

ansible/roles/profile_atlas/templates/atlas-zfs-scrub-timer.conf.j2

@@ -0,0 +1,6 @@
+[Timer]
+OnCalendar=
+OnCalendar={{ atlas_zfs_scrub_calendar }}
+Persistent=true
+RandomizedDelaySec=0
+AccuracySec=1min

ansible/roles/profile_atlas/templates/atlas-zfs-snapshot.sh.j2

@@ -0,0 +1,72 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+export LC_ALL=C
+export PATH=/usr/sbin:/usr/bin:/sbin:/bin
+
+if [[ $# -ne 1 ]]; then
+  printf 'Usage: %s <policy>\n' "$0" >&2
+  exit 64
+fi
+
+readonly pool={{ atlas_zfs_pool | quote }}
+readonly prefix={{ atlas_zfs_snapshot_prefix | quote }}
+readonly period="$1"
+
+case "$period" in
+{% for policy in atlas_zfs_snapshot_policies %}
+  {{ policy.name | quote }})
+    keep={{ policy.keep | int }}
+    ;;
+{% endfor %}
+  *)
+    printf 'Unknown Atlas ZFS snapshot policy: %s\n' "$period" >&2
+    exit 64
+    ;;
+esac
+readonly keep
+
+zpool list -H -o name "$pool" >/dev/null
+
+exec 9>/run/lock/atlas-zfs-snapshot.lock
+flock 9
+
+timestamp="$(date -u +%Y%m%dT%H%M%SZ)"
+readonly timestamp
+readonly snapshot_prefix="${pool}@${prefix}-${period}-"
+readonly snapshot="${snapshot_prefix}${timestamp}"
+
+zfs snapshot -r "$snapshot"
+printf 'Created recursive ZFS snapshot %s\n' "$snapshot"
+
+snapshot_listing="$(zfs list -H -t snapshot -o name -s creation -r "$pool")"
+managed_snapshots=()
+while IFS= read -r snapshot_name; do
+  if [[ "$snapshot_name" == "$snapshot_prefix"* ]]; then
+    snapshot_suffix="${snapshot_name#"$snapshot_prefix"}"
+    if [[ "$snapshot_suffix" =~ ^[0-9]{8}T[0-9]{6}Z$ ]]; then
+      managed_snapshots+=("$snapshot_name")
+    fi
+  fi
+done <<< "$snapshot_listing"
+
+{% raw %}
+managed_snapshot_count="${#managed_snapshots[@]}"
+{% endraw %}
+prune_count=$((managed_snapshot_count - keep))
+if ((prune_count <= 0)); then
+  printf 'Retaining %d of %d managed %s snapshots\n' \
+    "$managed_snapshot_count" "$keep" "$period"
+  exit 0
+fi
+
+for ((index = 0; index < prune_count; index++)); do
+  candidate="${managed_snapshots[$index]}"
+  if [[ "$candidate" != "$snapshot_prefix"* ]]; then
+    printf 'Refusing to destroy unexpected snapshot: %s\n' "$candidate" >&2
+    exit 65
+  fi
+
+  zfs destroy -r "$candidate"
+  printf 'Pruned recursive ZFS snapshot %s\n' "$candidate"
+done

ansible/roles/profile_atlas/templates/atlas-zfs-snapshot.timer.j2

@@ -0,0 +1,11 @@
+[Unit]
+Description=Schedule {{ item.name }} ZFS snapshots for {{ atlas_zfs_pool }}
+
+[Timer]
+OnCalendar={{ item.calendar }}
+Persistent=true
+AccuracySec=1min
+Unit=atlas-zfs-snapshot@{{ item.name }}.service
+
+[Install]
+WantedBy=timers.target

ansible/roles/profile_atlas/templates/atlas-zfs-snapshot@.service.j2

@@ -0,0 +1,25 @@
+[Unit]
+Description=Create and retain %i ZFS snapshots for {{ atlas_zfs_pool }}
+Documentation=man:zfs-snapshot(8) man:zfs-destroy(8)
+Requires=zfs.target
+After=zfs.target
+ConditionFileIsExecutable=/usr/local/sbin/atlas-zfs-snapshot
+
+[Service]
+Type=oneshot
+ExecStart=/usr/local/sbin/atlas-zfs-snapshot %i
+User=root
+Group=root
+UMask=0077
+Nice=10
+IOSchedulingClass=best-effort
+IOSchedulingPriority=7
+NoNewPrivileges=true
+PrivateTmp=true
+ProtectHome=true
+ProtectSystem=strict
+ProtectKernelTunables=true
+ProtectKernelModules=true
+ProtectControlGroups=true
+RestrictRealtime=true
+LockPersonality=true