mirror of
https://github.com/fscotto/infra.git
synced 2026-09-27 19:03:47 +00:00
Document Atlas NAS backups and monitoring
This commit is contained in:
Fabio Scotto di Santolo
@@ -108,6 +108,26 @@ atlas_manage_usb_reminder: false
|
||||
atlas_usb_reminder_calendar: ""
|
||||
atlas_usb_reminder_notifier: /opt/45drives/houston/houston-notify
|
||||
|
||||
# Read-only health probes and 45Drives Alerts; disabled outside Atlas host vars.
|
||||
atlas_manage_monitoring: false
|
||||
atlas_monitor_calendar: "*:0/30"
|
||||
atlas_monitor_notifier: "{{ atlas_usb_reminder_notifier }}"
|
||||
atlas_monitor_smart_devices: []
|
||||
atlas_monitor_timers: []
|
||||
atlas_monitor_failure_units: []
|
||||
atlas_monitor_remote_capacity: {}
|
||||
atlas_monitor_pool_warning_percent: 80
|
||||
atlas_monitor_pool_critical_percent: 90
|
||||
atlas_monitor_root_warning_percent: 80
|
||||
atlas_monitor_root_critical_percent: 90
|
||||
atlas_monitor_snapshot_warning_percent: 10
|
||||
atlas_monitor_snapshot_critical_percent: 20
|
||||
atlas_monitor_snapshot_growth_warning_gib_day: 100
|
||||
atlas_monitor_backup_growth_warning_gib_day: 100
|
||||
atlas_monitor_cpu_warning_c: 85
|
||||
atlas_monitor_cpu_critical_c: 95
|
||||
atlas_monitor_borg_max_runtime_days: 14
|
||||
|
||||
# Explicit post-restore relabeling only; never relabel datasets during ordinary runs.
|
||||
atlas_restorecon_paths: []
|
||||
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Turn Borg's JSON progress stream into bounded, readable journal entries."""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
import time
|
||||
@@ -12,6 +13,12 @@ def size(value):
|
||||
return f"{value / (1024 ** 3):.2f} GiB"
|
||||
|
||||
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--estimated-total-bytes", type=int, required=True)
|
||||
args = parser.parse_args()
|
||||
if args.estimated_total_bytes <= 0:
|
||||
parser.error("estimated total must be positive")
|
||||
|
||||
last_progress = 0.0
|
||||
for line in sys.stdin:
|
||||
try:
|
||||
@@ -28,10 +35,19 @@ for line in sys.stdin:
|
||||
path = event.get("path") or ""
|
||||
parts = path.split("/")
|
||||
dataset = parts[1] if len(parts) > 1 and parts[0] == "source" else "unknown"
|
||||
original_size = event.get("original_size")
|
||||
if isinstance(original_size, (int, float)) and original_size >= 0:
|
||||
percent = original_size / args.estimated_total_bytes * 100
|
||||
estimated_progress = (
|
||||
f"{percent:.1f}%" if percent < 100 else ">=100% (ZFS estimate exceeded)"
|
||||
)
|
||||
else:
|
||||
estimated_progress = "unknown"
|
||||
print(
|
||||
"Borg create progress: "
|
||||
f"dataset={dataset} files={event.get('nfiles', 'unknown')} "
|
||||
f"original={size(event.get('original_size'))} "
|
||||
f"estimated={estimated_progress} dataset={dataset} "
|
||||
f"files={event.get('nfiles', 'unknown')} "
|
||||
f"original={size(original_size)} "
|
||||
f"compressed={size(event.get('compressed_size'))} "
|
||||
f"deduplicated={size(event.get('deduplicated_size'))}",
|
||||
flush=True,
|
||||
|
||||
396
ansible/roles/profile_atlas/files/atlas-health-monitor.py
Normal file
396
ansible/roles/profile_atlas/files/atlas-health-monitor.py
Normal file
@@ -0,0 +1,396 @@
|
||||
#!/usr/bin/python3
|
||||
"""Read-only Atlas health probes with deduplicated 45Drives Alerts."""
|
||||
|
||||
import argparse
|
||||
import fcntl
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
CONFIG_PATH = Path("/etc/atlas-health-monitor.json")
|
||||
STATE_DIR = Path("/var/lib/atlas-health-monitor")
|
||||
STATE_PATH = STATE_DIR / "state.json"
|
||||
GIB = 1024**3
|
||||
|
||||
|
||||
def run(*argv, timeout=40):
|
||||
return subprocess.run(argv, capture_output=True, text=True, timeout=timeout, check=False)
|
||||
|
||||
|
||||
def issue(issues, key, severity, message):
|
||||
issues[key] = {"severity": severity, "message": message}
|
||||
|
||||
|
||||
def notify(config, event, severity, subject, message):
|
||||
now = datetime.now(timezone.utc)
|
||||
payload = {
|
||||
"timestamp": now.isoformat(timespec="seconds"),
|
||||
"unixtime": int(now.timestamp()),
|
||||
"event": event,
|
||||
"severity": severity,
|
||||
"subject": subject,
|
||||
"email_message": message,
|
||||
}
|
||||
result = run(config["notifier"], json.dumps(payload, ensure_ascii=False), timeout=30)
|
||||
if result.returncode:
|
||||
raise RuntimeError(f"45Drives notifier exited {result.returncode}: {result.stderr.strip()}")
|
||||
|
||||
|
||||
def parse_fields(text):
|
||||
return dict(line.split("=", 1) for line in text.splitlines() if "=" in line)
|
||||
|
||||
|
||||
def systemd_fields(unit, *properties):
|
||||
result = run("systemctl", "show", unit, *(f"-p{item}" for item in properties))
|
||||
if result.returncode:
|
||||
raise RuntimeError(f"systemctl show {unit} exited {result.returncode}")
|
||||
return parse_fields(result.stdout)
|
||||
|
||||
|
||||
def unix_time(text):
|
||||
if not text or text == "n/a":
|
||||
return None
|
||||
result = run("date", "-d", text, "+%s")
|
||||
if result.returncode:
|
||||
raise ValueError(f"Cannot parse systemd timestamp: {text}")
|
||||
return int(result.stdout.strip())
|
||||
|
||||
|
||||
def check_pool(config, issues, measurements):
|
||||
pool = config["pool"]
|
||||
listing = run("zpool", "list", "-H", "-p", "-o", "size,alloc,capacity,health", pool)
|
||||
if listing.returncode:
|
||||
issue(issues, "pool.probe", "critical", f"Cannot query ZFS pool {pool}")
|
||||
return
|
||||
try:
|
||||
size, alloc, capacity, health = listing.stdout.strip().split("\t")
|
||||
size, alloc, capacity = int(size), int(alloc), int(capacity)
|
||||
except (ValueError, TypeError):
|
||||
issue(issues, "pool.probe", "critical", "Invalid ZFS pool capacity response")
|
||||
return
|
||||
measurements.update(pool_size_bytes=size, pool_alloc_bytes=alloc, pool_capacity_percent=capacity)
|
||||
if health != "ONLINE":
|
||||
issue(issues, "pool.health", "critical", f"ZFS pool {pool} state is {health}")
|
||||
if capacity >= config["pool_critical_percent"]:
|
||||
issue(issues, "pool.capacity", "critical", f"ZFS pool {pool} is {capacity}% full")
|
||||
elif capacity >= config["pool_warning_percent"]:
|
||||
issue(issues, "pool.capacity", "warning", f"ZFS pool {pool} is {capacity}% full")
|
||||
|
||||
status = run("zpool", "status", "-P", pool)
|
||||
if status.returncode:
|
||||
issue(issues, "pool.status", "critical", f"Cannot query detailed ZFS status for {pool}")
|
||||
return
|
||||
bad_vdevs = []
|
||||
for line in status.stdout.splitlines():
|
||||
match = re.match(r"^\s*(\S+)\s+(ONLINE|DEGRADED|FAULTED|OFFLINE|UNAVAIL|REMOVED)\s+(\d+)\s+(\d+)\s+(\d+)", line)
|
||||
if match:
|
||||
name, state, reads, writes, checksums = match.groups()
|
||||
if state != "ONLINE" or any(int(value) for value in (reads, writes, checksums)):
|
||||
bad_vdevs.append(f"{name}: {state}, READ={reads}, WRITE={writes}, CKSUM={checksums}")
|
||||
if bad_vdevs:
|
||||
issue(issues, "pool.vdevs", "critical", "ZFS vdev errors: " + "; ".join(bad_vdevs))
|
||||
errors = re.search(r"^errors:\s*(.*)$", status.stdout, re.MULTILINE)
|
||||
if not errors or errors.group(1).strip() != "No known data errors":
|
||||
issue(issues, "pool.data_errors", "critical", "ZFS status reports data errors; inspect zpool status -v")
|
||||
if re.search(r"^\s*scan:\s*resilver in progress", status.stdout, re.MULTILINE | re.IGNORECASE):
|
||||
issue(issues, "pool.resilver", "warning", "ZFS resilver is in progress; inspect zpool status")
|
||||
scan = re.search(r"^\s*scan:\s*(.*)$", status.stdout, re.MULTILINE)
|
||||
if scan and re.search(r"\bwith [1-9][0-9]* errors\b", scan.group(1)):
|
||||
issue(issues, "pool.scan_errors", "critical", f"ZFS scan reported errors: {scan.group(1)}")
|
||||
|
||||
|
||||
def check_capacity(config, issues, measurements):
|
||||
pool = config["pool"]
|
||||
listing = run("zfs", "list", "-H", "-p", "-o", "name,usedbysnapshots", "-r", pool)
|
||||
if listing.returncode:
|
||||
issue(issues, "snapshot.probe", "warning", "Cannot query ZFS snapshot space")
|
||||
else:
|
||||
try:
|
||||
snapshots = sum(int(line.split("\t")[1]) for line in listing.stdout.splitlines())
|
||||
measurements["snapshots_bytes"] = snapshots
|
||||
size = measurements.get("pool_size_bytes")
|
||||
if size:
|
||||
percent = snapshots * 100 // size
|
||||
measurements["snapshots_percent"] = percent
|
||||
if percent >= config["snapshot_critical_percent"]:
|
||||
issue(issues, "snapshot.capacity", "critical", f"Snapshots use {percent}% of pool size")
|
||||
elif percent >= config["snapshot_warning_percent"]:
|
||||
issue(issues, "snapshot.capacity", "warning", f"Snapshots use {percent}% of pool size")
|
||||
except (ValueError, IndexError):
|
||||
issue(issues, "snapshot.probe", "warning", "Invalid ZFS snapshot-space response")
|
||||
backup = run("zfs", "list", "-H", "-p", "-o", "used", config["backup_dataset"])
|
||||
if backup.returncode:
|
||||
issue(issues, "backup.capacity_probe", "warning", "Cannot query local backup dataset space")
|
||||
else:
|
||||
try:
|
||||
measurements["backup_bytes"] = int(backup.stdout.strip())
|
||||
except ValueError:
|
||||
issue(issues, "backup.capacity_probe", "warning", "Invalid local backup space response")
|
||||
|
||||
try:
|
||||
filesystem = os.statvfs("/")
|
||||
total = filesystem.f_blocks * filesystem.f_frsize
|
||||
available = filesystem.f_bavail * filesystem.f_frsize
|
||||
used_percent = (total - available) * 100 // total
|
||||
measurements["root_capacity_percent"] = used_percent
|
||||
if used_percent >= config["root_critical_percent"]:
|
||||
issue(issues, "root.capacity", "critical", f"Atlas system filesystem is {used_percent}% full")
|
||||
elif used_percent >= config["root_warning_percent"]:
|
||||
issue(issues, "root.capacity", "warning", f"Atlas system filesystem is {used_percent}% full")
|
||||
except (OSError, ZeroDivisionError):
|
||||
issue(issues, "root.capacity_probe", "warning", "Cannot query Atlas system filesystem space")
|
||||
|
||||
|
||||
def check_remote_capacity(config, issues, measurements):
|
||||
"""Query only the Storage Box quota; do not open or inspect the Borg repository."""
|
||||
remote = config["remote_capacity"]
|
||||
try:
|
||||
result = run("runuser", "-u", remote["run_as"], "--", remote["ssh_wrapper"],
|
||||
f"{remote['user']}@{remote['host']}", "df", "-m", timeout=65)
|
||||
if result.returncode:
|
||||
raise ValueError(f"SSH df exited {result.returncode}")
|
||||
lines = result.stdout.strip().splitlines()
|
||||
if len(lines) != 2:
|
||||
raise ValueError("Unexpected Storage Box df output")
|
||||
fields = lines[1].split()
|
||||
if len(fields) < 5:
|
||||
raise ValueError("Incomplete Storage Box df output")
|
||||
total_mib, used_mib, available_mib = (int(value) for value in fields[1:4])
|
||||
percent = int(fields[4].rstrip("%"))
|
||||
if total_mib <= 0 or not 0 <= percent <= 100 or available_mib < 0:
|
||||
raise ValueError("Invalid Storage Box quota values")
|
||||
except (OSError, ValueError, subprocess.TimeoutExpired):
|
||||
issue(issues, "remote.capacity_probe", "warning", "Cannot query Hetzner Storage Box quota via pinned-key SSH")
|
||||
return
|
||||
measurements.update(remote_capacity_percent=percent, remote_bytes=used_mib * 1024**2,
|
||||
remote_available_bytes=available_mib * 1024**2)
|
||||
if percent >= remote["critical_percent"]:
|
||||
issue(issues, "remote.capacity", "critical", f"Hetzner Storage Box quota is {percent}% full")
|
||||
elif percent >= remote["warning_percent"]:
|
||||
issue(issues, "remote.capacity", "warning", f"Hetzner Storage Box quota is {percent}% full")
|
||||
|
||||
|
||||
def check_smart(config, issues, measurements):
|
||||
for device in config["smart_devices"]:
|
||||
name, path = device["name"], device["path"]
|
||||
try:
|
||||
result = run("smartctl", "-j", "-a", path, timeout=60)
|
||||
data = json.loads(result.stdout)
|
||||
status = int(data.get("smartctl", {}).get("exit_status", result.returncode))
|
||||
except (subprocess.TimeoutExpired, json.JSONDecodeError, ValueError) as exc:
|
||||
issue(issues, f"smart.{name}.probe", "critical", f"SMART probe failed for {name}: {type(exc).__name__}")
|
||||
continue
|
||||
if status:
|
||||
severity = "critical" if status & 0b00001111 else "warning"
|
||||
issue(issues, f"smart.{name}.status", severity, f"SMART reported exit status {status} for {name}")
|
||||
passed = data.get("smart_status", {}).get("passed")
|
||||
if passed is False:
|
||||
issue(issues, f"smart.{name}.health", "critical", f"SMART self-assessment failed for {name}")
|
||||
elif passed is None:
|
||||
issue(issues, f"smart.{name}.health", "warning", f"SMART self-assessment unavailable for {name}")
|
||||
temperature = data.get("temperature", {}).get("current")
|
||||
if isinstance(temperature, (int, float)):
|
||||
measurements[f"smart_{name}_c"] = temperature
|
||||
if temperature >= device["critical_c"]:
|
||||
issue(issues, f"smart.{name}.temperature", "critical", f"{name} temperature is {temperature} C")
|
||||
elif temperature >= device["warning_c"]:
|
||||
issue(issues, f"smart.{name}.temperature", "warning", f"{name} temperature is {temperature} C")
|
||||
else:
|
||||
issue(issues, f"smart.{name}.temperature", "warning", f"Temperature unavailable for {name}")
|
||||
for attribute in data.get("ata_smart_attributes", {}).get("table", []):
|
||||
attribute_id = attribute.get("id")
|
||||
if attribute_id in (5, 187, 197, 198):
|
||||
raw = attribute.get("raw", {}).get("value", 0)
|
||||
if isinstance(raw, int) and raw > 0:
|
||||
severity = "critical" if attribute_id in (197, 198) else "warning"
|
||||
issue(issues, f"smart.{name}.ata_{attribute_id}", severity,
|
||||
f"{name} SMART attribute {attribute_id} raw count is {raw}")
|
||||
nvme = data.get("nvme_smart_health_information_log", {})
|
||||
if isinstance(nvme, dict):
|
||||
if int(nvme.get("critical_warning", 0)):
|
||||
issue(issues, f"smart.{name}.nvme_warning", "critical", f"{name} NVMe critical warning is nonzero")
|
||||
if int(nvme.get("media_errors", 0)):
|
||||
issue(issues, f"smart.{name}.nvme_media", "critical", f"{name} NVMe media errors are nonzero")
|
||||
|
||||
|
||||
def check_cpu(config, issues, measurements):
|
||||
sensors = []
|
||||
for hwmon in Path("/sys/class/hwmon").glob("hwmon*"):
|
||||
try:
|
||||
if (hwmon / "name").read_text().strip() != "coretemp":
|
||||
continue
|
||||
sensors.extend(int(path.read_text().strip()) / 1000 for path in hwmon.glob("temp*_input"))
|
||||
except (OSError, ValueError):
|
||||
continue
|
||||
if not sensors:
|
||||
issue(issues, "cpu.temperature_probe", "warning", "CPU temperature sensors are unavailable")
|
||||
return
|
||||
hottest = max(sensors)
|
||||
measurements["cpu_max_c"] = hottest
|
||||
if hottest >= config["cpu_critical_c"]:
|
||||
issue(issues, "cpu.temperature", "critical", f"CPU temperature is {hottest:g} C")
|
||||
elif hottest >= config["cpu_warning_c"]:
|
||||
issue(issues, "cpu.temperature", "warning", f"CPU temperature is {hottest:g} C")
|
||||
|
||||
|
||||
def check_jobs(config, issues, measurements, now):
|
||||
for timer in config["timers"]:
|
||||
name = timer["name"]
|
||||
try:
|
||||
fields = systemd_fields(name, "ActiveState", "UnitFileState", "LastTriggerUSec", "ActiveEnterTimestamp")
|
||||
if fields.get("ActiveState") != "active" or fields.get("UnitFileState") != "enabled":
|
||||
issue(issues, f"timer.{name}", "critical", f"Timer {name} is not active and enabled")
|
||||
max_age = int(timer["max_age_hours"]) * 3600
|
||||
if max_age:
|
||||
last = unix_time(fields.get("LastTriggerUSec"))
|
||||
if last is None:
|
||||
last = unix_time(fields.get("ActiveEnterTimestamp"))
|
||||
if last is not None and now - last > max_age:
|
||||
issue(issues, f"timer.{name}.stale", "warning",
|
||||
f"Timer {name} has not fired in {int((now-last)/3600)} hours")
|
||||
except (RuntimeError, ValueError, subprocess.TimeoutExpired):
|
||||
issue(issues, f"timer.{name}.probe", "warning", f"Cannot query timer {name}")
|
||||
for unit in config["failure_units"]:
|
||||
if unit.endswith("@.service"):
|
||||
continue
|
||||
try:
|
||||
fields = systemd_fields(unit, "ActiveState", "Result", "ExecMainStartTimestamp")
|
||||
state = fields.get("ActiveState")
|
||||
if state == "failed" or (state == "inactive" and fields.get("Result") not in (None, "", "success")):
|
||||
issue(issues, f"service.{unit}", "critical", f"Service {unit} failed: {fields.get('Result')}")
|
||||
if unit == "atlas-borg-backup.service" and fields.get("ActiveState") == "activating":
|
||||
started = unix_time(fields.get("ExecMainStartTimestamp"))
|
||||
if started is not None and now - started > config["borg_max_runtime_days"] * 86400:
|
||||
issue(issues, "backup.borg_long_running", "warning",
|
||||
"Borg has run longer than its configured limit")
|
||||
except (RuntimeError, ValueError, subprocess.TimeoutExpired):
|
||||
issue(issues, f"service.{unit}.probe", "warning", f"Cannot query service {unit}")
|
||||
|
||||
|
||||
def check_growth(config, issues, measurements, samples, now):
|
||||
previous = [sample for sample in samples if 20 * 3600 <= now - sample.get("time", now) <= 48 * 3600]
|
||||
if previous:
|
||||
baseline = min(previous, key=lambda sample: abs(now - sample["time"] - 86400))
|
||||
days = (now - baseline["time"]) / 86400
|
||||
for name, threshold in (("snapshots", config["snapshot_growth_warning_gib_day"]),
|
||||
("backup", config["backup_growth_warning_gib_day"]),
|
||||
("remote", config["remote_capacity"]["growth_warning_gib_day"])):
|
||||
current, old = measurements.get(f"{name}_bytes"), baseline.get(f"{name}_bytes")
|
||||
if isinstance(current, int) and isinstance(old, int) and days > 0:
|
||||
growth_gib_day = (current - old) / GIB / days
|
||||
measurements[f"{name}_growth_gib_day"] = round(growth_gib_day, 1)
|
||||
if growth_gib_day >= threshold:
|
||||
issue(issues, f"{name}.growth", "warning",
|
||||
f"Local {name} usage grew {growth_gib_day:.1f} GiB/day over {days:.1f} days")
|
||||
|
||||
|
||||
def allowed_failure_unit(config, unit):
|
||||
for allowed in config["failure_units"]:
|
||||
if allowed == unit:
|
||||
return True
|
||||
if allowed.endswith("@.service") and unit.startswith(allowed[:-9] + "@") and unit.endswith(".service"):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def load_state():
|
||||
if not STATE_PATH.exists():
|
||||
return {"active": {}, "samples": []}
|
||||
with STATE_PATH.open(encoding="utf-8") as stream:
|
||||
state = json.load(stream)
|
||||
if not isinstance(state.get("active"), dict) or not isinstance(state.get("samples"), list):
|
||||
raise ValueError("Invalid Atlas monitor state; refusing to overwrite it")
|
||||
return state
|
||||
|
||||
|
||||
def save_state(state):
|
||||
with tempfile.NamedTemporaryFile("w", dir=STATE_DIR, prefix=".state-", delete=False,
|
||||
encoding="utf-8") as stream:
|
||||
path = Path(stream.name)
|
||||
os.chmod(path, 0o600)
|
||||
json.dump(state, stream, sort_keys=True)
|
||||
stream.write("\n")
|
||||
stream.flush()
|
||||
os.fsync(stream.fileno())
|
||||
os.replace(path, STATE_PATH)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--dry-run", action="store_true", help="probe without notifications or state changes")
|
||||
parser.add_argument("--test-notification", action="store_true", help="submit a labelled test alert")
|
||||
parser.add_argument("--job-failed", metavar="UNIT", help="notify about a failed configured service")
|
||||
args = parser.parse_args()
|
||||
with CONFIG_PATH.open(encoding="utf-8") as stream:
|
||||
config = json.load(stream)
|
||||
if args.test_notification:
|
||||
notify(config, "atlas_monitor_test", "warning", "Test monitoraggio Atlas",
|
||||
"Notifica di prova: il monitoraggio Atlas raggiunge 45Drives Alerts. Non conferma l'invio email.")
|
||||
print("Atlas monitor test submitted to 45Drives Alerts; email delivery is not verified.")
|
||||
return 0
|
||||
if args.job_failed:
|
||||
if not allowed_failure_unit(config, args.job_failed):
|
||||
raise ValueError("Unconfigured Atlas failure unit")
|
||||
notify(config, "atlas_job_failed", "critical", f"Job Atlas fallito: {args.job_failed}",
|
||||
f"Il servizio {args.job_failed} e' fallito. Controlla: "
|
||||
f"sudo journalctl -u {args.job_failed} -n 100 --no-pager")
|
||||
print(f"Atlas job failure submitted to 45Drives Alerts: {args.job_failed}")
|
||||
return 0
|
||||
|
||||
now = int(time.time())
|
||||
issues, measurements = {}, {}
|
||||
check_pool(config, issues, measurements)
|
||||
check_capacity(config, issues, measurements)
|
||||
check_remote_capacity(config, issues, measurements)
|
||||
check_smart(config, issues, measurements)
|
||||
check_cpu(config, issues, measurements)
|
||||
check_jobs(config, issues, measurements, now)
|
||||
if args.dry_run:
|
||||
print(json.dumps({"issues": issues, "measurements": measurements}, sort_keys=True))
|
||||
return 0
|
||||
|
||||
STATE_DIR.mkdir(mode=0o700, exist_ok=True)
|
||||
with (STATE_DIR / "monitor.lock").open("w") as lock:
|
||||
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
state = load_state()
|
||||
check_growth(config, issues, measurements, state["samples"], now)
|
||||
active, failed_notifications = state["active"], []
|
||||
for key, details in issues.items():
|
||||
old = active.get(key)
|
||||
if old is None or old.get("severity") != details["severity"]:
|
||||
try:
|
||||
notify(config, "atlas_health_issue", details["severity"],
|
||||
f"Atlas: {key}", details["message"])
|
||||
active[key] = details
|
||||
print(f"ALERT {details['severity']} {key}: {details['message']}", flush=True)
|
||||
except (RuntimeError, subprocess.TimeoutExpired) as exc:
|
||||
failed_notifications.append(key)
|
||||
print(f"NOTIFICATION FAILED {key}: {exc}", file=sys.stderr, flush=True)
|
||||
for key in set(active) - set(issues):
|
||||
print(f"RECOVERED {key}", flush=True)
|
||||
del active[key]
|
||||
state["samples"] = [sample for sample in state["samples"] if now - sample.get("time", 0) < 48 * 3600]
|
||||
state["samples"].append({"time": now, **{key: value for key, value in measurements.items()
|
||||
if key in ("snapshots_bytes", "backup_bytes", "remote_bytes")}})
|
||||
save_state(state)
|
||||
print(f"Atlas health: issues={len(issues)} notifications_failed={len(failed_notifications)} "
|
||||
f"pool={measurements.get('pool_capacity_percent', 'unknown')}% "
|
||||
f"remote={measurements.get('remote_capacity_percent', 'unknown')}% "
|
||||
f"snapshots={measurements.get('snapshots_bytes', 'unknown')} bytes "
|
||||
f"backup={measurements.get('backup_bytes', 'unknown')} bytes", flush=True)
|
||||
return 1 if failed_notifications else 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
sys.exit(main())
|
||||
except (OSError, RuntimeError, ValueError, subprocess.TimeoutExpired) as error:
|
||||
print(f"Atlas health monitor failed: {error}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
@@ -233,6 +233,16 @@
|
||||
mode: "0750"
|
||||
when: atlas_manage_borg_backup | bool
|
||||
|
||||
- name: Install the Atlas Borg snapshot cleanup helper
|
||||
tags: [atlas, storage, backup, borg, borg_logging]
|
||||
ansible.builtin.template:
|
||||
src: atlas-borg-snapshot-cleanup.sh.j2
|
||||
dest: /usr/local/sbin/atlas-borg-snapshot-cleanup
|
||||
owner: root
|
||||
group: root
|
||||
mode: "0750"
|
||||
when: atlas_manage_borg_backup | bool
|
||||
|
||||
- name: Install the Atlas Borg check helper
|
||||
tags: [atlas, storage, backup, borg]
|
||||
ansible.builtin.template:
|
||||
@@ -274,7 +284,7 @@
|
||||
when: atlas_manage_borg_backup | bool
|
||||
|
||||
- name: Install Atlas Borg systemd units
|
||||
tags: [atlas, storage, backup, borg]
|
||||
tags: [atlas, storage, backup, borg, borg_logging]
|
||||
ansible.builtin.template:
|
||||
src: "{{ item }}.j2"
|
||||
dest: "/etc/systemd/system/{{ item }}"
|
||||
|
||||
@@ -23,6 +23,9 @@
|
||||
- name: Import Atlas offline USB backup tasks
|
||||
ansible.builtin.import_tasks: usb_backup.yml
|
||||
|
||||
- name: Import Atlas health monitoring tasks
|
||||
ansible.builtin.import_tasks: monitoring.yml
|
||||
|
||||
- name: Import Atlas post-restore SELinux relabeling tasks
|
||||
ansible.builtin.import_tasks: restorecon.yml
|
||||
|
||||
|
||||
201
ansible/roles/profile_atlas/tasks/monitoring.yml
Normal file
201
ansible/roles/profile_atlas/tasks/monitoring.yml
Normal file
@@ -0,0 +1,201 @@
|
||||
---
|
||||
- name: Validate Atlas health monitoring policy
|
||||
tags: [atlas, monitoring]
|
||||
ansible.builtin.assert:
|
||||
that:
|
||||
- atlas_manage_storage | bool
|
||||
- atlas_zfs_pool != 'CHANGEME_ZFS_POOL'
|
||||
- atlas_monitor_calendar | length > 0
|
||||
- atlas_monitor_smart_devices | length > 0
|
||||
- atlas_monitor_timers | length > 0
|
||||
- atlas_monitor_failure_units | length > 0
|
||||
- atlas_monitor_remote_capacity.user == atlas_borg_repository_user
|
||||
- atlas_monitor_remote_capacity.host == atlas_borg_repository_host
|
||||
- atlas_monitor_remote_capacity.run_as == atlas_borg_username
|
||||
- atlas_monitor_remote_capacity.ssh_wrapper == atlas_borg_ssh_wrapper_path
|
||||
- >-
|
||||
0 < atlas_monitor_remote_capacity.warning_percent | int
|
||||
< atlas_monitor_remote_capacity.critical_percent | int < 100
|
||||
- atlas_monitor_remote_capacity.growth_warning_gib_day | int > 0
|
||||
- atlas_monitor_notifier.startswith('/opt/45drives/houston/')
|
||||
- 0 < atlas_monitor_pool_warning_percent | int < atlas_monitor_pool_critical_percent | int < 100
|
||||
- 0 < atlas_monitor_root_warning_percent | int < atlas_monitor_root_critical_percent | int < 100
|
||||
- 0 < atlas_monitor_snapshot_warning_percent | int < atlas_monitor_snapshot_critical_percent | int < 100
|
||||
- atlas_monitor_snapshot_growth_warning_gib_day | int > 0
|
||||
- atlas_monitor_backup_growth_warning_gib_day | int > 0
|
||||
- 0 < atlas_monitor_cpu_warning_c | int < atlas_monitor_cpu_critical_c | int
|
||||
- atlas_monitor_borg_max_runtime_days | int > 0
|
||||
fail_msg: >-
|
||||
Atlas health monitoring needs real devices, job units, a valid calendar,
|
||||
positive ordered thresholds, and the existing Houston notifier.
|
||||
when: atlas_manage_monitoring | bool
|
||||
|
||||
- name: Validate monitored Atlas SMART devices
|
||||
tags: [atlas, monitoring]
|
||||
ansible.builtin.assert:
|
||||
that:
|
||||
- item.name is match('^[a-z0-9][a-z0-9_-]*$')
|
||||
- item.path.startswith('/dev/disk/by-id/')
|
||||
- 0 < item.warning_c | int < item.critical_c | int
|
||||
fail_msg: "Every monitored disk needs a stable by-id path and ordered temperature thresholds."
|
||||
loop: "{{ atlas_monitor_smart_devices }}"
|
||||
loop_control:
|
||||
label: "{{ item.name }}"
|
||||
when: atlas_manage_monitoring | bool
|
||||
|
||||
- name: Validate monitored Atlas timer names and age thresholds
|
||||
tags: [atlas, monitoring]
|
||||
ansible.builtin.assert:
|
||||
that:
|
||||
- item.name is match('^[a-zA-Z0-9@_.-]+\\.timer$')
|
||||
- item.max_age_hours | int >= 0
|
||||
loop: "{{ atlas_monitor_timers }}"
|
||||
loop_control:
|
||||
label: "{{ item.name }}"
|
||||
when: atlas_manage_monitoring | bool
|
||||
|
||||
- name: Validate monitored Atlas failure unit names
|
||||
tags: [atlas, monitoring]
|
||||
ansible.builtin.assert:
|
||||
that:
|
||||
- item is match('^[a-zA-Z0-9@_.-]+\\.service$')
|
||||
loop: "{{ atlas_monitor_failure_units }}"
|
||||
when: atlas_manage_monitoring | bool
|
||||
|
||||
- name: Validate Atlas health monitor calendar
|
||||
tags: [atlas, monitoring]
|
||||
ansible.builtin.command:
|
||||
argv: [systemd-analyze, calendar, "{{ atlas_monitor_calendar }}"]
|
||||
changed_when: false
|
||||
check_mode: false
|
||||
when: atlas_manage_monitoring | bool
|
||||
|
||||
- name: Install SMART tooling for Atlas health checks
|
||||
tags: [atlas, monitoring, packages]
|
||||
ansible.builtin.dnf:
|
||||
name: smartmontools
|
||||
state: present
|
||||
when: atlas_manage_monitoring | bool
|
||||
|
||||
- name: Inspect the existing 45Drives notifier for monitoring
|
||||
tags: [atlas, monitoring]
|
||||
ansible.builtin.stat:
|
||||
path: "{{ atlas_monitor_notifier }}"
|
||||
register: atlas_monitor_notifier_file
|
||||
when: atlas_manage_monitoring | bool
|
||||
|
||||
- name: Require the existing 45Drives notifier for monitoring
|
||||
tags: [atlas, monitoring]
|
||||
ansible.builtin.assert:
|
||||
that:
|
||||
- atlas_monitor_notifier_file.stat.executable | default(false)
|
||||
fail_msg: "The existing 45Drives Houston notifier must be executable."
|
||||
when: atlas_manage_monitoring | bool
|
||||
|
||||
- name: Create private Atlas health monitor state directory
|
||||
tags: [atlas, monitoring]
|
||||
ansible.builtin.file:
|
||||
path: /var/lib/atlas-health-monitor
|
||||
state: directory
|
||||
owner: root
|
||||
group: root
|
||||
mode: "0700"
|
||||
when: atlas_manage_monitoring | bool
|
||||
|
||||
- name: Install Atlas health monitor configuration
|
||||
tags: [atlas, monitoring]
|
||||
ansible.builtin.template:
|
||||
src: atlas-health-monitor.json.j2
|
||||
dest: /etc/atlas-health-monitor.json
|
||||
owner: root
|
||||
group: root
|
||||
mode: "0600"
|
||||
when: atlas_manage_monitoring | bool
|
||||
|
||||
- name: Install Atlas health monitor helper
|
||||
tags: [atlas, monitoring]
|
||||
ansible.builtin.copy:
|
||||
src: atlas-health-monitor.py
|
||||
dest: /usr/local/libexec/atlas-health-monitor
|
||||
owner: root
|
||||
group: root
|
||||
mode: "0750"
|
||||
when: atlas_manage_monitoring | bool
|
||||
|
||||
- name: Install Atlas health monitoring units
|
||||
tags: [atlas, monitoring]
|
||||
ansible.builtin.template:
|
||||
src: "{{ item }}.j2"
|
||||
dest: "/etc/systemd/system/{{ item }}"
|
||||
owner: root
|
||||
group: root
|
||||
mode: "0644"
|
||||
loop:
|
||||
- atlas-health-monitor.service
|
||||
- atlas-health-monitor.timer
|
||||
- atlas-monitor-failure@.service
|
||||
when: atlas_manage_monitoring | bool
|
||||
|
||||
- name: Create failure hook directories for monitored Atlas jobs
|
||||
tags: [atlas, monitoring]
|
||||
ansible.builtin.file:
|
||||
path: "/etc/systemd/system/{{ item }}.d"
|
||||
state: directory
|
||||
owner: root
|
||||
group: root
|
||||
mode: "0755"
|
||||
loop: "{{ atlas_monitor_failure_units }}"
|
||||
when: atlas_manage_monitoring | bool
|
||||
|
||||
- name: Notify 45Drives Alerts when an Atlas job fails
|
||||
tags: [atlas, monitoring]
|
||||
ansible.builtin.template:
|
||||
src: atlas-monitor-failure.conf.j2
|
||||
dest: "/etc/systemd/system/{{ item }}.d/atlas-monitor.conf"
|
||||
owner: root
|
||||
group: root
|
||||
mode: "0644"
|
||||
loop: "{{ atlas_monitor_failure_units }}"
|
||||
when: atlas_manage_monitoring | bool
|
||||
|
||||
- name: Reload systemd after installing Atlas monitoring
|
||||
tags: [atlas, monitoring]
|
||||
ansible.builtin.systemd:
|
||||
daemon_reload: true
|
||||
when:
|
||||
- atlas_manage_monitoring | bool
|
||||
- not ansible_check_mode
|
||||
|
||||
- name: Enable the Atlas health monitoring timer
|
||||
tags: [atlas, monitoring]
|
||||
ansible.builtin.systemd:
|
||||
name: atlas-health-monitor.timer
|
||||
enabled: true
|
||||
state: started
|
||||
when:
|
||||
- atlas_manage_monitoring | bool
|
||||
- not ansible_check_mode
|
||||
|
||||
- name: Validate the deployed Atlas health monitoring units
|
||||
tags: [atlas, monitoring]
|
||||
ansible.builtin.command:
|
||||
argv:
|
||||
- systemd-analyze
|
||||
- verify
|
||||
- atlas-health-monitor.service
|
||||
- atlas-health-monitor.timer
|
||||
- atlas-monitor-failure@.service
|
||||
changed_when: false
|
||||
when:
|
||||
- atlas_manage_monitoring | bool
|
||||
- not ansible_check_mode
|
||||
|
||||
- name: Probe Atlas health without sending notifications
|
||||
tags: [atlas, monitoring]
|
||||
ansible.builtin.command:
|
||||
argv: [/usr/local/libexec/atlas-health-monitor, --dry-run]
|
||||
register: atlas_monitor_dry_run
|
||||
changed_when: false
|
||||
when:
|
||||
- atlas_manage_monitoring | bool
|
||||
- not ansible_check_mode
|
||||
@@ -14,6 +14,7 @@ ConditionPathExists={{ atlas_borg_known_hosts_path }}
|
||||
[Service]
|
||||
Type=oneshot
|
||||
ExecStart=/usr/local/sbin/atlas-borg-backup
|
||||
ExecStopPost=+/usr/local/sbin/atlas-borg-snapshot-cleanup
|
||||
User=root
|
||||
Group=root
|
||||
UMask=0077
|
||||
|
||||
@@ -17,6 +17,7 @@ readonly archive_prefix={{ atlas_borg_archive_prefix | quote }}
|
||||
readonly snapshot_prefix={{ atlas_borg_snapshot_prefix | quote }}
|
||||
readonly compression={{ atlas_borg_compression | quote }}
|
||||
readonly stage=/run/atlas-borg/source
|
||||
readonly snapshot_marker=/run/atlas-borg/snapshot-name
|
||||
readonly borg_user={{ atlas_borg_username | quote }}
|
||||
readonly borg_group={{ atlas_borg_group | quote }}
|
||||
readonly borg_home={{ atlas_borg_home | quote }}
|
||||
@@ -24,7 +25,6 @@ readonly borg_lock={{ atlas_borg_lock_path | quote }}
|
||||
readonly progress_filter=/usr/local/libexec/atlas-borg-progress
|
||||
|
||||
snapshot_name=""
|
||||
snapshot_created=false
|
||||
mounted_targets=()
|
||||
|
||||
# Invoked through the EXIT trap below.
|
||||
@@ -33,22 +33,31 @@ cleanup() {
|
||||
local status=$?
|
||||
local cleanup_status=0
|
||||
local index
|
||||
local source_mount_failed=false
|
||||
trap - EXIT HUP INT TERM
|
||||
set +e
|
||||
|
||||
{% raw %}
|
||||
for ((index = ${#mounted_targets[@]} - 1; index >= 0; index--)); do
|
||||
{% endraw %}
|
||||
if mountpoint -q "${mounted_targets[$index]}" && ! umount -R "${mounted_targets[$index]}"; then
|
||||
printf 'Source snapshot mount cleanup failed: %s\n' "${mounted_targets[$index]}" >&2
|
||||
source_mount_failed=true
|
||||
fi
|
||||
if mountpoint -q "${mounted_targets[$index]}"; then
|
||||
umount "${mounted_targets[$index]}" || cleanup_status=2
|
||||
printf 'Source snapshot mount is still active: %s\n' "${mounted_targets[$index]}" >&2
|
||||
source_mount_failed=true
|
||||
else
|
||||
rmdir -- "${mounted_targets[$index]}" 2>/dev/null || true
|
||||
fi
|
||||
done
|
||||
rm -rf "$stage" || cleanup_status=2
|
||||
|
||||
if [[ "$snapshot_created" == true ]]; then
|
||||
flock 9
|
||||
zfs destroy -r "${pool}@${snapshot_name}" || cleanup_status=2
|
||||
flock -u 9
|
||||
if [[ "$source_mount_failed" == false ]]; then
|
||||
if [[ -d "$stage" ]]; then
|
||||
rmdir -- "$stage" 2>/dev/null || cleanup_status=2
|
||||
fi
|
||||
else
|
||||
cleanup_status=2
|
||||
printf 'Source bind mount cleanup failed; keeping the snapshot for recovery\n' >&2
|
||||
fi
|
||||
|
||||
if ((status == 0 && cleanup_status != 0)); then
|
||||
@@ -97,8 +106,8 @@ timestamp="$(date -u +%Y%m%dT%H%M%SZ)"
|
||||
readonly timestamp
|
||||
snapshot_name="${snapshot_prefix}-${timestamp}"
|
||||
readonly snapshot_name
|
||||
printf '%s\n' "$snapshot_name" >"$snapshot_marker"
|
||||
zfs snapshot -r "${pool}@${snapshot_name}"
|
||||
snapshot_created=true
|
||||
flock -u 9
|
||||
printf 'Created recursive Borg source snapshot %s@%s\n' "$pool" "$snapshot_name"
|
||||
|
||||
@@ -117,10 +126,27 @@ while IFS=$'\t' read -r dataset dataset_mountpoint mounted; do
|
||||
target_path="${stage}${dataset_suffix}"
|
||||
mkdir -p "$target_path"
|
||||
mount --bind "$source_path" "$target_path"
|
||||
mount -o remount,bind,ro "$target_path"
|
||||
mounted_targets+=("$target_path")
|
||||
mount -o remount,bind,ro "$target_path"
|
||||
done < <(zfs list -H -o name,mountpoint,mounted -s name -r "$pool")
|
||||
|
||||
estimated_source_bytes=0
|
||||
while IFS=$'\t' read -r source_snapshot logical_bytes; do
|
||||
if [[ "$source_snapshot" == *"@${snapshot_name}" ]]; then
|
||||
[[ "$logical_bytes" =~ ^[0-9]+$ ]] || {
|
||||
printf 'Invalid logical size for Borg source snapshot %s\n' "$source_snapshot" >&2
|
||||
exit 74
|
||||
}
|
||||
estimated_source_bytes=$((estimated_source_bytes + logical_bytes))
|
||||
fi
|
||||
done < <(zfs list -H -p -t snapshot -o name,logicalreferenced -r "$pool")
|
||||
((estimated_source_bytes > 0)) || {
|
||||
printf 'Could not estimate the Borg source snapshot size\n' >&2
|
||||
exit 74
|
||||
}
|
||||
printf 'Estimated Borg source logical size: %s bytes (ZFS; progress percentage is approximate)\n' \
|
||||
"$estimated_source_bytes"
|
||||
|
||||
archive="${archive_prefix}-${timestamp}"
|
||||
readonly archive
|
||||
borg_status=0
|
||||
@@ -136,7 +162,7 @@ set +e
|
||||
--compression "$compression" \
|
||||
"${repository}::${archive}" \
|
||||
source 2>&1
|
||||
) | /usr/bin/python3 -u "$progress_filter"
|
||||
) | /usr/bin/python3 -u "$progress_filter" --estimated-total-bytes "$estimated_source_bytes"
|
||||
create_pipeline_status=("${PIPESTATUS[@]}")
|
||||
set -e
|
||||
create_status=${create_pipeline_status[0]}
|
||||
|
||||
@@ -0,0 +1,51 @@
|
||||
#!/usr/bin/env bash
|
||||
set -Eeuo pipefail
|
||||
|
||||
export PATH=/usr/sbin:/usr/bin:/sbin:/bin
|
||||
|
||||
readonly pool={{ atlas_zfs_pool | quote }}
|
||||
readonly mount_root={{ atlas_mount_root | quote }}
|
||||
readonly snapshot_prefix={{ atlas_borg_snapshot_prefix | quote }}
|
||||
readonly marker=/run/atlas-borg/snapshot-name
|
||||
|
||||
[[ -e "$marker" ]] || exit 0
|
||||
[[ -f "$marker" && ! -L "$marker" ]] || {
|
||||
printf 'Unsafe Atlas Borg snapshot marker; leaving snapshots unchanged\n' >&2
|
||||
exit 2
|
||||
}
|
||||
IFS= read -r snapshot_name <"$marker"
|
||||
[[ "$snapshot_name" =~ ^${snapshot_prefix}-[0-9]{8}T[0-9]{6}Z$ ]] || {
|
||||
printf 'Invalid Atlas Borg snapshot marker; leaving snapshots unchanged\n' >&2
|
||||
exit 2
|
||||
}
|
||||
|
||||
exec 9>/run/lock/atlas-zfs-snapshot.lock
|
||||
flock 9
|
||||
if zfs list -H -t snapshot -o name "${pool}@${snapshot_name}" >/dev/null 2>&1; then
|
||||
# The private bind mounts are gone, but ZFS may leave its on-demand
|
||||
# .zfs/snapshot mounts in the host namespace until explicitly unmounted.
|
||||
snapshot_mounts=()
|
||||
snapshot_sources=()
|
||||
while IFS=$'\t' read -r dataset dataset_mountpoint; do
|
||||
[[ "$dataset_mountpoint" == "$mount_root" || "$dataset_mountpoint" == "$mount_root/"* ]] || continue
|
||||
snapshot_mounts+=("${dataset_mountpoint}/.zfs/snapshot/${snapshot_name}")
|
||||
snapshot_sources+=("${dataset}@${snapshot_name}")
|
||||
done < <(zfs list -H -o name,mountpoint -s name -r "$pool")
|
||||
|
||||
{% raw %}
|
||||
for ((index = ${#snapshot_mounts[@]} - 1; index >= 0; index--)); do
|
||||
{% endraw %}
|
||||
mounted_source="$(findmnt -rn -M "${snapshot_mounts[$index]}" -o SOURCE || true)"
|
||||
[[ -n "$mounted_source" ]] || continue
|
||||
[[ "$mounted_source" == "${snapshot_sources[$index]}" ]] || {
|
||||
printf 'Unexpected source on Atlas Borg snapshot mount: %s\n' \
|
||||
"${snapshot_mounts[$index]}" >&2
|
||||
exit 2
|
||||
}
|
||||
umount "${snapshot_mounts[$index]}"
|
||||
done
|
||||
|
||||
zfs destroy -r "${pool}@${snapshot_name}"
|
||||
printf 'Removed recursive Atlas Borg source snapshot %s@%s after backup exit\n' \
|
||||
"$pool" "$snapshot_name"
|
||||
fi
|
||||
@@ -0,0 +1,20 @@
|
||||
{
|
||||
"pool": {{ atlas_zfs_pool | to_json }},
|
||||
"backup_dataset": {{ (atlas_zfs_pool ~ '/' ~ atlas_zfs_dataset_backup) | to_json }},
|
||||
"notifier": {{ atlas_monitor_notifier | to_json }},
|
||||
"smart_devices": {{ atlas_monitor_smart_devices | to_json }},
|
||||
"timers": {{ atlas_monitor_timers | to_json }},
|
||||
"failure_units": {{ atlas_monitor_failure_units | to_json }},
|
||||
"remote_capacity": {{ atlas_monitor_remote_capacity | to_json }},
|
||||
"pool_warning_percent": {{ atlas_monitor_pool_warning_percent | int }},
|
||||
"pool_critical_percent": {{ atlas_monitor_pool_critical_percent | int }},
|
||||
"root_warning_percent": {{ atlas_monitor_root_warning_percent | int }},
|
||||
"root_critical_percent": {{ atlas_monitor_root_critical_percent | int }},
|
||||
"snapshot_warning_percent": {{ atlas_monitor_snapshot_warning_percent | int }},
|
||||
"snapshot_critical_percent": {{ atlas_monitor_snapshot_critical_percent | int }},
|
||||
"snapshot_growth_warning_gib_day": {{ atlas_monitor_snapshot_growth_warning_gib_day | int }},
|
||||
"backup_growth_warning_gib_day": {{ atlas_monitor_backup_growth_warning_gib_day | int }},
|
||||
"cpu_warning_c": {{ atlas_monitor_cpu_warning_c | int }},
|
||||
"cpu_critical_c": {{ atlas_monitor_cpu_critical_c | int }},
|
||||
"borg_max_runtime_days": {{ atlas_monitor_borg_max_runtime_days | int }}
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
[Unit]
|
||||
Description=Check Atlas pool, disks, capacity, temperatures and maintenance jobs
|
||||
Wants=houston-dbus.service network-online.target
|
||||
After=zfs.target houston-dbus.service network-online.target
|
||||
ConditionFileIsExecutable=/usr/local/libexec/atlas-health-monitor
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
ExecStart=/usr/local/libexec/atlas-health-monitor
|
||||
User=root
|
||||
Group=root
|
||||
UMask=0077
|
||||
StateDirectory=atlas-health-monitor
|
||||
StateDirectoryMode=0700
|
||||
NoNewPrivileges=true
|
||||
PrivateTmp=true
|
||||
ProtectHome=true
|
||||
ProtectSystem=strict
|
||||
ReadWritePaths=/var/lib/atlas-health-monitor
|
||||
ProtectKernelTunables=true
|
||||
ProtectKernelModules=true
|
||||
ProtectControlGroups=true
|
||||
RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6
|
||||
RestrictRealtime=true
|
||||
LockPersonality=true
|
||||
@@ -0,0 +1,11 @@
|
||||
[Unit]
|
||||
Description=Schedule Atlas health checks
|
||||
|
||||
[Timer]
|
||||
OnCalendar={{ atlas_monitor_calendar }}
|
||||
Persistent=true
|
||||
RandomizedDelaySec=5min
|
||||
Unit=atlas-health-monitor.service
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
@@ -0,0 +1,2 @@
|
||||
[Unit]
|
||||
OnFailure=atlas-monitor-failure@%n.service
|
||||
@@ -0,0 +1,19 @@
|
||||
[Unit]
|
||||
Description=Submit a 45Drives Alert for failed Atlas job %I
|
||||
Requires=houston-dbus.service
|
||||
After=houston-dbus.service
|
||||
ConditionFileIsExecutable=/usr/local/libexec/atlas-health-monitor
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
ExecStart=/usr/local/libexec/atlas-health-monitor --job-failed %I
|
||||
User=root
|
||||
Group=root
|
||||
UMask=0077
|
||||
NoNewPrivileges=true
|
||||
PrivateTmp=true
|
||||
ProtectHome=true
|
||||
ProtectSystem=strict
|
||||
RestrictAddressFamilies=AF_UNIX
|
||||
RestrictRealtime=true
|
||||
LockPersonality=true
|
||||
Reference in New Issue
Block a user