declarative-data-backup 0.1.0__py3-none-any.whl

declarative_data_backup/__init__.py

@@ -0,0 +1 @@
+"""Declarative Borg-based data backups."""

declarative_data_backup/__main__.py

@@ -0,0 +1,6 @@
+import sys
+
+from .backup import main
+
+if __name__ == "__main__":
+    sys.exit(main())

declarative_data_backup/backup.py

@@ -0,0 +1,111 @@
+import logging
+import os
+import uuid
+from collections.abc import Mapping
+from datetime import UTC, datetime
+
+from .borg import (
+    backup_folder,
+    check_keyfile,
+    prune,
+    repository_lock,
+)
+from .borg.borg_cli import ensure_borg_binary
+from .config import configure_logging, load_yaml, parse_backup_definitions
+from .config.backup_config.models import BackupDefinition
+from .environment import Environment
+from .errors import BackupError, BorgWarningError
+
+logger = logging.getLogger(__name__)
+
+
+def check_settings(settings: Environment) -> None:
+    ensure_borg_binary()
+    if not settings.borg_base_dir.is_dir():
+        raise BackupError(
+            f"BORG_BASE_DIR does not exist or is not a directory: "
+            f"{settings.borg_base_dir}."
+        )
+    if not os.access(settings.borg_base_dir, os.R_OK | os.W_OK | os.X_OK):
+        raise BackupError(
+            f"BORG_BASE_DIR is not accessible for Borg state: {settings.borg_base_dir}."
+        )
+    check_keyfile(settings)
+
+
+def load_definitions(settings: Environment) -> list[BackupDefinition]:
+    return parse_backup_definitions(
+        load_yaml(settings.config_path),
+        settings.server_root,
+    )
+
+
+def run_backup(
+    settings: Environment,
+    definitions: list[BackupDefinition],
+) -> None:
+    total_folders = sum(len(definition.folders) for definition in definitions)
+    logger.info(
+        "Validated %d backup groups containing %d folders",
+        len(definitions),
+        total_folders,
+    )
+
+    run_id = uuid.uuid4().hex
+    with repository_lock(settings.borg_base_dir / "backup.lock"):
+        created_at = datetime.now(UTC)
+        for definition in definitions:
+            logger.info(
+                "Starting backup group '%s' (%d folders).",
+                definition.name,
+                len(definition.folders),
+            )
+            for folder in definition.folders:
+                backup_folder(
+                    folder,
+                    definition.compression,
+                    definition.name,
+                    settings,
+                    created_at,
+                    run_id,
+                )
+
+        for definition in definitions:
+            for folder in definition.folders:
+                prune(
+                    folder,
+                    definition.retention_days,
+                    definition.name,
+                    settings,
+                )
+
+
+def main(environ: Mapping[str, str] | None = None) -> int:
+    configure_logging()
+    try:
+        settings = Environment.from_env(environ)
+        logger.info("Starting backup using configuration %s", settings.config_path)
+        check_settings(settings)
+        run_backup(settings, load_definitions(settings))
+        logger.info("Backup completed successfully.")
+        return 0
+
+    except BorgWarningError as exc:
+        logger.error(
+            "Backup stopped because Borg reported warnings; "
+            "pruning was not completed: %s",
+            exc,
+        )
+        return 1
+    except BackupError as exc:
+        logger.error("%s", exc)
+        return 1
+    except OSError as exc:
+        logger.error(
+            "Unexpected I/O failure: %s.",
+            exc.strerror or "I/O error",
+        )
+        return 1
+    except KeyboardInterrupt:
+        logger.error("Interrupted.")
+        return 130

declarative_data_backup/borg/__init__.py

@@ -0,0 +1,13 @@
+from .borg import (
+    backup_folder,
+    check_keyfile,
+    prune,
+    repository_lock,
+)
+
+__all__ = [
+    "backup_folder",
+    "check_keyfile",
+    "prune",
+    "repository_lock",
+]

declarative_data_backup/borg/archive_naming.py

@@ -0,0 +1,48 @@
+import hashlib
+import re
+import uuid
+from datetime import UTC, datetime
+from pathlib import Path
+
+from ..errors import fail
+
+APPLICATION_ARCHIVE_PREFIX = "bbkp__"
+_SAFE_ARCHIVE_ID = re.compile(r"[a-f0-9]{32}\Z")
+_MAX_ARCHIVE_NAME_LENGTH = 255
+_MAX_SOURCE_SLUG_LENGTH = 120
+
+
+def archive_series_id(path: Path) -> str:
+    canonical = str(path.resolve())
+    escaped = canonical.strip("/").replace("/", "__")
+    escaped = re.sub(r"[^A-Za-z0-9._-]", "_", escaped)
+    escaped = escaped[:_MAX_SOURCE_SLUG_LENGTH].rstrip("_") or "root"
+    digest = hashlib.sha256(
+        canonical.encode("utf-8", errors="surrogateescape")
+    ).hexdigest()[:24]
+    return f"{APPLICATION_ARCHIVE_PREFIX}{escaped}__{digest}"
+
+
+def archive_name_escaper(
+    path: Path,
+    created_at: datetime | None = None,
+    run_id: str | None = None,
+) -> str:
+    if created_at is None:
+        created_at = datetime.now(UTC)
+    if created_at.tzinfo is None or created_at.utcoffset() is None:
+        fail("Archive timestamps must include a timezone.")
+
+    timestamp = created_at.astimezone(UTC).strftime("%Y%m%dT%H%M%SZ")
+    unique_id = uuid.uuid4().hex if run_id is None else run_id
+    if not _SAFE_ARCHIVE_ID.fullmatch(unique_id):
+        fail("Archive run identifiers must be 32 lowercase hexadecimal characters.")
+
+    name = f"{archive_series_id(path)}__{timestamp}__{unique_id}"
+    if len(name) > _MAX_ARCHIVE_NAME_LENGTH:
+        fail("Generated Borg archive name exceeds the supported length.")
+    return name
+
+
+def archive_series_glob(path: Path) -> str:
+    return f"{archive_series_id(path)}__*"

declarative_data_backup/borg/borg.py

@@ -0,0 +1,166 @@
+import fcntl
+import logging
+import os
+import stat
+import time
+from collections.abc import Generator
+from contextlib import contextmanager
+from datetime import datetime
+from pathlib import Path
+
+from ..environment import (
+    REPOSITORY_LOCK_TIMEOUT_SECONDS,
+    Environment,
+)
+from ..errors import BackupError, fail
+from .archive_naming import (
+    archive_name_escaper,
+    archive_series_glob,
+)
+from .borg_cli import run_borg
+
+logger = logging.getLogger(__name__)
+
+
+def backup_folder(
+    folder: Path,
+    compression: str,
+    group: str,
+    settings: Environment,
+    created_at: datetime,
+    run_id: str,
+) -> None:
+    name = archive_name_escaper(
+        folder,
+        created_at=created_at,
+        run_id=run_id,
+    )
+    context = f"group={group!r} source={folder}"
+    started_at = time.monotonic()
+    logger.info("Starting backup for %s as archive %s.", context, name)
+    try:
+        run_borg(
+            [
+                "create",
+                "--verbose",
+                "--stats",
+                "--compression",
+                compression,
+                f"{settings.borg_repo}::{name}",
+                str(folder),
+            ],
+            settings,
+            "create",
+            context,
+        )
+    except BackupError:
+        logger.error(
+            "Backup failed for %s (elapsed_seconds=%.2f).",
+            context,
+            time.monotonic() - started_at,
+        )
+        raise
+    logger.info(
+        "Backup succeeded for %s (elapsed_seconds=%.2f).",
+        context,
+        time.monotonic() - started_at,
+    )
+
+
+def prune(
+    source: Path,
+    retention_days: int,
+    group: str,
+    settings: Environment,
+) -> None:
+    selector = archive_series_glob(source)
+    context = f"group={group!r} source={source}"
+    started_at = time.monotonic()
+    logger.info(
+        "Starting prune for %s (keep_daily=%d).",
+        context,
+        retention_days,
+    )
+    try:
+        run_borg(
+            [
+                "prune",
+                "--verbose",
+                "--list",
+                "--glob-archives",
+                selector,
+                "--keep-daily",
+                str(retention_days),
+                settings.borg_repo,
+            ],
+            settings,
+            "prune",
+            context,
+        )
+    except BackupError:
+        logger.error(
+            "Prune failed for %s (elapsed_seconds=%.2f).",
+            context,
+            time.monotonic() - started_at,
+        )
+        raise
+    logger.info(
+        "Prune succeeded for %s (elapsed_seconds=%.2f).",
+        context,
+        time.monotonic() - started_at,
+    )
+
+
+def check_keyfile(settings: Environment) -> None:
+    try:
+        if not settings.key_file.is_file():
+            fail(f"Borg keyfile does not exist: {settings.key_file}.")
+        if not os.access(settings.key_file, os.R_OK):
+            fail(f"Borg keyfile is not readable: {settings.key_file}.")
+        permissions = stat.S_IMODE(settings.key_file.stat().st_mode)
+    except OSError:
+        fail(f"Borg keyfile cannot be accessed: {settings.key_file}.")
+
+    if permissions & 0o077:
+        logger.warning(
+            "Borg keyfile permissions are broader than 0600: %s.",
+            settings.key_file,
+        )
+
+
+@contextmanager
+def repository_lock(
+    lock_path: Path,
+    timeout_seconds: int = REPOSITORY_LOCK_TIMEOUT_SECONDS,
+) -> Generator[None]:
+    started_at = time.monotonic()
+    try:
+        lock_file = lock_path.open("a", encoding="utf-8")
+    except OSError as exc:
+        fail(
+            f"Unable to open repository coordination lock: {exc.strerror or 'I/O error'}."
+        )
+
+    acquired = False
+    try:
+        while not acquired:
+            try:
+                fcntl.flock(
+                    lock_file.fileno(),
+                    fcntl.LOCK_EX | fcntl.LOCK_NB,
+                )
+                acquired = True
+            except BlockingIOError:
+                if time.monotonic() - started_at >= timeout_seconds:
+                    fail(
+                        "Timed out waiting for the repository coordination lock "
+                        f"after {timeout_seconds} seconds."
+                    )
+                time.sleep(0.25)
+
+        logger.info("Acquired repository coordination lock.")
+        yield
+    finally:
+        if acquired:
+            fcntl.flock(lock_file.fileno(), fcntl.LOCK_UN)
+        lock_file.close()

declarative_data_backup/borg/borg_cli.py

@@ -0,0 +1,69 @@
+import logging
+import os
+import shutil
+import subprocess
+
+from ..environment import Environment
+from ..errors import BorgWarningError, fail
+
+logger = logging.getLogger(__name__)
+
+
+def ensure_borg_binary() -> None:
+    if shutil.which("borg") is None:
+        fail("Required executable 'borg' was not found on PATH.")
+
+
+def borg_environment(settings: Environment) -> dict[str, str]:
+    environment = os.environ.copy()
+    environment.update(
+        {
+            "BORG_REPO": settings.borg_repo,
+            "BORG_BASE_DIR": str(settings.borg_base_dir),
+            "BORG_KEY_FILE": str(settings.key_file),
+            "BORG_RSH": settings.borg_rsh,
+            "BORG_PASSCOMMAND": settings.borg_passcommand,
+        }
+    )
+    if settings.borg_exit_codes is not None:
+        environment["BORG_EXIT_CODES"] = settings.borg_exit_codes
+    return environment
+
+
+def run_borg(
+    args: list[str],
+    settings: Environment,
+    operation: str,
+    context: str,
+) -> None:
+    logger.info("Running Borg %s for %s.", operation, context)
+    command = ["borg"]
+    remote_path = settings.borg_remote_path
+    if remote_path is not None:
+        if not remote_path.strip() or "\0" in remote_path:
+            fail("BORG_REMOTE_PATH must be a nonempty Borg executable path.")
+        command.extend(("--remote-path", remote_path))
+    command.extend(args)
+
+    try:
+        result = subprocess.run(
+            command,
+            env=borg_environment(settings),
+            check=False,
+        )
+    except FileNotFoundError:
+        fail("Required executable 'borg' could not be started.")
+    except OSError as exc:
+        fail(
+            f"Unable to start Borg {operation} for {context}: {exc.strerror or 'I/O error'}."
+        )
+
+    if result.returncode == 0:
+        return
+
+    if result.returncode == 1:
+        message = f"Borg {operation} reported warnings for {context} (exit code 1)."
+        logger.warning("%s", message)
+        raise BorgWarningError(message)
+
+    fail(f"Borg {operation} failed for {context} (exit code {result.returncode}).")

declarative_data_backup/config/__init__.py

@@ -0,0 +1,11 @@
+from .backup_config.utils import read_mount_points
+from .backup_config.yaml_parser import parse_backup_definitions
+from .backup_config.yaml_raw_loader import load_yaml
+from .logging import configure_logging
+
+__all__ = [
+    "configure_logging",
+    "load_yaml",
+    "parse_backup_definitions",
+    "read_mount_points",
+]

declarative_data_backup/config/backup_config/__init__.py

@@ -0,0 +1 @@
+__all__: list[str] = []

declarative_data_backup/config/backup_config/models.py

@@ -0,0 +1,12 @@
+from dataclasses import dataclass
+from pathlib import Path
+
+
+@dataclass(frozen=True)
+class BackupDefinition:
+    name: str
+    folders: tuple[Path, ...]
+    compression: str
+    retention_days: int
+    required_mounts: tuple[Path, ...] = ()
+    allow_empty: bool = True

declarative_data_backup/config/backup_config/utils.py

@@ -0,0 +1,141 @@
+import os
+import re
+from pathlib import Path
+from typing import Any
+
+from ...errors import fail
+from .models import BackupDefinition
+
+_MOUNT_ESCAPE = re.compile(r"\\([0-7]{3})")
+
+
+def resolve_path(raw_path: Any, server_root: Path, label: str) -> Path:
+    if not isinstance(raw_path, str) or not raw_path:
+        fail(f"{label} must be a nonempty string.")
+
+    path = Path(raw_path)
+    if not path.is_absolute():
+        fail(f"{label} must be absolute: {raw_path!r}.")
+    if ".." in path.parts:
+        fail(f"{label} must not contain '..': {raw_path!r}.")
+    if not path.is_relative_to(server_root):
+        fail(f"{label} must be lexically inside SERVER_ROOT: {raw_path!r}.")
+
+    try:
+        resolved = path.resolve(strict=True)
+    except (OSError, RuntimeError, ValueError):
+        fail(f"{label} does not exist or cannot be resolved: {raw_path!r}.")
+
+    if not resolved.is_relative_to(server_root):
+        fail(f"{label} resolves outside SERVER_ROOT: {raw_path!r}.")
+    return resolved
+
+
+def parse_folders(
+    raw_folders: Any,
+    server_root: Path,
+    group: str,
+    allow_empty: bool,
+) -> tuple[Path, ...]:
+    if not isinstance(raw_folders, list) or not raw_folders:
+        fail(f"'{group}.folders' must be a nonempty YAML list.")
+
+    folders = []
+    for raw_folder in raw_folders:
+        folder = resolve_path(raw_folder, server_root, f"'{group}.folders' entry")
+        if not folder.is_dir():
+            fail(f"Backup source is not a directory: {folder}.")
+        if not os.access(folder, os.R_OK | os.X_OK):
+            fail(f"Backup source is not readable: {folder}.")
+        if not allow_empty:
+            try:
+                is_empty = next(folder.iterdir(), None) is None
+            except OSError:
+                fail(f"Unable to inspect backup source: {folder}.")
+            if is_empty:
+                fail(
+                    f"Backup source is empty and '{group}.allow_empty' is false: {folder}."
+                )
+        folders.append(folder)
+    return tuple(folders)
+
+
+def parse_required_mounts(
+    raw_mounts: Any,
+    server_root: Path,
+    group: str,
+) -> tuple[Path, ...]:
+    if not isinstance(raw_mounts, list):
+        fail(f"'{group}.required_mounts' must be a YAML list.")
+
+    mounts = tuple(
+        resolve_path(raw, server_root, f"'{group}.required_mounts' entry")
+        for raw in raw_mounts
+    )
+    if len(set(mounts)) != len(mounts):
+        fail(f"'{group}.required_mounts' contains a duplicate path.")
+    for mount in mounts:
+        if not mount.is_dir():
+            fail(f"Required mount is not a directory: {mount}.")
+    return mounts
+
+
+def check_folders_for_overlappings(definitions: list[BackupDefinition]) -> None:
+    owners: dict[Path, str] = {}
+    for definition in definitions:
+        for folder in definition.folders:
+            for existing, owner in owners.items():
+                if folder == existing:
+                    fail(
+                        f"Duplicate canonical backup source {folder} "
+                        f"(already configured by '{owner}')."
+                    )
+                if folder in existing.parents or existing in folder.parents:
+                    fail(
+                        f"Overlapping backup sources are not allowed: "
+                        f"'{owner}' uses {existing}, while "
+                        f"'{definition.name}' uses {folder}."
+                    )
+            owners[folder] = definition.name
+
+
+def _mountinfo_path(raw_mount: str) -> Path:
+    decoded = _MOUNT_ESCAPE.sub(
+        lambda match: chr(int(match.group(1), 8)),
+        raw_mount,
+    )
+    return Path(decoded)
+
+
+def read_mount_points(mountinfo_path: Path) -> set[Path]:
+    try:
+        lines = mountinfo_path.read_text(encoding="utf-8").splitlines()
+    except (OSError, UnicodeError) as exc:
+        fail(
+            f"Unable to read mount information: {getattr(exc, 'strerror', None) or 'I/O error'}."
+        )
+
+    mount_points: set[Path] = set()
+    for line_number, line in enumerate(lines, start=1):
+        fields = line.split()
+        if len(fields) < 10 or "-" not in fields:
+            fail(f"Malformed mount information at line {line_number}.")
+        mount_points.add(_mountinfo_path(fields[4]))
+    return mount_points
+
+
+def check_required_mounts_are_mounted(
+    definitions: list[BackupDefinition],
+    mountinfo_path: Path,
+) -> None:
+    required = {
+        mount for definition in definitions for mount in definition.required_mounts
+    }
+    if required:
+        missing = sorted(required - read_mount_points(mountinfo_path))
+        if missing:
+            fail(
+                "Required filesystem mount point(s) are not mounted: "
+                + ", ".join(map(str, missing))
+                + "."
+            )

declarative_data_backup/config/backup_config/yaml_parser.py

@@ -0,0 +1,136 @@
+import re
+from pathlib import Path
+from typing import Any
+
+from ...environment import DEFAULT_COMPRESSION, DEFAULT_RETENTION_DAYS
+from ...errors import fail
+from .models import BackupDefinition
+from .utils import (
+    check_folders_for_overlappings,
+    check_required_mounts_are_mounted,
+    parse_folders,
+    parse_required_mounts,
+)
+
+_GROUP_NAME = re.compile(r"[A-Za-z0-9][A-Za-z0-9_-]{0,63}\Z")
+_GROUP_KEYS = {
+    "folders",
+    "compression",
+    "retention_days",
+    "required_mounts",
+    "allow_empty",
+}
+_GROUP_DEFAULTS = {
+    "compression": DEFAULT_COMPRESSION,
+    "retention_days": DEFAULT_RETENTION_DAYS,
+    "required_mounts": [],
+    "allow_empty": True,
+}
+
+
+def _validate_compression(value: Any, group: str) -> str:
+    if not isinstance(value, str):
+        fail(f"'{group}.compression' is not a supported Borg 1.4 compression value.")
+
+    parts = value.split(",")
+    index = 0
+    while index < len(parts):
+        codec = parts[index]
+        if codec in {"none", "lz4"}:
+            index += 1
+            break
+        elif codec in {"zstd", "zlib", "lzma"}:
+            index += 1
+            if index < len(parts) and parts[index].isdigit():
+                level = int(parts[index])
+                minimum, maximum = {
+                    "zstd": (1, 22),
+                    "zlib": (0, 9),
+                    "lzma": (0, 9),
+                }[codec]
+                if not minimum <= level <= maximum:
+                    fail(
+                        f"'{group}.compression' level for {codec} "
+                        f"must be between {minimum} and {maximum}."
+                    )
+                index += 1
+            break
+        elif codec == "auto":
+            index += 1
+        elif codec == "obfuscate":
+            if index + 2 >= len(parts) or not parts[index + 1].isdigit():
+                fail(f"'{group}.compression' has an invalid obfuscation specifier.")
+            obfuscation_spec = int(parts[index + 1])
+            if (
+                obfuscation_spec not in range(1, 7)
+                and obfuscation_spec not in range(110, 124)
+                and obfuscation_spec != 250
+            ):
+                fail(f"'{group}.compression' has an unsupported obfuscation specifier.")
+            index += 2
+        else:
+            fail(
+                f"'{group}.compression' is not a supported Borg 1.4 compression value."
+            )
+
+    if index != len(parts):
+        fail(f"'{group}.compression' contains unexpected compression parameters.")
+    return value
+
+
+def _parse_definition(name: Any, raw: Any, server_root: Path) -> BackupDefinition:
+    if not isinstance(name, str) or not _GROUP_NAME.fullmatch(name):
+        fail("Backup group names must contain only letters, digits, '_' or '-'.")
+    if not isinstance(raw, dict):
+        fail(f"'{name}' must be a YAML mapping.")
+
+    unknown_keys = set(raw) - _GROUP_KEYS
+    if unknown_keys:
+        fail(f"'{name}' contains unsupported configuration keys.")
+
+    group = _GROUP_DEFAULTS | raw
+    compression = _validate_compression(group["compression"], name)
+    retention_days = group["retention_days"]
+    if (
+        isinstance(retention_days, bool)
+        or not isinstance(retention_days, int)
+        or retention_days < 1
+    ):
+        fail(f"'{name}.retention_days' must be a positive integer.")
+
+    allow_empty = group["allow_empty"]
+    if not isinstance(allow_empty, bool):
+        fail(f"'{name}.allow_empty' must be a boolean.")
+
+    return BackupDefinition(
+        name=name,
+        folders=parse_folders(group.get("folders"), server_root, name, allow_empty),
+        compression=compression,
+        retention_days=retention_days,
+        required_mounts=parse_required_mounts(
+            group["required_mounts"],
+            server_root,
+            name,
+        ),
+        allow_empty=allow_empty,
+    )
+
+
+def parse_backup_definitions(
+    data: Any,
+    server_root: Path,
+    mountinfo_path: Path = Path("/proc/self/mountinfo"),
+) -> list[BackupDefinition]:
+    if not isinstance(data, dict):
+        fail("Top-level configuration must be a YAML mapping.")
+    if not data:
+        fail("Backup configuration is empty.")
+    if not server_root.is_dir():
+        fail(f"SERVER_ROOT does not exist or is not a directory: {server_root}.")
+
+    definitions = [
+        _parse_definition(name, raw, server_root) for name, raw in data.items()
+    ]
+    check_folders_for_overlappings(definitions)
+    check_required_mounts_are_mounted(definitions, mountinfo_path)
+    return definitions

declarative_data_backup/config/backup_config/yaml_raw_loader.py

@@ -0,0 +1,79 @@
+from pathlib import Path
+from typing import Any
+
+import yaml
+from yaml.constructor import ConstructorError
+from yaml.nodes import MappingNode
+
+from ...errors import fail
+
+
+class UniqueKeyLoader(yaml.SafeLoader):
+    def construct_mapping(
+        self,
+        node: MappingNode,
+        deep: bool = False,
+    ) -> dict[Any, Any]:
+        if not isinstance(node, MappingNode):
+            raise ConstructorError(
+                None,
+                None,
+                "expected a YAML mapping",
+                node.start_mark,
+            )
+
+        self.flatten_mapping(node)
+        mapping: dict[Any, Any] = {}
+        for key_node, value_node in node.value:
+            key = self.construct_object(key_node, deep=deep)
+            try:
+                exists = key in mapping
+            except TypeError as exc:
+                raise ConstructorError(
+                    "while constructing a mapping",
+                    node.start_mark,
+                    "found an unhashable mapping key",
+                    key_node.start_mark,
+                ) from exc
+
+            if exists:
+                raise ConstructorError(
+                    "while constructing a mapping",
+                    node.start_mark,
+                    "found a duplicate mapping key",
+                    key_node.start_mark,
+                )
+
+            mapping[key] = self.construct_object(value_node, deep=deep)
+
+        return mapping
+
+
+def load_yaml(path: Path) -> dict[str, Any]:
+    try:
+        with path.open("r", encoding="utf-8") as file:
+            data = yaml.load(file, Loader=UniqueKeyLoader)
+    except yaml.YAMLError as exc:
+        mark = getattr(exc, "problem_mark", None)
+        location = (
+            f" at line {mark.line + 1}, column {mark.column + 1}"
+            if mark is not None
+            else ""
+        )
+        duplicate = (
+            "duplicate YAML mapping key"
+            if getattr(exc, "problem", "") == "found a duplicate mapping key"
+            else "malformed YAML"
+        )
+        fail(f"Invalid YAML in {path}{location}: {duplicate}.")
+    except (OSError, UnicodeError) as exc:
+        fail(
+            f"Unable to read configuration file {path}: "
+            f"{getattr(exc, 'strerror', None) or 'I/O error'}."
+        )
+
+    if not isinstance(data, dict):
+        fail("Top-level configuration must be a YAML mapping.")
+    if not data:
+        fail("Backup configuration is empty.")
+    return data

declarative_data_backup/config/logging.py

@@ -0,0 +1,8 @@
+import logging
+
+
+def configure_logging() -> None:
+    logging.basicConfig(
+        level=logging.INFO,
+        format="%(asctime)s %(levelname)s %(message)s",
+    )

declarative_data_backup/environment.py

@@ -0,0 +1,70 @@
+import os
+from collections.abc import Mapping
+from dataclasses import dataclass
+from pathlib import Path
+
+from .errors import fail
+
+
+@dataclass(frozen=True)
+class Environment:
+    config_path: Path
+    key_file: Path
+    borg_repo: str
+    server_root: Path
+    borg_base_dir: Path
+    borg_rsh: str
+    borg_passcommand: str
+    borg_remote_path: str | None
+    borg_exit_codes: str | None
+
+    @classmethod
+    def from_env(
+        cls,
+        environ: Mapping[str, str] | None = None,
+    ) -> "Environment":
+        source = os.environ if environ is None else environ
+
+        def required(name: str) -> str:
+            value = source.get(name)
+            if value is None or not value.strip():
+                fail(f"{name} environment variable is required.")
+            return value
+
+        def required_path(name: str) -> Path:
+            raw_value = required(name)
+            if "\0" in raw_value:
+                fail(f"{name} must be a valid absolute path.")
+            value = Path(raw_value)
+            if not value.is_absolute():
+                fail(f"{name} must be an absolute path.")
+            return value
+
+        repo = required("BORG_REPO")
+        if "\0" in repo:
+            fail("BORG_REPO must be a valid repository location.")
+        borg_exit_codes = source.get("BORG_EXIT_CODES")
+        if borg_exit_codes not in (None, "legacy", "modern"):
+            fail("BORG_EXIT_CODES must be either 'legacy' or 'modern'.")
+
+        try:
+            server_root = required_path("SERVER_ROOT").resolve(strict=True)
+        except (OSError, RuntimeError, ValueError):
+            fail("SERVER_ROOT does not exist or cannot be accessed.")
+
+        return cls(
+            config_path=required_path("BACKUP_CONFIG_PATH"),
+            key_file=required_path("BORG_KEY_FILE"),
+            borg_repo=repo,
+            server_root=server_root,
+            borg_base_dir=required_path("BORG_BASE_DIR"),
+            borg_rsh=required("BORG_RSH"),
+            borg_passcommand=required("BORG_PASSCOMMAND"),
+            borg_remote_path=source.get("BORG_REMOTE_PATH"),
+            borg_exit_codes=borg_exit_codes,
+        )
+
+
+DEFAULT_COMPRESSION = "auto,zstd,22"
+DEFAULT_RETENTION_DAYS = 30
+REPOSITORY_LOCK_TIMEOUT_SECONDS = 3600

declarative_data_backup/errors.py

@@ -0,0 +1,13 @@
+from typing import NoReturn
+
+
+class BackupError(RuntimeError):
+    """Raised when the backup configuration or execution is invalid."""
+
+
+class BorgWarningError(BackupError):
+    """Raised when Borg completes with warnings that make the run unsafe."""
+
+
+def fail(message: str) -> NoReturn:
+    raise BackupError(message)

declarative_data_backup-0.1.0.dist-info/METADATA

@@ -0,0 +1,319 @@
+Metadata-Version: 2.4
+Name: declarative-data-backup
+Version: 0.1.0
+Summary: Declarative Borg-based data backups
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown
+Requires-Dist: PyYAML<7,>=6
+Provides-Extra: dev
+Requires-Dist: pre-commit<5,>=4; extra == "dev"
+Requires-Dist: pyright<2,>=1.1.400; extra == "dev"
+Requires-Dist: pytest<10,>=8; extra == "dev"
+Requires-Dist: ruff<1,>=0.11; extra == "dev"
+
+# Declarative Data Backup
+
+Declarative Data Backup creates encrypted Borg archives for configured source
+directories. Each source gets its own archive series in one Borg repository,
+so Borg can deduplicate chunks across all sources. The application validates
+the complete configuration before invoking Borg and prunes a source's archives
+only after every create operation in the run has succeeded.
+
+## Requirements
+
+- Python 3.11 or newer.
+- Borg 1.4.x installed separately and available as `borg` on `PATH`.
+- A Borg repository that has already been initialized in keyfile encryption
+  mode, its matching keyfile, and a protected passphrase source.
+- For remote repositories, OpenSSH and a trusted `known_hosts` file.
+
+The application never initializes a repository. Borg 2 is not supported.
+
+## Installation
+
+Install the package from PyPI:
+
+```shell
+python -m pip install declarative-data-backup
+```
+
+Or install a checkout with the development dependencies:
+
+```shell
+python -m pip install -e ".[dev]"
+```
+
+The PyPI distribution is named `declarative-data-backup`; its importable
+package is `declarative_data_backup`.
+
+## GitLab release automation
+
+The pipeline in [.gitlab-ci.yml](.gitlab-ci.yml) publishes releases when a Git
+tag in `major.minor.patch` format is pushed, for example `1.2.3`. Use the same
+version for the GitLab Release name and tag. Tags with a `v` prefix, prerelease
+suffix, or leading zeros are not published. Branch pushes do not run this
+release pipeline.
+
+For each version tag, the pipeline:
+
+1. Sets the package version in the CI checkout's `pyproject.toml`, builds a
+   wheel and source distribution, and checks them with Twine.
+2. Uploads `declarative-data-backup` to PyPI.
+3. Builds the existing Python 3.13 Dockerfile with the same package version,
+   verifies the installed version, and pushes
+   `${CI_REGISTRY_IMAGE}/declarative-data-backup:1.2.3-python3.13` to the
+   project's GitLab Container Registry.
+
+The source checkout's version is not committed back to the repository.
+The two publishing jobs run independently after the package build succeeds;
+publishing is not atomic across PyPI and the container registry.
+
+Before the first release:
+
+- Enable the project's GitLab Container Registry.
+- Add `PYPI_API_TOKEN` in GitLab **Settings > CI/CD > Variables** as a masked,
+  protected variable. Use a PyPI API token authorized to publish
+  `declarative-data-backup`; for a new PyPI project, its first upload may
+  require an account-wide token, which should then be replaced with a
+  project-scoped token.
+- Protect release tags in GitLab (for example, `*.*.*`) and restrict who can
+  create them. Protected variables are only available on protected refs.
+- Use a trusted Docker-executor runner configured for Docker-in-Docker with
+  TLS: `privileged = true` and `volumes = ["/certs/client", "/cache"]` in its
+  `[runners.docker]` configuration. GitLab.com instance runners support this
+  setup. Privileged runners should not execute untrusted code. If your runner
+  requires a tag, add it to `publish_container`.
+
+Registry authentication uses GitLab's built-in `CI_REGISTRY_USER` and
+`CI_REGISTRY_PASSWORD`; no separate registry password is needed.
+
+Create and push a release tag from the commit you want to publish:
+
+```shell
+git tag 1.2.3
+git push origin 1.2.3
+```
+
+Creating a GitLab Release with a new `1.2.3` tag also triggers the tag pipeline.
+Creating or editing a Release on an already existing tag does not trigger a
+new pipeline. PyPI versions cannot be overwritten: do not move or reuse
+release tags. If only the container job fails, retry that job rather than
+rerunning a successful PyPI upload.
+
+## Configure and run
+
+Create a YAML file accessible to the backup process:
+
+```yaml
+media_backup:
+  folders:
+    - /server_root/media/photos
+    - /server_root/media/music
+  compression: auto,zstd,22
+  retention_days: 30
+  required_mounts:
+    - /server_root/media
+  allow_empty: true
+
+postgres_backup:
+  folders:
+    - /server_root/path/to/postgres-backups
+  required_mounts:
+    - /server_root/path/to/postgres-backups
+```
+
+Set the required environment variables before starting the process:
+
+| Variable | Purpose |
+| --- | --- |
+| `BACKUP_CONFIG_PATH` | Absolute path to the YAML configuration. |
+| `SERVER_ROOT` | Absolute path to the root containing all configured source directories. |
+| `BORG_REPO` | Existing Borg repository location, local or remote. |
+| `BORG_KEY_FILE` | Path to the external keyfile for the encrypted repository. |
+| `BORG_BASE_DIR` | Writable directory for Borg state and the backup coordination lock. |
+| `BORG_RSH` | SSH command and options used by Borg for a remote repository. |
+| `BORG_PASSCOMMAND` | Command Borg runs to obtain the repository passphrase. |
+
+Optional variables:
+
+| Variable | Purpose |
+| --- | --- |
+| `BORG_REMOTE_PATH` | Remote Borg executable path, passed as Borg's `--remote-path`. Leave unset to use Borg's default. |
+| `BORG_EXIT_CODES` | Borg exit-code mode: `legacy` or `modern`. |
+
+Provide credentials through a secret manager or protected files; do not put
+passphrases, private keys, or credentials in the YAML file, source control, or
+shell history. For example, in a controlled environment where these variables
+have already been securely populated:
+
+```shell
+python -m declarative_data_backup
+```
+
+The command exits with status `0` on success, nonzero on configuration or Borg
+failure, and `130` when interrupted. Borg's exit code `1` indicates warnings:
+the run fails and pruning is skipped.
+
+## Configuration reference
+
+The top level must be a nonempty YAML mapping of backup group names to group
+mappings. Group names may contain ASCII letters, digits, `_`, and `-`, and
+must be at most 64 characters. Duplicate YAML keys, unknown group fields,
+invalid values, and malformed or empty configuration are rejected before any
+create or prune operation.
+
+Each group supports:
+
+- `folders` (required): a nonempty list of absolute directory paths. Each
+  source must exist, be readable, and resolve inside `SERVER_ROOT`. Duplicate
+  canonical paths and parent/child overlaps across groups are rejected.
+- `compression` (optional): defaults to `auto,zstd,22`. Supported Borg 1.4
+  codecs include `none`, `lz4`, `zstd`, `zlib`, and `lzma`; `auto` may precede
+  a codec. Codec levels are checked against supported ranges: zstd `1`-`22`,
+  zlib `0`-`9`, and lzma `0`-`9`. Borg's `obfuscate` compression specifier is
+  also accepted for specifier values `1`-`6`, `110`-`123`, or `250`; it is
+  useful only with repository encryption.
+- `retention_days` (optional): positive integer, default `30`. This controls
+  Borg's `--keep-daily` policy.
+- `required_mounts` (optional): list of exact mount points that must appear
+  in the process's `/proc/self/mountinfo`.
+- `allow_empty` (optional): boolean, default `true`. If set to `false`, an
+  empty source is rejected.
+
+Configure `required_mounts` explicitly for every source that depends on a
+separately mounted filesystem. The program does not infer mount requirements
+from source paths. In a container or Kubernetes pod, mounting `/server_root`
+does not prove that nested host filesystems are mounted. In particular,
+`allow_empty: true` does not prove that a disk is present: an empty source may
+be legitimate and can be archived.
+
+Paths are resolved before backup. A top-level source symlink is allowed only
+when its canonical target remains inside `SERVER_ROOT`; symlinks within a
+source are stored as links rather than followed.
+
+## Archives, retention, and compaction
+
+The application creates one archive per configured source in the single
+`BORG_REPO`. Names have this form:
+
+```text
+bbkp__<source-series>__<UTC timestamp>__<run UUID>
+```
+
+The stable source series is derived from the canonical source path and includes
+a SHA-256 suffix. Borg receives the canonical absolute path as the source, so
+the original directory hierarchy is retained inside the archive.
+
+The application creates all configured archives before pruning any source.
+If a create fails or Borg reports warnings, the run stops and no pruning is
+performed. Retries use a new run UUID and may create another archive for that
+day; archives are never overwritten to handle a collision.
+
+`--keep-daily N` retains the latest archive in up to `N` daily buckets. It does
+not guarantee `N` successful calendar days of recovery if scheduled runs fail
+or are missed. Removing a source from the YAML does not delete its old
+archives. The source-series selector does not match archives made with the
+previous non-namespaced naming scheme; retain those archives unless a
+separately reviewed migration or cleanup is performed.
+
+Pruning marks repository data for deletion; it does not reclaim the space.
+Repository compaction is a separate maintenance operation and is not run by
+this application. Any scheduled compact job or manual maintenance must first
+acquire an exclusive `flock` on:
+
+```text
+<BORG_BASE_DIR>/backup.lock
+```
+
+Use the same persistent state volume and filesystem as the backup job, with
+working advisory-lock semantics. The backup process waits up to 3600 seconds
+for this lock. Kubernetes `concurrencyPolicy: Forbid` does not coordinate
+separate CronJobs, and Borg's own repository lock does not replace this
+application-level lock. Never unlink or replace the lock file while any
+process could hold it.
+
+## Deployment and operational notes
+
+The Dockerfile builds an image with Borg 1.4.1 and the Python package. The
+application itself does not set CPU or I/O priority; a scheduled deployment
+may wrap the command with `nice -n 19` and `ionice -c 3`. For Kubernetes or
+another container runtime:
+
+- Run as a non-root user and confirm it can read the keyfile and credentials
+  and write the Borg state volume.
+- Mount source data read-only. Keep Borg cache, configuration, security state,
+  keyfile, and the coordination lock on persistent writable storage.
+- For nested source filesystems, configure and verify the corresponding
+  `required_mounts`.
+- Use a versioned image tag or digest and confirm private-registry pull
+  credentials have only the required read access.
+- Do not mount a service-account token unless the workload needs one.
+- Ensure any compact job mounts the same state volume and honors the lock
+  contract above.
+- Monitor failed and warning backup jobs. Retention cannot make up for days
+  without successful archives.
+- A fixed schedule does not ensure upstream jobs, such as Unmanic processing,
+  have completed; establish that dependency separately.
+
+The default `auto,zstd,22` may be slow or exceed a 1 GiB memory limit on some
+data sets. It has not been benchmarked under that limit and may be terminated
+for memory use. Compression can be overridden per group without recreating
+the repository.
+
+Kubernetes Secret projection permissions, storage ownership, mount behavior,
+node placement, registry access, and workload scheduling must be verified in
+the target cluster. They are not verified by this Python package or its
+container integration test.
+
+## Testing
+
+Run the unit tests:
+
+```shell
+python -m pytest -m "not integration"
+```
+
+The opt-in local Borg integration test uses a temporary encrypted repository:
+
+```shell
+RUN_BORG_LOCAL_INTEGRATION=1 python -m pytest -m integration \
+  tests/integration/test_local_borg_integration.py
+```
+
+The opt-in Docker SSH integration test uses disposable containers and a
+temporary repository; it does not use production credentials or BorgBase:
+
+```shell
+RUN_BORG_DOCKER_INTEGRATION=1 python -m pytest -m integration \
+  tests/integration/test_docker_integration.py
+```
+
+Do not treat skipped integration tests as successful restore validation.
+
+## Recovery
+
+Recovery requires the repository and the matching external keyfile and
+passphrase. Keep independent recovery copies of the keyfile and passphrase;
+the repository must not be their only copy. Also retain the SSH private key
+and verified `known_hosts` entry for a remote repository.
+
+With the required Borg environment and credentials securely configured, list
+archives and inspect the desired archive:
+
+```shell
+borg list "$BORG_REPO"
+borg list "$BORG_REPO"::'bbkp__<source-series>__<timestamp>__<run-uuid>'
+```
+
+Extract into an isolated recovery directory and verify the restored data
+before returning it to service:
+
+```shell
+mkdir -p /restore
+cd /restore
+borg extract "$BORG_REPO"::'bbkp__<source-series>__<timestamp>__<run-uuid>'
+```
+
+Test recovery periodically using a disposable or approved recovery
+environment. This repository's tests do not verify access to a live remote
+repository or actual Kubernetes Secret projection and storage behavior.

declarative_data_backup-0.1.0.dist-info/RECORD

@@ -0,0 +1,20 @@
+declarative_data_backup/__init__.py,sha256=fzbCEZKBtlf2q6gIQT-EpOjIcRXS9tvwHwhbpkfw5zo,43
+declarative_data_backup/__main__.py,sha256=ITJgzyMyGjFKmAXg_t3lgvBk06Mra4CDKHmZJKeoCJg,86
+declarative_data_backup/backup.py,sha256=PL635OglOFRpj9DmwVsfCBXejbbVEtVDv_UIi3rN-yE,3332
+declarative_data_backup/environment.py,sha256=7CSKXtJhvN2-P35ykpsAJ2yg58A5eiqH895k2n5L-aA,2251
+declarative_data_backup/errors.py,sha256=QzAqBWnwm_TsRI1g_An9iwBOQAB-rIrHQV6ur5l3bCk,320
+declarative_data_backup/borg/__init__.py,sha256=kSSwLAXTsMbio6PopLLbdC-0C6gnQqub5bMpAc5dwKQ,185
+declarative_data_backup/borg/archive_naming.py,sha256=Sugp3-bjHoijT6DYSFCwJAOUpwOiu5iJun351J-4SJA,1585
+declarative_data_backup/borg/borg.py,sha256=qOp33-wxg3Zz9I5b0exeV5RsSZ5qlws2PJSFLu5Kkvk,4410
+declarative_data_backup/borg/borg_cli.py,sha256=TLAh2plWqYNwTDmNrDSpknTrB4_wkFzJGOhvGi6Se3w,2062
+declarative_data_backup/config/__init__.py,sha256=IuTiEReE4DV9fDWGb1sS9ip5MDsxtS_6RZ541RCSv88,321
+declarative_data_backup/config/logging.py,sha256=GKjdt26vvIWsQHek7KaALTYmv104TyTVLtAI29GqDv8,165
+declarative_data_backup/config/backup_config/__init__.py,sha256=juqd9PbXs4yg45zMJ7BHAOPQjb7sgEbWE9InBtGZhfo,24
+declarative_data_backup/config/backup_config/models.py,sha256=npHkm-6H1UgTNE8UTV7Zhg4mCqKQC8N7iAF-f3eaWXQ,270
+declarative_data_backup/config/backup_config/utils.py,sha256=cN4W36z3a4Vg4dQceNHGSpMzkcVjMQFJdf3XQAUxywU,4766
+declarative_data_backup/config/backup_config/yaml_parser.py,sha256=JluOZZEikFwyMY8XBb7Owth1iJtnabanSkZega2I1oU,4533
+declarative_data_backup/config/backup_config/yaml_raw_loader.py,sha256=t_BXlrA0bwYZRVRSdH23SljyQj6QXj_f04Q6YouZRnY,2460
+declarative_data_backup-0.1.0.dist-info/METADATA,sha256=T-SGGBVnH0_8uscsh-uRnA8knY6qWucoSLLTUr9QZ8A,13027
+declarative_data_backup-0.1.0.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
+declarative_data_backup-0.1.0.dist-info/top_level.txt,sha256=nwO5fKvUWgOJAFwiGXK89jQlJP4EP5r-2KaxKZtBrVg,24
+declarative_data_backup-0.1.0.dist-info/RECORD,,

declarative_data_backup-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (84.2.1)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

declarative_data_backup-0.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+declarative_data_backup