dorestic 0.4.4__py3-none-any.whl

dorestic/__init__.py

@@ -0,0 +1,39 @@
+from dorestic.backup import (
+    TeeStream as TeeStream,
+    acquire_lock as acquire_lock,
+    backup_container as backup_container,
+    backup_host_group as backup_host_group,
+    run_backup as run_backup,
+    run_hook as run_hook,
+)
+from dorestic.config import find_config as find_config, load_config as load_config
+from dorestic.docker import (
+    discover_targets as discover_targets,
+    docker_cp as docker_cp,
+    resolve_container_path as resolve_container_path,
+    resolve_container_paths as resolve_container_paths,
+    run_docker_exec as run_docker_exec,
+)
+from dorestic.models import (
+    DEFAULT_CONTAINER_SHELL as DEFAULT_CONTAINER_SHELL,
+    DEFAULT_LABEL_PREFIX as DEFAULT_LABEL_PREFIX,
+    DEFAULT_RESTIC_IMAGE as DEFAULT_RESTIC_IMAGE,
+    EXIT_ON_START_FAILED as EXIT_ON_START_FAILED,
+    BackupConfig as BackupConfig,
+    ContainerTarget as ContainerTarget,
+    HostGroup as HostGroup,
+    RetentionPolicy as RetentionPolicy,
+    ScopeConfig as ScopeConfig,
+    ScopeResult as ScopeResult,
+)
+from dorestic.paths import (
+    expand_depth_limited_path as expand_depth_limited_path,
+    parse_comma_list as parse_comma_list,
+    resolve_host_path_spec as resolve_host_path_spec,
+    resolve_host_paths as resolve_host_paths,
+)
+from dorestic.restic import (
+    make_restic_hostname as make_restic_hostname,
+    run_restic as run_restic,
+    run_scope_backup as run_scope_backup,
+)

dorestic/__main__.py

@@ -0,0 +1,4 @@
+from dorestic.cli import main
+
+if __name__ == "__main__":
+    main()

dorestic/backup.py

@@ -0,0 +1,356 @@
+from __future__ import annotations
+
+import fcntl
+import hashlib
+import logging
+import os
+import shutil
+import subprocess
+import sys
+import tempfile
+from io import TextIOBase
+from pathlib import Path
+from typing import IO
+
+import docker
+
+from dorestic.config import load_config
+from dorestic.docker import (
+    discover_targets,
+    resolve_container_paths,
+    run_docker_exec,
+)
+from dorestic.models import (
+    BackupConfig,
+    ContainerTarget,
+    EXIT_ON_START_FAILED,
+    HostGroup,
+    ScopeResult,
+)
+from dorestic.paths import resolve_host_paths
+from dorestic.restic import make_restic_hostname, run_restic, run_scope_backup
+
+log = logging.getLogger("backup")
+
+
+class TeeStream(TextIOBase):
+    """Writes to both a file and the original stream."""
+
+    def __init__(self, original: IO[str], log_file: IO[str]) -> None:
+        self.original = original
+        self.log_file = log_file
+
+    def write(self, s: str) -> int:
+        self.original.write(s)
+        self.log_file.write(s)
+        return len(s)
+
+    def flush(self) -> None:
+        self.original.flush()
+        self.log_file.flush()
+
+
+def _lock_path_for(config: BackupConfig) -> Path:
+    """Derive a per-repository lock file path."""
+    repo_hash = hashlib.sha256(config.repository.encode()).hexdigest()[:16]
+    return Path(tempfile.gettempdir()) / f"dorestic-{repo_hash}.lock"
+
+
+def acquire_lock(config: BackupConfig) -> IO[str]:
+    lock_path = _lock_path_for(config)
+    lock_fd: IO[str] = open(lock_path, "w")
+    try:
+        fcntl.flock(lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
+    except OSError:
+        log.error("Another backup is already running (lock held on %s)", lock_path)
+        sys.exit(1)
+    return lock_fd
+
+
+def run_hook(command: str, env: dict[str, str] | None = None) -> int:
+    hook_env = os.environ.copy()
+    if env:
+        hook_env.update(env)
+    result = subprocess.run(["sh", "-c", command], env=hook_env)
+    return result.returncode
+
+
+def backup_container(
+    target: ContainerTarget,
+    config: BackupConfig,
+    staging_dir: Path | None = None,
+) -> tuple[ScopeResult, ScopeResult]:
+    log.info("")
+    log.info("=== %s ===", target.name)
+
+    container_paths = resolve_container_paths(target, staging_dir=staging_dir)
+    host_paths = resolve_host_paths(target)
+
+    tag_env = {"DORESTIC_TAG": target.name}
+
+    container_on_start_ok = True
+    if target.container_scope and target.container_scope.on_start:
+        log.info("  container.on_start: %s", target.container_scope.on_start)
+        code, output = run_docker_exec(
+            target.container, target.container_scope.on_start,
+            env=tag_env, shell=target.container_scope.shell,
+        )
+        if output:
+            for line in output.splitlines():
+                log.info("    %s", line)
+        if code != 0:
+            log.error(
+                "  container.on_start failed (exit %d), skipping container backup",
+                code,
+            )
+            container_on_start_ok = False
+
+    host_on_start_ok = True
+    if target.host_scope and target.host_scope.on_start:
+        log.info("  host.on_start: %s", target.host_scope.on_start)
+        code = run_hook(target.host_scope.on_start, env=tag_env)
+        if code != 0:
+            log.error(
+                "  host.on_start failed (exit %d), skipping host backup", code
+            )
+            host_on_start_ok = False
+
+    container_result = ScopeResult(exit_code=0, skipped=True)
+    if container_paths and container_on_start_ok:
+        container_tag = f"{target.name}:container"
+        exit_code = run_scope_backup(
+            container_tag,
+            container_paths,
+            target.container_scope.exclude if target.container_scope else [],
+            config=config,
+            hostname=make_restic_hostname("container", target.name),
+        )
+        container_result = ScopeResult(exit_code=exit_code)
+        if exit_code == 0:
+            log.info("  container backup OK")
+        else:
+            log.error("  container backup FAILED (exit %d)", exit_code)
+    elif container_paths and not container_on_start_ok:
+        container_result = ScopeResult(exit_code=EXIT_ON_START_FAILED, skipped=True)
+
+    host_result = ScopeResult(exit_code=0, skipped=True)
+    if host_paths and host_on_start_ok:
+        host_tag = f"{target.name}:host"
+        exit_code = run_scope_backup(
+            host_tag,
+            host_paths,
+            target.host_scope.exclude if target.host_scope else [],
+            config=config,
+            hostname=make_restic_hostname("host", target.name),
+        )
+        host_result = ScopeResult(exit_code=exit_code)
+        if exit_code == 0:
+            log.info("  host backup OK")
+        else:
+            log.error("  host backup FAILED (exit %d)", exit_code)
+    elif host_paths and not host_on_start_ok:
+        host_result = ScopeResult(exit_code=EXIT_ON_START_FAILED, skipped=True)
+
+    if target.container_scope and target.container_scope.on_complete:
+        log.info("  container.on_complete: %s", target.container_scope.on_complete)
+        code, output = run_docker_exec(
+            target.container, target.container_scope.on_complete,
+            env={**tag_env, "DORESTIC_EXIT_CODE": str(container_result.exit_code)},
+            shell=target.container_scope.shell,
+        )
+        if output:
+            for line in output.splitlines():
+                log.info("    %s", line)
+        if code != 0:
+            log.warning("  container.on_complete failed (exit %d)", code)
+
+    if target.host_scope and target.host_scope.on_complete:
+        log.info("  host.on_complete: %s", target.host_scope.on_complete)
+        code = run_hook(
+            target.host_scope.on_complete,
+            env={**tag_env, "DORESTIC_EXIT_CODE": str(host_result.exit_code)},
+        )
+        if code != 0:
+            log.warning("  host.on_complete failed (exit %d)", code)
+
+    return container_result, host_result
+
+
+def backup_host_group(group: HostGroup, config: BackupConfig) -> ScopeResult:
+    log.info("")
+    log.info("=== host:%s ===", group.tag)
+
+    resolved_paths: list[Path] = []
+    for raw in group.paths:
+        full = Path(raw)
+        if full.exists():
+            resolved_paths.append(full)
+        else:
+            log.warning("  host path %s does not exist", full)
+
+    if not resolved_paths:
+        log.info("  no valid paths, skipping")
+        return ScopeResult(exit_code=0, skipped=True)
+
+    tag_env = {"DORESTIC_TAG": group.tag}
+
+    if group.on_start:
+        log.info("  on_start: %s", group.on_start)
+        code = run_hook(group.on_start, env=tag_env)
+        if code != 0:
+            log.error("  on_start failed (exit %d), skipping backup", code)
+            result = ScopeResult(exit_code=EXIT_ON_START_FAILED, skipped=True)
+            if group.on_complete:
+                log.info("  on_complete: %s", group.on_complete)
+                run_hook(group.on_complete, env={**tag_env, "DORESTIC_EXIT_CODE": str(result.exit_code)})
+            return result
+
+    exit_code = run_scope_backup(
+        group.tag, resolved_paths, group.exclude, config=config,
+        hostname=make_restic_hostname("host", group.tag),
+    )
+    result = ScopeResult(exit_code=exit_code)
+    if exit_code == 0:
+        log.info("  backup OK")
+    else:
+        log.error("  backup FAILED (exit %d)", exit_code)
+
+    if group.on_complete:
+        log.info("  on_complete: %s", group.on_complete)
+        code = run_hook(group.on_complete, env={**tag_env, "DORESTIC_EXIT_CODE": str(result.exit_code)})
+        if code != 0:
+            log.warning("  on_complete failed (exit %d)", code)
+
+    return result
+
+
+def run_backup(config_path: str) -> None:
+    logging.basicConfig(
+        level=logging.INFO,
+        format="%(asctime)s %(levelname)s %(message)s",
+        datefmt="%Y-%m-%d %H:%M:%S",
+    )
+
+    log_file = tempfile.NamedTemporaryFile(
+        mode="w", prefix="backup-", suffix=".log", delete=False
+    )
+    log_path = log_file.name
+    original_stdout = sys.stdout
+    original_stderr = sys.stderr
+    overall_exit = 1
+    lock_fd: IO[str] | None = None
+    staging_dir: Path | None = None
+
+    tee_stdout = TeeStream(original_stdout, log_file)
+    tee_stderr = TeeStream(original_stderr, log_file)
+
+    root_logger = logging.getLogger()
+    for handler in root_logger.handlers:
+        if isinstance(handler, logging.StreamHandler):
+            handler.stream = tee_stdout
+
+    sys.stdout = tee_stdout  # type: ignore[assignment]
+    sys.stderr = tee_stderr  # type: ignore[assignment]
+
+    try:
+        config = load_config(config_path)
+        lock_fd = acquire_lock(config)
+
+        if config.on_start:
+            log.info("Running on_start: %s", config.on_start)
+            start_code = run_hook(config.on_start)
+            if start_code != 0:
+                log.error("on_start failed (exit %d), aborting backup", start_code)
+                sys.exit(1)
+
+        log.info("=== Initializing repository if needed ===")
+        init_code, init_output = run_restic("init", config=config, capture=True)
+        if init_code == 0:
+            log.info("Initialized new repository at %s", config.repository)
+        else:
+            check_code, _ = run_restic("cat", "config", config=config, capture=True)
+            if check_code != 0:
+                log.error("Repository init failed at %s:\n%s", config.repository, init_output)
+                sys.exit(1)
+
+        client = docker.DockerClient.from_env()
+        errors = 0
+        staging_dir = Path(tempfile.mkdtemp(prefix="backup-staging-"))
+
+        log.info("")
+        log.info("=== Discovering backup-enabled containers ===")
+        targets = discover_targets(client)
+
+        if not targets:
+            log.info("  No backup-enabled containers found")
+
+        for target in targets:
+            container_result, host_result = backup_container(
+                target, config=config, staging_dir=staging_dir,
+            )
+            if container_result.exit_code != 0:
+                errors += 1
+            if host_result.exit_code != 0:
+                errors += 1
+
+        if config.host_groups:
+            log.info("")
+            log.info("=== Processing host backup groups ===")
+            for group in config.host_groups:
+                group_result = backup_host_group(group, config=config)
+                if group_result.exit_code != 0:
+                    errors += 1
+
+        log.info("")
+        log.info("=== Forgetting old snapshots and pruning ===")
+        run_restic(
+            "forget",
+            "--group-by", "host,tags",
+            "--keep-daily", str(config.retention.daily),
+            "--keep-weekly", str(config.retention.weekly),
+            "--keep-monthly", str(config.retention.monthly),
+            "--prune",
+            config=config,
+        )
+
+        log.info("")
+        log.info("=== Checking repository integrity ===")
+        run_restic("check", config=config)
+
+        overall_exit = 1 if errors > 0 else 0
+
+        log.info("")
+        log.info("=== Backup complete ===")
+        if errors > 0:
+            log.warning("%d backup(s) had errors", errors)
+
+        if config.on_complete:
+            log_file.flush()
+            log.info("Running on_complete: %s", config.on_complete)
+            run_hook(config.on_complete, env={
+                "DORESTIC_EXIT_CODE": str(overall_exit),
+                "DORESTIC_LOGFILE": log_path,
+            })
+
+    except SystemExit:
+        raise
+    except Exception:
+        log.exception("Backup failed with unexpected error")
+        overall_exit = 1
+    finally:
+        sys.stdout = original_stdout
+        sys.stderr = original_stderr
+        for handler in root_logger.handlers:
+            if isinstance(handler, logging.StreamHandler):
+                handler.stream = original_stderr
+        if not log_file.closed:
+            log_file.close()
+        os.unlink(log_path)
+        if staging_dir is not None:
+            shutil.rmtree(staging_dir, ignore_errors=True)
+        if lock_fd is not None:
+            lock_fd.close()
+
+    sys.exit(overall_exit)
+
+

dorestic/cli.py

@@ -0,0 +1,58 @@
+from __future__ import annotations
+
+import argparse
+import importlib.resources
+import sys
+from pathlib import Path
+
+from dorestic.config import find_config
+
+
+def write_example_config(dest: str) -> None:
+    """Write the bundled config.yml.example to the given path."""
+    dest_path = Path(dest)
+
+    if dest_path.suffix != ".yml":
+        dest_path = dest_path / "config.yml"
+
+    if dest_path.exists():
+        print(f"Error: {dest_path} already exists", file=sys.stderr)
+        sys.exit(1)
+
+    dest_path.parent.mkdir(parents=True, exist_ok=True)
+
+    ref = importlib.resources.files("dorestic").joinpath("config.yml.example")
+    content = ref.read_text(encoding="utf-8")
+    dest_path.write_text(content)
+    print(f"Wrote example config to {dest_path}")
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(
+        prog="dorestic",
+        description="Label-driven Docker backup using restic.",
+    )
+    parser.add_argument(
+        "config",
+        nargs="?",
+        default=None,
+        help="path to config.yml (default: auto-discover)",
+    )
+    parser.add_argument(
+        "--init",
+        metavar="PATH",
+        nargs="?",
+        const=".",
+        help="write example config.yml to PATH (default: current directory)",
+    )
+
+    args = parser.parse_args()
+
+    if args.init is not None:
+        write_example_config(args.init)
+        return
+
+    config_path: str = args.config if args.config is not None else find_config()
+
+    from dorestic.backup import run_backup
+    run_backup(config_path)

dorestic/config.py

@@ -0,0 +1,110 @@
+from __future__ import annotations
+
+import os
+from pathlib import Path
+from typing import Any
+
+import yaml
+
+from dorestic.models import (
+    BackupConfig,
+    DEFAULT_RESTIC_IMAGE,
+    HostGroup,
+    RetentionPolicy,
+)
+
+CONFIG_FILENAME = "config.yml"
+
+
+def find_config() -> str:
+    """Search for config.yml in standard locations.
+
+    Order: ./config.yml, then $XDG_CONFIG_HOME/dorestic/config.yml
+    (defaulting to ~/.config/dorestic/config.yml).
+    """
+    local = Path(CONFIG_FILENAME)
+    if local.is_file():
+        return str(local)
+
+    xdg = os.environ.get("XDG_CONFIG_HOME", "")
+    config_dir = Path(xdg) if xdg else Path.home() / ".config"
+    xdg_path = config_dir / "dorestic" / CONFIG_FILENAME
+    if xdg_path.is_file():
+        return str(xdg_path)
+
+    raise FileNotFoundError(
+        f"No config.yml found. Searched:\n"
+        f"  ./{CONFIG_FILENAME}\n"
+        f"  {xdg_path}\n"
+        f"Run 'dorestic --init' to create one, or pass a path: dorestic /path/to/config.yml"
+    )
+
+
+def _as_dict(value: Any, msg: str) -> dict[str, Any]:
+    """Validate that a YAML value is a mapping and return it typed."""
+    if not hasattr(value, "keys"):
+        raise ValueError(msg)
+    result: dict[str, Any] = value
+    return result
+
+
+def load_config(path: str) -> BackupConfig:
+    with open(path) as f:
+        raw = yaml.safe_load(f)
+
+    data: dict[str, Any] = _as_dict(raw, f"Config file {path} must be a YAML mapping")
+
+    if "repository" not in data:
+        raise ValueError(f"Config file {path}: missing required field 'repository'")
+    if "password_file" not in data:
+        raise ValueError(f"Config file {path}: missing required field 'password_file'")
+    if "excludes" in data:
+        raise ValueError("Config uses 'excludes' (plural) — use 'exclude' instead")
+
+    pw_path = Path(str(data["password_file"]))
+    if not pw_path.exists():
+        raise ValueError(f"password_file does not exist: {pw_path}")
+
+    retention = RetentionPolicy()
+    raw_retention = data.get("retention")
+    if raw_retention is not None:
+        ret = _as_dict(raw_retention, "retention must be a mapping")
+        if "daily" in ret:
+            retention.daily = int(ret["daily"])
+        if "weekly" in ret:
+            retention.weekly = int(ret["weekly"])
+        if "monthly" in ret:
+            retention.monthly = int(ret["monthly"])
+
+    host_groups: list[HostGroup] = []
+    for entry in data.get("host_groups", []):
+        group_data: dict[str, Any] = entry
+        if "excludes" in group_data:
+            raise ValueError(
+                f"host_groups entry '{group_data.get('tag', '?')}' uses 'excludes' "
+                "(plural) — use 'exclude' instead"
+            )
+        on_start_val = group_data.get("on_start")
+        on_complete_val = group_data.get("on_complete")
+        host_groups.append(
+            HostGroup(
+                tag=str(group_data["tag"]),
+                paths=[str(p) for p in group_data["paths"]],
+                exclude=[str(e) for e in group_data.get("exclude", [])],
+                on_start=str(on_start_val) if on_start_val is not None else None,
+                on_complete=str(on_complete_val) if on_complete_val is not None else None,
+            )
+        )
+
+    on_start_val = data.get("on_start")
+    on_complete_val = data.get("on_complete")
+
+    return BackupConfig(
+        repository=str(data["repository"]),
+        password_file=str(data["password_file"]),
+        restic_image=str(data.get("restic_image", DEFAULT_RESTIC_IMAGE)),
+        on_start=str(on_start_val) if on_start_val is not None else None,
+        on_complete=str(on_complete_val) if on_complete_val is not None else None,
+        retention=retention,
+        host_groups=host_groups,
+    )

dorestic/config.yml.example

@@ -0,0 +1,50 @@
+# Restic backup configuration
+#
+# Edit this file and fill in your values.
+
+# Path to the restic repository (required)
+repository: /mnt/backup/backup1
+
+# Path to a file containing the restic repository password (required)
+# This is mounted into the restic container via RESTIC_PASSWORD_FILE —
+# the password never appears on the command line.
+# A trailing newline is fine (restic strips it).
+password_file: /etc/backup/restic-password
+
+# Restic Docker image (optional, default: restic/restic:latest)
+# restic_image: restic/restic:latest
+
+# Command to run before the backup starts (optional)
+# If it exits non-zero, the entire backup is aborted.
+# All on_start/on_complete hooks are run via sh -c.
+# on_start: /path/to/on_start.sh
+
+# Command to run after the entire backup completes (optional)
+# Environment: $DORESTIC_EXIT_CODE, $DORESTIC_LOGFILE
+# The log file is deleted after on_complete returns — copy or upload it
+# inside the command if you want to keep it.
+# on_complete: /path/to/on_complete.sh
+
+# Snapshot retention policy (optional, these are the defaults)
+# retention:
+#   daily: 7
+#   weekly: 4
+#   monthly: 12
+
+# Host-only backup groups — not tied to any Docker container (optional)
+# Each group gets its own tagged restic snapshot.
+# host_groups:
+#   - tag: documents
+#     paths:
+#       - /mnt/fileserver/share
+#       - /mnt/fileserver/share-private
+#     exclude:
+#       - "*.tmp"
+#
+#   - tag: docker-config
+#     paths:
+#       - /mnt/fileserver/docker
+#     exclude:
+#       - "aosp-mirror/mirror"
+#     # on_start: /path/to/pre-backup.sh     # env: $DORESTIC_TAG
+#     # on_complete: /path/to/post-backup.sh  # env: $DORESTIC_TAG, $DORESTIC_EXIT_CODE