declarative-data-backup 0.2.1__tar.gz → 0.3.1__tar.gz

{declarative_data_backup-0.2.1 → declarative_data_backup-0.3.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: declarative-data-backup
-Version: 0.2.1
+Version: 0.3.1
 Summary: Declarative Borg-based data backups
 Requires-Python: >=3.11
 Description-Content-Type: text/markdown
@@ -149,6 +149,8 @@ Optional variables:
 | --- | --- |
 | `BORG_REMOTE_PATH` | Remote Borg executable path, passed as Borg's `--remote-path`. Leave unset to use Borg's default. |
 | `BORG_EXIT_CODES` | Borg exit-code mode: `legacy` or `modern`. |
+| `BACKUP_PROGRESS` | Set to `true` to log the latest Borg progress every 30 seconds during archive creation. Defaults to `false`. |
+| `BACKUP_VERBOSE` | Set to `true` to show Borg's per-file create and per-archive prune output. Defaults to `false` for concise logs. |
 
 Provide credentials through a secret manager or protected files; do not put
 passphrases, private keys, or credentials in the YAML file, source control, or
@@ -161,7 +163,15 @@ python -m declarative_data_backup
 
 The command exits with status `0` on success, nonzero on configuration or Borg
 failure, and `130` when interrupted. Borg's exit code `1` indicates warnings:
-the run fails and pruning is skipped.
+the run fails and pruning is skipped. Logs include group and source start/end
+messages, while Borg's per-file and per-archive output is disabled by default.
+Set `BACKUP_VERBOSE=true` for detailed Borg output or `BACKUP_PROGRESS=true`
+for live progress during archive creation; either can increase log volume.
+Progress is logged as plain status lines, not a terminal progress bar. Each
+line includes the group, source, elapsed time, and latest reported file/byte
+counters or Borg phase. Unchanged counters do not prove forward progress;
+before Borg reports a phase, the status only indicates that the process is
+running. Warnings and statistics are not throttled.
 
 ## Configuration reference
 
@@ -289,23 +299,24 @@ container integration test.
 
 ## Testing
 
-Run the unit tests:
+Run the unit tests, integration tests, or both:
 
 ```shell
-uv run pytest -m "not integration"
+make test-unit
+make test-integration
+make test
 ```
 
-The opt-in local Borg integration test uses a temporary encrypted repository:
+The integration target opts in to both local Borg tests (which use temporary
+encrypted repositories) and the Docker SSH test (which uses disposable
+containers and a temporary repository). It does not use production credentials
+or BorgBase. Docker must be installed and its daemon available.
+
+To run either integration suite individually:
 
 ```shell
 RUN_BORG_LOCAL_INTEGRATION=1 uv run pytest -m integration \
   tests/integration/test_local_borg_integration.py
-```
-
-The opt-in Docker SSH integration test uses disposable containers and a
-temporary repository; it does not use production credentials or BorgBase:
-
-```shell
 RUN_BORG_DOCKER_INTEGRATION=1 uv run pytest -m integration \
   tests/integration/test_docker_integration.py
 ```

{declarative_data_backup-0.2.1 → declarative_data_backup-0.3.1}/README.md

@@ -141,6 +141,8 @@ Optional variables:
 | --- | --- |
 | `BORG_REMOTE_PATH` | Remote Borg executable path, passed as Borg's `--remote-path`. Leave unset to use Borg's default. |
 | `BORG_EXIT_CODES` | Borg exit-code mode: `legacy` or `modern`. |
+| `BACKUP_PROGRESS` | Set to `true` to log the latest Borg progress every 30 seconds during archive creation. Defaults to `false`. |
+| `BACKUP_VERBOSE` | Set to `true` to show Borg's per-file create and per-archive prune output. Defaults to `false` for concise logs. |
 
 Provide credentials through a secret manager or protected files; do not put
 passphrases, private keys, or credentials in the YAML file, source control, or
@@ -153,7 +155,15 @@ python -m declarative_data_backup
 
 The command exits with status `0` on success, nonzero on configuration or Borg
 failure, and `130` when interrupted. Borg's exit code `1` indicates warnings:
-the run fails and pruning is skipped.
+the run fails and pruning is skipped. Logs include group and source start/end
+messages, while Borg's per-file and per-archive output is disabled by default.
+Set `BACKUP_VERBOSE=true` for detailed Borg output or `BACKUP_PROGRESS=true`
+for live progress during archive creation; either can increase log volume.
+Progress is logged as plain status lines, not a terminal progress bar. Each
+line includes the group, source, elapsed time, and latest reported file/byte
+counters or Borg phase. Unchanged counters do not prove forward progress;
+before Borg reports a phase, the status only indicates that the process is
+running. Warnings and statistics are not throttled.
 
 ## Configuration reference
 
@@ -281,23 +291,24 @@ container integration test.
 
 ## Testing
 
-Run the unit tests:
+Run the unit tests, integration tests, or both:
 
 ```shell
-uv run pytest -m "not integration"
+make test-unit
+make test-integration
+make test
 ```
 
-The opt-in local Borg integration test uses a temporary encrypted repository:
+The integration target opts in to both local Borg tests (which use temporary
+encrypted repositories) and the Docker SSH test (which uses disposable
+containers and a temporary repository). It does not use production credentials
+or BorgBase. Docker must be installed and its daemon available.
+
+To run either integration suite individually:
 
 ```shell
 RUN_BORG_LOCAL_INTEGRATION=1 uv run pytest -m integration \
   tests/integration/test_local_borg_integration.py
-```
-
-The opt-in Docker SSH integration test uses disposable containers and a
-temporary repository; it does not use production credentials or BorgBase:
-
-```shell
 RUN_BORG_DOCKER_INTEGRATION=1 uv run pytest -m integration \
   tests/integration/test_docker_integration.py
 ```

{declarative_data_backup-0.2.1 → declarative_data_backup-0.3.1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "declarative-data-backup"
-version = "0.2.1"
+version = "0.3.1"
 description = "Declarative Borg-based data backups"
 readme = "README.md"
 requires-python = ">=3.11"

{declarative_data_backup-0.2.1 → declarative_data_backup-0.3.1}/src/declarative_data_backup/backup.py

@@ -70,8 +70,14 @@ def run_backup(
                     run_id,
                     definition.excludes,
                 )
+            logger.info("Backup group '%s' completed.", definition.name)
 
         for definition in definitions:
+            logger.info(
+                "Starting retention pruning for group '%s' (%d sources).",
+                definition.name,
+                len(definition.sources),
+            )
             for folder in definition.sources:
                 prune(
                     folder,
@@ -79,6 +85,7 @@ def run_backup(
                     definition.name,
                     settings,
                 )
+            logger.info("Retention pruning for group '%s' completed.", definition.name)
 
 
 def main(environ: Mapping[str, str] | None = None) -> int:

{declarative_data_backup-0.2.1 → declarative_data_backup-0.3.1}/src/declarative_data_backup/borg/borg.py

@@ -41,11 +41,14 @@ def backup_folder(
     logger.info("Starting backup for %s as archive %s.", context, name)
     args = [
         "create",
-        "--verbose",
         "--stats",
         "--compression",
         compression,
     ]
+    if settings.backup_verbose:
+        args.append("--verbose")
+    if settings.backup_progress:
+        args.append("--progress")
     for pattern in excludes:
         args.extend(("--exclude", pattern))
     args.extend((f"{settings.borg_repo}::{name}", str(folder)))
@@ -85,17 +88,20 @@ def prune(
         retention_days,
     )
     try:
-        run_borg(
-            [
-                "prune",
-                "--verbose",
-                "--list",
+        args = ["prune"]
+        if settings.backup_verbose:
+            args.extend(("--verbose", "--list"))
+        args.extend(
+            (
                 "--glob-archives",
                 selector,
                 "--keep-daily",
                 str(retention_days),
                 settings.borg_repo,
-            ],
+            )
+        )
+        run_borg(
+            args,
             settings,
             "prune",
             context,

declarative_data_backup-0.3.1/src/declarative_data_backup/borg/borg_cli.py

@@ -0,0 +1,164 @@
+import json
+import logging
+import os
+import selectors
+import shutil
+import subprocess
+import sys
+import time
+
+from ..environment import Environment
+from ..errors import BorgWarningError, fail
+
+logger = logging.getLogger(__name__)
+PROGRESS_INTERVAL_SECONDS = 30
+
+
+def run_with_progress(
+    command: list[str], environment: dict[str, str], context: str
+) -> int:
+    started_at = time.monotonic()
+    next_report = started_at + PROGRESS_INTERVAL_SECONDS
+    latest_progress = "Waiting for Borg progress (process running)."
+
+    def handle_line(line: bytes) -> None:
+        nonlocal latest_progress
+        text = line.decode("utf-8", errors="replace")
+        try:
+            event = json.loads(text)
+        except json.JSONDecodeError:
+            print(text, file=sys.stderr, flush=True)
+            return
+        if not isinstance(event, dict):
+            print(text, file=sys.stderr, flush=True)
+            return
+        kind = event.get("type")
+        if kind in ("archive_progress", "progress_message", "progress_percent"):
+            if event.get("finished"):
+                return
+            if kind == "archive_progress":
+                latest_progress = (
+                    f"files={event.get('nfiles')} "
+                    f"original_bytes={event.get('original_size')} "
+                    f"compressed_bytes={event.get('compressed_size')} "
+                    f"deduplicated_bytes={event.get('deduplicated_size')}"
+                )
+            else:
+                latest_progress = str(event.get("message", text))
+            return
+        if kind == "log_message":
+            print(event.get("message", text), file=sys.stderr, flush=True)
+        else:
+            print(text, file=sys.stderr, flush=True)
+
+    with subprocess.Popen(
+        command,
+        env=environment,
+        stderr=subprocess.PIPE,
+    ) as process:
+        assert process.stderr is not None
+        try:
+            with selectors.DefaultSelector() as selector:
+                selector.register(process.stderr, selectors.EVENT_READ)
+                pending = b""
+                while selector.get_map():
+                    for key, _ in selector.select(
+                        max(0, next_report - time.monotonic())
+                    ):
+                        chunk = os.read(key.fd, 65536)
+                        if not chunk:
+                            selector.unregister(key.fileobj)
+                            if pending:
+                                handle_line(pending)
+                            break
+                        pending += chunk
+                        while b"\n" in pending:
+                            line, pending = pending.split(b"\n", 1)
+                            handle_line(line)
+                    now = time.monotonic()
+                    if now >= next_report and selector.get_map():
+                        logger.info(
+                            "Borg progress for %s (elapsed_seconds=%.0f; "
+                            "latest reported: %s).",
+                            context,
+                            now - started_at,
+                            latest_progress,
+                        )
+                        next_report = now + PROGRESS_INTERVAL_SECONDS
+            return process.wait()
+        except BaseException:
+            process.terminate()
+            try:
+                process.wait(timeout=10)
+            except subprocess.TimeoutExpired:
+                process.kill()
+                process.wait()
+            raise
+
+
+def ensure_borg_binary() -> None:
+    if shutil.which("borg") is None:
+        fail("Required executable 'borg' was not found on PATH.")
+
+
+def borg_environment(settings: Environment) -> dict[str, str]:
+    environment = os.environ.copy()
+    environment.update(
+        {
+            "BORG_REPO": settings.borg_repo,
+            "BORG_BASE_DIR": str(settings.borg_base_dir),
+            "BORG_KEY_FILE": str(settings.key_file),
+            "BORG_RSH": settings.borg_rsh,
+            "BORG_PASSCOMMAND": settings.borg_passcommand,
+        }
+    )
+    if settings.borg_exit_codes is not None:
+        environment["BORG_EXIT_CODES"] = settings.borg_exit_codes
+    return environment
+
+
+def run_borg(
+    args: list[str],
+    settings: Environment,
+    operation: str,
+    context: str,
+) -> None:
+    logger.info("Running Borg %s for %s.", operation, context)
+    command = ["borg"]
+    remote_path = settings.borg_remote_path
+    if remote_path is not None:
+        if not remote_path.strip() or "\0" in remote_path:
+            fail("BORG_REMOTE_PATH must be a nonempty Borg executable path.")
+        command.extend(("--remote-path", remote_path))
+    command.extend(args)
+
+    try:
+        if settings.backup_progress and operation == "create":
+            command.insert(1, "--log-json")
+            returncode = run_with_progress(
+                command,
+                borg_environment(settings),
+                context,
+            )
+        else:
+            returncode = subprocess.run(
+                command,
+                env=borg_environment(settings),
+                check=False,
+            ).returncode
+    except FileNotFoundError:
+        fail("Required executable 'borg' could not be started.")
+    except OSError as exc:
+        fail(
+            f"Unable to start Borg {operation} for {context}: {exc.strerror or 'I/O error'}."
+        )
+
+    if returncode == 0:
+        return
+
+    if returncode == 1:
+        message = f"Borg {operation} reported warnings for {context} (exit code 1)."
+        logger.warning("%s", message)
+        raise BorgWarningError(message)
+
+    fail(f"Borg {operation} failed for {context} (exit code {returncode}).")

{declarative_data_backup-0.2.1 → declarative_data_backup-0.3.1}/src/declarative_data_backup/environment.py

@@ -17,6 +17,8 @@ class Environment:
     borg_passcommand: str
     borg_remote_path: str | None
     borg_exit_codes: str | None
+    backup_progress: bool
+    backup_verbose: bool
 
     @classmethod
     def from_env(
@@ -46,6 +48,12 @@ class Environment:
         borg_exit_codes = source.get("BORG_EXIT_CODES")
         if borg_exit_codes not in (None, "legacy", "modern"):
             fail("BORG_EXIT_CODES must be either 'legacy' or 'modern'.")
+        backup_progress_value = source.get("BACKUP_PROGRESS", "false").lower()
+        if backup_progress_value not in ("true", "false"):
+            fail("BACKUP_PROGRESS must be either 'true' or 'false'.")
+        backup_verbose_value = source.get("BACKUP_VERBOSE", "false").lower()
+        if backup_verbose_value not in ("true", "false"):
+            fail("BACKUP_VERBOSE must be either 'true' or 'false'.")
 
         try:
             server_root = required_path("SERVER_ROOT").resolve(strict=True)
@@ -62,6 +70,8 @@ class Environment:
             borg_passcommand=required("BORG_PASSCOMMAND"),
             borg_remote_path=source.get("BORG_REMOTE_PATH"),
             borg_exit_codes=borg_exit_codes,
+            backup_progress=backup_progress_value == "true",
+            backup_verbose=backup_verbose_value == "true",
         )
 
 

{declarative_data_backup-0.2.1 → declarative_data_backup-0.3.1}/src/declarative_data_backup.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: declarative-data-backup
-Version: 0.2.1
+Version: 0.3.1
 Summary: Declarative Borg-based data backups
 Requires-Python: >=3.11
 Description-Content-Type: text/markdown
@@ -149,6 +149,8 @@ Optional variables:
 | --- | --- |
 | `BORG_REMOTE_PATH` | Remote Borg executable path, passed as Borg's `--remote-path`. Leave unset to use Borg's default. |
 | `BORG_EXIT_CODES` | Borg exit-code mode: `legacy` or `modern`. |
+| `BACKUP_PROGRESS` | Set to `true` to log the latest Borg progress every 30 seconds during archive creation. Defaults to `false`. |
+| `BACKUP_VERBOSE` | Set to `true` to show Borg's per-file create and per-archive prune output. Defaults to `false` for concise logs. |
 
 Provide credentials through a secret manager or protected files; do not put
 passphrases, private keys, or credentials in the YAML file, source control, or
@@ -161,7 +163,15 @@ python -m declarative_data_backup
 
 The command exits with status `0` on success, nonzero on configuration or Borg
 failure, and `130` when interrupted. Borg's exit code `1` indicates warnings:
-the run fails and pruning is skipped.
+the run fails and pruning is skipped. Logs include group and source start/end
+messages, while Borg's per-file and per-archive output is disabled by default.
+Set `BACKUP_VERBOSE=true` for detailed Borg output or `BACKUP_PROGRESS=true`
+for live progress during archive creation; either can increase log volume.
+Progress is logged as plain status lines, not a terminal progress bar. Each
+line includes the group, source, elapsed time, and latest reported file/byte
+counters or Borg phase. Unchanged counters do not prove forward progress;
+before Borg reports a phase, the status only indicates that the process is
+running. Warnings and statistics are not throttled.
 
 ## Configuration reference
 
@@ -289,23 +299,24 @@ container integration test.
 
 ## Testing
 
-Run the unit tests:
+Run the unit tests, integration tests, or both:
 
 ```shell
-uv run pytest -m "not integration"
+make test-unit
+make test-integration
+make test
 ```
 
-The opt-in local Borg integration test uses a temporary encrypted repository:
+The integration target opts in to both local Borg tests (which use temporary
+encrypted repositories) and the Docker SSH test (which uses disposable
+containers and a temporary repository). It does not use production credentials
+or BorgBase. Docker must be installed and its daemon available.
+
+To run either integration suite individually:
 
 ```shell
 RUN_BORG_LOCAL_INTEGRATION=1 uv run pytest -m integration \
   tests/integration/test_local_borg_integration.py
-```
-
-The opt-in Docker SSH integration test uses disposable containers and a
-temporary repository; it does not use production credentials or BorgBase:
-
-```shell
 RUN_BORG_DOCKER_INTEGRATION=1 uv run pytest -m integration \
   tests/integration/test_docker_integration.py
 ```

{declarative_data_backup-0.2.1 → declarative_data_backup-0.3.1}/tests/test_backup.py

@@ -29,6 +29,7 @@ def _configure_preflight(monkeypatch):
 def test_main_runs_all_creates_before_any_prune(
     settings_factory,
     monkeypatch,
+    caplog,
 ):
     settings = settings_factory()
     media = settings.server_root / "media"
@@ -62,6 +63,7 @@ def test_main_runs_all_creates_before_any_prune(
     monkeypatch.setattr(
         backup.uuid, "uuid4", lambda: type("UUID", (), {"hex": "a" * 32})()
     )
+    caplog.set_level(logging.INFO)
 
     assert backup.main(_settings_environment(settings)) == 0
     assert [event[0] for event in events] == [
@@ -73,6 +75,8 @@ def test_main_runs_all_creates_before_any_prune(
     assert events[0][3] == "zstd,5"
     assert events[1][3] == "auto,zstd,22"
     assert events[2][3] == 30
+    assert "Backup group 'media_backup' completed." in caplog.text
+    assert "Retention pruning for group 'media_backup' completed." in caplog.text
 
 
 def test_main_warning_aborts_remaining_creates_and_all_pruning(

declarative_data_backup-0.2.1/src/declarative_data_backup/borg/borg_cli.py

@@ -1,69 +0,0 @@
-import logging
-import os
-import shutil
-import subprocess
-
-from ..environment import Environment
-from ..errors import BorgWarningError, fail
-
-logger = logging.getLogger(__name__)
-
-
-def ensure_borg_binary() -> None:
-    if shutil.which("borg") is None:
-        fail("Required executable 'borg' was not found on PATH.")
-
-
-def borg_environment(settings: Environment) -> dict[str, str]:
-    environment = os.environ.copy()
-    environment.update(
-        {
-            "BORG_REPO": settings.borg_repo,
-            "BORG_BASE_DIR": str(settings.borg_base_dir),
-            "BORG_KEY_FILE": str(settings.key_file),
-            "BORG_RSH": settings.borg_rsh,
-            "BORG_PASSCOMMAND": settings.borg_passcommand,
-        }
-    )
-    if settings.borg_exit_codes is not None:
-        environment["BORG_EXIT_CODES"] = settings.borg_exit_codes
-    return environment
-
-
-def run_borg(
-    args: list[str],
-    settings: Environment,
-    operation: str,
-    context: str,
-) -> None:
-    logger.info("Running Borg %s for %s.", operation, context)
-    command = ["borg"]
-    remote_path = settings.borg_remote_path
-    if remote_path is not None:
-        if not remote_path.strip() or "\0" in remote_path:
-            fail("BORG_REMOTE_PATH must be a nonempty Borg executable path.")
-        command.extend(("--remote-path", remote_path))
-    command.extend(args)
-
-    try:
-        result = subprocess.run(
-            command,
-            env=borg_environment(settings),
-            check=False,
-        )
-    except FileNotFoundError:
-        fail("Required executable 'borg' could not be started.")
-    except OSError as exc:
-        fail(
-            f"Unable to start Borg {operation} for {context}: {exc.strerror or 'I/O error'}."
-        )
-
-    if result.returncode == 0:
-        return
-
-    if result.returncode == 1:
-        message = f"Borg {operation} reported warnings for {context} (exit code 1)."
-        logger.warning("%s", message)
-        raise BorgWarningError(message)
-
-    fail(f"Borg {operation} failed for {context} (exit code {result.returncode}).")