declarative-data-backup 0.3.1__tar.gz → 0.4.1__tar.gz

{declarative_data_backup-0.3.1/src/declarative_data_backup.egg-info → declarative_data_backup-0.4.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: declarative-data-backup
-Version: 0.3.1
+Version: 0.4.1
 Summary: Declarative Borg-based data backups
 Requires-Python: >=3.11
 Description-Content-Type: text/markdown
@@ -173,6 +173,16 @@ counters or Borg phase. Unchanged counters do not prove forward progress;
 before Borg reports a phase, the status only indicates that the process is
 running. Warnings and statistics are not throttled.
 
+On SIGTERM (including normal Kubernetes pod termination) or Ctrl+C, the
+wrapper asks Borg to stop and waits up to 20 seconds for cleanup before
+releasing its coordination lock. No further sources or pruning are started.
+The exit status is 143 for SIGTERM or 130 for Ctrl+C. Repeated termination
+signals are ignored during cleanup. If Borg does not stop within the timeout,
+its process group is killed and an error warns that Borg locks may remain.
+Use a pod termination grace period of at least 30 seconds (60 recommended);
+forced deletion or SIGKILL cannot be handled gracefully. Never break Borg
+locks without first confirming that no repository operation is still running.
+
 ## Configuration reference
 
 The top level must be a nonempty YAML mapping of backup group names to group
@@ -236,10 +246,12 @@ The stable source series is derived from the canonical source path and includes
 a SHA-256 suffix. Borg receives the canonical absolute path as the source, so
 the original directory hierarchy is retained inside the archive.
 
-The application creates all configured archives before pruning any source.
-If a create fails or Borg reports warnings, the run stops and no pruning is
-performed. Retries use a new run UUID and may create another archive for that
-day; archives are never overwritten to handle a collision.
+The application attempts every configured backup group before pruning any
+source. If a group has a create failure or Borg reports warnings, pruning is
+skipped for that group, but other groups that completed successfully are still
+pruned. Failures are reported and make the overall run fail. Retries use a new
+run UUID and may create another archive for that day; archives are never
+overwritten to handle a collision.
 
 `--keep-daily N` retains the latest archive in up to `N` daily buckets. It does
 not guarantee `N` successful calendar days of recovery if scheduled runs fail

declarative_data_backup-0.3.1/PKG-INFO → declarative_data_backup-0.4.1/README.md

@@ -1,11 +1,3 @@
-Metadata-Version: 2.4
-Name: declarative-data-backup
-Version: 0.3.1
-Summary: Declarative Borg-based data backups
-Requires-Python: >=3.11
-Description-Content-Type: text/markdown
-Requires-Dist: PyYAML<7,>=6
-
 # Declarative Data Backup
 
 Declarative Data Backup creates encrypted Borg archives for configured source
@@ -173,6 +165,16 @@ counters or Borg phase. Unchanged counters do not prove forward progress;
 before Borg reports a phase, the status only indicates that the process is
 running. Warnings and statistics are not throttled.
 
+On SIGTERM (including normal Kubernetes pod termination) or Ctrl+C, the
+wrapper asks Borg to stop and waits up to 20 seconds for cleanup before
+releasing its coordination lock. No further sources or pruning are started.
+The exit status is 143 for SIGTERM or 130 for Ctrl+C. Repeated termination
+signals are ignored during cleanup. If Borg does not stop within the timeout,
+its process group is killed and an error warns that Borg locks may remain.
+Use a pod termination grace period of at least 30 seconds (60 recommended);
+forced deletion or SIGKILL cannot be handled gracefully. Never break Borg
+locks without first confirming that no repository operation is still running.
+
 ## Configuration reference
 
 The top level must be a nonempty YAML mapping of backup group names to group
@@ -236,10 +238,12 @@ The stable source series is derived from the canonical source path and includes
 a SHA-256 suffix. Borg receives the canonical absolute path as the source, so
 the original directory hierarchy is retained inside the archive.
 
-The application creates all configured archives before pruning any source.
-If a create fails or Borg reports warnings, the run stops and no pruning is
-performed. Retries use a new run UUID and may create another archive for that
-day; archives are never overwritten to handle a collision.
+The application attempts every configured backup group before pruning any
+source. If a group has a create failure or Borg reports warnings, pruning is
+skipped for that group, but other groups that completed successfully are still
+pruned. Failures are reported and make the overall run fail. Retries use a new
+run UUID and may create another archive for that day; archives are never
+overwritten to handle a collision.
 
 `--keep-daily N` retains the latest archive in up to `N` daily buckets. It does
 not guarantee `N` successful calendar days of recovery if scheduled runs fail

{declarative_data_backup-0.3.1 → declarative_data_backup-0.4.1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "declarative-data-backup"
-version = "0.3.1"
+version = "0.4.1"
 description = "Declarative Borg-based data backups"
 readme = "README.md"
 requires-python = ">=3.11"

{declarative_data_backup-0.3.1 → declarative_data_backup-0.4.1}/src/declarative_data_backup/backup.py

@@ -1,8 +1,10 @@
 import logging
 import os
+import signal
 import uuid
 from collections.abc import Mapping
 from datetime import UTC, datetime
+from types import FrameType
 
 from .borg import (
     backup_folder,
@@ -14,7 +16,7 @@ from .borg.borg_cli import ensure_borg_binary
 from .config import configure_logging, load_yaml, parse_backup_definitions
 from .config.backup_config.models import BackupDefinition
 from .environment import Environment
-from .errors import BackupError, BorgWarningError
+from .errors import BackupError, BackupInterrupted, BorgWarningError
 
 logger = logging.getLogger(__name__)
 
@@ -54,42 +56,86 @@ def run_backup(
     run_id = uuid.uuid4().hex
     with repository_lock(settings.borg_base_dir / "backup.lock"):
         created_at = datetime.now(UTC)
+        successful_definitions = []
+        failures: list[BackupError] = []
         for definition in definitions:
             logger.info(
                 "Starting backup group '%s' (%d sources).",
                 definition.name,
                 len(definition.sources),
             )
-            for folder in definition.sources:
-                backup_folder(
-                    folder,
-                    definition.compression,
+            try:
+                for folder in definition.sources:
+                    backup_folder(
+                        folder,
+                        definition.compression,
+                        definition.name,
+                        settings,
+                        created_at,
+                        run_id,
+                        definition.excludes,
+                    )
+            except BackupError as exc:
+                logger.error(
+                    "Backup group '%s' failed; retention pruning will be skipped "
+                    "for this group: %s",
                     definition.name,
-                    settings,
-                    created_at,
-                    run_id,
-                    definition.excludes,
+                    exc,
                 )
+                failures.append(exc)
+                continue
+
             logger.info("Backup group '%s' completed.", definition.name)
+            successful_definitions.append(definition)
 
-        for definition in definitions:
+        for definition in successful_definitions:
             logger.info(
                 "Starting retention pruning for group '%s' (%d sources).",
                 definition.name,
                 len(definition.sources),
             )
+            pruning_succeeded = True
             for folder in definition.sources:
-                prune(
-                    folder,
-                    definition.retention_days,
-                    definition.name,
-                    settings,
+                try:
+                    prune(
+                        folder,
+                        definition.retention_days,
+                        definition.name,
+                        settings,
+                    )
+                except BackupError as exc:
+                    logger.error(
+                        "Retention pruning failed for group '%s' source '%s': %s",
+                        definition.name,
+                        folder,
+                        exc,
+                    )
+                    failures.append(exc)
+                    pruning_succeeded = False
+
+            if pruning_succeeded:
+                logger.info(
+                    "Retention pruning for group '%s' completed.", definition.name
                 )
-            logger.info("Retention pruning for group '%s' completed.", definition.name)
+
+        if failures:
+            message = "; ".join(str(failure) for failure in failures)
+            if any(isinstance(failure, BorgWarningError) for failure in failures):
+                raise BorgWarningError(message)
+            raise BackupError(message)
 
 
 def main(environ: Mapping[str, str] | None = None) -> int:
     configure_logging()
+
+    def interrupt(signum: int, frame: FrameType | None) -> None:
+        # Repeated signals must not interrupt Borg cleanup or lock release.
+        signal.signal(signal.SIGTERM, signal.SIG_IGN)
+        signal.signal(signal.SIGINT, signal.SIG_IGN)
+        raise BackupInterrupted(signum)
+
+    previous_sigterm = signal.signal(signal.SIGTERM, interrupt)
+    previous_sigint = signal.signal(signal.SIGINT, interrupt)
     try:
         settings = Environment.from_env(environ)
         logger.info("Starting backup using configuration %s", settings.config_path)
@@ -100,8 +146,8 @@ def main(environ: Mapping[str, str] | None = None) -> int:
 
     except BorgWarningError as exc:
         logger.error(
-            "Backup stopped because Borg reported warnings; "
-            "pruning was not completed: %s",
+            "Backup failed because Borg reported warnings; "
+            "retention pruning may have completed for other groups: %s",
             exc,
         )
         return 1
@@ -114,6 +160,15 @@ def main(environ: Mapping[str, str] | None = None) -> int:
             exc.strerror or "I/O error",
         )
         return 1
+    except BackupInterrupted as exc:
+        logger.error(
+            "Interrupted by %s; no further backups or pruning will run.",
+            signal.Signals(exc.signum).name,
+        )
+        return 128 + exc.signum
     except KeyboardInterrupt:
         logger.error("Interrupted.")
         return 130
+    finally:
+        signal.signal(signal.SIGTERM, previous_sigterm)
+        signal.signal(signal.SIGINT, previous_sigint)

{declarative_data_backup-0.3.1 → declarative_data_backup-0.4.1}/src/declarative_data_backup/borg/borg_cli.py

@@ -3,15 +3,63 @@ import logging
 import os
 import selectors
 import shutil
+import signal
 import subprocess
 import sys
 import time
+from collections.abc import Generator
+from contextlib import contextmanager
 
 from ..environment import Environment
 from ..errors import BorgWarningError, fail
 
 logger = logging.getLogger(__name__)
 PROGRESS_INTERVAL_SECONDS = 30
+SHUTDOWN_TIMEOUT_SECONDS = 20
+
+
+@contextmanager
+def borg_process(
+    command: list[str],
+    environment: dict[str, str],
+    *,
+    capture_stderr: bool = False,
+) -> Generator[subprocess.Popen[bytes], None, None]:
+    with subprocess.Popen(
+        command,
+        env=environment,
+        start_new_session=True,
+        stderr=subprocess.PIPE if capture_stderr else None,
+    ) as process:
+        try:
+            yield process
+        except BaseException:
+            if process.poll() is None:
+                logger.warning(
+                    "Stopping Borg (pid=%d); waiting up to %d seconds for cleanup.",
+                    process.pid,
+                    SHUTDOWN_TIMEOUT_SECONDS,
+                )
+                process.terminate()
+                try:
+                    _, stderr = process.communicate(timeout=SHUTDOWN_TIMEOUT_SECONDS)
+                except subprocess.TimeoutExpired:
+                    logger.error(
+                        "Borg did not stop in time; killing its process group. "
+                        "Stale Borg locks may require inspection before retrying."
+                    )
+                    os.killpg(process.pid, signal.SIGKILL)
+                    _, stderr = process.communicate()
+                if stderr:
+                    sys.stderr.write(stderr.decode("utf-8", errors="replace"))
+                    sys.stderr.flush()
+                logger.warning("Borg stopped (exit code %d).", process.returncode)
+            raise
+
+
+def run_without_progress(command: list[str], environment: dict[str, str]) -> int:
+    with borg_process(command, environment) as process:
+        return process.wait()
 
 
 def run_with_progress(
@@ -51,49 +99,34 @@ def run_with_progress(
         else:
             print(text, file=sys.stderr, flush=True)
 
-    with subprocess.Popen(
-        command,
-        env=environment,
-        stderr=subprocess.PIPE,
-    ) as process:
+    with borg_process(command, environment, capture_stderr=True) as process:
         assert process.stderr is not None
-        try:
-            with selectors.DefaultSelector() as selector:
-                selector.register(process.stderr, selectors.EVENT_READ)
-                pending = b""
-                while selector.get_map():
-                    for key, _ in selector.select(
-                        max(0, next_report - time.monotonic())
-                    ):
-                        chunk = os.read(key.fd, 65536)
-                        if not chunk:
-                            selector.unregister(key.fileobj)
-                            if pending:
-                                handle_line(pending)
-                            break
-                        pending += chunk
-                        while b"\n" in pending:
-                            line, pending = pending.split(b"\n", 1)
-                            handle_line(line)
-                    now = time.monotonic()
-                    if now >= next_report and selector.get_map():
-                        logger.info(
-                            "Borg progress for %s (elapsed_seconds=%.0f; "
-                            "latest reported: %s).",
-                            context,
-                            now - started_at,
-                            latest_progress,
-                        )
-                        next_report = now + PROGRESS_INTERVAL_SECONDS
-            return process.wait()
-        except BaseException:
-            process.terminate()
-            try:
-                process.wait(timeout=10)
-            except subprocess.TimeoutExpired:
-                process.kill()
-                process.wait()
-            raise
+        with selectors.DefaultSelector() as selector:
+            selector.register(process.stderr, selectors.EVENT_READ)
+            pending = b""
+            while selector.get_map():
+                for key, _ in selector.select(max(0, next_report - time.monotonic())):
+                    chunk = os.read(key.fd, 65536)
+                    if not chunk:
+                        selector.unregister(key.fileobj)
+                        if pending:
+                            handle_line(pending)
+                        break
+                    pending += chunk
+                    while b"\n" in pending:
+                        line, pending = pending.split(b"\n", 1)
+                        handle_line(line)
+                now = time.monotonic()
+                if now >= next_report and selector.get_map():
+                    logger.info(
+                        "Borg progress for %s (elapsed_seconds=%.0f; "
+                        "latest reported: %s).",
+                        context,
+                        now - started_at,
+                        latest_progress,
+                    )
+                    next_report = now + PROGRESS_INTERVAL_SECONDS
+        return process.wait()
 
 
 def ensure_borg_binary() -> None:
@@ -141,11 +174,7 @@ def run_borg(
                 context,
             )
         else:
-            returncode = subprocess.run(
-                command,
-                env=borg_environment(settings),
-                check=False,
-            ).returncode
+            returncode = run_without_progress(command, borg_environment(settings))
     except FileNotFoundError:
         fail("Required executable 'borg' could not be started.")
     except OSError as exc:

{declarative_data_backup-0.3.1 → declarative_data_backup-0.4.1}/src/declarative_data_backup/errors.py

@@ -9,5 +9,11 @@ class BorgWarningError(BackupError):
     """Raised when Borg completes with warnings that make the run unsafe."""
 
 
+class BackupInterrupted(KeyboardInterrupt):
+    def __init__(self, signum: int) -> None:
+        self.signum = signum
+        super().__init__(f"Interrupted by signal {signum}.")
+
+
 def fail(message: str) -> NoReturn:
     raise BackupError(message)

declarative_data_backup-0.3.1/README.md → declarative_data_backup-0.4.1/src/declarative_data_backup.egg-info/PKG-INFO

@@ -1,3 +1,11 @@
+Metadata-Version: 2.4
+Name: declarative-data-backup
+Version: 0.4.1
+Summary: Declarative Borg-based data backups
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown
+Requires-Dist: PyYAML<7,>=6
+
 # Declarative Data Backup
 
 Declarative Data Backup creates encrypted Borg archives for configured source
@@ -165,6 +173,16 @@ counters or Borg phase. Unchanged counters do not prove forward progress;
 before Borg reports a phase, the status only indicates that the process is
 running. Warnings and statistics are not throttled.
 
+On SIGTERM (including normal Kubernetes pod termination) or Ctrl+C, the
+wrapper asks Borg to stop and waits up to 20 seconds for cleanup before
+releasing its coordination lock. No further sources or pruning are started.
+The exit status is 143 for SIGTERM or 130 for Ctrl+C. Repeated termination
+signals are ignored during cleanup. If Borg does not stop within the timeout,
+its process group is killed and an error warns that Borg locks may remain.
+Use a pod termination grace period of at least 30 seconds (60 recommended);
+forced deletion or SIGKILL cannot be handled gracefully. Never break Borg
+locks without first confirming that no repository operation is still running.
+
 ## Configuration reference
 
 The top level must be a nonempty YAML mapping of backup group names to group
@@ -228,10 +246,12 @@ The stable source series is derived from the canonical source path and includes
 a SHA-256 suffix. Borg receives the canonical absolute path as the source, so
 the original directory hierarchy is retained inside the archive.
 
-The application creates all configured archives before pruning any source.
-If a create fails or Borg reports warnings, the run stops and no pruning is
-performed. Retries use a new run UUID and may create another archive for that
-day; archives are never overwritten to handle a collision.
+The application attempts every configured backup group before pruning any
+source. If a group has a create failure or Borg reports warnings, pruning is
+skipped for that group, but other groups that completed successfully are still
+pruned. Failures are reported and make the overall run fail. Retries use a new
+run UUID and may create another archive for that day; archives are never
+overwritten to handle a collision.
 
 `--keep-daily N` retains the latest archive in up to `N` daily buckets. It does
 not guarantee `N` successful calendar days of recovery if scheduled runs fail

{declarative_data_backup-0.3.1 → declarative_data_backup-0.4.1}/tests/test_backup.py

@@ -79,7 +79,7 @@ def test_main_runs_all_creates_before_any_prune(
     assert "Retention pruning for group 'media_backup' completed." in caplog.text
 
 
-def test_main_warning_aborts_remaining_creates_and_all_pruning(
+def test_main_warning_skips_failed_group_but_prunes_successful_groups(
     settings_factory,
     monkeypatch,
     caplog,
@@ -91,14 +91,15 @@ def test_main_warning_aborts_remaining_creates_and_all_pruning(
     second.mkdir()
     _write_config(
         settings,
-        f"media:\n  folders: [{first}, {second}]\n",
+        f"failed:\n  folders: [{first}]\nsucceeded:\n  folders: [{second}]\n",
     )
     _configure_preflight(monkeypatch)
     calls = []
 
     def warning(folder, *args):
         calls.append(("create", folder))
-        raise BorgWarningError(f"warning for {folder}")
+        if folder == first:
+            raise BorgWarningError(f"warning for {folder}")
 
     monkeypatch.setattr(backup, "backup_folder", warning)
     monkeypatch.setattr(
@@ -109,7 +110,11 @@ def test_main_warning_aborts_remaining_creates_and_all_pruning(
     caplog.set_level(logging.ERROR)
 
     assert backup.main(_settings_environment(settings)) == 1
-    assert calls == [("create", first)]
+    assert calls == [
+        ("create", first),
+        ("create", second),
+        ("prune", second),
+    ]
     assert "warnings" in caplog.text
     assert str(first) in caplog.text
 
@@ -269,6 +274,7 @@ def test_main_prune_failure_is_nonzero_and_scoped(
         ("create", first),
         ("create", second),
         ("prune", first),
+        ("prune", second),
     ]