declarative-data-backup 0.4.1__tar.gz → 0.5.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (26) hide show
  1. {declarative_data_backup-0.4.1 → declarative_data_backup-0.5.1}/PKG-INFO +4 -1
  2. {declarative_data_backup-0.4.1 → declarative_data_backup-0.5.1}/README.md +3 -0
  3. {declarative_data_backup-0.4.1 → declarative_data_backup-0.5.1}/pyproject.toml +1 -1
  4. {declarative_data_backup-0.4.1 → declarative_data_backup-0.5.1}/src/declarative_data_backup/backup.py +42 -0
  5. {declarative_data_backup-0.4.1 → declarative_data_backup-0.5.1}/src/declarative_data_backup/borg/borg_cli.py +2 -0
  6. {declarative_data_backup-0.4.1 → declarative_data_backup-0.5.1}/src/declarative_data_backup/environment.py +35 -0
  7. {declarative_data_backup-0.4.1 → declarative_data_backup-0.5.1}/src/declarative_data_backup.egg-info/PKG-INFO +4 -1
  8. {declarative_data_backup-0.4.1 → declarative_data_backup-0.5.1}/tests/test_backup.py +59 -11
  9. {declarative_data_backup-0.4.1 → declarative_data_backup-0.5.1}/setup.cfg +0 -0
  10. {declarative_data_backup-0.4.1 → declarative_data_backup-0.5.1}/src/declarative_data_backup/__init__.py +0 -0
  11. {declarative_data_backup-0.4.1 → declarative_data_backup-0.5.1}/src/declarative_data_backup/__main__.py +0 -0
  12. {declarative_data_backup-0.4.1 → declarative_data_backup-0.5.1}/src/declarative_data_backup/borg/__init__.py +0 -0
  13. {declarative_data_backup-0.4.1 → declarative_data_backup-0.5.1}/src/declarative_data_backup/borg/archive_naming.py +0 -0
  14. {declarative_data_backup-0.4.1 → declarative_data_backup-0.5.1}/src/declarative_data_backup/borg/borg.py +0 -0
  15. {declarative_data_backup-0.4.1 → declarative_data_backup-0.5.1}/src/declarative_data_backup/config/__init__.py +0 -0
  16. {declarative_data_backup-0.4.1 → declarative_data_backup-0.5.1}/src/declarative_data_backup/config/backup_config/__init__.py +0 -0
  17. {declarative_data_backup-0.4.1 → declarative_data_backup-0.5.1}/src/declarative_data_backup/config/backup_config/models.py +0 -0
  18. {declarative_data_backup-0.4.1 → declarative_data_backup-0.5.1}/src/declarative_data_backup/config/backup_config/utils.py +0 -0
  19. {declarative_data_backup-0.4.1 → declarative_data_backup-0.5.1}/src/declarative_data_backup/config/backup_config/yaml_parser.py +0 -0
  20. {declarative_data_backup-0.4.1 → declarative_data_backup-0.5.1}/src/declarative_data_backup/config/backup_config/yaml_raw_loader.py +0 -0
  21. {declarative_data_backup-0.4.1 → declarative_data_backup-0.5.1}/src/declarative_data_backup/config/logging.py +0 -0
  22. {declarative_data_backup-0.4.1 → declarative_data_backup-0.5.1}/src/declarative_data_backup/errors.py +0 -0
  23. {declarative_data_backup-0.4.1 → declarative_data_backup-0.5.1}/src/declarative_data_backup.egg-info/SOURCES.txt +0 -0
  24. {declarative_data_backup-0.4.1 → declarative_data_backup-0.5.1}/src/declarative_data_backup.egg-info/dependency_links.txt +0 -0
  25. {declarative_data_backup-0.4.1 → declarative_data_backup-0.5.1}/src/declarative_data_backup.egg-info/requires.txt +0 -0
  26. {declarative_data_backup-0.4.1 → declarative_data_backup-0.5.1}/src/declarative_data_backup.egg-info/top_level.txt +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: declarative-data-backup
3
- Version: 0.4.1
3
+ Version: 0.5.1
4
4
  Summary: Declarative Borg-based data backups
5
5
  Requires-Python: >=3.11
6
6
  Description-Content-Type: text/markdown
@@ -149,6 +149,9 @@ Optional variables:
149
149
  | --- | --- |
150
150
  | `BORG_REMOTE_PATH` | Remote Borg executable path, passed as Borg's `--remote-path`. Leave unset to use Borg's default. |
151
151
  | `BORG_EXIT_CODES` | Borg exit-code mode: `legacy` or `modern`. |
152
+ | `BACKUP_FILES_CACHE_TTL_MARGIN` | Extra archive creations to retain file-cache entries beyond the number of configured sources. The app sets Borg's TTL to source count plus this margin; defaults to `20`. |
153
+ | `BORG_FILES_CACHE_TTL` | Optional absolute Borg TTL override; it must be at least the number of configured sources. Usually leave unset to use the source-count-based TTL and configured margin. |
154
+ | `BACKUP_FAILURES_PATH` | Optional absolute path where failed backup group names are written as JSON. |
152
155
  | `BACKUP_PROGRESS` | Set to `true` to log the latest Borg progress every 30 seconds during archive creation. Defaults to `false`. |
153
156
  | `BACKUP_VERBOSE` | Set to `true` to show Borg's per-file create and per-archive prune output. Defaults to `false` for concise logs. |
154
157
 
@@ -141,6 +141,9 @@ Optional variables:
141
141
  | --- | --- |
142
142
  | `BORG_REMOTE_PATH` | Remote Borg executable path, passed as Borg's `--remote-path`. Leave unset to use Borg's default. |
143
143
  | `BORG_EXIT_CODES` | Borg exit-code mode: `legacy` or `modern`. |
144
+ | `BACKUP_FILES_CACHE_TTL_MARGIN` | Extra archive creations to retain file-cache entries beyond the number of configured sources. The app sets Borg's TTL to source count plus this margin; defaults to `20`. |
145
+ | `BORG_FILES_CACHE_TTL` | Optional absolute Borg TTL override; it must be at least the number of configured sources. Usually leave unset to use the source-count-based TTL and configured margin. |
146
+ | `BACKUP_FAILURES_PATH` | Optional absolute path where failed backup group names are written as JSON. |
144
147
  | `BACKUP_PROGRESS` | Set to `true` to log the latest Borg progress every 30 seconds during archive creation. Defaults to `false`. |
145
148
  | `BACKUP_VERBOSE` | Set to `true` to show Borg's per-file create and per-archive prune output. Defaults to `false` for concise logs. |
146
149
 
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "declarative-data-backup"
7
- version = "0.4.1"
7
+ version = "0.5.1"
8
8
  description = "Declarative Borg-based data backups"
9
9
  readme = "README.md"
10
10
  requires-python = ">=3.11"
@@ -1,8 +1,10 @@
1
+ import json
1
2
  import logging
2
3
  import os
3
4
  import signal
4
5
  import uuid
5
6
  from collections.abc import Mapping
7
+ from dataclasses import replace
6
8
  from datetime import UTC, datetime
7
9
  from types import FrameType
8
10
 
@@ -21,6 +23,20 @@ from .errors import BackupError, BackupInterrupted, BorgWarningError
21
23
  logger = logging.getLogger(__name__)
22
24
 
23
25
 
26
+ def write_failed_groups(settings: Environment, groups: set[str]) -> None:
27
+ path = settings.failure_groups_path
28
+ if path is None or not groups:
29
+ return
30
+ try:
31
+ path.write_text(json.dumps(sorted(groups)), encoding="utf-8")
32
+ except OSError as exc:
33
+ logger.warning(
34
+ "Unable to write failed backup group summary to %s: %s",
35
+ path,
36
+ exc.strerror or "I/O error",
37
+ )
38
+
39
+
24
40
  def check_settings(settings: Environment) -> None:
25
41
  ensure_borg_binary()
26
42
  if not settings.borg_base_dir.is_dir():
@@ -52,12 +68,35 @@ def run_backup(
52
68
  len(definitions),
53
69
  total_sources,
54
70
  )
71
+ if (
72
+ settings.borg_files_cache_ttl is not None
73
+ and settings.borg_files_cache_ttl < total_sources
74
+ ):
75
+ raise BackupError(
76
+ "BORG_FILES_CACHE_TTL must be at least the number of configured "
77
+ f"source creates ({total_sources})."
78
+ )
79
+ if settings.borg_files_cache_ttl is None:
80
+ files_cache_ttl = total_sources + settings.backup_files_cache_ttl_margin
81
+ settings = replace(settings, borg_files_cache_ttl=files_cache_ttl)
82
+ logger.info(
83
+ "Using Borg files-cache TTL %d (%d source creates + %d margin).",
84
+ files_cache_ttl,
85
+ total_sources,
86
+ settings.backup_files_cache_ttl_margin,
87
+ )
88
+ else:
89
+ logger.info(
90
+ "Using explicit Borg files-cache TTL %d.",
91
+ settings.borg_files_cache_ttl,
92
+ )
55
93
 
56
94
  run_id = uuid.uuid4().hex
57
95
  with repository_lock(settings.borg_base_dir / "backup.lock"):
58
96
  created_at = datetime.now(UTC)
59
97
  successful_definitions = []
60
98
  failures: list[BackupError] = []
99
+ failed_groups: set[str] = set()
61
100
  for definition in definitions:
62
101
  logger.info(
63
102
  "Starting backup group '%s' (%d sources).",
@@ -83,6 +122,7 @@ def run_backup(
83
122
  exc,
84
123
  )
85
124
  failures.append(exc)
125
+ failed_groups.add(definition.name)
86
126
  continue
87
127
 
88
128
  logger.info("Backup group '%s' completed.", definition.name)
@@ -111,6 +151,7 @@ def run_backup(
111
151
  exc,
112
152
  )
113
153
  failures.append(exc)
154
+ failed_groups.add(definition.name)
114
155
  pruning_succeeded = False
115
156
 
116
157
  if pruning_succeeded:
@@ -119,6 +160,7 @@ def run_backup(
119
160
  )
120
161
 
121
162
  if failures:
163
+ write_failed_groups(settings, failed_groups)
122
164
  message = "; ".join(str(failure) for failure in failures)
123
165
  if any(isinstance(failure, BorgWarningError) for failure in failures):
124
166
  raise BorgWarningError(message)
@@ -147,6 +147,8 @@ def borg_environment(settings: Environment) -> dict[str, str]:
147
147
  )
148
148
  if settings.borg_exit_codes is not None:
149
149
  environment["BORG_EXIT_CODES"] = settings.borg_exit_codes
150
+ if settings.borg_files_cache_ttl is not None:
151
+ environment["BORG_FILES_CACHE_TTL"] = str(settings.borg_files_cache_ttl)
150
152
  return environment
151
153
 
152
154
 
@@ -19,6 +19,9 @@ class Environment:
19
19
  borg_exit_codes: str | None
20
20
  backup_progress: bool
21
21
  backup_verbose: bool
22
+ failure_groups_path: Path | None = None
23
+ backup_files_cache_ttl_margin: int = 20
24
+ borg_files_cache_ttl: int | None = None
22
25
 
23
26
  @classmethod
24
27
  def from_env(
@@ -42,6 +45,17 @@ class Environment:
42
45
  fail(f"{name} must be an absolute path.")
43
46
  return value
44
47
 
48
+ def optional_path(name: str) -> Path | None:
49
+ raw_value = source.get(name)
50
+ if raw_value is None:
51
+ return None
52
+ if "\0" in raw_value:
53
+ fail(f"{name} must be a valid absolute path.")
54
+ value = Path(raw_value)
55
+ if not value.is_absolute():
56
+ fail(f"{name} must be an absolute path.")
57
+ return value
58
+
45
59
  repo = required("BORG_REPO")
46
60
  if "\0" in repo:
47
61
  fail("BORG_REPO must be a valid repository location.")
@@ -54,6 +68,24 @@ class Environment:
54
68
  backup_verbose_value = source.get("BACKUP_VERBOSE", "false").lower()
55
69
  if backup_verbose_value not in ("true", "false"):
56
70
  fail("BACKUP_VERBOSE must be either 'true' or 'false'.")
71
+ try:
72
+ backup_files_cache_ttl_margin = int(
73
+ source.get("BACKUP_FILES_CACHE_TTL_MARGIN", "20")
74
+ )
75
+ except ValueError:
76
+ fail("BACKUP_FILES_CACHE_TTL_MARGIN must be a nonnegative integer.")
77
+ if backup_files_cache_ttl_margin < 0:
78
+ fail("BACKUP_FILES_CACHE_TTL_MARGIN must be a nonnegative integer.")
79
+ raw_borg_files_cache_ttl = source.get("BORG_FILES_CACHE_TTL")
80
+ if raw_borg_files_cache_ttl is None:
81
+ borg_files_cache_ttl = None
82
+ else:
83
+ try:
84
+ borg_files_cache_ttl = int(raw_borg_files_cache_ttl)
85
+ except ValueError:
86
+ fail("BORG_FILES_CACHE_TTL must be a positive integer.")
87
+ if borg_files_cache_ttl < 1:
88
+ fail("BORG_FILES_CACHE_TTL must be a positive integer.")
57
89
 
58
90
  try:
59
91
  server_root = required_path("SERVER_ROOT").resolve(strict=True)
@@ -72,6 +104,9 @@ class Environment:
72
104
  borg_exit_codes=borg_exit_codes,
73
105
  backup_progress=backup_progress_value == "true",
74
106
  backup_verbose=backup_verbose_value == "true",
107
+ failure_groups_path=optional_path("BACKUP_FAILURES_PATH"),
108
+ backup_files_cache_ttl_margin=backup_files_cache_ttl_margin,
109
+ borg_files_cache_ttl=borg_files_cache_ttl,
75
110
  )
76
111
 
77
112
 
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: declarative-data-backup
3
- Version: 0.4.1
3
+ Version: 0.5.1
4
4
  Summary: Declarative Borg-based data backups
5
5
  Requires-Python: >=3.11
6
6
  Description-Content-Type: text/markdown
@@ -149,6 +149,9 @@ Optional variables:
149
149
  | --- | --- |
150
150
  | `BORG_REMOTE_PATH` | Remote Borg executable path, passed as Borg's `--remote-path`. Leave unset to use Borg's default. |
151
151
  | `BORG_EXIT_CODES` | Borg exit-code mode: `legacy` or `modern`. |
152
+ | `BACKUP_FILES_CACHE_TTL_MARGIN` | Extra archive creations to retain file-cache entries beyond the number of configured sources. The app sets Borg's TTL to source count plus this margin; defaults to `20`. |
153
+ | `BORG_FILES_CACHE_TTL` | Optional absolute Borg TTL override; it must be at least the number of configured sources. Usually leave unset to use the source-count-based TTL and configured margin. |
154
+ | `BACKUP_FAILURES_PATH` | Optional absolute path where failed backup group names are written as JSON. |
152
155
  | `BACKUP_PROGRESS` | Set to `true` to log the latest Borg progress every 30 seconds during archive creation. Defaults to `false`. |
153
156
  | `BACKUP_VERBOSE` | Set to `true` to show Borg's per-file create and per-archive prune output. Defaults to `false` for concise logs. |
154
157
 
@@ -1,12 +1,15 @@
1
+ import json
1
2
  import logging
2
3
 
4
+ import pytest
5
+
3
6
  import declarative_data_backup.backup as backup
4
7
  from declarative_data_backup.environment import Environment
5
8
  from declarative_data_backup.errors import BackupError, BorgWarningError
6
9
 
7
10
 
8
11
  def _settings_environment(settings: Environment) -> dict[str, str]:
9
- return {
12
+ environment = {
10
13
  "BACKUP_CONFIG_PATH": str(settings.config_path),
11
14
  "BORG_KEY_FILE": str(settings.key_file),
12
15
  "BORG_REPO": settings.borg_repo,
@@ -15,6 +18,14 @@ def _settings_environment(settings: Environment) -> dict[str, str]:
15
18
  "BORG_RSH": settings.borg_rsh,
16
19
  "BORG_PASSCOMMAND": settings.borg_passcommand,
17
20
  }
21
+ if settings.failure_groups_path is not None:
22
+ environment["BACKUP_FAILURES_PATH"] = str(settings.failure_groups_path)
23
+ environment["BACKUP_FILES_CACHE_TTL_MARGIN"] = str(
24
+ settings.backup_files_cache_ttl_margin
25
+ )
26
+ if settings.borg_files_cache_ttl is not None:
27
+ environment["BORG_FILES_CACHE_TTL"] = str(settings.borg_files_cache_ttl)
28
+ return environment
18
29
 
19
30
 
20
31
  def _write_config(settings, contents: str) -> None:
@@ -26,12 +37,17 @@ def _configure_preflight(monkeypatch):
26
37
  monkeypatch.setattr(backup, "check_keyfile", lambda settings: None)
27
38
 
28
39
 
40
+ @pytest.mark.parametrize(("margin", "expected_ttl"), [(0, 2), (7, 9)])
29
41
  def test_main_runs_all_creates_before_any_prune(
30
42
  settings_factory,
31
43
  monkeypatch,
32
44
  caplog,
45
+ margin,
46
+ expected_ttl,
33
47
  ):
34
- settings = settings_factory()
48
+ settings = settings_factory(
49
+ environment={"BACKUP_FILES_CACHE_TTL_MARGIN": str(margin)}
50
+ )
35
51
  media = settings.server_root / "media"
36
52
  postgres = settings.server_root / "postgres"
37
53
  media.mkdir()
@@ -46,13 +62,13 @@ def test_main_runs_all_creates_before_any_prune(
46
62
  )
47
63
  _configure_preflight(monkeypatch)
48
64
  events = []
49
- monkeypatch.setattr(
50
- backup,
51
- "backup_folder",
52
- lambda folder, compression, group, settings, created_at, run_id, excludes: (
53
- events.append(("create", group, folder, compression, run_id, excludes))
54
- ),
55
- )
65
+ files_cache_ttls = []
66
+
67
+ def create(folder, compression, group, settings, created_at, run_id, excludes):
68
+ files_cache_ttls.append(settings.borg_files_cache_ttl)
69
+ events.append(("create", group, folder, compression, run_id, excludes))
70
+
71
+ monkeypatch.setattr(backup, "backup_folder", create)
56
72
  monkeypatch.setattr(
57
73
  backup,
58
74
  "prune",
@@ -75,6 +91,7 @@ def test_main_runs_all_creates_before_any_prune(
75
91
  assert events[0][3] == "zstd,5"
76
92
  assert events[1][3] == "auto,zstd,22"
77
93
  assert events[2][3] == 30
94
+ assert files_cache_ttls == [expected_ttl, expected_ttl]
78
95
  assert "Backup group 'media_backup' completed." in caplog.text
79
96
  assert "Retention pruning for group 'media_backup' completed." in caplog.text
80
97
 
@@ -83,8 +100,12 @@ def test_main_warning_skips_failed_group_but_prunes_successful_groups(
83
100
  settings_factory,
84
101
  monkeypatch,
85
102
  caplog,
103
+ tmp_path,
86
104
  ):
87
- settings = settings_factory()
105
+ failure_groups_path = tmp_path / "failed-groups.json"
106
+ settings = settings_factory(
107
+ environment={"BACKUP_FAILURES_PATH": str(failure_groups_path)}
108
+ )
88
109
  first = settings.server_root / "first"
89
110
  second = settings.server_root / "second"
90
111
  first.mkdir()
@@ -117,6 +138,7 @@ def test_main_warning_skips_failed_group_but_prunes_successful_groups(
117
138
  ]
118
139
  assert "warnings" in caplog.text
119
140
  assert str(first) in caplog.text
141
+ assert json.loads(failure_groups_path.read_text(encoding="utf-8")) == ["failed"]
120
142
 
121
143
 
122
144
  def test_main_interruption_returns_130_without_pruning(
@@ -215,6 +237,27 @@ def test_main_validation_failure_runs_no_borg_operations(
215
237
  assert not (settings.borg_base_dir / "backup.lock").exists()
216
238
 
217
239
 
240
+ def test_main_rejects_files_cache_ttl_below_source_count(
241
+ settings_factory,
242
+ monkeypatch,
243
+ caplog,
244
+ ):
245
+ settings = settings_factory(environment={"BORG_FILES_CACHE_TTL": "1"})
246
+ first = settings.server_root / "first"
247
+ second = settings.server_root / "second"
248
+ first.mkdir()
249
+ second.mkdir()
250
+ _write_config(settings, f"media:\n folders: [{first}, {second}]\n")
251
+ _configure_preflight(monkeypatch)
252
+ calls = []
253
+ monkeypatch.setattr(backup, "backup_folder", lambda *args: calls.append(args))
254
+ caplog.set_level(logging.ERROR)
255
+
256
+ assert backup.main(_settings_environment(settings)) == 1
257
+ assert calls == []
258
+ assert "must be at least the number of configured source creates (2)" in caplog.text
259
+
260
+
218
261
  def test_main_missing_environment_is_reported_without_traceback(caplog):
219
262
  caplog.set_level(logging.ERROR)
220
263
  assert backup.main({}) == 1
@@ -245,8 +288,12 @@ def test_main_missing_borg_binary_returns_nonzero(
245
288
  def test_main_prune_failure_is_nonzero_and_scoped(
246
289
  settings_factory,
247
290
  monkeypatch,
291
+ tmp_path,
248
292
  ):
249
- settings = settings_factory()
293
+ failure_groups_path = tmp_path / "failed-groups.json"
294
+ settings = settings_factory(
295
+ environment={"BACKUP_FAILURES_PATH": str(failure_groups_path)}
296
+ )
250
297
  first = settings.server_root / "first"
251
298
  second = settings.server_root / "second"
252
299
  first.mkdir()
@@ -276,6 +323,7 @@ def test_main_prune_failure_is_nonzero_and_scoped(
276
323
  ("prune", first),
277
324
  ("prune", second),
278
325
  ]
326
+ assert json.loads(failure_groups_path.read_text(encoding="utf-8")) == ["media"]
279
327
 
280
328
 
281
329
  def test_exclusions_are_passed_only_to_their_backup_group(