declarative-data-backup 0.5.0__tar.gz → 0.5.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (26) hide show
  1. {declarative_data_backup-0.5.0 → declarative_data_backup-0.5.2}/PKG-INFO +3 -2
  2. {declarative_data_backup-0.5.0 → declarative_data_backup-0.5.2}/README.md +2 -1
  3. {declarative_data_backup-0.5.0 → declarative_data_backup-0.5.2}/pyproject.toml +1 -1
  4. {declarative_data_backup-0.5.0 → declarative_data_backup-0.5.2}/src/declarative_data_backup/backup.py +23 -0
  5. {declarative_data_backup-0.5.0 → declarative_data_backup-0.5.2}/src/declarative_data_backup/borg/borg_cli.py +6 -2
  6. {declarative_data_backup-0.5.0 → declarative_data_backup-0.5.2}/src/declarative_data_backup/environment.py +22 -0
  7. {declarative_data_backup-0.5.0 → declarative_data_backup-0.5.2}/src/declarative_data_backup.egg-info/PKG-INFO +3 -2
  8. {declarative_data_backup-0.5.0 → declarative_data_backup-0.5.2}/tests/test_backup.py +42 -8
  9. {declarative_data_backup-0.5.0 → declarative_data_backup-0.5.2}/setup.cfg +0 -0
  10. {declarative_data_backup-0.5.0 → declarative_data_backup-0.5.2}/src/declarative_data_backup/__init__.py +0 -0
  11. {declarative_data_backup-0.5.0 → declarative_data_backup-0.5.2}/src/declarative_data_backup/__main__.py +0 -0
  12. {declarative_data_backup-0.5.0 → declarative_data_backup-0.5.2}/src/declarative_data_backup/borg/__init__.py +0 -0
  13. {declarative_data_backup-0.5.0 → declarative_data_backup-0.5.2}/src/declarative_data_backup/borg/archive_naming.py +0 -0
  14. {declarative_data_backup-0.5.0 → declarative_data_backup-0.5.2}/src/declarative_data_backup/borg/borg.py +0 -0
  15. {declarative_data_backup-0.5.0 → declarative_data_backup-0.5.2}/src/declarative_data_backup/config/__init__.py +0 -0
  16. {declarative_data_backup-0.5.0 → declarative_data_backup-0.5.2}/src/declarative_data_backup/config/backup_config/__init__.py +0 -0
  17. {declarative_data_backup-0.5.0 → declarative_data_backup-0.5.2}/src/declarative_data_backup/config/backup_config/models.py +0 -0
  18. {declarative_data_backup-0.5.0 → declarative_data_backup-0.5.2}/src/declarative_data_backup/config/backup_config/utils.py +0 -0
  19. {declarative_data_backup-0.5.0 → declarative_data_backup-0.5.2}/src/declarative_data_backup/config/backup_config/yaml_parser.py +0 -0
  20. {declarative_data_backup-0.5.0 → declarative_data_backup-0.5.2}/src/declarative_data_backup/config/backup_config/yaml_raw_loader.py +0 -0
  21. {declarative_data_backup-0.5.0 → declarative_data_backup-0.5.2}/src/declarative_data_backup/config/logging.py +0 -0
  22. {declarative_data_backup-0.5.0 → declarative_data_backup-0.5.2}/src/declarative_data_backup/errors.py +0 -0
  23. {declarative_data_backup-0.5.0 → declarative_data_backup-0.5.2}/src/declarative_data_backup.egg-info/SOURCES.txt +0 -0
  24. {declarative_data_backup-0.5.0 → declarative_data_backup-0.5.2}/src/declarative_data_backup.egg-info/dependency_links.txt +0 -0
  25. {declarative_data_backup-0.5.0 → declarative_data_backup-0.5.2}/src/declarative_data_backup.egg-info/requires.txt +0 -0
  26. {declarative_data_backup-0.5.0 → declarative_data_backup-0.5.2}/src/declarative_data_backup.egg-info/top_level.txt +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: declarative-data-backup
3
- Version: 0.5.0
3
+ Version: 0.5.2
4
4
  Summary: Declarative Borg-based data backups
5
5
  Requires-Python: >=3.11
6
6
  Description-Content-Type: text/markdown
@@ -149,7 +149,8 @@ Optional variables:
149
149
  | --- | --- |
150
150
  | `BORG_REMOTE_PATH` | Remote Borg executable path, passed as Borg's `--remote-path`. Leave unset to use Borg's default. |
151
151
  | `BORG_EXIT_CODES` | Borg exit-code mode: `legacy` or `modern`. |
152
- | `BORG_FILES_CACHE_TTL` | How many archive creations can pass without a file appearing before Borg forgets its cached metadata. The default is `20`; raise it if more than 20 archives are created between backups of the same source, so unchanged files can still be recognized. |
152
+ | `BACKUP_FILES_CACHE_TTL_MARGIN` | Extra archive creations to retain file-cache entries beyond the number of configured sources. The app sets Borg's TTL to source count plus this margin; defaults to `20`. |
153
+ | `BORG_FILES_CACHE_TTL` | Optional absolute Borg TTL override; it must be at least the number of configured sources. Usually leave unset to use the source-count-based TTL and configured margin. |
153
154
  | `BACKUP_FAILURES_PATH` | Optional absolute path where failed backup group names are written as JSON. |
154
155
  | `BACKUP_PROGRESS` | Set to `true` to log the latest Borg progress every 30 seconds during archive creation. Defaults to `false`. |
155
156
  | `BACKUP_VERBOSE` | Set to `true` to show Borg's per-file create and per-archive prune output. Defaults to `false` for concise logs. |
@@ -141,7 +141,8 @@ Optional variables:
141
141
  | --- | --- |
142
142
  | `BORG_REMOTE_PATH` | Remote Borg executable path, passed as Borg's `--remote-path`. Leave unset to use Borg's default. |
143
143
  | `BORG_EXIT_CODES` | Borg exit-code mode: `legacy` or `modern`. |
144
- | `BORG_FILES_CACHE_TTL` | How many archive creations can pass without a file appearing before Borg forgets its cached metadata. The default is `20`; raise it if more than 20 archives are created between backups of the same source, so unchanged files can still be recognized. |
144
+ | `BACKUP_FILES_CACHE_TTL_MARGIN` | Extra archive creations to retain file-cache entries beyond the number of configured sources. The app sets Borg's TTL to source count plus this margin; defaults to `20`. |
145
+ | `BORG_FILES_CACHE_TTL` | Optional absolute Borg TTL override; it must be at least the number of configured sources. Usually leave unset to use the source-count-based TTL and configured margin. |
145
146
  | `BACKUP_FAILURES_PATH` | Optional absolute path where failed backup group names are written as JSON. |
146
147
  | `BACKUP_PROGRESS` | Set to `true` to log the latest Borg progress every 30 seconds during archive creation. Defaults to `false`. |
147
148
  | `BACKUP_VERBOSE` | Set to `true` to show Borg's per-file create and per-archive prune output. Defaults to `false` for concise logs. |
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "declarative-data-backup"
7
- version = "0.5.0"
7
+ version = "0.5.2"
8
8
  description = "Declarative Borg-based data backups"
9
9
  readme = "README.md"
10
10
  requires-python = ">=3.11"
@@ -4,6 +4,7 @@ import os
4
4
  import signal
5
5
  import uuid
6
6
  from collections.abc import Mapping
7
+ from dataclasses import replace
7
8
  from datetime import UTC, datetime
8
9
  from types import FrameType
9
10
 
@@ -67,6 +68,28 @@ def run_backup(
67
68
  len(definitions),
68
69
  total_sources,
69
70
  )
71
+ if (
72
+ settings.borg_files_cache_ttl is not None
73
+ and settings.borg_files_cache_ttl < total_sources
74
+ ):
75
+ raise BackupError(
76
+ "BORG_FILES_CACHE_TTL must be at least the number of configured "
77
+ f"source creates ({total_sources})."
78
+ )
79
+ if settings.borg_files_cache_ttl is None:
80
+ files_cache_ttl = total_sources + settings.backup_files_cache_ttl_margin
81
+ settings = replace(settings, borg_files_cache_ttl=files_cache_ttl)
82
+ logger.info(
83
+ "Using Borg files-cache TTL %d (%d source creates + %d margin).",
84
+ files_cache_ttl,
85
+ total_sources,
86
+ settings.backup_files_cache_ttl_margin,
87
+ )
88
+ else:
89
+ logger.info(
90
+ "Using explicit Borg files-cache TTL %d.",
91
+ settings.borg_files_cache_ttl,
92
+ )
70
93
 
71
94
  run_id = uuid.uuid4().hex
72
95
  with repository_lock(settings.borg_base_dir / "backup.lock"):
@@ -100,9 +100,11 @@ def run_with_progress(
100
100
  print(text, file=sys.stderr, flush=True)
101
101
 
102
102
  with borg_process(command, environment, capture_stderr=True) as process:
103
- assert process.stderr is not None
103
+ stderr = process.stderr
104
+ if stderr is None:
105
+ fail("Borg stderr was not captured for progress reporting.")
104
106
  with selectors.DefaultSelector() as selector:
105
- selector.register(process.stderr, selectors.EVENT_READ)
107
+ selector.register(stderr, selectors.EVENT_READ)
106
108
  pending = b""
107
109
  while selector.get_map():
108
110
  for key, _ in selector.select(max(0, next_report - time.monotonic())):
@@ -147,6 +149,8 @@ def borg_environment(settings: Environment) -> dict[str, str]:
147
149
  )
148
150
  if settings.borg_exit_codes is not None:
149
151
  environment["BORG_EXIT_CODES"] = settings.borg_exit_codes
152
+ if settings.borg_files_cache_ttl is not None:
153
+ environment["BORG_FILES_CACHE_TTL"] = str(settings.borg_files_cache_ttl)
150
154
  return environment
151
155
 
152
156
 
@@ -20,6 +20,8 @@ class Environment:
20
20
  backup_progress: bool
21
21
  backup_verbose: bool
22
22
  failure_groups_path: Path | None = None
23
+ backup_files_cache_ttl_margin: int = 20
24
+ borg_files_cache_ttl: int | None = None
23
25
 
24
26
  @classmethod
25
27
  def from_env(
@@ -66,6 +68,24 @@ class Environment:
66
68
  backup_verbose_value = source.get("BACKUP_VERBOSE", "false").lower()
67
69
  if backup_verbose_value not in ("true", "false"):
68
70
  fail("BACKUP_VERBOSE must be either 'true' or 'false'.")
71
+ try:
72
+ backup_files_cache_ttl_margin = int(
73
+ source.get("BACKUP_FILES_CACHE_TTL_MARGIN", "20")
74
+ )
75
+ except ValueError:
76
+ fail("BACKUP_FILES_CACHE_TTL_MARGIN must be a nonnegative integer.")
77
+ if backup_files_cache_ttl_margin < 0:
78
+ fail("BACKUP_FILES_CACHE_TTL_MARGIN must be a nonnegative integer.")
79
+ raw_borg_files_cache_ttl = source.get("BORG_FILES_CACHE_TTL")
80
+ if raw_borg_files_cache_ttl is None:
81
+ borg_files_cache_ttl = None
82
+ else:
83
+ try:
84
+ borg_files_cache_ttl = int(raw_borg_files_cache_ttl)
85
+ except ValueError:
86
+ fail("BORG_FILES_CACHE_TTL must be a positive integer.")
87
+ if borg_files_cache_ttl < 1:
88
+ fail("BORG_FILES_CACHE_TTL must be a positive integer.")
69
89
 
70
90
  try:
71
91
  server_root = required_path("SERVER_ROOT").resolve(strict=True)
@@ -85,6 +105,8 @@ class Environment:
85
105
  backup_progress=backup_progress_value == "true",
86
106
  backup_verbose=backup_verbose_value == "true",
87
107
  failure_groups_path=optional_path("BACKUP_FAILURES_PATH"),
108
+ backup_files_cache_ttl_margin=backup_files_cache_ttl_margin,
109
+ borg_files_cache_ttl=borg_files_cache_ttl,
88
110
  )
89
111
 
90
112
 
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: declarative-data-backup
3
- Version: 0.5.0
3
+ Version: 0.5.2
4
4
  Summary: Declarative Borg-based data backups
5
5
  Requires-Python: >=3.11
6
6
  Description-Content-Type: text/markdown
@@ -149,7 +149,8 @@ Optional variables:
149
149
  | --- | --- |
150
150
  | `BORG_REMOTE_PATH` | Remote Borg executable path, passed as Borg's `--remote-path`. Leave unset to use Borg's default. |
151
151
  | `BORG_EXIT_CODES` | Borg exit-code mode: `legacy` or `modern`. |
152
- | `BORG_FILES_CACHE_TTL` | How many archive creations can pass without a file appearing before Borg forgets its cached metadata. The default is `20`; raise it if more than 20 archives are created between backups of the same source, so unchanged files can still be recognized. |
152
+ | `BACKUP_FILES_CACHE_TTL_MARGIN` | Extra archive creations to retain file-cache entries beyond the number of configured sources. The app sets Borg's TTL to source count plus this margin; defaults to `20`. |
153
+ | `BORG_FILES_CACHE_TTL` | Optional absolute Borg TTL override; it must be at least the number of configured sources. Usually leave unset to use the source-count-based TTL and configured margin. |
153
154
  | `BACKUP_FAILURES_PATH` | Optional absolute path where failed backup group names are written as JSON. |
154
155
  | `BACKUP_PROGRESS` | Set to `true` to log the latest Borg progress every 30 seconds during archive creation. Defaults to `false`. |
155
156
  | `BACKUP_VERBOSE` | Set to `true` to show Borg's per-file create and per-archive prune output. Defaults to `false` for concise logs. |
@@ -1,6 +1,8 @@
1
1
  import json
2
2
  import logging
3
3
 
4
+ import pytest
5
+
4
6
  import declarative_data_backup.backup as backup
5
7
  from declarative_data_backup.environment import Environment
6
8
  from declarative_data_backup.errors import BackupError, BorgWarningError
@@ -18,6 +20,11 @@ def _settings_environment(settings: Environment) -> dict[str, str]:
18
20
  }
19
21
  if settings.failure_groups_path is not None:
20
22
  environment["BACKUP_FAILURES_PATH"] = str(settings.failure_groups_path)
23
+ environment["BACKUP_FILES_CACHE_TTL_MARGIN"] = str(
24
+ settings.backup_files_cache_ttl_margin
25
+ )
26
+ if settings.borg_files_cache_ttl is not None:
27
+ environment["BORG_FILES_CACHE_TTL"] = str(settings.borg_files_cache_ttl)
21
28
  return environment
22
29
 
23
30
 
@@ -30,12 +37,17 @@ def _configure_preflight(monkeypatch):
30
37
  monkeypatch.setattr(backup, "check_keyfile", lambda settings: None)
31
38
 
32
39
 
40
+ @pytest.mark.parametrize(("margin", "expected_ttl"), [(0, 2), (7, 9)])
33
41
  def test_main_runs_all_creates_before_any_prune(
34
42
  settings_factory,
35
43
  monkeypatch,
36
44
  caplog,
45
+ margin,
46
+ expected_ttl,
37
47
  ):
38
- settings = settings_factory()
48
+ settings = settings_factory(
49
+ environment={"BACKUP_FILES_CACHE_TTL_MARGIN": str(margin)}
50
+ )
39
51
  media = settings.server_root / "media"
40
52
  postgres = settings.server_root / "postgres"
41
53
  media.mkdir()
@@ -50,13 +62,13 @@ def test_main_runs_all_creates_before_any_prune(
50
62
  )
51
63
  _configure_preflight(monkeypatch)
52
64
  events = []
53
- monkeypatch.setattr(
54
- backup,
55
- "backup_folder",
56
- lambda folder, compression, group, settings, created_at, run_id, excludes: (
57
- events.append(("create", group, folder, compression, run_id, excludes))
58
- ),
59
- )
65
+ files_cache_ttls = []
66
+
67
+ def create(folder, compression, group, settings, created_at, run_id, excludes):
68
+ files_cache_ttls.append(settings.borg_files_cache_ttl)
69
+ events.append(("create", group, folder, compression, run_id, excludes))
70
+
71
+ monkeypatch.setattr(backup, "backup_folder", create)
60
72
  monkeypatch.setattr(
61
73
  backup,
62
74
  "prune",
@@ -79,6 +91,7 @@ def test_main_runs_all_creates_before_any_prune(
79
91
  assert events[0][3] == "zstd,5"
80
92
  assert events[1][3] == "auto,zstd,22"
81
93
  assert events[2][3] == 30
94
+ assert files_cache_ttls == [expected_ttl, expected_ttl]
82
95
  assert "Backup group 'media_backup' completed." in caplog.text
83
96
  assert "Retention pruning for group 'media_backup' completed." in caplog.text
84
97
 
@@ -224,6 +237,27 @@ def test_main_validation_failure_runs_no_borg_operations(
224
237
  assert not (settings.borg_base_dir / "backup.lock").exists()
225
238
 
226
239
 
240
+ def test_main_rejects_files_cache_ttl_below_source_count(
241
+ settings_factory,
242
+ monkeypatch,
243
+ caplog,
244
+ ):
245
+ settings = settings_factory(environment={"BORG_FILES_CACHE_TTL": "1"})
246
+ first = settings.server_root / "first"
247
+ second = settings.server_root / "second"
248
+ first.mkdir()
249
+ second.mkdir()
250
+ _write_config(settings, f"media:\n folders: [{first}, {second}]\n")
251
+ _configure_preflight(monkeypatch)
252
+ calls = []
253
+ monkeypatch.setattr(backup, "backup_folder", lambda *args: calls.append(args))
254
+ caplog.set_level(logging.ERROR)
255
+
256
+ assert backup.main(_settings_environment(settings)) == 1
257
+ assert calls == []
258
+ assert "must be at least the number of configured source creates (2)" in caplog.text
259
+
260
+
227
261
  def test_main_missing_environment_is_reported_without_traceback(caplog):
228
262
  caplog.set_level(logging.ERROR)
229
263
  assert backup.main({}) == 1