declarative-data-backup 0.6.0__tar.gz → 0.7.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (30) hide show
  1. {declarative_data_backup-0.6.0 → declarative_data_backup-0.7.0}/PKG-INFO +2 -1
  2. {declarative_data_backup-0.6.0 → declarative_data_backup-0.7.0}/README.md +1 -0
  3. {declarative_data_backup-0.6.0 → declarative_data_backup-0.7.0}/pyproject.toml +1 -1
  4. {declarative_data_backup-0.6.0 → declarative_data_backup-0.7.0}/src/declarative_data_backup/backup.py +15 -7
  5. {declarative_data_backup-0.6.0 → declarative_data_backup-0.7.0}/src/declarative_data_backup/environment.py +2 -0
  6. {declarative_data_backup-0.6.0 → declarative_data_backup-0.7.0}/src/declarative_data_backup.egg-info/PKG-INFO +2 -1
  7. {declarative_data_backup-0.6.0 → declarative_data_backup-0.7.0}/tests/test_backup.py +59 -3
  8. {declarative_data_backup-0.6.0 → declarative_data_backup-0.7.0}/setup.cfg +0 -0
  9. {declarative_data_backup-0.6.0 → declarative_data_backup-0.7.0}/src/declarative_data_backup/__init__.py +0 -0
  10. {declarative_data_backup-0.6.0 → declarative_data_backup-0.7.0}/src/declarative_data_backup/__main__.py +0 -0
  11. {declarative_data_backup-0.6.0 → declarative_data_backup-0.7.0}/src/declarative_data_backup/borg/__init__.py +0 -0
  12. {declarative_data_backup-0.6.0 → declarative_data_backup-0.7.0}/src/declarative_data_backup/borg/archive_naming.py +0 -0
  13. {declarative_data_backup-0.6.0 → declarative_data_backup-0.7.0}/src/declarative_data_backup/borg/borg.py +0 -0
  14. {declarative_data_backup-0.6.0 → declarative_data_backup-0.7.0}/src/declarative_data_backup/borg/borg_cli.py +0 -0
  15. {declarative_data_backup-0.6.0 → declarative_data_backup-0.7.0}/src/declarative_data_backup/config/__init__.py +0 -0
  16. {declarative_data_backup-0.6.0 → declarative_data_backup-0.7.0}/src/declarative_data_backup/config/backup_config/__init__.py +0 -0
  17. {declarative_data_backup-0.6.0 → declarative_data_backup-0.7.0}/src/declarative_data_backup/config/backup_config/models.py +0 -0
  18. {declarative_data_backup-0.6.0 → declarative_data_backup-0.7.0}/src/declarative_data_backup/config/backup_config/utils.py +0 -0
  19. {declarative_data_backup-0.6.0 → declarative_data_backup-0.7.0}/src/declarative_data_backup/config/backup_config/yaml_parser.py +0 -0
  20. {declarative_data_backup-0.6.0 → declarative_data_backup-0.7.0}/src/declarative_data_backup/config/backup_config/yaml_raw_loader.py +0 -0
  21. {declarative_data_backup-0.6.0 → declarative_data_backup-0.7.0}/src/declarative_data_backup/config/logging.py +0 -0
  22. {declarative_data_backup-0.6.0 → declarative_data_backup-0.7.0}/src/declarative_data_backup/dirty_groups.py +0 -0
  23. {declarative_data_backup-0.6.0 → declarative_data_backup-0.7.0}/src/declarative_data_backup/errors.py +0 -0
  24. {declarative_data_backup-0.6.0 → declarative_data_backup-0.7.0}/src/declarative_data_backup/exclusions.py +0 -0
  25. {declarative_data_backup-0.6.0 → declarative_data_backup-0.7.0}/src/declarative_data_backup.egg-info/SOURCES.txt +0 -0
  26. {declarative_data_backup-0.6.0 → declarative_data_backup-0.7.0}/src/declarative_data_backup.egg-info/dependency_links.txt +0 -0
  27. {declarative_data_backup-0.6.0 → declarative_data_backup-0.7.0}/src/declarative_data_backup.egg-info/requires.txt +0 -0
  28. {declarative_data_backup-0.6.0 → declarative_data_backup-0.7.0}/src/declarative_data_backup.egg-info/top_level.txt +0 -0
  29. {declarative_data_backup-0.6.0 → declarative_data_backup-0.7.0}/tests/test_dirty_groups.py +0 -0
  30. {declarative_data_backup-0.6.0 → declarative_data_backup-0.7.0}/tests/test_exclusions.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: declarative-data-backup
3
- Version: 0.6.0
3
+ Version: 0.7.0
4
4
  Summary: Declarative Borg-based data backups
5
5
  Requires-Python: >=3.11
6
6
  Description-Content-Type: text/markdown
@@ -167,6 +167,7 @@ Optional variables:
167
167
  | `BACKUP_FILES_CACHE_TTL_MARGIN` | Extra archive creations to retain file-cache entries beyond the number of configured sources. The app sets Borg's TTL to source count plus this margin; defaults to `20`. |
168
168
  | `BORG_FILES_CACHE_TTL` | Optional absolute Borg TTL override; it must be at least the number of configured sources. Usually leave unset to use the source-count-based TTL and configured margin. |
169
169
  | `BACKUP_FAILURES_PATH` | Optional absolute path where failed backup group names are written as JSON. |
170
+ | `BACKUP_SUCCESSES_PATH` | Optional absolute path where successfully backed-up group names are written as JSON, including groups whose subsequent pruning failed. |
170
171
  | `BACKUP_PROGRESS` | Set to `true` to log the latest Borg progress every 30 seconds during archive creation. Defaults to `false`. |
171
172
  | `BACKUP_VERBOSE` | Set to `true` to show Borg's per-file create and per-archive prune output. Defaults to `false` for concise logs. |
172
173
  | `BACKUP_MODE` | `full` (default) backs up every configured group; `shadow` reports the dirty selection but backs up all groups; `dirty` consumes `ddb:v1:dirty-groups` from Redis. Shadow and dirty modes require the Redis variables below. |
@@ -158,6 +158,7 @@ Optional variables:
158
158
  | `BACKUP_FILES_CACHE_TTL_MARGIN` | Extra archive creations to retain file-cache entries beyond the number of configured sources. The app sets Borg's TTL to source count plus this margin; defaults to `20`. |
159
159
  | `BORG_FILES_CACHE_TTL` | Optional absolute Borg TTL override; it must be at least the number of configured sources. Usually leave unset to use the source-count-based TTL and configured margin. |
160
160
  | `BACKUP_FAILURES_PATH` | Optional absolute path where failed backup group names are written as JSON. |
161
+ | `BACKUP_SUCCESSES_PATH` | Optional absolute path where successfully backed-up group names are written as JSON, including groups whose subsequent pruning failed. |
161
162
  | `BACKUP_PROGRESS` | Set to `true` to log the latest Borg progress every 30 seconds during archive creation. Defaults to `false`. |
162
163
  | `BACKUP_VERBOSE` | Set to `true` to show Borg's per-file create and per-archive prune output. Defaults to `false` for concise logs. |
163
164
  | `BACKUP_MODE` | `full` (default) backs up every configured group; `shadow` reports the dirty selection but backs up all groups; `dirty` consumes `ddb:v1:dirty-groups` from Redis. Shadow and dirty modes require the Redis variables below. |
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "declarative-data-backup"
7
- version = "0.6.0"
7
+ version = "0.7.0"
8
8
  description = "Declarative Borg-based data backups"
9
9
  readme = "README.md"
10
10
  requires-python = ">=3.11"
@@ -6,6 +6,7 @@ import uuid
6
6
  from collections.abc import Mapping
7
7
  from dataclasses import replace
8
8
  from datetime import UTC, datetime
9
+ from pathlib import Path
9
10
  from types import FrameType
10
11
 
11
12
  from .borg import (
@@ -25,15 +26,14 @@ from .exclusions import translate_excludes
25
26
  logger = logging.getLogger(__name__)
26
27
 
27
28
 
28
- def write_failed_groups(settings: Environment, groups: set[str]) -> None:
29
- path = settings.failure_groups_path
30
- if path is None or not groups:
29
+ def write_group_summary(path: Path | None, groups: set[str]) -> None:
30
+ if path is None:
31
31
  return
32
32
  try:
33
33
  path.write_text(json.dumps(sorted(groups)), encoding="utf-8")
34
34
  except OSError as exc:
35
35
  logger.warning(
36
- "Unable to write failed backup group summary to %s: %s",
36
+ "Unable to write backup group summary to %s: %s",
37
37
  path,
38
38
  exc.strerror or "I/O error",
39
39
  )
@@ -95,6 +95,8 @@ def run_backup(
95
95
 
96
96
  run_id = uuid.uuid4().hex
97
97
  with repository_lock(settings.borg_base_dir / "backup.lock"):
98
+ backed_up_groups: set[str] = set()
99
+ write_group_summary(settings.backed_up_groups_path, backed_up_groups)
98
100
  dirty_groups = (
99
101
  DirtyGroupConsumer(settings)
100
102
  if settings.backup_mode in {"shadow", "dirty"}
@@ -150,7 +152,9 @@ def run_backup(
150
152
  dirty_groups.requeue(definition.name)
151
153
  except BackupError as state_error:
152
154
  failed_groups.add(definition.name)
153
- write_failed_groups(settings, failed_groups)
155
+ write_group_summary(
156
+ settings.failure_groups_path, failed_groups
157
+ )
154
158
  raise BackupError(f"{exc}; {state_error}") from state_error
155
159
  logger.error(
156
160
  "Backup group '%s' failed; retention pruning will be skipped "
@@ -160,10 +164,13 @@ def run_backup(
160
164
  )
161
165
  failures.append(exc)
162
166
  failed_groups.add(definition.name)
167
+ write_group_summary(settings.failure_groups_path, failed_groups)
163
168
  continue
164
169
 
165
170
  logger.info("Backup group '%s' completed.", definition.name)
166
171
  successful_definitions.append(definition)
172
+ backed_up_groups.add(definition.name)
173
+ write_group_summary(settings.backed_up_groups_path, backed_up_groups)
167
174
 
168
175
  for definition in successful_definitions:
169
176
  logger.info(
@@ -189,6 +196,7 @@ def run_backup(
189
196
  )
190
197
  failures.append(exc)
191
198
  failed_groups.add(definition.name)
199
+ write_group_summary(settings.failure_groups_path, failed_groups)
192
200
  pruning_succeeded = False
193
201
 
194
202
  if settings.backup_mode == "dirty" and dirty_groups is not None:
@@ -199,7 +207,7 @@ def run_backup(
199
207
  dirty_groups.requeue(definition.name)
200
208
  except BackupError as state_error:
201
209
  failed_groups.add(definition.name)
202
- write_failed_groups(settings, failed_groups)
210
+ write_group_summary(settings.failure_groups_path, failed_groups)
203
211
  raise state_error
204
212
 
205
213
  if pruning_succeeded:
@@ -209,7 +217,7 @@ def run_backup(
209
217
  )
210
218
 
211
219
  if failures:
212
- write_failed_groups(settings, failed_groups)
220
+ write_group_summary(settings.failure_groups_path, failed_groups)
213
221
  message = "; ".join(str(failure) for failure in failures)
214
222
  if any(isinstance(failure, BorgWarningError) for failure in failures):
215
223
  raise BorgWarningError(message)
@@ -21,6 +21,7 @@ class Environment:
21
21
  backup_progress: bool
22
22
  backup_verbose: bool
23
23
  failure_groups_path: Path | None = None
24
+ backed_up_groups_path: Path | None = None
24
25
  backup_files_cache_ttl_margin: int = 20
25
26
  borg_files_cache_ttl: int | None = None
26
27
  backup_mode: str = "full"
@@ -164,6 +165,7 @@ class Environment:
164
165
  backup_progress=backup_progress_value == "true",
165
166
  backup_verbose=backup_verbose_value == "true",
166
167
  failure_groups_path=optional_path("BACKUP_FAILURES_PATH"),
168
+ backed_up_groups_path=optional_path("BACKUP_SUCCESSES_PATH"),
167
169
  backup_files_cache_ttl_margin=backup_files_cache_ttl_margin,
168
170
  borg_files_cache_ttl=borg_files_cache_ttl,
169
171
  backup_mode=backup_mode,
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: declarative-data-backup
3
- Version: 0.6.0
3
+ Version: 0.7.0
4
4
  Summary: Declarative Borg-based data backups
5
5
  Requires-Python: >=3.11
6
6
  Description-Content-Type: text/markdown
@@ -167,6 +167,7 @@ Optional variables:
167
167
  | `BACKUP_FILES_CACHE_TTL_MARGIN` | Extra archive creations to retain file-cache entries beyond the number of configured sources. The app sets Borg's TTL to source count plus this margin; defaults to `20`. |
168
168
  | `BORG_FILES_CACHE_TTL` | Optional absolute Borg TTL override; it must be at least the number of configured sources. Usually leave unset to use the source-count-based TTL and configured margin. |
169
169
  | `BACKUP_FAILURES_PATH` | Optional absolute path where failed backup group names are written as JSON. |
170
+ | `BACKUP_SUCCESSES_PATH` | Optional absolute path where successfully backed-up group names are written as JSON, including groups whose subsequent pruning failed. |
170
171
  | `BACKUP_PROGRESS` | Set to `true` to log the latest Borg progress every 30 seconds during archive creation. Defaults to `false`. |
171
172
  | `BACKUP_VERBOSE` | Set to `true` to show Borg's per-file create and per-archive prune output. Defaults to `false` for concise logs. |
172
173
  | `BACKUP_MODE` | `full` (default) backs up every configured group; `shadow` reports the dirty selection but backs up all groups; `dirty` consumes `ddb:v1:dirty-groups` from Redis. Shadow and dirty modes require the Redis variables below. |
@@ -22,6 +22,8 @@ def _settings_environment(settings: Environment) -> dict[str, str]:
22
22
  }
23
23
  if settings.failure_groups_path is not None:
24
24
  environment["BACKUP_FAILURES_PATH"] = str(settings.failure_groups_path)
25
+ if settings.backed_up_groups_path is not None:
26
+ environment["BACKUP_SUCCESSES_PATH"] = str(settings.backed_up_groups_path)
25
27
  environment["BACKUP_FILES_CACHE_TTL_MARGIN"] = str(
26
28
  settings.backup_files_cache_ttl_margin
27
29
  )
@@ -111,11 +113,16 @@ def test_main_runs_all_creates_before_any_prune(
111
113
  settings_factory,
112
114
  monkeypatch,
113
115
  caplog,
116
+ tmp_path,
114
117
  margin,
115
118
  expected_ttl,
116
119
  ):
120
+ backed_up_groups_path = tmp_path / "backed-up-groups.json"
117
121
  settings = settings_factory(
118
- environment={"BACKUP_FILES_CACHE_TTL_MARGIN": str(margin)}
122
+ environment={
123
+ "BACKUP_FILES_CACHE_TTL_MARGIN": str(margin),
124
+ "BACKUP_SUCCESSES_PATH": str(backed_up_groups_path),
125
+ }
119
126
  )
120
127
  media = settings.server_root / "media"
121
128
  postgres = settings.server_root / "postgres"
@@ -163,6 +170,10 @@ def test_main_runs_all_creates_before_any_prune(
163
170
  assert files_cache_ttls == [expected_ttl, expected_ttl]
164
171
  assert "Backup group 'media_backup' completed." in caplog.text
165
172
  assert "Retention pruning for group 'media_backup' completed." in caplog.text
173
+ assert json.loads(backed_up_groups_path.read_text(encoding="utf-8")) == [
174
+ "media_backup",
175
+ "postgres_backup",
176
+ ]
166
177
 
167
178
 
168
179
  def test_main_warning_skips_failed_group_but_prunes_successful_groups(
@@ -172,8 +183,12 @@ def test_main_warning_skips_failed_group_but_prunes_successful_groups(
172
183
  tmp_path,
173
184
  ):
174
185
  failure_groups_path = tmp_path / "failed-groups.json"
186
+ backed_up_groups_path = tmp_path / "backed-up-groups.json"
175
187
  settings = settings_factory(
176
- environment={"BACKUP_FAILURES_PATH": str(failure_groups_path)}
188
+ environment={
189
+ "BACKUP_FAILURES_PATH": str(failure_groups_path),
190
+ "BACKUP_SUCCESSES_PATH": str(backed_up_groups_path),
191
+ }
177
192
  )
178
193
  first = settings.server_root / "first"
179
194
  second = settings.server_root / "second"
@@ -208,6 +223,9 @@ def test_main_warning_skips_failed_group_but_prunes_successful_groups(
208
223
  assert "warnings" in caplog.text
209
224
  assert str(first) in caplog.text
210
225
  assert json.loads(failure_groups_path.read_text(encoding="utf-8")) == ["failed"]
226
+ assert json.loads(backed_up_groups_path.read_text(encoding="utf-8")) == [
227
+ "succeeded"
228
+ ]
211
229
 
212
230
 
213
231
  def test_main_interruption_returns_130_without_pruning(
@@ -232,6 +250,31 @@ def test_main_interruption_returns_130_without_pruning(
232
250
  assert calls == ["create"]
233
251
 
234
252
 
253
+ def test_interruption_preserves_completed_group_summary(
254
+ settings_factory,
255
+ monkeypatch,
256
+ tmp_path,
257
+ ):
258
+ groups_path = tmp_path / "groups.json"
259
+ settings = settings_factory(environment={"BACKUP_SUCCESSES_PATH": str(groups_path)})
260
+ first = settings.server_root / "first"
261
+ second = settings.server_root / "second"
262
+ first.mkdir()
263
+ second.mkdir()
264
+ _write_config(
265
+ settings, f"first:\n folders: [{first}]\nsecond:\n folders: [{second}]\n"
266
+ )
267
+ _configure_preflight(monkeypatch)
268
+
269
+ def create(folder, *args):
270
+ if folder == second:
271
+ raise KeyboardInterrupt
272
+
273
+ monkeypatch.setattr(backup, "backup_folder", create)
274
+ assert backup.main(_settings_environment(settings)) == 130
275
+ assert json.loads(groups_path.read_text(encoding="utf-8")) == ["first"]
276
+
277
+
235
278
  def test_retry_after_partial_create_uses_a_new_run_id_before_pruning(
236
279
  settings_factory,
237
280
  monkeypatch,
@@ -360,8 +403,12 @@ def test_main_prune_failure_is_nonzero_and_scoped(
360
403
  tmp_path,
361
404
  ):
362
405
  failure_groups_path = tmp_path / "failed-groups.json"
406
+ backed_up_groups_path = tmp_path / "backed-up-groups.json"
363
407
  settings = settings_factory(
364
- environment={"BACKUP_FAILURES_PATH": str(failure_groups_path)}
408
+ environment={
409
+ "BACKUP_FAILURES_PATH": str(failure_groups_path),
410
+ "BACKUP_SUCCESSES_PATH": str(backed_up_groups_path),
411
+ }
365
412
  )
366
413
  first = settings.server_root / "first"
367
414
  second = settings.server_root / "second"
@@ -393,6 +440,7 @@ def test_main_prune_failure_is_nonzero_and_scoped(
393
440
  ("prune", second),
394
441
  ]
395
442
  assert json.loads(failure_groups_path.read_text(encoding="utf-8")) == ["media"]
443
+ assert json.loads(backed_up_groups_path.read_text(encoding="utf-8")) == ["media"]
396
444
 
397
445
 
398
446
  def test_exclusions_are_passed_only_to_their_backup_group(
@@ -437,9 +485,12 @@ def test_exclusions_are_passed_only_to_their_backup_group(
437
485
  def test_dirty_mode_processes_only_claimed_groups_and_preserves_create_order(
438
486
  settings_factory,
439
487
  monkeypatch,
488
+ tmp_path,
440
489
  ):
490
+ backed_up_groups_path = tmp_path / "backed-up-groups.json"
441
491
  settings = settings_factory(
442
492
  environment={
493
+ "BACKUP_SUCCESSES_PATH": str(backed_up_groups_path),
443
494
  "BACKUP_MODE": "dirty",
444
495
  "REDIS_URL": "redis://redis.example:6379",
445
496
  "REDIS_DBINDEX": "15",
@@ -484,14 +535,18 @@ def test_dirty_mode_processes_only_claimed_groups_and_preserves_create_order(
484
535
  assert cache_ttls == [23, 23]
485
536
  assert state.dirty == set()
486
537
  assert state.inflight == set()
538
+ assert json.loads(backed_up_groups_path.read_text(encoding="utf-8")) == ["selected"]
487
539
 
488
540
 
489
541
  def test_empty_dirty_set_succeeds_without_borg_operations(
490
542
  settings_factory,
491
543
  monkeypatch,
544
+ tmp_path,
492
545
  ):
546
+ backed_up_groups_path = tmp_path / "backed-up-groups.json"
493
547
  settings = settings_factory(
494
548
  environment={
549
+ "BACKUP_SUCCESSES_PATH": str(backed_up_groups_path),
495
550
  "BACKUP_MODE": "dirty",
496
551
  "REDIS_URL": "redis://redis.example:6379",
497
552
  "REDIS_DBINDEX": "15",
@@ -511,6 +566,7 @@ def test_empty_dirty_set_succeeds_without_borg_operations(
511
566
  assert backup.main(_settings_environment(settings)) == 0
512
567
  assert events == []
513
568
  assert state.claimed == ["source"]
569
+ assert json.loads(backed_up_groups_path.read_text(encoding="utf-8")) == []
514
570
 
515
571
 
516
572
  def test_shadow_mode_reports_selection_but_backs_up_all_groups(