declarative-data-backup 0.4.0__tar.gz → 0.5.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (26) hide show
  1. {declarative_data_backup-0.4.0 → declarative_data_backup-0.5.0}/PKG-INFO +9 -5
  2. {declarative_data_backup-0.4.0 → declarative_data_backup-0.5.0}/README.md +8 -4
  3. {declarative_data_backup-0.4.0 → declarative_data_backup-0.5.0}/pyproject.toml +1 -1
  4. {declarative_data_backup-0.4.0 → declarative_data_backup-0.5.0}/src/declarative_data_backup/backup.py +71 -17
  5. {declarative_data_backup-0.4.0 → declarative_data_backup-0.5.0}/src/declarative_data_backup/environment.py +13 -0
  6. {declarative_data_backup-0.4.0 → declarative_data_backup-0.5.0}/src/declarative_data_backup.egg-info/PKG-INFO +9 -5
  7. {declarative_data_backup-0.4.0 → declarative_data_backup-0.5.0}/tests/test_backup.py +27 -7
  8. {declarative_data_backup-0.4.0 → declarative_data_backup-0.5.0}/setup.cfg +0 -0
  9. {declarative_data_backup-0.4.0 → declarative_data_backup-0.5.0}/src/declarative_data_backup/__init__.py +0 -0
  10. {declarative_data_backup-0.4.0 → declarative_data_backup-0.5.0}/src/declarative_data_backup/__main__.py +0 -0
  11. {declarative_data_backup-0.4.0 → declarative_data_backup-0.5.0}/src/declarative_data_backup/borg/__init__.py +0 -0
  12. {declarative_data_backup-0.4.0 → declarative_data_backup-0.5.0}/src/declarative_data_backup/borg/archive_naming.py +0 -0
  13. {declarative_data_backup-0.4.0 → declarative_data_backup-0.5.0}/src/declarative_data_backup/borg/borg.py +0 -0
  14. {declarative_data_backup-0.4.0 → declarative_data_backup-0.5.0}/src/declarative_data_backup/borg/borg_cli.py +0 -0
  15. {declarative_data_backup-0.4.0 → declarative_data_backup-0.5.0}/src/declarative_data_backup/config/__init__.py +0 -0
  16. {declarative_data_backup-0.4.0 → declarative_data_backup-0.5.0}/src/declarative_data_backup/config/backup_config/__init__.py +0 -0
  17. {declarative_data_backup-0.4.0 → declarative_data_backup-0.5.0}/src/declarative_data_backup/config/backup_config/models.py +0 -0
  18. {declarative_data_backup-0.4.0 → declarative_data_backup-0.5.0}/src/declarative_data_backup/config/backup_config/utils.py +0 -0
  19. {declarative_data_backup-0.4.0 → declarative_data_backup-0.5.0}/src/declarative_data_backup/config/backup_config/yaml_parser.py +0 -0
  20. {declarative_data_backup-0.4.0 → declarative_data_backup-0.5.0}/src/declarative_data_backup/config/backup_config/yaml_raw_loader.py +0 -0
  21. {declarative_data_backup-0.4.0 → declarative_data_backup-0.5.0}/src/declarative_data_backup/config/logging.py +0 -0
  22. {declarative_data_backup-0.4.0 → declarative_data_backup-0.5.0}/src/declarative_data_backup/errors.py +0 -0
  23. {declarative_data_backup-0.4.0 → declarative_data_backup-0.5.0}/src/declarative_data_backup.egg-info/SOURCES.txt +0 -0
  24. {declarative_data_backup-0.4.0 → declarative_data_backup-0.5.0}/src/declarative_data_backup.egg-info/dependency_links.txt +0 -0
  25. {declarative_data_backup-0.4.0 → declarative_data_backup-0.5.0}/src/declarative_data_backup.egg-info/requires.txt +0 -0
  26. {declarative_data_backup-0.4.0 → declarative_data_backup-0.5.0}/src/declarative_data_backup.egg-info/top_level.txt +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: declarative-data-backup
3
- Version: 0.4.0
3
+ Version: 0.5.0
4
4
  Summary: Declarative Borg-based data backups
5
5
  Requires-Python: >=3.11
6
6
  Description-Content-Type: text/markdown
@@ -149,6 +149,8 @@ Optional variables:
149
149
  | --- | --- |
150
150
  | `BORG_REMOTE_PATH` | Remote Borg executable path, passed as Borg's `--remote-path`. Leave unset to use Borg's default. |
151
151
  | `BORG_EXIT_CODES` | Borg exit-code mode: `legacy` or `modern`. |
152
+ | `BORG_FILES_CACHE_TTL` | How many archive creations can pass without a file appearing before Borg forgets its cached metadata. The default is `20`; raise it if more than 20 archives are created between backups of the same source, so unchanged files can still be recognized. |
153
+ | `BACKUP_FAILURES_PATH` | Optional absolute path where failed backup group names are written as JSON. |
152
154
  | `BACKUP_PROGRESS` | Set to `true` to log the latest Borg progress every 30 seconds during archive creation. Defaults to `false`. |
153
155
  | `BACKUP_VERBOSE` | Set to `true` to show Borg's per-file create and per-archive prune output. Defaults to `false` for concise logs. |
154
156
 
@@ -246,10 +248,12 @@ The stable source series is derived from the canonical source path and includes
246
248
  a SHA-256 suffix. Borg receives the canonical absolute path as the source, so
247
249
  the original directory hierarchy is retained inside the archive.
248
250
 
249
- The application creates all configured archives before pruning any source.
250
- If a create fails or Borg reports warnings, the run stops and no pruning is
251
- performed. Retries use a new run UUID and may create another archive for that
252
- day; archives are never overwritten to handle a collision.
251
+ The application attempts every configured backup group before pruning any
252
+ source. If a group has a create failure or Borg reports warnings, pruning is
253
+ skipped for that group, but other groups that completed successfully are still
254
+ pruned. Failures are reported and make the overall run fail. Retries use a new
255
+ run UUID and may create another archive for that day; archives are never
256
+ overwritten to handle a collision.
253
257
 
254
258
  `--keep-daily N` retains the latest archive in up to `N` daily buckets. It does
255
259
  not guarantee `N` successful calendar days of recovery if scheduled runs fail
@@ -141,6 +141,8 @@ Optional variables:
141
141
  | --- | --- |
142
142
  | `BORG_REMOTE_PATH` | Remote Borg executable path, passed as Borg's `--remote-path`. Leave unset to use Borg's default. |
143
143
  | `BORG_EXIT_CODES` | Borg exit-code mode: `legacy` or `modern`. |
144
+ | `BORG_FILES_CACHE_TTL` | How many archive creations can pass without a file appearing before Borg forgets its cached metadata. The default is `20`; raise it if more than 20 archives are created between backups of the same source, so unchanged files can still be recognized. |
145
+ | `BACKUP_FAILURES_PATH` | Optional absolute path where failed backup group names are written as JSON. |
144
146
  | `BACKUP_PROGRESS` | Set to `true` to log the latest Borg progress every 30 seconds during archive creation. Defaults to `false`. |
145
147
  | `BACKUP_VERBOSE` | Set to `true` to show Borg's per-file create and per-archive prune output. Defaults to `false` for concise logs. |
146
148
 
@@ -238,10 +240,12 @@ The stable source series is derived from the canonical source path and includes
238
240
  a SHA-256 suffix. Borg receives the canonical absolute path as the source, so
239
241
  the original directory hierarchy is retained inside the archive.
240
242
 
241
- The application creates all configured archives before pruning any source.
242
- If a create fails or Borg reports warnings, the run stops and no pruning is
243
- performed. Retries use a new run UUID and may create another archive for that
244
- day; archives are never overwritten to handle a collision.
243
+ The application attempts every configured backup group before pruning any
244
+ source. If a group has a create failure or Borg reports warnings, pruning is
245
+ skipped for that group, but other groups that completed successfully are still
246
+ pruned. Failures are reported and make the overall run fail. Retries use a new
247
+ run UUID and may create another archive for that day; archives are never
248
+ overwritten to handle a collision.
245
249
 
246
250
  `--keep-daily N` retains the latest archive in up to `N` daily buckets. It does
247
251
  not guarantee `N` successful calendar days of recovery if scheduled runs fail
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "declarative-data-backup"
7
- version = "0.4.0"
7
+ version = "0.5.0"
8
8
  description = "Declarative Borg-based data backups"
9
9
  readme = "README.md"
10
10
  requires-python = ">=3.11"
@@ -1,3 +1,4 @@
1
+ import json
1
2
  import logging
2
3
  import os
3
4
  import signal
@@ -21,6 +22,20 @@ from .errors import BackupError, BackupInterrupted, BorgWarningError
21
22
  logger = logging.getLogger(__name__)
22
23
 
23
24
 
25
+ def write_failed_groups(settings: Environment, groups: set[str]) -> None:
26
+ path = settings.failure_groups_path
27
+ if path is None or not groups:
28
+ return
29
+ try:
30
+ path.write_text(json.dumps(sorted(groups)), encoding="utf-8")
31
+ except OSError as exc:
32
+ logger.warning(
33
+ "Unable to write failed backup group summary to %s: %s",
34
+ path,
35
+ exc.strerror or "I/O error",
36
+ )
37
+
38
+
24
39
  def check_settings(settings: Environment) -> None:
25
40
  ensure_borg_binary()
26
41
  if not settings.borg_base_dir.is_dir():
@@ -56,38 +71,77 @@ def run_backup(
56
71
  run_id = uuid.uuid4().hex
57
72
  with repository_lock(settings.borg_base_dir / "backup.lock"):
58
73
  created_at = datetime.now(UTC)
74
+ successful_definitions = []
75
+ failures: list[BackupError] = []
76
+ failed_groups: set[str] = set()
59
77
  for definition in definitions:
60
78
  logger.info(
61
79
  "Starting backup group '%s' (%d sources).",
62
80
  definition.name,
63
81
  len(definition.sources),
64
82
  )
65
- for folder in definition.sources:
66
- backup_folder(
67
- folder,
68
- definition.compression,
83
+ try:
84
+ for folder in definition.sources:
85
+ backup_folder(
86
+ folder,
87
+ definition.compression,
88
+ definition.name,
89
+ settings,
90
+ created_at,
91
+ run_id,
92
+ definition.excludes,
93
+ )
94
+ except BackupError as exc:
95
+ logger.error(
96
+ "Backup group '%s' failed; retention pruning will be skipped "
97
+ "for this group: %s",
69
98
  definition.name,
70
- settings,
71
- created_at,
72
- run_id,
73
- definition.excludes,
99
+ exc,
74
100
  )
101
+ failures.append(exc)
102
+ failed_groups.add(definition.name)
103
+ continue
104
+
75
105
  logger.info("Backup group '%s' completed.", definition.name)
106
+ successful_definitions.append(definition)
76
107
 
77
- for definition in definitions:
108
+ for definition in successful_definitions:
78
109
  logger.info(
79
110
  "Starting retention pruning for group '%s' (%d sources).",
80
111
  definition.name,
81
112
  len(definition.sources),
82
113
  )
114
+ pruning_succeeded = True
83
115
  for folder in definition.sources:
84
- prune(
85
- folder,
86
- definition.retention_days,
87
- definition.name,
88
- settings,
116
+ try:
117
+ prune(
118
+ folder,
119
+ definition.retention_days,
120
+ definition.name,
121
+ settings,
122
+ )
123
+ except BackupError as exc:
124
+ logger.error(
125
+ "Retention pruning failed for group '%s' source '%s': %s",
126
+ definition.name,
127
+ folder,
128
+ exc,
129
+ )
130
+ failures.append(exc)
131
+ failed_groups.add(definition.name)
132
+ pruning_succeeded = False
133
+
134
+ if pruning_succeeded:
135
+ logger.info(
136
+ "Retention pruning for group '%s' completed.", definition.name
89
137
  )
90
- logger.info("Retention pruning for group '%s' completed.", definition.name)
138
+
139
+ if failures:
140
+ write_failed_groups(settings, failed_groups)
141
+ message = "; ".join(str(failure) for failure in failures)
142
+ if any(isinstance(failure, BorgWarningError) for failure in failures):
143
+ raise BorgWarningError(message)
144
+ raise BackupError(message)
91
145
 
92
146
 
93
147
  def main(environ: Mapping[str, str] | None = None) -> int:
@@ -111,8 +165,8 @@ def main(environ: Mapping[str, str] | None = None) -> int:
111
165
 
112
166
  except BorgWarningError as exc:
113
167
  logger.error(
114
- "Backup stopped because Borg reported warnings; "
115
- "pruning was not completed: %s",
168
+ "Backup failed because Borg reported warnings; "
169
+ "retention pruning may have completed for other groups: %s",
116
170
  exc,
117
171
  )
118
172
  return 1
@@ -19,6 +19,7 @@ class Environment:
19
19
  borg_exit_codes: str | None
20
20
  backup_progress: bool
21
21
  backup_verbose: bool
22
+ failure_groups_path: Path | None = None
22
23
 
23
24
  @classmethod
24
25
  def from_env(
@@ -42,6 +43,17 @@ class Environment:
42
43
  fail(f"{name} must be an absolute path.")
43
44
  return value
44
45
 
46
+ def optional_path(name: str) -> Path | None:
47
+ raw_value = source.get(name)
48
+ if raw_value is None:
49
+ return None
50
+ if "\0" in raw_value:
51
+ fail(f"{name} must be a valid absolute path.")
52
+ value = Path(raw_value)
53
+ if not value.is_absolute():
54
+ fail(f"{name} must be an absolute path.")
55
+ return value
56
+
45
57
  repo = required("BORG_REPO")
46
58
  if "\0" in repo:
47
59
  fail("BORG_REPO must be a valid repository location.")
@@ -72,6 +84,7 @@ class Environment:
72
84
  borg_exit_codes=borg_exit_codes,
73
85
  backup_progress=backup_progress_value == "true",
74
86
  backup_verbose=backup_verbose_value == "true",
87
+ failure_groups_path=optional_path("BACKUP_FAILURES_PATH"),
75
88
  )
76
89
 
77
90
 
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: declarative-data-backup
3
- Version: 0.4.0
3
+ Version: 0.5.0
4
4
  Summary: Declarative Borg-based data backups
5
5
  Requires-Python: >=3.11
6
6
  Description-Content-Type: text/markdown
@@ -149,6 +149,8 @@ Optional variables:
149
149
  | --- | --- |
150
150
  | `BORG_REMOTE_PATH` | Remote Borg executable path, passed as Borg's `--remote-path`. Leave unset to use Borg's default. |
151
151
  | `BORG_EXIT_CODES` | Borg exit-code mode: `legacy` or `modern`. |
152
+ | `BORG_FILES_CACHE_TTL` | How many archive creations can pass without a file appearing before Borg forgets its cached metadata. The default is `20`; raise it if more than 20 archives are created between backups of the same source, so unchanged files can still be recognized. |
153
+ | `BACKUP_FAILURES_PATH` | Optional absolute path where failed backup group names are written as JSON. |
152
154
  | `BACKUP_PROGRESS` | Set to `true` to log the latest Borg progress every 30 seconds during archive creation. Defaults to `false`. |
153
155
  | `BACKUP_VERBOSE` | Set to `true` to show Borg's per-file create and per-archive prune output. Defaults to `false` for concise logs. |
154
156
 
@@ -246,10 +248,12 @@ The stable source series is derived from the canonical source path and includes
246
248
  a SHA-256 suffix. Borg receives the canonical absolute path as the source, so
247
249
  the original directory hierarchy is retained inside the archive.
248
250
 
249
- The application creates all configured archives before pruning any source.
250
- If a create fails or Borg reports warnings, the run stops and no pruning is
251
- performed. Retries use a new run UUID and may create another archive for that
252
- day; archives are never overwritten to handle a collision.
251
+ The application attempts every configured backup group before pruning any
252
+ source. If a group has a create failure or Borg reports warnings, pruning is
253
+ skipped for that group, but other groups that completed successfully are still
254
+ pruned. Failures are reported and make the overall run fail. Retries use a new
255
+ run UUID and may create another archive for that day; archives are never
256
+ overwritten to handle a collision.
253
257
 
254
258
  `--keep-daily N` retains the latest archive in up to `N` daily buckets. It does
255
259
  not guarantee `N` successful calendar days of recovery if scheduled runs fail
@@ -1,3 +1,4 @@
1
+ import json
1
2
  import logging
2
3
 
3
4
  import declarative_data_backup.backup as backup
@@ -6,7 +7,7 @@ from declarative_data_backup.errors import BackupError, BorgWarningError
6
7
 
7
8
 
8
9
  def _settings_environment(settings: Environment) -> dict[str, str]:
9
- return {
10
+ environment = {
10
11
  "BACKUP_CONFIG_PATH": str(settings.config_path),
11
12
  "BORG_KEY_FILE": str(settings.key_file),
12
13
  "BORG_REPO": settings.borg_repo,
@@ -15,6 +16,9 @@ def _settings_environment(settings: Environment) -> dict[str, str]:
15
16
  "BORG_RSH": settings.borg_rsh,
16
17
  "BORG_PASSCOMMAND": settings.borg_passcommand,
17
18
  }
19
+ if settings.failure_groups_path is not None:
20
+ environment["BACKUP_FAILURES_PATH"] = str(settings.failure_groups_path)
21
+ return environment
18
22
 
19
23
 
20
24
  def _write_config(settings, contents: str) -> None:
@@ -79,26 +83,31 @@ def test_main_runs_all_creates_before_any_prune(
79
83
  assert "Retention pruning for group 'media_backup' completed." in caplog.text
80
84
 
81
85
 
82
- def test_main_warning_aborts_remaining_creates_and_all_pruning(
86
+ def test_main_warning_skips_failed_group_but_prunes_successful_groups(
83
87
  settings_factory,
84
88
  monkeypatch,
85
89
  caplog,
90
+ tmp_path,
86
91
  ):
87
- settings = settings_factory()
92
+ failure_groups_path = tmp_path / "failed-groups.json"
93
+ settings = settings_factory(
94
+ environment={"BACKUP_FAILURES_PATH": str(failure_groups_path)}
95
+ )
88
96
  first = settings.server_root / "first"
89
97
  second = settings.server_root / "second"
90
98
  first.mkdir()
91
99
  second.mkdir()
92
100
  _write_config(
93
101
  settings,
94
- f"media:\n folders: [{first}, {second}]\n",
102
+ f"failed:\n folders: [{first}]\nsucceeded:\n folders: [{second}]\n",
95
103
  )
96
104
  _configure_preflight(monkeypatch)
97
105
  calls = []
98
106
 
99
107
  def warning(folder, *args):
100
108
  calls.append(("create", folder))
101
- raise BorgWarningError(f"warning for {folder}")
109
+ if folder == first:
110
+ raise BorgWarningError(f"warning for {folder}")
102
111
 
103
112
  monkeypatch.setattr(backup, "backup_folder", warning)
104
113
  monkeypatch.setattr(
@@ -109,9 +118,14 @@ def test_main_warning_aborts_remaining_creates_and_all_pruning(
109
118
  caplog.set_level(logging.ERROR)
110
119
 
111
120
  assert backup.main(_settings_environment(settings)) == 1
112
- assert calls == [("create", first)]
121
+ assert calls == [
122
+ ("create", first),
123
+ ("create", second),
124
+ ("prune", second),
125
+ ]
113
126
  assert "warnings" in caplog.text
114
127
  assert str(first) in caplog.text
128
+ assert json.loads(failure_groups_path.read_text(encoding="utf-8")) == ["failed"]
115
129
 
116
130
 
117
131
  def test_main_interruption_returns_130_without_pruning(
@@ -240,8 +254,12 @@ def test_main_missing_borg_binary_returns_nonzero(
240
254
  def test_main_prune_failure_is_nonzero_and_scoped(
241
255
  settings_factory,
242
256
  monkeypatch,
257
+ tmp_path,
243
258
  ):
244
- settings = settings_factory()
259
+ failure_groups_path = tmp_path / "failed-groups.json"
260
+ settings = settings_factory(
261
+ environment={"BACKUP_FAILURES_PATH": str(failure_groups_path)}
262
+ )
245
263
  first = settings.server_root / "first"
246
264
  second = settings.server_root / "second"
247
265
  first.mkdir()
@@ -269,7 +287,9 @@ def test_main_prune_failure_is_nonzero_and_scoped(
269
287
  ("create", first),
270
288
  ("create", second),
271
289
  ("prune", first),
290
+ ("prune", second),
272
291
  ]
292
+ assert json.loads(failure_groups_path.read_text(encoding="utf-8")) == ["media"]
273
293
 
274
294
 
275
295
  def test_exclusions_are_passed_only_to_their_backup_group(