declarative-data-backup 0.4.0__tar.gz → 0.5.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {declarative_data_backup-0.4.0 → declarative_data_backup-0.5.0}/PKG-INFO +9 -5
- {declarative_data_backup-0.4.0 → declarative_data_backup-0.5.0}/README.md +8 -4
- {declarative_data_backup-0.4.0 → declarative_data_backup-0.5.0}/pyproject.toml +1 -1
- {declarative_data_backup-0.4.0 → declarative_data_backup-0.5.0}/src/declarative_data_backup/backup.py +71 -17
- {declarative_data_backup-0.4.0 → declarative_data_backup-0.5.0}/src/declarative_data_backup/environment.py +13 -0
- {declarative_data_backup-0.4.0 → declarative_data_backup-0.5.0}/src/declarative_data_backup.egg-info/PKG-INFO +9 -5
- {declarative_data_backup-0.4.0 → declarative_data_backup-0.5.0}/tests/test_backup.py +27 -7
- {declarative_data_backup-0.4.0 → declarative_data_backup-0.5.0}/setup.cfg +0 -0
- {declarative_data_backup-0.4.0 → declarative_data_backup-0.5.0}/src/declarative_data_backup/__init__.py +0 -0
- {declarative_data_backup-0.4.0 → declarative_data_backup-0.5.0}/src/declarative_data_backup/__main__.py +0 -0
- {declarative_data_backup-0.4.0 → declarative_data_backup-0.5.0}/src/declarative_data_backup/borg/__init__.py +0 -0
- {declarative_data_backup-0.4.0 → declarative_data_backup-0.5.0}/src/declarative_data_backup/borg/archive_naming.py +0 -0
- {declarative_data_backup-0.4.0 → declarative_data_backup-0.5.0}/src/declarative_data_backup/borg/borg.py +0 -0
- {declarative_data_backup-0.4.0 → declarative_data_backup-0.5.0}/src/declarative_data_backup/borg/borg_cli.py +0 -0
- {declarative_data_backup-0.4.0 → declarative_data_backup-0.5.0}/src/declarative_data_backup/config/__init__.py +0 -0
- {declarative_data_backup-0.4.0 → declarative_data_backup-0.5.0}/src/declarative_data_backup/config/backup_config/__init__.py +0 -0
- {declarative_data_backup-0.4.0 → declarative_data_backup-0.5.0}/src/declarative_data_backup/config/backup_config/models.py +0 -0
- {declarative_data_backup-0.4.0 → declarative_data_backup-0.5.0}/src/declarative_data_backup/config/backup_config/utils.py +0 -0
- {declarative_data_backup-0.4.0 → declarative_data_backup-0.5.0}/src/declarative_data_backup/config/backup_config/yaml_parser.py +0 -0
- {declarative_data_backup-0.4.0 → declarative_data_backup-0.5.0}/src/declarative_data_backup/config/backup_config/yaml_raw_loader.py +0 -0
- {declarative_data_backup-0.4.0 → declarative_data_backup-0.5.0}/src/declarative_data_backup/config/logging.py +0 -0
- {declarative_data_backup-0.4.0 → declarative_data_backup-0.5.0}/src/declarative_data_backup/errors.py +0 -0
- {declarative_data_backup-0.4.0 → declarative_data_backup-0.5.0}/src/declarative_data_backup.egg-info/SOURCES.txt +0 -0
- {declarative_data_backup-0.4.0 → declarative_data_backup-0.5.0}/src/declarative_data_backup.egg-info/dependency_links.txt +0 -0
- {declarative_data_backup-0.4.0 → declarative_data_backup-0.5.0}/src/declarative_data_backup.egg-info/requires.txt +0 -0
- {declarative_data_backup-0.4.0 → declarative_data_backup-0.5.0}/src/declarative_data_backup.egg-info/top_level.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: declarative-data-backup
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.5.0
|
|
4
4
|
Summary: Declarative Borg-based data backups
|
|
5
5
|
Requires-Python: >=3.11
|
|
6
6
|
Description-Content-Type: text/markdown
|
|
@@ -149,6 +149,8 @@ Optional variables:
|
|
|
149
149
|
| --- | --- |
|
|
150
150
|
| `BORG_REMOTE_PATH` | Remote Borg executable path, passed as Borg's `--remote-path`. Leave unset to use Borg's default. |
|
|
151
151
|
| `BORG_EXIT_CODES` | Borg exit-code mode: `legacy` or `modern`. |
|
|
152
|
+
| `BORG_FILES_CACHE_TTL` | How many archive creations can pass without a file appearing before Borg forgets its cached metadata. The default is `20`; raise it if more than 20 archives are created between backups of the same source, so unchanged files can still be recognized. |
|
|
153
|
+
| `BACKUP_FAILURES_PATH` | Optional absolute path where failed backup group names are written as JSON. |
|
|
152
154
|
| `BACKUP_PROGRESS` | Set to `true` to log the latest Borg progress every 30 seconds during archive creation. Defaults to `false`. |
|
|
153
155
|
| `BACKUP_VERBOSE` | Set to `true` to show Borg's per-file create and per-archive prune output. Defaults to `false` for concise logs. |
|
|
154
156
|
|
|
@@ -246,10 +248,12 @@ The stable source series is derived from the canonical source path and includes
|
|
|
246
248
|
a SHA-256 suffix. Borg receives the canonical absolute path as the source, so
|
|
247
249
|
the original directory hierarchy is retained inside the archive.
|
|
248
250
|
|
|
249
|
-
The application
|
|
250
|
-
If a create
|
|
251
|
-
|
|
252
|
-
|
|
251
|
+
The application attempts every configured backup group before pruning any
|
|
252
|
+
source. If a group has a create failure or Borg reports warnings, pruning is
|
|
253
|
+
skipped for that group, but other groups that completed successfully are still
|
|
254
|
+
pruned. Failures are reported and make the overall run fail. Retries use a new
|
|
255
|
+
run UUID and may create another archive for that day; archives are never
|
|
256
|
+
overwritten to handle a collision.
|
|
253
257
|
|
|
254
258
|
`--keep-daily N` retains the latest archive in up to `N` daily buckets. It does
|
|
255
259
|
not guarantee `N` successful calendar days of recovery if scheduled runs fail
|
|
@@ -141,6 +141,8 @@ Optional variables:
|
|
|
141
141
|
| --- | --- |
|
|
142
142
|
| `BORG_REMOTE_PATH` | Remote Borg executable path, passed as Borg's `--remote-path`. Leave unset to use Borg's default. |
|
|
143
143
|
| `BORG_EXIT_CODES` | Borg exit-code mode: `legacy` or `modern`. |
|
|
144
|
+
| `BORG_FILES_CACHE_TTL` | How many archive creations can pass without a file appearing before Borg forgets its cached metadata. The default is `20`; raise it if more than 20 archives are created between backups of the same source, so unchanged files can still be recognized. |
|
|
145
|
+
| `BACKUP_FAILURES_PATH` | Optional absolute path where failed backup group names are written as JSON. |
|
|
144
146
|
| `BACKUP_PROGRESS` | Set to `true` to log the latest Borg progress every 30 seconds during archive creation. Defaults to `false`. |
|
|
145
147
|
| `BACKUP_VERBOSE` | Set to `true` to show Borg's per-file create and per-archive prune output. Defaults to `false` for concise logs. |
|
|
146
148
|
|
|
@@ -238,10 +240,12 @@ The stable source series is derived from the canonical source path and includes
|
|
|
238
240
|
a SHA-256 suffix. Borg receives the canonical absolute path as the source, so
|
|
239
241
|
the original directory hierarchy is retained inside the archive.
|
|
240
242
|
|
|
241
|
-
The application
|
|
242
|
-
If a create
|
|
243
|
-
|
|
244
|
-
|
|
243
|
+
The application attempts every configured backup group before pruning any
|
|
244
|
+
source. If a group has a create failure or Borg reports warnings, pruning is
|
|
245
|
+
skipped for that group, but other groups that completed successfully are still
|
|
246
|
+
pruned. Failures are reported and make the overall run fail. Retries use a new
|
|
247
|
+
run UUID and may create another archive for that day; archives are never
|
|
248
|
+
overwritten to handle a collision.
|
|
245
249
|
|
|
246
250
|
`--keep-daily N` retains the latest archive in up to `N` daily buckets. It does
|
|
247
251
|
not guarantee `N` successful calendar days of recovery if scheduled runs fail
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import json
|
|
1
2
|
import logging
|
|
2
3
|
import os
|
|
3
4
|
import signal
|
|
@@ -21,6 +22,20 @@ from .errors import BackupError, BackupInterrupted, BorgWarningError
|
|
|
21
22
|
logger = logging.getLogger(__name__)
|
|
22
23
|
|
|
23
24
|
|
|
25
|
+
def write_failed_groups(settings: Environment, groups: set[str]) -> None:
|
|
26
|
+
path = settings.failure_groups_path
|
|
27
|
+
if path is None or not groups:
|
|
28
|
+
return
|
|
29
|
+
try:
|
|
30
|
+
path.write_text(json.dumps(sorted(groups)), encoding="utf-8")
|
|
31
|
+
except OSError as exc:
|
|
32
|
+
logger.warning(
|
|
33
|
+
"Unable to write failed backup group summary to %s: %s",
|
|
34
|
+
path,
|
|
35
|
+
exc.strerror or "I/O error",
|
|
36
|
+
)
|
|
37
|
+
|
|
38
|
+
|
|
24
39
|
def check_settings(settings: Environment) -> None:
|
|
25
40
|
ensure_borg_binary()
|
|
26
41
|
if not settings.borg_base_dir.is_dir():
|
|
@@ -56,38 +71,77 @@ def run_backup(
|
|
|
56
71
|
run_id = uuid.uuid4().hex
|
|
57
72
|
with repository_lock(settings.borg_base_dir / "backup.lock"):
|
|
58
73
|
created_at = datetime.now(UTC)
|
|
74
|
+
successful_definitions = []
|
|
75
|
+
failures: list[BackupError] = []
|
|
76
|
+
failed_groups: set[str] = set()
|
|
59
77
|
for definition in definitions:
|
|
60
78
|
logger.info(
|
|
61
79
|
"Starting backup group '%s' (%d sources).",
|
|
62
80
|
definition.name,
|
|
63
81
|
len(definition.sources),
|
|
64
82
|
)
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
83
|
+
try:
|
|
84
|
+
for folder in definition.sources:
|
|
85
|
+
backup_folder(
|
|
86
|
+
folder,
|
|
87
|
+
definition.compression,
|
|
88
|
+
definition.name,
|
|
89
|
+
settings,
|
|
90
|
+
created_at,
|
|
91
|
+
run_id,
|
|
92
|
+
definition.excludes,
|
|
93
|
+
)
|
|
94
|
+
except BackupError as exc:
|
|
95
|
+
logger.error(
|
|
96
|
+
"Backup group '%s' failed; retention pruning will be skipped "
|
|
97
|
+
"for this group: %s",
|
|
69
98
|
definition.name,
|
|
70
|
-
|
|
71
|
-
created_at,
|
|
72
|
-
run_id,
|
|
73
|
-
definition.excludes,
|
|
99
|
+
exc,
|
|
74
100
|
)
|
|
101
|
+
failures.append(exc)
|
|
102
|
+
failed_groups.add(definition.name)
|
|
103
|
+
continue
|
|
104
|
+
|
|
75
105
|
logger.info("Backup group '%s' completed.", definition.name)
|
|
106
|
+
successful_definitions.append(definition)
|
|
76
107
|
|
|
77
|
-
for definition in
|
|
108
|
+
for definition in successful_definitions:
|
|
78
109
|
logger.info(
|
|
79
110
|
"Starting retention pruning for group '%s' (%d sources).",
|
|
80
111
|
definition.name,
|
|
81
112
|
len(definition.sources),
|
|
82
113
|
)
|
|
114
|
+
pruning_succeeded = True
|
|
83
115
|
for folder in definition.sources:
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
116
|
+
try:
|
|
117
|
+
prune(
|
|
118
|
+
folder,
|
|
119
|
+
definition.retention_days,
|
|
120
|
+
definition.name,
|
|
121
|
+
settings,
|
|
122
|
+
)
|
|
123
|
+
except BackupError as exc:
|
|
124
|
+
logger.error(
|
|
125
|
+
"Retention pruning failed for group '%s' source '%s': %s",
|
|
126
|
+
definition.name,
|
|
127
|
+
folder,
|
|
128
|
+
exc,
|
|
129
|
+
)
|
|
130
|
+
failures.append(exc)
|
|
131
|
+
failed_groups.add(definition.name)
|
|
132
|
+
pruning_succeeded = False
|
|
133
|
+
|
|
134
|
+
if pruning_succeeded:
|
|
135
|
+
logger.info(
|
|
136
|
+
"Retention pruning for group '%s' completed.", definition.name
|
|
89
137
|
)
|
|
90
|
-
|
|
138
|
+
|
|
139
|
+
if failures:
|
|
140
|
+
write_failed_groups(settings, failed_groups)
|
|
141
|
+
message = "; ".join(str(failure) for failure in failures)
|
|
142
|
+
if any(isinstance(failure, BorgWarningError) for failure in failures):
|
|
143
|
+
raise BorgWarningError(message)
|
|
144
|
+
raise BackupError(message)
|
|
91
145
|
|
|
92
146
|
|
|
93
147
|
def main(environ: Mapping[str, str] | None = None) -> int:
|
|
@@ -111,8 +165,8 @@ def main(environ: Mapping[str, str] | None = None) -> int:
|
|
|
111
165
|
|
|
112
166
|
except BorgWarningError as exc:
|
|
113
167
|
logger.error(
|
|
114
|
-
"Backup
|
|
115
|
-
"pruning
|
|
168
|
+
"Backup failed because Borg reported warnings; "
|
|
169
|
+
"retention pruning may have completed for other groups: %s",
|
|
116
170
|
exc,
|
|
117
171
|
)
|
|
118
172
|
return 1
|
|
@@ -19,6 +19,7 @@ class Environment:
|
|
|
19
19
|
borg_exit_codes: str | None
|
|
20
20
|
backup_progress: bool
|
|
21
21
|
backup_verbose: bool
|
|
22
|
+
failure_groups_path: Path | None = None
|
|
22
23
|
|
|
23
24
|
@classmethod
|
|
24
25
|
def from_env(
|
|
@@ -42,6 +43,17 @@ class Environment:
|
|
|
42
43
|
fail(f"{name} must be an absolute path.")
|
|
43
44
|
return value
|
|
44
45
|
|
|
46
|
+
def optional_path(name: str) -> Path | None:
|
|
47
|
+
raw_value = source.get(name)
|
|
48
|
+
if raw_value is None:
|
|
49
|
+
return None
|
|
50
|
+
if "\0" in raw_value:
|
|
51
|
+
fail(f"{name} must be a valid absolute path.")
|
|
52
|
+
value = Path(raw_value)
|
|
53
|
+
if not value.is_absolute():
|
|
54
|
+
fail(f"{name} must be an absolute path.")
|
|
55
|
+
return value
|
|
56
|
+
|
|
45
57
|
repo = required("BORG_REPO")
|
|
46
58
|
if "\0" in repo:
|
|
47
59
|
fail("BORG_REPO must be a valid repository location.")
|
|
@@ -72,6 +84,7 @@ class Environment:
|
|
|
72
84
|
borg_exit_codes=borg_exit_codes,
|
|
73
85
|
backup_progress=backup_progress_value == "true",
|
|
74
86
|
backup_verbose=backup_verbose_value == "true",
|
|
87
|
+
failure_groups_path=optional_path("BACKUP_FAILURES_PATH"),
|
|
75
88
|
)
|
|
76
89
|
|
|
77
90
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: declarative-data-backup
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.5.0
|
|
4
4
|
Summary: Declarative Borg-based data backups
|
|
5
5
|
Requires-Python: >=3.11
|
|
6
6
|
Description-Content-Type: text/markdown
|
|
@@ -149,6 +149,8 @@ Optional variables:
|
|
|
149
149
|
| --- | --- |
|
|
150
150
|
| `BORG_REMOTE_PATH` | Remote Borg executable path, passed as Borg's `--remote-path`. Leave unset to use Borg's default. |
|
|
151
151
|
| `BORG_EXIT_CODES` | Borg exit-code mode: `legacy` or `modern`. |
|
|
152
|
+
| `BORG_FILES_CACHE_TTL` | How many archive creations can pass without a file appearing before Borg forgets its cached metadata. The default is `20`; raise it if more than 20 archives are created between backups of the same source, so unchanged files can still be recognized. |
|
|
153
|
+
| `BACKUP_FAILURES_PATH` | Optional absolute path where failed backup group names are written as JSON. |
|
|
152
154
|
| `BACKUP_PROGRESS` | Set to `true` to log the latest Borg progress every 30 seconds during archive creation. Defaults to `false`. |
|
|
153
155
|
| `BACKUP_VERBOSE` | Set to `true` to show Borg's per-file create and per-archive prune output. Defaults to `false` for concise logs. |
|
|
154
156
|
|
|
@@ -246,10 +248,12 @@ The stable source series is derived from the canonical source path and includes
|
|
|
246
248
|
a SHA-256 suffix. Borg receives the canonical absolute path as the source, so
|
|
247
249
|
the original directory hierarchy is retained inside the archive.
|
|
248
250
|
|
|
249
|
-
The application
|
|
250
|
-
If a create
|
|
251
|
-
|
|
252
|
-
|
|
251
|
+
The application attempts every configured backup group before pruning any
|
|
252
|
+
source. If a group has a create failure or Borg reports warnings, pruning is
|
|
253
|
+
skipped for that group, but other groups that completed successfully are still
|
|
254
|
+
pruned. Failures are reported and make the overall run fail. Retries use a new
|
|
255
|
+
run UUID and may create another archive for that day; archives are never
|
|
256
|
+
overwritten to handle a collision.
|
|
253
257
|
|
|
254
258
|
`--keep-daily N` retains the latest archive in up to `N` daily buckets. It does
|
|
255
259
|
not guarantee `N` successful calendar days of recovery if scheduled runs fail
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import json
|
|
1
2
|
import logging
|
|
2
3
|
|
|
3
4
|
import declarative_data_backup.backup as backup
|
|
@@ -6,7 +7,7 @@ from declarative_data_backup.errors import BackupError, BorgWarningError
|
|
|
6
7
|
|
|
7
8
|
|
|
8
9
|
def _settings_environment(settings: Environment) -> dict[str, str]:
|
|
9
|
-
|
|
10
|
+
environment = {
|
|
10
11
|
"BACKUP_CONFIG_PATH": str(settings.config_path),
|
|
11
12
|
"BORG_KEY_FILE": str(settings.key_file),
|
|
12
13
|
"BORG_REPO": settings.borg_repo,
|
|
@@ -15,6 +16,9 @@ def _settings_environment(settings: Environment) -> dict[str, str]:
|
|
|
15
16
|
"BORG_RSH": settings.borg_rsh,
|
|
16
17
|
"BORG_PASSCOMMAND": settings.borg_passcommand,
|
|
17
18
|
}
|
|
19
|
+
if settings.failure_groups_path is not None:
|
|
20
|
+
environment["BACKUP_FAILURES_PATH"] = str(settings.failure_groups_path)
|
|
21
|
+
return environment
|
|
18
22
|
|
|
19
23
|
|
|
20
24
|
def _write_config(settings, contents: str) -> None:
|
|
@@ -79,26 +83,31 @@ def test_main_runs_all_creates_before_any_prune(
|
|
|
79
83
|
assert "Retention pruning for group 'media_backup' completed." in caplog.text
|
|
80
84
|
|
|
81
85
|
|
|
82
|
-
def
|
|
86
|
+
def test_main_warning_skips_failed_group_but_prunes_successful_groups(
|
|
83
87
|
settings_factory,
|
|
84
88
|
monkeypatch,
|
|
85
89
|
caplog,
|
|
90
|
+
tmp_path,
|
|
86
91
|
):
|
|
87
|
-
|
|
92
|
+
failure_groups_path = tmp_path / "failed-groups.json"
|
|
93
|
+
settings = settings_factory(
|
|
94
|
+
environment={"BACKUP_FAILURES_PATH": str(failure_groups_path)}
|
|
95
|
+
)
|
|
88
96
|
first = settings.server_root / "first"
|
|
89
97
|
second = settings.server_root / "second"
|
|
90
98
|
first.mkdir()
|
|
91
99
|
second.mkdir()
|
|
92
100
|
_write_config(
|
|
93
101
|
settings,
|
|
94
|
-
f"
|
|
102
|
+
f"failed:\n folders: [{first}]\nsucceeded:\n folders: [{second}]\n",
|
|
95
103
|
)
|
|
96
104
|
_configure_preflight(monkeypatch)
|
|
97
105
|
calls = []
|
|
98
106
|
|
|
99
107
|
def warning(folder, *args):
|
|
100
108
|
calls.append(("create", folder))
|
|
101
|
-
|
|
109
|
+
if folder == first:
|
|
110
|
+
raise BorgWarningError(f"warning for {folder}")
|
|
102
111
|
|
|
103
112
|
monkeypatch.setattr(backup, "backup_folder", warning)
|
|
104
113
|
monkeypatch.setattr(
|
|
@@ -109,9 +118,14 @@ def test_main_warning_aborts_remaining_creates_and_all_pruning(
|
|
|
109
118
|
caplog.set_level(logging.ERROR)
|
|
110
119
|
|
|
111
120
|
assert backup.main(_settings_environment(settings)) == 1
|
|
112
|
-
assert calls == [
|
|
121
|
+
assert calls == [
|
|
122
|
+
("create", first),
|
|
123
|
+
("create", second),
|
|
124
|
+
("prune", second),
|
|
125
|
+
]
|
|
113
126
|
assert "warnings" in caplog.text
|
|
114
127
|
assert str(first) in caplog.text
|
|
128
|
+
assert json.loads(failure_groups_path.read_text(encoding="utf-8")) == ["failed"]
|
|
115
129
|
|
|
116
130
|
|
|
117
131
|
def test_main_interruption_returns_130_without_pruning(
|
|
@@ -240,8 +254,12 @@ def test_main_missing_borg_binary_returns_nonzero(
|
|
|
240
254
|
def test_main_prune_failure_is_nonzero_and_scoped(
|
|
241
255
|
settings_factory,
|
|
242
256
|
monkeypatch,
|
|
257
|
+
tmp_path,
|
|
243
258
|
):
|
|
244
|
-
|
|
259
|
+
failure_groups_path = tmp_path / "failed-groups.json"
|
|
260
|
+
settings = settings_factory(
|
|
261
|
+
environment={"BACKUP_FAILURES_PATH": str(failure_groups_path)}
|
|
262
|
+
)
|
|
245
263
|
first = settings.server_root / "first"
|
|
246
264
|
second = settings.server_root / "second"
|
|
247
265
|
first.mkdir()
|
|
@@ -269,7 +287,9 @@ def test_main_prune_failure_is_nonzero_and_scoped(
|
|
|
269
287
|
("create", first),
|
|
270
288
|
("create", second),
|
|
271
289
|
("prune", first),
|
|
290
|
+
("prune", second),
|
|
272
291
|
]
|
|
292
|
+
assert json.loads(failure_groups_path.read_text(encoding="utf-8")) == ["media"]
|
|
273
293
|
|
|
274
294
|
|
|
275
295
|
def test_exclusions_are_passed_only_to_their_backup_group(
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|