declarative-data-backup 0.5.0__tar.gz → 0.5.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {declarative_data_backup-0.5.0 → declarative_data_backup-0.5.2}/PKG-INFO +3 -2
- {declarative_data_backup-0.5.0 → declarative_data_backup-0.5.2}/README.md +2 -1
- {declarative_data_backup-0.5.0 → declarative_data_backup-0.5.2}/pyproject.toml +1 -1
- {declarative_data_backup-0.5.0 → declarative_data_backup-0.5.2}/src/declarative_data_backup/backup.py +23 -0
- {declarative_data_backup-0.5.0 → declarative_data_backup-0.5.2}/src/declarative_data_backup/borg/borg_cli.py +6 -2
- {declarative_data_backup-0.5.0 → declarative_data_backup-0.5.2}/src/declarative_data_backup/environment.py +22 -0
- {declarative_data_backup-0.5.0 → declarative_data_backup-0.5.2}/src/declarative_data_backup.egg-info/PKG-INFO +3 -2
- {declarative_data_backup-0.5.0 → declarative_data_backup-0.5.2}/tests/test_backup.py +42 -8
- {declarative_data_backup-0.5.0 → declarative_data_backup-0.5.2}/setup.cfg +0 -0
- {declarative_data_backup-0.5.0 → declarative_data_backup-0.5.2}/src/declarative_data_backup/__init__.py +0 -0
- {declarative_data_backup-0.5.0 → declarative_data_backup-0.5.2}/src/declarative_data_backup/__main__.py +0 -0
- {declarative_data_backup-0.5.0 → declarative_data_backup-0.5.2}/src/declarative_data_backup/borg/__init__.py +0 -0
- {declarative_data_backup-0.5.0 → declarative_data_backup-0.5.2}/src/declarative_data_backup/borg/archive_naming.py +0 -0
- {declarative_data_backup-0.5.0 → declarative_data_backup-0.5.2}/src/declarative_data_backup/borg/borg.py +0 -0
- {declarative_data_backup-0.5.0 → declarative_data_backup-0.5.2}/src/declarative_data_backup/config/__init__.py +0 -0
- {declarative_data_backup-0.5.0 → declarative_data_backup-0.5.2}/src/declarative_data_backup/config/backup_config/__init__.py +0 -0
- {declarative_data_backup-0.5.0 → declarative_data_backup-0.5.2}/src/declarative_data_backup/config/backup_config/models.py +0 -0
- {declarative_data_backup-0.5.0 → declarative_data_backup-0.5.2}/src/declarative_data_backup/config/backup_config/utils.py +0 -0
- {declarative_data_backup-0.5.0 → declarative_data_backup-0.5.2}/src/declarative_data_backup/config/backup_config/yaml_parser.py +0 -0
- {declarative_data_backup-0.5.0 → declarative_data_backup-0.5.2}/src/declarative_data_backup/config/backup_config/yaml_raw_loader.py +0 -0
- {declarative_data_backup-0.5.0 → declarative_data_backup-0.5.2}/src/declarative_data_backup/config/logging.py +0 -0
- {declarative_data_backup-0.5.0 → declarative_data_backup-0.5.2}/src/declarative_data_backup/errors.py +0 -0
- {declarative_data_backup-0.5.0 → declarative_data_backup-0.5.2}/src/declarative_data_backup.egg-info/SOURCES.txt +0 -0
- {declarative_data_backup-0.5.0 → declarative_data_backup-0.5.2}/src/declarative_data_backup.egg-info/dependency_links.txt +0 -0
- {declarative_data_backup-0.5.0 → declarative_data_backup-0.5.2}/src/declarative_data_backup.egg-info/requires.txt +0 -0
- {declarative_data_backup-0.5.0 → declarative_data_backup-0.5.2}/src/declarative_data_backup.egg-info/top_level.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: declarative-data-backup
|
|
3
|
-
Version: 0.5.
|
|
3
|
+
Version: 0.5.2
|
|
4
4
|
Summary: Declarative Borg-based data backups
|
|
5
5
|
Requires-Python: >=3.11
|
|
6
6
|
Description-Content-Type: text/markdown
|
|
@@ -149,7 +149,8 @@ Optional variables:
|
|
|
149
149
|
| --- | --- |
|
|
150
150
|
| `BORG_REMOTE_PATH` | Remote Borg executable path, passed as Borg's `--remote-path`. Leave unset to use Borg's default. |
|
|
151
151
|
| `BORG_EXIT_CODES` | Borg exit-code mode: `legacy` or `modern`. |
|
|
152
|
-
| `
|
|
152
|
+
| `BACKUP_FILES_CACHE_TTL_MARGIN` | Extra archive creations to retain file-cache entries beyond the number of configured sources. The app sets Borg's TTL to source count plus this margin; defaults to `20`. |
|
|
153
|
+
| `BORG_FILES_CACHE_TTL` | Optional absolute Borg TTL override; it must be at least the number of configured sources. Usually leave unset to use the source-count-based TTL and configured margin. |
|
|
153
154
|
| `BACKUP_FAILURES_PATH` | Optional absolute path where failed backup group names are written as JSON. |
|
|
154
155
|
| `BACKUP_PROGRESS` | Set to `true` to log the latest Borg progress every 30 seconds during archive creation. Defaults to `false`. |
|
|
155
156
|
| `BACKUP_VERBOSE` | Set to `true` to show Borg's per-file create and per-archive prune output. Defaults to `false` for concise logs. |
|
|
@@ -141,7 +141,8 @@ Optional variables:
|
|
|
141
141
|
| --- | --- |
|
|
142
142
|
| `BORG_REMOTE_PATH` | Remote Borg executable path, passed as Borg's `--remote-path`. Leave unset to use Borg's default. |
|
|
143
143
|
| `BORG_EXIT_CODES` | Borg exit-code mode: `legacy` or `modern`. |
|
|
144
|
-
| `
|
|
144
|
+
| `BACKUP_FILES_CACHE_TTL_MARGIN` | Extra archive creations to retain file-cache entries beyond the number of configured sources. The app sets Borg's TTL to source count plus this margin; defaults to `20`. |
|
|
145
|
+
| `BORG_FILES_CACHE_TTL` | Optional absolute Borg TTL override; it must be at least the number of configured sources. Usually leave unset to use the source-count-based TTL and configured margin. |
|
|
145
146
|
| `BACKUP_FAILURES_PATH` | Optional absolute path where failed backup group names are written as JSON. |
|
|
146
147
|
| `BACKUP_PROGRESS` | Set to `true` to log the latest Borg progress every 30 seconds during archive creation. Defaults to `false`. |
|
|
147
148
|
| `BACKUP_VERBOSE` | Set to `true` to show Borg's per-file create and per-archive prune output. Defaults to `false` for concise logs. |
|
|
@@ -4,6 +4,7 @@ import os
|
|
|
4
4
|
import signal
|
|
5
5
|
import uuid
|
|
6
6
|
from collections.abc import Mapping
|
|
7
|
+
from dataclasses import replace
|
|
7
8
|
from datetime import UTC, datetime
|
|
8
9
|
from types import FrameType
|
|
9
10
|
|
|
@@ -67,6 +68,28 @@ def run_backup(
|
|
|
67
68
|
len(definitions),
|
|
68
69
|
total_sources,
|
|
69
70
|
)
|
|
71
|
+
if (
|
|
72
|
+
settings.borg_files_cache_ttl is not None
|
|
73
|
+
and settings.borg_files_cache_ttl < total_sources
|
|
74
|
+
):
|
|
75
|
+
raise BackupError(
|
|
76
|
+
"BORG_FILES_CACHE_TTL must be at least the number of configured "
|
|
77
|
+
f"source creates ({total_sources})."
|
|
78
|
+
)
|
|
79
|
+
if settings.borg_files_cache_ttl is None:
|
|
80
|
+
files_cache_ttl = total_sources + settings.backup_files_cache_ttl_margin
|
|
81
|
+
settings = replace(settings, borg_files_cache_ttl=files_cache_ttl)
|
|
82
|
+
logger.info(
|
|
83
|
+
"Using Borg files-cache TTL %d (%d source creates + %d margin).",
|
|
84
|
+
files_cache_ttl,
|
|
85
|
+
total_sources,
|
|
86
|
+
settings.backup_files_cache_ttl_margin,
|
|
87
|
+
)
|
|
88
|
+
else:
|
|
89
|
+
logger.info(
|
|
90
|
+
"Using explicit Borg files-cache TTL %d.",
|
|
91
|
+
settings.borg_files_cache_ttl,
|
|
92
|
+
)
|
|
70
93
|
|
|
71
94
|
run_id = uuid.uuid4().hex
|
|
72
95
|
with repository_lock(settings.borg_base_dir / "backup.lock"):
|
|
@@ -100,9 +100,11 @@ def run_with_progress(
|
|
|
100
100
|
print(text, file=sys.stderr, flush=True)
|
|
101
101
|
|
|
102
102
|
with borg_process(command, environment, capture_stderr=True) as process:
|
|
103
|
-
|
|
103
|
+
stderr = process.stderr
|
|
104
|
+
if stderr is None:
|
|
105
|
+
fail("Borg stderr was not captured for progress reporting.")
|
|
104
106
|
with selectors.DefaultSelector() as selector:
|
|
105
|
-
selector.register(
|
|
107
|
+
selector.register(stderr, selectors.EVENT_READ)
|
|
106
108
|
pending = b""
|
|
107
109
|
while selector.get_map():
|
|
108
110
|
for key, _ in selector.select(max(0, next_report - time.monotonic())):
|
|
@@ -147,6 +149,8 @@ def borg_environment(settings: Environment) -> dict[str, str]:
|
|
|
147
149
|
)
|
|
148
150
|
if settings.borg_exit_codes is not None:
|
|
149
151
|
environment["BORG_EXIT_CODES"] = settings.borg_exit_codes
|
|
152
|
+
if settings.borg_files_cache_ttl is not None:
|
|
153
|
+
environment["BORG_FILES_CACHE_TTL"] = str(settings.borg_files_cache_ttl)
|
|
150
154
|
return environment
|
|
151
155
|
|
|
152
156
|
|
|
@@ -20,6 +20,8 @@ class Environment:
|
|
|
20
20
|
backup_progress: bool
|
|
21
21
|
backup_verbose: bool
|
|
22
22
|
failure_groups_path: Path | None = None
|
|
23
|
+
backup_files_cache_ttl_margin: int = 20
|
|
24
|
+
borg_files_cache_ttl: int | None = None
|
|
23
25
|
|
|
24
26
|
@classmethod
|
|
25
27
|
def from_env(
|
|
@@ -66,6 +68,24 @@ class Environment:
|
|
|
66
68
|
backup_verbose_value = source.get("BACKUP_VERBOSE", "false").lower()
|
|
67
69
|
if backup_verbose_value not in ("true", "false"):
|
|
68
70
|
fail("BACKUP_VERBOSE must be either 'true' or 'false'.")
|
|
71
|
+
try:
|
|
72
|
+
backup_files_cache_ttl_margin = int(
|
|
73
|
+
source.get("BACKUP_FILES_CACHE_TTL_MARGIN", "20")
|
|
74
|
+
)
|
|
75
|
+
except ValueError:
|
|
76
|
+
fail("BACKUP_FILES_CACHE_TTL_MARGIN must be a nonnegative integer.")
|
|
77
|
+
if backup_files_cache_ttl_margin < 0:
|
|
78
|
+
fail("BACKUP_FILES_CACHE_TTL_MARGIN must be a nonnegative integer.")
|
|
79
|
+
raw_borg_files_cache_ttl = source.get("BORG_FILES_CACHE_TTL")
|
|
80
|
+
if raw_borg_files_cache_ttl is None:
|
|
81
|
+
borg_files_cache_ttl = None
|
|
82
|
+
else:
|
|
83
|
+
try:
|
|
84
|
+
borg_files_cache_ttl = int(raw_borg_files_cache_ttl)
|
|
85
|
+
except ValueError:
|
|
86
|
+
fail("BORG_FILES_CACHE_TTL must be a positive integer.")
|
|
87
|
+
if borg_files_cache_ttl < 1:
|
|
88
|
+
fail("BORG_FILES_CACHE_TTL must be a positive integer.")
|
|
69
89
|
|
|
70
90
|
try:
|
|
71
91
|
server_root = required_path("SERVER_ROOT").resolve(strict=True)
|
|
@@ -85,6 +105,8 @@ class Environment:
|
|
|
85
105
|
backup_progress=backup_progress_value == "true",
|
|
86
106
|
backup_verbose=backup_verbose_value == "true",
|
|
87
107
|
failure_groups_path=optional_path("BACKUP_FAILURES_PATH"),
|
|
108
|
+
backup_files_cache_ttl_margin=backup_files_cache_ttl_margin,
|
|
109
|
+
borg_files_cache_ttl=borg_files_cache_ttl,
|
|
88
110
|
)
|
|
89
111
|
|
|
90
112
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: declarative-data-backup
|
|
3
|
-
Version: 0.5.
|
|
3
|
+
Version: 0.5.2
|
|
4
4
|
Summary: Declarative Borg-based data backups
|
|
5
5
|
Requires-Python: >=3.11
|
|
6
6
|
Description-Content-Type: text/markdown
|
|
@@ -149,7 +149,8 @@ Optional variables:
|
|
|
149
149
|
| --- | --- |
|
|
150
150
|
| `BORG_REMOTE_PATH` | Remote Borg executable path, passed as Borg's `--remote-path`. Leave unset to use Borg's default. |
|
|
151
151
|
| `BORG_EXIT_CODES` | Borg exit-code mode: `legacy` or `modern`. |
|
|
152
|
-
| `
|
|
152
|
+
| `BACKUP_FILES_CACHE_TTL_MARGIN` | Extra archive creations to retain file-cache entries beyond the number of configured sources. The app sets Borg's TTL to source count plus this margin; defaults to `20`. |
|
|
153
|
+
| `BORG_FILES_CACHE_TTL` | Optional absolute Borg TTL override; it must be at least the number of configured sources. Usually leave unset to use the source-count-based TTL and configured margin. |
|
|
153
154
|
| `BACKUP_FAILURES_PATH` | Optional absolute path where failed backup group names are written as JSON. |
|
|
154
155
|
| `BACKUP_PROGRESS` | Set to `true` to log the latest Borg progress every 30 seconds during archive creation. Defaults to `false`. |
|
|
155
156
|
| `BACKUP_VERBOSE` | Set to `true` to show Borg's per-file create and per-archive prune output. Defaults to `false` for concise logs. |
|
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
import json
|
|
2
2
|
import logging
|
|
3
3
|
|
|
4
|
+
import pytest
|
|
5
|
+
|
|
4
6
|
import declarative_data_backup.backup as backup
|
|
5
7
|
from declarative_data_backup.environment import Environment
|
|
6
8
|
from declarative_data_backup.errors import BackupError, BorgWarningError
|
|
@@ -18,6 +20,11 @@ def _settings_environment(settings: Environment) -> dict[str, str]:
|
|
|
18
20
|
}
|
|
19
21
|
if settings.failure_groups_path is not None:
|
|
20
22
|
environment["BACKUP_FAILURES_PATH"] = str(settings.failure_groups_path)
|
|
23
|
+
environment["BACKUP_FILES_CACHE_TTL_MARGIN"] = str(
|
|
24
|
+
settings.backup_files_cache_ttl_margin
|
|
25
|
+
)
|
|
26
|
+
if settings.borg_files_cache_ttl is not None:
|
|
27
|
+
environment["BORG_FILES_CACHE_TTL"] = str(settings.borg_files_cache_ttl)
|
|
21
28
|
return environment
|
|
22
29
|
|
|
23
30
|
|
|
@@ -30,12 +37,17 @@ def _configure_preflight(monkeypatch):
|
|
|
30
37
|
monkeypatch.setattr(backup, "check_keyfile", lambda settings: None)
|
|
31
38
|
|
|
32
39
|
|
|
40
|
+
@pytest.mark.parametrize(("margin", "expected_ttl"), [(0, 2), (7, 9)])
|
|
33
41
|
def test_main_runs_all_creates_before_any_prune(
|
|
34
42
|
settings_factory,
|
|
35
43
|
monkeypatch,
|
|
36
44
|
caplog,
|
|
45
|
+
margin,
|
|
46
|
+
expected_ttl,
|
|
37
47
|
):
|
|
38
|
-
settings = settings_factory(
|
|
48
|
+
settings = settings_factory(
|
|
49
|
+
environment={"BACKUP_FILES_CACHE_TTL_MARGIN": str(margin)}
|
|
50
|
+
)
|
|
39
51
|
media = settings.server_root / "media"
|
|
40
52
|
postgres = settings.server_root / "postgres"
|
|
41
53
|
media.mkdir()
|
|
@@ -50,13 +62,13 @@ def test_main_runs_all_creates_before_any_prune(
|
|
|
50
62
|
)
|
|
51
63
|
_configure_preflight(monkeypatch)
|
|
52
64
|
events = []
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
)
|
|
65
|
+
files_cache_ttls = []
|
|
66
|
+
|
|
67
|
+
def create(folder, compression, group, settings, created_at, run_id, excludes):
|
|
68
|
+
files_cache_ttls.append(settings.borg_files_cache_ttl)
|
|
69
|
+
events.append(("create", group, folder, compression, run_id, excludes))
|
|
70
|
+
|
|
71
|
+
monkeypatch.setattr(backup, "backup_folder", create)
|
|
60
72
|
monkeypatch.setattr(
|
|
61
73
|
backup,
|
|
62
74
|
"prune",
|
|
@@ -79,6 +91,7 @@ def test_main_runs_all_creates_before_any_prune(
|
|
|
79
91
|
assert events[0][3] == "zstd,5"
|
|
80
92
|
assert events[1][3] == "auto,zstd,22"
|
|
81
93
|
assert events[2][3] == 30
|
|
94
|
+
assert files_cache_ttls == [expected_ttl, expected_ttl]
|
|
82
95
|
assert "Backup group 'media_backup' completed." in caplog.text
|
|
83
96
|
assert "Retention pruning for group 'media_backup' completed." in caplog.text
|
|
84
97
|
|
|
@@ -224,6 +237,27 @@ def test_main_validation_failure_runs_no_borg_operations(
|
|
|
224
237
|
assert not (settings.borg_base_dir / "backup.lock").exists()
|
|
225
238
|
|
|
226
239
|
|
|
240
|
+
def test_main_rejects_files_cache_ttl_below_source_count(
|
|
241
|
+
settings_factory,
|
|
242
|
+
monkeypatch,
|
|
243
|
+
caplog,
|
|
244
|
+
):
|
|
245
|
+
settings = settings_factory(environment={"BORG_FILES_CACHE_TTL": "1"})
|
|
246
|
+
first = settings.server_root / "first"
|
|
247
|
+
second = settings.server_root / "second"
|
|
248
|
+
first.mkdir()
|
|
249
|
+
second.mkdir()
|
|
250
|
+
_write_config(settings, f"media:\n folders: [{first}, {second}]\n")
|
|
251
|
+
_configure_preflight(monkeypatch)
|
|
252
|
+
calls = []
|
|
253
|
+
monkeypatch.setattr(backup, "backup_folder", lambda *args: calls.append(args))
|
|
254
|
+
caplog.set_level(logging.ERROR)
|
|
255
|
+
|
|
256
|
+
assert backup.main(_settings_environment(settings)) == 1
|
|
257
|
+
assert calls == []
|
|
258
|
+
assert "must be at least the number of configured source creates (2)" in caplog.text
|
|
259
|
+
|
|
260
|
+
|
|
227
261
|
def test_main_missing_environment_is_reported_without_traceback(caplog):
|
|
228
262
|
caplog.set_level(logging.ERROR)
|
|
229
263
|
assert backup.main({}) == 1
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|