declarative-data-backup 0.1.0__tar.gz → 0.2.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {declarative_data_backup-0.1.0 → declarative_data_backup-0.2.1}/PKG-INFO +42 -20
- {declarative_data_backup-0.1.0 → declarative_data_backup-0.2.1}/README.md +41 -14
- {declarative_data_backup-0.1.0 → declarative_data_backup-0.2.1}/pyproject.toml +2 -2
- {declarative_data_backup-0.1.0 → declarative_data_backup-0.2.1}/src/declarative_data_backup/backup.py +8 -7
- {declarative_data_backup-0.1.0 → declarative_data_backup-0.2.1}/src/declarative_data_backup/borg/borg.py +12 -9
- {declarative_data_backup-0.1.0 → declarative_data_backup-0.2.1}/src/declarative_data_backup/config/backup_config/models.py +6 -0
- {declarative_data_backup-0.1.0 → declarative_data_backup-0.2.1}/src/declarative_data_backup/config/backup_config/utils.py +50 -9
- {declarative_data_backup-0.1.0 → declarative_data_backup-0.2.1}/src/declarative_data_backup/config/backup_config/yaml_parser.py +23 -3
- {declarative_data_backup-0.1.0 → declarative_data_backup-0.2.1}/src/declarative_data_backup.egg-info/PKG-INFO +42 -20
- declarative_data_backup-0.2.1/src/declarative_data_backup.egg-info/requires.txt +1 -0
- {declarative_data_backup-0.1.0 → declarative_data_backup-0.2.1}/tests/test_backup.py +38 -3
- declarative_data_backup-0.1.0/src/declarative_data_backup.egg-info/requires.txt +0 -7
- {declarative_data_backup-0.1.0 → declarative_data_backup-0.2.1}/setup.cfg +0 -0
- {declarative_data_backup-0.1.0 → declarative_data_backup-0.2.1}/src/declarative_data_backup/__init__.py +0 -0
- {declarative_data_backup-0.1.0 → declarative_data_backup-0.2.1}/src/declarative_data_backup/__main__.py +0 -0
- {declarative_data_backup-0.1.0 → declarative_data_backup-0.2.1}/src/declarative_data_backup/borg/__init__.py +0 -0
- {declarative_data_backup-0.1.0 → declarative_data_backup-0.2.1}/src/declarative_data_backup/borg/archive_naming.py +0 -0
- {declarative_data_backup-0.1.0 → declarative_data_backup-0.2.1}/src/declarative_data_backup/borg/borg_cli.py +0 -0
- {declarative_data_backup-0.1.0 → declarative_data_backup-0.2.1}/src/declarative_data_backup/config/__init__.py +0 -0
- {declarative_data_backup-0.1.0 → declarative_data_backup-0.2.1}/src/declarative_data_backup/config/backup_config/__init__.py +0 -0
- {declarative_data_backup-0.1.0 → declarative_data_backup-0.2.1}/src/declarative_data_backup/config/backup_config/yaml_raw_loader.py +0 -0
- {declarative_data_backup-0.1.0 → declarative_data_backup-0.2.1}/src/declarative_data_backup/config/logging.py +0 -0
- {declarative_data_backup-0.1.0 → declarative_data_backup-0.2.1}/src/declarative_data_backup/environment.py +0 -0
- {declarative_data_backup-0.1.0 → declarative_data_backup-0.2.1}/src/declarative_data_backup/errors.py +0 -0
- {declarative_data_backup-0.1.0 → declarative_data_backup-0.2.1}/src/declarative_data_backup.egg-info/SOURCES.txt +0 -0
- {declarative_data_backup-0.1.0 → declarative_data_backup-0.2.1}/src/declarative_data_backup.egg-info/dependency_links.txt +0 -0
- {declarative_data_backup-0.1.0 → declarative_data_backup-0.2.1}/src/declarative_data_backup.egg-info/top_level.txt +0 -0
|
@@ -1,15 +1,10 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: declarative-data-backup
|
|
3
|
-
Version: 0.1
|
|
3
|
+
Version: 0.2.1
|
|
4
4
|
Summary: Declarative Borg-based data backups
|
|
5
5
|
Requires-Python: >=3.11
|
|
6
6
|
Description-Content-Type: text/markdown
|
|
7
7
|
Requires-Dist: PyYAML<7,>=6
|
|
8
|
-
Provides-Extra: dev
|
|
9
|
-
Requires-Dist: pre-commit<5,>=4; extra == "dev"
|
|
10
|
-
Requires-Dist: pyright<2,>=1.1.400; extra == "dev"
|
|
11
|
-
Requires-Dist: pytest<10,>=8; extra == "dev"
|
|
12
|
-
Requires-Dist: ruff<1,>=0.11; extra == "dev"
|
|
13
8
|
|
|
14
9
|
# Declarative Data Backup
|
|
15
10
|
|
|
@@ -31,18 +26,20 @@ The application never initializes a repository. Borg 2 is not supported.
|
|
|
31
26
|
|
|
32
27
|
## Installation
|
|
33
28
|
|
|
34
|
-
Install the package from PyPI:
|
|
29
|
+
Install the package from PyPI into a `uv`-managed environment:
|
|
35
30
|
|
|
36
31
|
```shell
|
|
37
|
-
|
|
32
|
+
uv venv
|
|
33
|
+
uv pip install declarative-data-backup
|
|
38
34
|
```
|
|
39
35
|
|
|
40
|
-
|
|
36
|
+
For a checkout, install the development dependencies with:
|
|
41
37
|
|
|
42
38
|
```shell
|
|
43
|
-
|
|
39
|
+
make install
|
|
44
40
|
```
|
|
45
41
|
|
|
42
|
+
To also install the repository's pre-commit hooks, run `make setup`.
|
|
46
43
|
The PyPI distribution is named `declarative-data-backup`; its importable
|
|
47
44
|
package is `declarative_data_backup`.
|
|
48
45
|
|
|
@@ -56,8 +53,8 @@ release pipeline.
|
|
|
56
53
|
|
|
57
54
|
For each version tag, the pipeline:
|
|
58
55
|
|
|
59
|
-
1. Sets the package version in the CI checkout's `pyproject.toml
|
|
60
|
-
|
|
56
|
+
1. Sets the package version in the CI checkout's `pyproject.toml` and uses
|
|
57
|
+
`uv build` to create a wheel and source distribution.
|
|
61
58
|
2. Uploads `declarative-data-backup` to PyPI.
|
|
62
59
|
3. Builds the existing Python 3.13 Dockerfile with the same package version,
|
|
63
60
|
verifies the installed version, and pushes
|
|
@@ -120,6 +117,18 @@ postgres_backup:
|
|
|
120
117
|
- /server_root/path/to/postgres-backups
|
|
121
118
|
required_mounts:
|
|
122
119
|
- /server_root/path/to/postgres-backups
|
|
120
|
+
|
|
121
|
+
apprise_backup:
|
|
122
|
+
files:
|
|
123
|
+
- /server_root/data/apprise/apprise.yml
|
|
124
|
+
|
|
125
|
+
jellyfin_backup:
|
|
126
|
+
folders:
|
|
127
|
+
- /server_root/data/jellyfin
|
|
128
|
+
exclude:
|
|
129
|
+
- "**/cache/**"
|
|
130
|
+
- "**/log/**"
|
|
131
|
+
- "**/transcodes/**"
|
|
123
132
|
```
|
|
124
133
|
|
|
125
134
|
Set the required environment variables before starting the process:
|
|
@@ -159,14 +168,22 @@ the run fails and pruning is skipped.
|
|
|
159
168
|
The top level must be a nonempty YAML mapping of backup group names to group
|
|
160
169
|
mappings. Group names may contain ASCII letters, digits, `_`, and `-`, and
|
|
161
170
|
must be at most 64 characters. Duplicate YAML keys, unknown group fields,
|
|
162
|
-
invalid
|
|
163
|
-
create or prune operation.
|
|
171
|
+
invalid types, unsupported pattern selectors, and empty configuration are
|
|
172
|
+
rejected before any create or prune operation.
|
|
164
173
|
|
|
165
174
|
Each group supports:
|
|
166
175
|
|
|
167
|
-
- `folders` (
|
|
168
|
-
|
|
169
|
-
|
|
176
|
+
- `folders` (optional): list of absolute directory paths.
|
|
177
|
+
- `files` (optional): list of absolute regular-file paths. Folders and files
|
|
178
|
+
share the same containment, readability, duplicate, and overlap checks;
|
|
179
|
+
configure at least one source per group.
|
|
180
|
+
- `exclude` (optional): list of Borg 1.4 glob patterns, scoped to this group
|
|
181
|
+
and passed directly to Borg for folder sources. To avoid approximating
|
|
182
|
+
Borg's pattern matching, a group with explicit `files` cannot also configure
|
|
183
|
+
`exclude`; this is rejected before any Borg operation.
|
|
184
|
+
Patterns are otherwise passed unchanged to Borg, which evaluates their
|
|
185
|
+
matching semantics. If Borg rejects a pattern during create, the run stops
|
|
186
|
+
before pruning any source.
|
|
170
187
|
- `compression` (optional): defaults to `auto,zstd,22`. Supported Borg 1.4
|
|
171
188
|
codecs include `none`, `lz4`, `zstd`, `zlib`, and `lzma`; `auto` may precede
|
|
172
189
|
a codec. Codec levels are checked against supported ranges: zstd `1`-`22`,
|
|
@@ -187,6 +204,11 @@ does not prove that nested host filesystems are mounted. In particular,
|
|
|
187
204
|
`allow_empty: true` does not prove that a disk is present: an empty source may
|
|
188
205
|
be legitimate and can be archived.
|
|
189
206
|
|
|
207
|
+
Backing up a live SQLite database with `files` or `folders` does not produce a
|
|
208
|
+
consistent database snapshot. Use an application-supported export, backup API,
|
|
209
|
+
or coordinated filesystem snapshot for live database data. Exclusions do not
|
|
210
|
+
make a database backup consistent.
|
|
211
|
+
|
|
190
212
|
Paths are resolved before backup. A top-level source symlink is allowed only
|
|
191
213
|
when its canonical target remains inside `SERVER_ROOT`; symlinks within a
|
|
192
214
|
source are stored as links rather than followed.
|
|
@@ -270,13 +292,13 @@ container integration test.
|
|
|
270
292
|
Run the unit tests:
|
|
271
293
|
|
|
272
294
|
```shell
|
|
273
|
-
|
|
295
|
+
uv run pytest -m "not integration"
|
|
274
296
|
```
|
|
275
297
|
|
|
276
298
|
The opt-in local Borg integration test uses a temporary encrypted repository:
|
|
277
299
|
|
|
278
300
|
```shell
|
|
279
|
-
RUN_BORG_LOCAL_INTEGRATION=1
|
|
301
|
+
RUN_BORG_LOCAL_INTEGRATION=1 uv run pytest -m integration \
|
|
280
302
|
tests/integration/test_local_borg_integration.py
|
|
281
303
|
```
|
|
282
304
|
|
|
@@ -284,7 +306,7 @@ The opt-in Docker SSH integration test uses disposable containers and a
|
|
|
284
306
|
temporary repository; it does not use production credentials or BorgBase:
|
|
285
307
|
|
|
286
308
|
```shell
|
|
287
|
-
RUN_BORG_DOCKER_INTEGRATION=1
|
|
309
|
+
RUN_BORG_DOCKER_INTEGRATION=1 uv run pytest -m integration \
|
|
288
310
|
tests/integration/test_docker_integration.py
|
|
289
311
|
```
|
|
290
312
|
|
|
@@ -18,18 +18,20 @@ The application never initializes a repository. Borg 2 is not supported.
|
|
|
18
18
|
|
|
19
19
|
## Installation
|
|
20
20
|
|
|
21
|
-
Install the package from PyPI:
|
|
21
|
+
Install the package from PyPI into a `uv`-managed environment:
|
|
22
22
|
|
|
23
23
|
```shell
|
|
24
|
-
|
|
24
|
+
uv venv
|
|
25
|
+
uv pip install declarative-data-backup
|
|
25
26
|
```
|
|
26
27
|
|
|
27
|
-
|
|
28
|
+
For a checkout, install the development dependencies with:
|
|
28
29
|
|
|
29
30
|
```shell
|
|
30
|
-
|
|
31
|
+
make install
|
|
31
32
|
```
|
|
32
33
|
|
|
34
|
+
To also install the repository's pre-commit hooks, run `make setup`.
|
|
33
35
|
The PyPI distribution is named `declarative-data-backup`; its importable
|
|
34
36
|
package is `declarative_data_backup`.
|
|
35
37
|
|
|
@@ -43,8 +45,8 @@ release pipeline.
|
|
|
43
45
|
|
|
44
46
|
For each version tag, the pipeline:
|
|
45
47
|
|
|
46
|
-
1. Sets the package version in the CI checkout's `pyproject.toml
|
|
47
|
-
|
|
48
|
+
1. Sets the package version in the CI checkout's `pyproject.toml` and uses
|
|
49
|
+
`uv build` to create a wheel and source distribution.
|
|
48
50
|
2. Uploads `declarative-data-backup` to PyPI.
|
|
49
51
|
3. Builds the existing Python 3.13 Dockerfile with the same package version,
|
|
50
52
|
verifies the installed version, and pushes
|
|
@@ -107,6 +109,18 @@ postgres_backup:
|
|
|
107
109
|
- /server_root/path/to/postgres-backups
|
|
108
110
|
required_mounts:
|
|
109
111
|
- /server_root/path/to/postgres-backups
|
|
112
|
+
|
|
113
|
+
apprise_backup:
|
|
114
|
+
files:
|
|
115
|
+
- /server_root/data/apprise/apprise.yml
|
|
116
|
+
|
|
117
|
+
jellyfin_backup:
|
|
118
|
+
folders:
|
|
119
|
+
- /server_root/data/jellyfin
|
|
120
|
+
exclude:
|
|
121
|
+
- "**/cache/**"
|
|
122
|
+
- "**/log/**"
|
|
123
|
+
- "**/transcodes/**"
|
|
110
124
|
```
|
|
111
125
|
|
|
112
126
|
Set the required environment variables before starting the process:
|
|
@@ -146,14 +160,22 @@ the run fails and pruning is skipped.
|
|
|
146
160
|
The top level must be a nonempty YAML mapping of backup group names to group
|
|
147
161
|
mappings. Group names may contain ASCII letters, digits, `_`, and `-`, and
|
|
148
162
|
must be at most 64 characters. Duplicate YAML keys, unknown group fields,
|
|
149
|
-
invalid
|
|
150
|
-
create or prune operation.
|
|
163
|
+
invalid types, unsupported pattern selectors, and empty configuration are
|
|
164
|
+
rejected before any create or prune operation.
|
|
151
165
|
|
|
152
166
|
Each group supports:
|
|
153
167
|
|
|
154
|
-
- `folders` (
|
|
155
|
-
|
|
156
|
-
|
|
168
|
+
- `folders` (optional): list of absolute directory paths.
|
|
169
|
+
- `files` (optional): list of absolute regular-file paths. Folders and files
|
|
170
|
+
share the same containment, readability, duplicate, and overlap checks;
|
|
171
|
+
configure at least one source per group.
|
|
172
|
+
- `exclude` (optional): list of Borg 1.4 glob patterns, scoped to this group
|
|
173
|
+
and passed directly to Borg for folder sources. To avoid approximating
|
|
174
|
+
Borg's pattern matching, a group with explicit `files` cannot also configure
|
|
175
|
+
`exclude`; this is rejected before any Borg operation.
|
|
176
|
+
Patterns are otherwise passed unchanged to Borg, which evaluates their
|
|
177
|
+
matching semantics. If Borg rejects a pattern during create, the run stops
|
|
178
|
+
before pruning any source.
|
|
157
179
|
- `compression` (optional): defaults to `auto,zstd,22`. Supported Borg 1.4
|
|
158
180
|
codecs include `none`, `lz4`, `zstd`, `zlib`, and `lzma`; `auto` may precede
|
|
159
181
|
a codec. Codec levels are checked against supported ranges: zstd `1`-`22`,
|
|
@@ -174,6 +196,11 @@ does not prove that nested host filesystems are mounted. In particular,
|
|
|
174
196
|
`allow_empty: true` does not prove that a disk is present: an empty source may
|
|
175
197
|
be legitimate and can be archived.
|
|
176
198
|
|
|
199
|
+
Backing up a live SQLite database with `files` or `folders` does not produce a
|
|
200
|
+
consistent database snapshot. Use an application-supported export, backup API,
|
|
201
|
+
or coordinated filesystem snapshot for live database data. Exclusions do not
|
|
202
|
+
make a database backup consistent.
|
|
203
|
+
|
|
177
204
|
Paths are resolved before backup. A top-level source symlink is allowed only
|
|
178
205
|
when its canonical target remains inside `SERVER_ROOT`; symlinks within a
|
|
179
206
|
source are stored as links rather than followed.
|
|
@@ -257,13 +284,13 @@ container integration test.
|
|
|
257
284
|
Run the unit tests:
|
|
258
285
|
|
|
259
286
|
```shell
|
|
260
|
-
|
|
287
|
+
uv run pytest -m "not integration"
|
|
261
288
|
```
|
|
262
289
|
|
|
263
290
|
The opt-in local Borg integration test uses a temporary encrypted repository:
|
|
264
291
|
|
|
265
292
|
```shell
|
|
266
|
-
RUN_BORG_LOCAL_INTEGRATION=1
|
|
293
|
+
RUN_BORG_LOCAL_INTEGRATION=1 uv run pytest -m integration \
|
|
267
294
|
tests/integration/test_local_borg_integration.py
|
|
268
295
|
```
|
|
269
296
|
|
|
@@ -271,7 +298,7 @@ The opt-in Docker SSH integration test uses disposable containers and a
|
|
|
271
298
|
temporary repository; it does not use production credentials or BorgBase:
|
|
272
299
|
|
|
273
300
|
```shell
|
|
274
|
-
RUN_BORG_DOCKER_INTEGRATION=1
|
|
301
|
+
RUN_BORG_DOCKER_INTEGRATION=1 uv run pytest -m integration \
|
|
275
302
|
tests/integration/test_docker_integration.py
|
|
276
303
|
```
|
|
277
304
|
|
|
@@ -4,13 +4,13 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "declarative-data-backup"
|
|
7
|
-
version = "0.1
|
|
7
|
+
version = "0.2.1"
|
|
8
8
|
description = "Declarative Borg-based data backups"
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.11"
|
|
11
11
|
dependencies = ["PyYAML>=6,<7"]
|
|
12
12
|
|
|
13
|
-
[
|
|
13
|
+
[dependency-groups]
|
|
14
14
|
dev = [
|
|
15
15
|
"pre-commit>=4,<5",
|
|
16
16
|
"pyright>=1.1.400,<2",
|
|
@@ -44,11 +44,11 @@ def run_backup(
|
|
|
44
44
|
settings: Environment,
|
|
45
45
|
definitions: list[BackupDefinition],
|
|
46
46
|
) -> None:
|
|
47
|
-
|
|
47
|
+
total_sources = sum(len(definition.sources) for definition in definitions)
|
|
48
48
|
logger.info(
|
|
49
|
-
"Validated %d backup groups containing %d
|
|
49
|
+
"Validated %d backup groups containing %d sources",
|
|
50
50
|
len(definitions),
|
|
51
|
-
|
|
51
|
+
total_sources,
|
|
52
52
|
)
|
|
53
53
|
|
|
54
54
|
run_id = uuid.uuid4().hex
|
|
@@ -56,11 +56,11 @@ def run_backup(
|
|
|
56
56
|
created_at = datetime.now(UTC)
|
|
57
57
|
for definition in definitions:
|
|
58
58
|
logger.info(
|
|
59
|
-
"Starting backup group '%s' (%d
|
|
59
|
+
"Starting backup group '%s' (%d sources).",
|
|
60
60
|
definition.name,
|
|
61
|
-
len(definition.
|
|
61
|
+
len(definition.sources),
|
|
62
62
|
)
|
|
63
|
-
for folder in definition.
|
|
63
|
+
for folder in definition.sources:
|
|
64
64
|
backup_folder(
|
|
65
65
|
folder,
|
|
66
66
|
definition.compression,
|
|
@@ -68,10 +68,11 @@ def run_backup(
|
|
|
68
68
|
settings,
|
|
69
69
|
created_at,
|
|
70
70
|
run_id,
|
|
71
|
+
definition.excludes,
|
|
71
72
|
)
|
|
72
73
|
|
|
73
74
|
for definition in definitions:
|
|
74
|
-
for folder in definition.
|
|
75
|
+
for folder in definition.sources:
|
|
75
76
|
prune(
|
|
76
77
|
folder,
|
|
77
78
|
definition.retention_days,
|
|
@@ -29,6 +29,7 @@ def backup_folder(
|
|
|
29
29
|
settings: Environment,
|
|
30
30
|
created_at: datetime,
|
|
31
31
|
run_id: str,
|
|
32
|
+
excludes: tuple[str, ...] = (),
|
|
32
33
|
) -> None:
|
|
33
34
|
name = archive_name_escaper(
|
|
34
35
|
folder,
|
|
@@ -38,17 +39,19 @@ def backup_folder(
|
|
|
38
39
|
context = f"group={group!r} source={folder}"
|
|
39
40
|
started_at = time.monotonic()
|
|
40
41
|
logger.info("Starting backup for %s as archive %s.", context, name)
|
|
42
|
+
args = [
|
|
43
|
+
"create",
|
|
44
|
+
"--verbose",
|
|
45
|
+
"--stats",
|
|
46
|
+
"--compression",
|
|
47
|
+
compression,
|
|
48
|
+
]
|
|
49
|
+
for pattern in excludes:
|
|
50
|
+
args.extend(("--exclude", pattern))
|
|
51
|
+
args.extend((f"{settings.borg_repo}::{name}", str(folder)))
|
|
41
52
|
try:
|
|
42
53
|
run_borg(
|
|
43
|
-
|
|
44
|
-
"create",
|
|
45
|
-
"--verbose",
|
|
46
|
-
"--stats",
|
|
47
|
-
"--compression",
|
|
48
|
-
compression,
|
|
49
|
-
f"{settings.borg_repo}::{name}",
|
|
50
|
-
str(folder),
|
|
51
|
-
],
|
|
54
|
+
args,
|
|
52
55
|
settings,
|
|
53
56
|
"create",
|
|
54
57
|
context,
|
|
@@ -10,3 +10,9 @@ class BackupDefinition:
|
|
|
10
10
|
retention_days: int
|
|
11
11
|
required_mounts: tuple[Path, ...] = ()
|
|
12
12
|
allow_empty: bool = True
|
|
13
|
+
files: tuple[Path, ...] = ()
|
|
14
|
+
excludes: tuple[str, ...] = ()
|
|
15
|
+
|
|
16
|
+
@property
|
|
17
|
+
def sources(self) -> tuple[Path, ...]:
|
|
18
|
+
return self.folders + self.files
|
|
@@ -37,8 +37,8 @@ def parse_folders(
|
|
|
37
37
|
group: str,
|
|
38
38
|
allow_empty: bool,
|
|
39
39
|
) -> tuple[Path, ...]:
|
|
40
|
-
if not isinstance(raw_folders, list)
|
|
41
|
-
fail(f"'{group}.folders' must be a
|
|
40
|
+
if not isinstance(raw_folders, list):
|
|
41
|
+
fail(f"'{group}.folders' must be a YAML list.")
|
|
42
42
|
|
|
43
43
|
folders = []
|
|
44
44
|
for raw_folder in raw_folders:
|
|
@@ -60,6 +60,47 @@ def parse_folders(
|
|
|
60
60
|
return tuple(folders)
|
|
61
61
|
|
|
62
62
|
|
|
63
|
+
def parse_files(
|
|
64
|
+
raw_files: Any,
|
|
65
|
+
server_root: Path,
|
|
66
|
+
group: str,
|
|
67
|
+
) -> tuple[Path, ...]:
|
|
68
|
+
if not isinstance(raw_files, list):
|
|
69
|
+
fail(f"'{group}.files' must be a YAML list.")
|
|
70
|
+
|
|
71
|
+
files = []
|
|
72
|
+
for raw_file in raw_files:
|
|
73
|
+
file = resolve_path(raw_file, server_root, f"'{group}.files' entry")
|
|
74
|
+
if not file.is_file():
|
|
75
|
+
fail(f"Backup source is not a regular file: {file}.")
|
|
76
|
+
if not os.access(file, os.R_OK):
|
|
77
|
+
fail(f"Backup source is not readable: {file}.")
|
|
78
|
+
files.append(file)
|
|
79
|
+
return tuple(files)
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def parse_excludes(raw_excludes: Any, group: str) -> tuple[str, ...]:
|
|
83
|
+
if not isinstance(raw_excludes, list):
|
|
84
|
+
fail(f"'{group}.exclude' must be a YAML list.")
|
|
85
|
+
|
|
86
|
+
excludes = []
|
|
87
|
+
for pattern in raw_excludes:
|
|
88
|
+
if not isinstance(pattern, str) or not pattern.strip() or "\0" in pattern:
|
|
89
|
+
fail(f"'{group}.exclude' entries must be nonempty Borg glob patterns.")
|
|
90
|
+
|
|
91
|
+
pattern_type, separator, pattern_body = pattern.partition(":")
|
|
92
|
+
if separator and len(pattern_type) == 2:
|
|
93
|
+
if pattern_type not in {"fm", "sh", "pf", "pp"}:
|
|
94
|
+
fail(
|
|
95
|
+
f"'{group}.exclude' entries must use Borg glob patterns "
|
|
96
|
+
"(fm:, sh:, pf:, or pp:)."
|
|
97
|
+
)
|
|
98
|
+
if not pattern_body:
|
|
99
|
+
fail(f"'{group}.exclude' entries must be nonempty Borg patterns.")
|
|
100
|
+
excludes.append(pattern)
|
|
101
|
+
return tuple(excludes)
|
|
102
|
+
|
|
103
|
+
|
|
63
104
|
def parse_required_mounts(
|
|
64
105
|
raw_mounts: Any,
|
|
65
106
|
server_root: Path,
|
|
@@ -80,23 +121,23 @@ def parse_required_mounts(
|
|
|
80
121
|
return mounts
|
|
81
122
|
|
|
82
123
|
|
|
83
|
-
def
|
|
124
|
+
def check_sources_for_overlappings(definitions: list[BackupDefinition]) -> None:
|
|
84
125
|
owners: dict[Path, str] = {}
|
|
85
126
|
for definition in definitions:
|
|
86
|
-
for
|
|
127
|
+
for source in definition.sources:
|
|
87
128
|
for existing, owner in owners.items():
|
|
88
|
-
if
|
|
129
|
+
if source == existing:
|
|
89
130
|
fail(
|
|
90
|
-
f"Duplicate canonical backup source {
|
|
131
|
+
f"Duplicate canonical backup source {source} "
|
|
91
132
|
f"(already configured by '{owner}')."
|
|
92
133
|
)
|
|
93
|
-
if
|
|
134
|
+
if source in existing.parents or existing in source.parents:
|
|
94
135
|
fail(
|
|
95
136
|
f"Overlapping backup sources are not allowed: "
|
|
96
137
|
f"'{owner}' uses {existing}, while "
|
|
97
|
-
f"'{definition.name}' uses {
|
|
138
|
+
f"'{definition.name}' uses {source}."
|
|
98
139
|
)
|
|
99
|
-
owners[
|
|
140
|
+
owners[source] = definition.name
|
|
100
141
|
|
|
101
142
|
|
|
102
143
|
def _mountinfo_path(raw_mount: str) -> Path:
|
|
@@ -6,8 +6,10 @@ from ...environment import DEFAULT_COMPRESSION, DEFAULT_RETENTION_DAYS
|
|
|
6
6
|
from ...errors import fail
|
|
7
7
|
from .models import BackupDefinition
|
|
8
8
|
from .utils import (
|
|
9
|
-
check_folders_for_overlappings,
|
|
10
9
|
check_required_mounts_are_mounted,
|
|
10
|
+
check_sources_for_overlappings,
|
|
11
|
+
parse_excludes,
|
|
12
|
+
parse_files,
|
|
11
13
|
parse_folders,
|
|
12
14
|
parse_required_mounts,
|
|
13
15
|
)
|
|
@@ -15,12 +17,17 @@ from .utils import (
|
|
|
15
17
|
_GROUP_NAME = re.compile(r"[A-Za-z0-9][A-Za-z0-9_-]{0,63}\Z")
|
|
16
18
|
_GROUP_KEYS = {
|
|
17
19
|
"folders",
|
|
20
|
+
"files",
|
|
21
|
+
"exclude",
|
|
18
22
|
"compression",
|
|
19
23
|
"retention_days",
|
|
20
24
|
"required_mounts",
|
|
21
25
|
"allow_empty",
|
|
22
26
|
}
|
|
23
27
|
_GROUP_DEFAULTS = {
|
|
28
|
+
"folders": [],
|
|
29
|
+
"files": [],
|
|
30
|
+
"exclude": [],
|
|
24
31
|
"compression": DEFAULT_COMPRESSION,
|
|
25
32
|
"retention_days": DEFAULT_RETENTION_DAYS,
|
|
26
33
|
"required_mounts": [],
|
|
@@ -102,9 +109,20 @@ def _parse_definition(name: Any, raw: Any, server_root: Path) -> BackupDefinitio
|
|
|
102
109
|
if not isinstance(allow_empty, bool):
|
|
103
110
|
fail(f"'{name}.allow_empty' must be a boolean.")
|
|
104
111
|
|
|
112
|
+
folders = parse_folders(group["folders"], server_root, name, allow_empty)
|
|
113
|
+
files = parse_files(group["files"], server_root, name)
|
|
114
|
+
excludes = parse_excludes(group["exclude"], name)
|
|
115
|
+
if not folders and not files:
|
|
116
|
+
fail(f"'{name}' must configure at least one folder or file source.")
|
|
117
|
+
if files and excludes:
|
|
118
|
+
fail(
|
|
119
|
+
f"'{name}' cannot combine 'files' and 'exclude': Borg exclusion "
|
|
120
|
+
"of an explicitly selected file cannot be verified safely."
|
|
121
|
+
)
|
|
122
|
+
|
|
105
123
|
return BackupDefinition(
|
|
106
124
|
name=name,
|
|
107
|
-
folders=
|
|
125
|
+
folders=folders,
|
|
108
126
|
compression=compression,
|
|
109
127
|
retention_days=retention_days,
|
|
110
128
|
required_mounts=parse_required_mounts(
|
|
@@ -113,6 +131,8 @@ def _parse_definition(name: Any, raw: Any, server_root: Path) -> BackupDefinitio
|
|
|
113
131
|
name,
|
|
114
132
|
),
|
|
115
133
|
allow_empty=allow_empty,
|
|
134
|
+
files=files,
|
|
135
|
+
excludes=excludes,
|
|
116
136
|
)
|
|
117
137
|
|
|
118
138
|
|
|
@@ -131,6 +151,6 @@ def parse_backup_definitions(
|
|
|
131
151
|
definitions = [
|
|
132
152
|
_parse_definition(name, raw, server_root) for name, raw in data.items()
|
|
133
153
|
]
|
|
134
|
-
|
|
154
|
+
check_sources_for_overlappings(definitions)
|
|
135
155
|
check_required_mounts_are_mounted(definitions, mountinfo_path)
|
|
136
156
|
return definitions
|
|
@@ -1,15 +1,10 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: declarative-data-backup
|
|
3
|
-
Version: 0.1
|
|
3
|
+
Version: 0.2.1
|
|
4
4
|
Summary: Declarative Borg-based data backups
|
|
5
5
|
Requires-Python: >=3.11
|
|
6
6
|
Description-Content-Type: text/markdown
|
|
7
7
|
Requires-Dist: PyYAML<7,>=6
|
|
8
|
-
Provides-Extra: dev
|
|
9
|
-
Requires-Dist: pre-commit<5,>=4; extra == "dev"
|
|
10
|
-
Requires-Dist: pyright<2,>=1.1.400; extra == "dev"
|
|
11
|
-
Requires-Dist: pytest<10,>=8; extra == "dev"
|
|
12
|
-
Requires-Dist: ruff<1,>=0.11; extra == "dev"
|
|
13
8
|
|
|
14
9
|
# Declarative Data Backup
|
|
15
10
|
|
|
@@ -31,18 +26,20 @@ The application never initializes a repository. Borg 2 is not supported.
|
|
|
31
26
|
|
|
32
27
|
## Installation
|
|
33
28
|
|
|
34
|
-
Install the package from PyPI:
|
|
29
|
+
Install the package from PyPI into a `uv`-managed environment:
|
|
35
30
|
|
|
36
31
|
```shell
|
|
37
|
-
|
|
32
|
+
uv venv
|
|
33
|
+
uv pip install declarative-data-backup
|
|
38
34
|
```
|
|
39
35
|
|
|
40
|
-
|
|
36
|
+
For a checkout, install the development dependencies with:
|
|
41
37
|
|
|
42
38
|
```shell
|
|
43
|
-
|
|
39
|
+
make install
|
|
44
40
|
```
|
|
45
41
|
|
|
42
|
+
To also install the repository's pre-commit hooks, run `make setup`.
|
|
46
43
|
The PyPI distribution is named `declarative-data-backup`; its importable
|
|
47
44
|
package is `declarative_data_backup`.
|
|
48
45
|
|
|
@@ -56,8 +53,8 @@ release pipeline.
|
|
|
56
53
|
|
|
57
54
|
For each version tag, the pipeline:
|
|
58
55
|
|
|
59
|
-
1. Sets the package version in the CI checkout's `pyproject.toml
|
|
60
|
-
|
|
56
|
+
1. Sets the package version in the CI checkout's `pyproject.toml` and uses
|
|
57
|
+
`uv build` to create a wheel and source distribution.
|
|
61
58
|
2. Uploads `declarative-data-backup` to PyPI.
|
|
62
59
|
3. Builds the existing Python 3.13 Dockerfile with the same package version,
|
|
63
60
|
verifies the installed version, and pushes
|
|
@@ -120,6 +117,18 @@ postgres_backup:
|
|
|
120
117
|
- /server_root/path/to/postgres-backups
|
|
121
118
|
required_mounts:
|
|
122
119
|
- /server_root/path/to/postgres-backups
|
|
120
|
+
|
|
121
|
+
apprise_backup:
|
|
122
|
+
files:
|
|
123
|
+
- /server_root/data/apprise/apprise.yml
|
|
124
|
+
|
|
125
|
+
jellyfin_backup:
|
|
126
|
+
folders:
|
|
127
|
+
- /server_root/data/jellyfin
|
|
128
|
+
exclude:
|
|
129
|
+
- "**/cache/**"
|
|
130
|
+
- "**/log/**"
|
|
131
|
+
- "**/transcodes/**"
|
|
123
132
|
```
|
|
124
133
|
|
|
125
134
|
Set the required environment variables before starting the process:
|
|
@@ -159,14 +168,22 @@ the run fails and pruning is skipped.
|
|
|
159
168
|
The top level must be a nonempty YAML mapping of backup group names to group
|
|
160
169
|
mappings. Group names may contain ASCII letters, digits, `_`, and `-`, and
|
|
161
170
|
must be at most 64 characters. Duplicate YAML keys, unknown group fields,
|
|
162
|
-
invalid
|
|
163
|
-
create or prune operation.
|
|
171
|
+
invalid types, unsupported pattern selectors, and empty configuration are
|
|
172
|
+
rejected before any create or prune operation.
|
|
164
173
|
|
|
165
174
|
Each group supports:
|
|
166
175
|
|
|
167
|
-
- `folders` (
|
|
168
|
-
|
|
169
|
-
|
|
176
|
+
- `folders` (optional): list of absolute directory paths.
|
|
177
|
+
- `files` (optional): list of absolute regular-file paths. Folders and files
|
|
178
|
+
share the same containment, readability, duplicate, and overlap checks;
|
|
179
|
+
configure at least one source per group.
|
|
180
|
+
- `exclude` (optional): list of Borg 1.4 glob patterns, scoped to this group
|
|
181
|
+
and passed directly to Borg for folder sources. To avoid approximating
|
|
182
|
+
Borg's pattern matching, a group with explicit `files` cannot also configure
|
|
183
|
+
`exclude`; this is rejected before any Borg operation.
|
|
184
|
+
Patterns are otherwise passed unchanged to Borg, which evaluates their
|
|
185
|
+
matching semantics. If Borg rejects a pattern during create, the run stops
|
|
186
|
+
before pruning any source.
|
|
170
187
|
- `compression` (optional): defaults to `auto,zstd,22`. Supported Borg 1.4
|
|
171
188
|
codecs include `none`, `lz4`, `zstd`, `zlib`, and `lzma`; `auto` may precede
|
|
172
189
|
a codec. Codec levels are checked against supported ranges: zstd `1`-`22`,
|
|
@@ -187,6 +204,11 @@ does not prove that nested host filesystems are mounted. In particular,
|
|
|
187
204
|
`allow_empty: true` does not prove that a disk is present: an empty source may
|
|
188
205
|
be legitimate and can be archived.
|
|
189
206
|
|
|
207
|
+
Backing up a live SQLite database with `files` or `folders` does not produce a
|
|
208
|
+
consistent database snapshot. Use an application-supported export, backup API,
|
|
209
|
+
or coordinated filesystem snapshot for live database data. Exclusions do not
|
|
210
|
+
make a database backup consistent.
|
|
211
|
+
|
|
190
212
|
Paths are resolved before backup. A top-level source symlink is allowed only
|
|
191
213
|
when its canonical target remains inside `SERVER_ROOT`; symlinks within a
|
|
192
214
|
source are stored as links rather than followed.
|
|
@@ -270,13 +292,13 @@ container integration test.
|
|
|
270
292
|
Run the unit tests:
|
|
271
293
|
|
|
272
294
|
```shell
|
|
273
|
-
|
|
295
|
+
uv run pytest -m "not integration"
|
|
274
296
|
```
|
|
275
297
|
|
|
276
298
|
The opt-in local Borg integration test uses a temporary encrypted repository:
|
|
277
299
|
|
|
278
300
|
```shell
|
|
279
|
-
RUN_BORG_LOCAL_INTEGRATION=1
|
|
301
|
+
RUN_BORG_LOCAL_INTEGRATION=1 uv run pytest -m integration \
|
|
280
302
|
tests/integration/test_local_borg_integration.py
|
|
281
303
|
```
|
|
282
304
|
|
|
@@ -284,7 +306,7 @@ The opt-in Docker SSH integration test uses disposable containers and a
|
|
|
284
306
|
temporary repository; it does not use production credentials or BorgBase:
|
|
285
307
|
|
|
286
308
|
```shell
|
|
287
|
-
RUN_BORG_DOCKER_INTEGRATION=1
|
|
309
|
+
RUN_BORG_DOCKER_INTEGRATION=1 uv run pytest -m integration \
|
|
288
310
|
tests/integration/test_docker_integration.py
|
|
289
311
|
```
|
|
290
312
|
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
PyYAML<7,>=6
|
|
@@ -48,8 +48,8 @@ def test_main_runs_all_creates_before_any_prune(
|
|
|
48
48
|
monkeypatch.setattr(
|
|
49
49
|
backup,
|
|
50
50
|
"backup_folder",
|
|
51
|
-
lambda folder, compression, group, settings, created_at, run_id:
|
|
52
|
-
("create", group, folder, compression, run_id)
|
|
51
|
+
lambda folder, compression, group, settings, created_at, run_id, excludes: (
|
|
52
|
+
events.append(("create", group, folder, compression, run_id, excludes))
|
|
53
53
|
),
|
|
54
54
|
)
|
|
55
55
|
monkeypatch.setattr(
|
|
@@ -153,7 +153,7 @@ def test_retry_after_partial_create_uses_a_new_run_id_before_pruning(
|
|
|
153
153
|
prunes = []
|
|
154
154
|
fail_second = True
|
|
155
155
|
|
|
156
|
-
def create(folder, compression, group, settings, created_at, run_id):
|
|
156
|
+
def create(folder, compression, group, settings, created_at, run_id, excludes):
|
|
157
157
|
nonlocal fail_second
|
|
158
158
|
creates.append((folder, run_id))
|
|
159
159
|
if fail_second and folder == second:
|
|
@@ -268,6 +268,41 @@ def test_main_prune_failure_is_nonzero_and_scoped(
|
|
|
268
268
|
]
|
|
269
269
|
|
|
270
270
|
|
|
271
|
+
def test_exclusions_are_passed_only_to_their_backup_group(
|
|
272
|
+
settings_factory,
|
|
273
|
+
monkeypatch,
|
|
274
|
+
):
|
|
275
|
+
settings = settings_factory()
|
|
276
|
+
first = settings.server_root / "first"
|
|
277
|
+
first.mkdir()
|
|
278
|
+
second = settings.server_root / "second.yml"
|
|
279
|
+
second.write_text("config\n", encoding="utf-8")
|
|
280
|
+
_write_config(
|
|
281
|
+
settings,
|
|
282
|
+
"first:\n"
|
|
283
|
+
f" folders: [{first}]\n"
|
|
284
|
+
" exclude: ['**/cache/**']\n"
|
|
285
|
+
"second:\n"
|
|
286
|
+
f" files: [{second}]\n",
|
|
287
|
+
)
|
|
288
|
+
_configure_preflight(monkeypatch)
|
|
289
|
+
calls = []
|
|
290
|
+
monkeypatch.setattr(
|
|
291
|
+
backup,
|
|
292
|
+
"backup_folder",
|
|
293
|
+
lambda source, compression, group, settings, created_at, run_id, excludes: (
|
|
294
|
+
calls.append((group, source, excludes))
|
|
295
|
+
),
|
|
296
|
+
)
|
|
297
|
+
monkeypatch.setattr(backup, "prune", lambda *args: None)
|
|
298
|
+
|
|
299
|
+
assert backup.main(_settings_environment(settings)) == 0
|
|
300
|
+
assert calls == [
|
|
301
|
+
("first", first, ("**/cache/**",)),
|
|
302
|
+
("second", second, ()),
|
|
303
|
+
]
|
|
304
|
+
|
|
305
|
+
|
|
271
306
|
def test_main_reports_unexpected_io_errors_cleanly(
|
|
272
307
|
settings_factory,
|
|
273
308
|
monkeypatch,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|