declarative-data-backup 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- declarative_data_backup-0.1.0/PKG-INFO +319 -0
- declarative_data_backup-0.1.0/README.md +306 -0
- declarative_data_backup-0.1.0/pyproject.toml +30 -0
- declarative_data_backup-0.1.0/setup.cfg +4 -0
- declarative_data_backup-0.1.0/src/declarative_data_backup/__init__.py +1 -0
- declarative_data_backup-0.1.0/src/declarative_data_backup/__main__.py +6 -0
- declarative_data_backup-0.1.0/src/declarative_data_backup/backup.py +111 -0
- declarative_data_backup-0.1.0/src/declarative_data_backup/borg/__init__.py +13 -0
- declarative_data_backup-0.1.0/src/declarative_data_backup/borg/archive_naming.py +48 -0
- declarative_data_backup-0.1.0/src/declarative_data_backup/borg/borg.py +166 -0
- declarative_data_backup-0.1.0/src/declarative_data_backup/borg/borg_cli.py +69 -0
- declarative_data_backup-0.1.0/src/declarative_data_backup/config/__init__.py +11 -0
- declarative_data_backup-0.1.0/src/declarative_data_backup/config/backup_config/__init__.py +1 -0
- declarative_data_backup-0.1.0/src/declarative_data_backup/config/backup_config/models.py +12 -0
- declarative_data_backup-0.1.0/src/declarative_data_backup/config/backup_config/utils.py +141 -0
- declarative_data_backup-0.1.0/src/declarative_data_backup/config/backup_config/yaml_parser.py +136 -0
- declarative_data_backup-0.1.0/src/declarative_data_backup/config/backup_config/yaml_raw_loader.py +79 -0
- declarative_data_backup-0.1.0/src/declarative_data_backup/config/logging.py +8 -0
- declarative_data_backup-0.1.0/src/declarative_data_backup/environment.py +70 -0
- declarative_data_backup-0.1.0/src/declarative_data_backup/errors.py +13 -0
- declarative_data_backup-0.1.0/src/declarative_data_backup.egg-info/PKG-INFO +319 -0
- declarative_data_backup-0.1.0/src/declarative_data_backup.egg-info/SOURCES.txt +24 -0
- declarative_data_backup-0.1.0/src/declarative_data_backup.egg-info/dependency_links.txt +1 -0
- declarative_data_backup-0.1.0/src/declarative_data_backup.egg-info/requires.txt +7 -0
- declarative_data_backup-0.1.0/src/declarative_data_backup.egg-info/top_level.txt +1 -0
- declarative_data_backup-0.1.0/tests/test_backup.py +287 -0
|
@@ -0,0 +1,319 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: declarative-data-backup
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Declarative Borg-based data backups
|
|
5
|
+
Requires-Python: >=3.11
|
|
6
|
+
Description-Content-Type: text/markdown
|
|
7
|
+
Requires-Dist: PyYAML<7,>=6
|
|
8
|
+
Provides-Extra: dev
|
|
9
|
+
Requires-Dist: pre-commit<5,>=4; extra == "dev"
|
|
10
|
+
Requires-Dist: pyright<2,>=1.1.400; extra == "dev"
|
|
11
|
+
Requires-Dist: pytest<10,>=8; extra == "dev"
|
|
12
|
+
Requires-Dist: ruff<1,>=0.11; extra == "dev"
|
|
13
|
+
|
|
14
|
+
# Declarative Data Backup
|
|
15
|
+
|
|
16
|
+
Declarative Data Backup creates encrypted Borg archives for configured source
|
|
17
|
+
directories. Each source gets its own archive series in one Borg repository,
|
|
18
|
+
so Borg can deduplicate chunks across all sources. The application validates
|
|
19
|
+
the complete configuration before invoking Borg and prunes a source's archives
|
|
20
|
+
only after every create operation in the run has succeeded.
|
|
21
|
+
|
|
22
|
+
## Requirements
|
|
23
|
+
|
|
24
|
+
- Python 3.11 or newer.
|
|
25
|
+
- Borg 1.4.x installed separately and available as `borg` on `PATH`.
|
|
26
|
+
- A Borg repository that has already been initialized in keyfile encryption
|
|
27
|
+
mode, its matching keyfile, and a protected passphrase source.
|
|
28
|
+
- For remote repositories, OpenSSH and a trusted `known_hosts` file.
|
|
29
|
+
|
|
30
|
+
The application never initializes a repository. Borg 2 is not supported.
|
|
31
|
+
|
|
32
|
+
## Installation
|
|
33
|
+
|
|
34
|
+
Install the package from PyPI:
|
|
35
|
+
|
|
36
|
+
```shell
|
|
37
|
+
python -m pip install declarative-data-backup
|
|
38
|
+
```
|
|
39
|
+
|
|
40
|
+
Or install a checkout with the development dependencies:
|
|
41
|
+
|
|
42
|
+
```shell
|
|
43
|
+
python -m pip install -e ".[dev]"
|
|
44
|
+
```
|
|
45
|
+
|
|
46
|
+
The PyPI distribution is named `declarative-data-backup`; its importable
|
|
47
|
+
package is `declarative_data_backup`.
|
|
48
|
+
|
|
49
|
+
## GitLab release automation
|
|
50
|
+
|
|
51
|
+
The pipeline in [.gitlab-ci.yml](.gitlab-ci.yml) publishes releases when a Git
|
|
52
|
+
tag in `major.minor.patch` format is pushed, for example `1.2.3`. Use the same
|
|
53
|
+
version for the GitLab Release name and tag. Tags with a `v` prefix, prerelease
|
|
54
|
+
suffix, or leading zeros are not published. Branch pushes do not run this
|
|
55
|
+
release pipeline.
|
|
56
|
+
|
|
57
|
+
For each version tag, the pipeline:
|
|
58
|
+
|
|
59
|
+
1. Sets the package version in the CI checkout's `pyproject.toml`, builds a
|
|
60
|
+
wheel and source distribution, and checks them with Twine.
|
|
61
|
+
2. Uploads `declarative-data-backup` to PyPI.
|
|
62
|
+
3. Builds the existing Python 3.13 Dockerfile with the same package version,
|
|
63
|
+
verifies the installed version, and pushes
|
|
64
|
+
`${CI_REGISTRY_IMAGE}/declarative-data-backup:1.2.3-python3.13` to the
|
|
65
|
+
project's GitLab Container Registry.
|
|
66
|
+
|
|
67
|
+
The source checkout's version is not committed back to the repository.
|
|
68
|
+
The two publishing jobs run independently after the package build succeeds;
|
|
69
|
+
publishing is not atomic across PyPI and the container registry.
|
|
70
|
+
|
|
71
|
+
Before the first release:
|
|
72
|
+
|
|
73
|
+
- Enable the project's GitLab Container Registry.
|
|
74
|
+
- Add `PYPI_API_TOKEN` in GitLab **Settings > CI/CD > Variables** as a masked,
|
|
75
|
+
protected variable. Use a PyPI API token authorized to publish
|
|
76
|
+
`declarative-data-backup`; for a new PyPI project, its first upload may
|
|
77
|
+
require an account-wide token, which should then be replaced with a
|
|
78
|
+
project-scoped token.
|
|
79
|
+
- Protect release tags in GitLab (for example, `*.*.*`) and restrict who can
|
|
80
|
+
create them. Protected variables are only available on protected refs.
|
|
81
|
+
- Use a trusted Docker-executor runner configured for Docker-in-Docker with
|
|
82
|
+
TLS: `privileged = true` and `volumes = ["/certs/client", "/cache"]` in its
|
|
83
|
+
`[runners.docker]` configuration. GitLab.com instance runners support this
|
|
84
|
+
setup. Privileged runners should not execute untrusted code. If your runner
|
|
85
|
+
requires a tag, add it to `publish_container`.
|
|
86
|
+
|
|
87
|
+
Registry authentication uses GitLab's built-in `CI_REGISTRY_USER` and
|
|
88
|
+
`CI_REGISTRY_PASSWORD`; no separate registry password is needed.
|
|
89
|
+
|
|
90
|
+
Create and push a release tag from the commit you want to publish:
|
|
91
|
+
|
|
92
|
+
```shell
|
|
93
|
+
git tag 1.2.3
|
|
94
|
+
git push origin 1.2.3
|
|
95
|
+
```
|
|
96
|
+
|
|
97
|
+
Creating a GitLab Release with a new `1.2.3` tag also triggers the tag pipeline.
|
|
98
|
+
Creating or editing a Release on an already existing tag does not trigger a
|
|
99
|
+
new pipeline. PyPI versions cannot be overwritten: do not move or reuse
|
|
100
|
+
release tags. If only the container job fails, retry that job rather than
|
|
101
|
+
rerunning a successful PyPI upload.
|
|
102
|
+
|
|
103
|
+
## Configure and run
|
|
104
|
+
|
|
105
|
+
Create a YAML file accessible to the backup process:
|
|
106
|
+
|
|
107
|
+
```yaml
|
|
108
|
+
media_backup:
|
|
109
|
+
folders:
|
|
110
|
+
- /server_root/media/photos
|
|
111
|
+
- /server_root/media/music
|
|
112
|
+
compression: auto,zstd,22
|
|
113
|
+
retention_days: 30
|
|
114
|
+
required_mounts:
|
|
115
|
+
- /server_root/media
|
|
116
|
+
allow_empty: true
|
|
117
|
+
|
|
118
|
+
postgres_backup:
|
|
119
|
+
folders:
|
|
120
|
+
- /server_root/path/to/postgres-backups
|
|
121
|
+
required_mounts:
|
|
122
|
+
- /server_root/path/to/postgres-backups
|
|
123
|
+
```
|
|
124
|
+
|
|
125
|
+
Set the required environment variables before starting the process:
|
|
126
|
+
|
|
127
|
+
| Variable | Purpose |
|
|
128
|
+
| --- | --- |
|
|
129
|
+
| `BACKUP_CONFIG_PATH` | Absolute path to the YAML configuration. |
|
|
130
|
+
| `SERVER_ROOT` | Absolute path to the root containing all configured source directories. |
|
|
131
|
+
| `BORG_REPO` | Existing Borg repository location, local or remote. |
|
|
132
|
+
| `BORG_KEY_FILE` | Path to the external keyfile for the encrypted repository. |
|
|
133
|
+
| `BORG_BASE_DIR` | Writable directory for Borg state and the backup coordination lock. |
|
|
134
|
+
| `BORG_RSH` | SSH command and options used by Borg for a remote repository. |
|
|
135
|
+
| `BORG_PASSCOMMAND` | Command Borg runs to obtain the repository passphrase. |
|
|
136
|
+
|
|
137
|
+
Optional variables:
|
|
138
|
+
|
|
139
|
+
| Variable | Purpose |
|
|
140
|
+
| --- | --- |
|
|
141
|
+
| `BORG_REMOTE_PATH` | Remote Borg executable path, passed as Borg's `--remote-path`. Leave unset to use Borg's default. |
|
|
142
|
+
| `BORG_EXIT_CODES` | Borg exit-code mode: `legacy` or `modern`. |
|
|
143
|
+
|
|
144
|
+
Provide credentials through a secret manager or protected files; do not put
|
|
145
|
+
passphrases, private keys, or credentials in the YAML file, source control, or
|
|
146
|
+
shell history. For example, in a controlled environment where these variables
|
|
147
|
+
have already been securely populated:
|
|
148
|
+
|
|
149
|
+
```shell
|
|
150
|
+
python -m declarative_data_backup
|
|
151
|
+
```
|
|
152
|
+
|
|
153
|
+
The command exits with status `0` on success, nonzero on configuration or Borg
|
|
154
|
+
failure, and `130` when interrupted. Borg's exit code `1` indicates warnings:
|
|
155
|
+
the run fails and pruning is skipped.
|
|
156
|
+
|
|
157
|
+
## Configuration reference
|
|
158
|
+
|
|
159
|
+
The top level must be a nonempty YAML mapping of backup group names to group
|
|
160
|
+
mappings. Group names may contain ASCII letters, digits, `_`, and `-`, and
|
|
161
|
+
must be at most 64 characters. Duplicate YAML keys, unknown group fields,
|
|
162
|
+
invalid values, and malformed or empty configuration are rejected before any
|
|
163
|
+
create or prune operation.
|
|
164
|
+
|
|
165
|
+
Each group supports:
|
|
166
|
+
|
|
167
|
+
- `folders` (required): a nonempty list of absolute directory paths. Each
|
|
168
|
+
source must exist, be readable, and resolve inside `SERVER_ROOT`. Duplicate
|
|
169
|
+
canonical paths and parent/child overlaps across groups are rejected.
|
|
170
|
+
- `compression` (optional): defaults to `auto,zstd,22`. Supported Borg 1.4
|
|
171
|
+
codecs include `none`, `lz4`, `zstd`, `zlib`, and `lzma`; `auto` may precede
|
|
172
|
+
a codec. Codec levels are checked against supported ranges: zstd `1`-`22`,
|
|
173
|
+
zlib `0`-`9`, and lzma `0`-`9`. Borg's `obfuscate` compression specifier is
|
|
174
|
+
also accepted for specifier values `1`-`6`, `110`-`123`, or `250`; it is
|
|
175
|
+
useful only with repository encryption.
|
|
176
|
+
- `retention_days` (optional): positive integer, default `30`. This controls
|
|
177
|
+
Borg's `--keep-daily` policy.
|
|
178
|
+
- `required_mounts` (optional): list of exact mount points that must appear
|
|
179
|
+
in the process's `/proc/self/mountinfo`.
|
|
180
|
+
- `allow_empty` (optional): boolean, default `true`. If set to `false`, an
|
|
181
|
+
empty source is rejected.
|
|
182
|
+
|
|
183
|
+
Configure `required_mounts` explicitly for every source that depends on a
|
|
184
|
+
separately mounted filesystem. The program does not infer mount requirements
|
|
185
|
+
from source paths. In a container or Kubernetes pod, mounting `/server_root`
|
|
186
|
+
does not prove that nested host filesystems are mounted. In particular,
|
|
187
|
+
`allow_empty: true` does not prove that a disk is present: an empty source may
|
|
188
|
+
be legitimate and can be archived.
|
|
189
|
+
|
|
190
|
+
Paths are resolved before backup. A top-level source symlink is allowed only
|
|
191
|
+
when its canonical target remains inside `SERVER_ROOT`; symlinks within a
|
|
192
|
+
source are stored as links rather than followed.
|
|
193
|
+
|
|
194
|
+
## Archives, retention, and compaction
|
|
195
|
+
|
|
196
|
+
The application creates one archive per configured source in the single
|
|
197
|
+
`BORG_REPO`. Names have this form:
|
|
198
|
+
|
|
199
|
+
```text
|
|
200
|
+
bbkp__<source-series>__<UTC timestamp>__<run UUID>
|
|
201
|
+
```
|
|
202
|
+
|
|
203
|
+
The stable source series is derived from the canonical source path and includes
|
|
204
|
+
a SHA-256 suffix. Borg receives the canonical absolute path as the source, so
|
|
205
|
+
the original directory hierarchy is retained inside the archive.
|
|
206
|
+
|
|
207
|
+
The application creates all configured archives before pruning any source.
|
|
208
|
+
If a create fails or Borg reports warnings, the run stops and no pruning is
|
|
209
|
+
performed. Retries use a new run UUID and may create another archive for that
|
|
210
|
+
day; archives are never overwritten to handle a collision.
|
|
211
|
+
|
|
212
|
+
`--keep-daily N` retains the latest archive in up to `N` daily buckets. It does
|
|
213
|
+
not guarantee `N` successful calendar days of recovery if scheduled runs fail
|
|
214
|
+
or are missed. Removing a source from the YAML does not delete its old
|
|
215
|
+
archives. The source-series selector does not match archives made with the
|
|
216
|
+
previous non-namespaced naming scheme; retain those archives unless a
|
|
217
|
+
separately reviewed migration or cleanup is performed.
|
|
218
|
+
|
|
219
|
+
Pruning marks repository data for deletion; it does not reclaim the space.
|
|
220
|
+
Repository compaction is a separate maintenance operation and is not run by
|
|
221
|
+
this application. Any scheduled compact job or manual maintenance must first
|
|
222
|
+
acquire an exclusive `flock` on:
|
|
223
|
+
|
|
224
|
+
```text
|
|
225
|
+
<BORG_BASE_DIR>/backup.lock
|
|
226
|
+
```
|
|
227
|
+
|
|
228
|
+
Use the same persistent state volume and filesystem as the backup job, with
|
|
229
|
+
working advisory-lock semantics. The backup process waits up to 3600 seconds
|
|
230
|
+
for this lock. Kubernetes `concurrencyPolicy: Forbid` does not coordinate
|
|
231
|
+
separate CronJobs, and Borg's own repository lock does not replace this
|
|
232
|
+
application-level lock. Never unlink or replace the lock file while any
|
|
233
|
+
process could hold it.
|
|
234
|
+
|
|
235
|
+
## Deployment and operational notes
|
|
236
|
+
|
|
237
|
+
The Dockerfile builds an image with Borg 1.4.1 and the Python package. The
|
|
238
|
+
application itself does not set CPU or I/O priority; a scheduled deployment
|
|
239
|
+
may wrap the command with `nice -n 19` and `ionice -c 3`. For Kubernetes or
|
|
240
|
+
another container runtime:
|
|
241
|
+
|
|
242
|
+
- Run as a non-root user and confirm it can read the keyfile and credentials
|
|
243
|
+
and write the Borg state volume.
|
|
244
|
+
- Mount source data read-only. Keep Borg cache, configuration, security state,
|
|
245
|
+
keyfile, and the coordination lock on persistent writable storage.
|
|
246
|
+
- For nested source filesystems, configure and verify the corresponding
|
|
247
|
+
`required_mounts`.
|
|
248
|
+
- Use a versioned image tag or digest and confirm private-registry pull
|
|
249
|
+
credentials have only the required read access.
|
|
250
|
+
- Do not mount a service-account token unless the workload needs one.
|
|
251
|
+
- Ensure any compact job mounts the same state volume and honors the lock
|
|
252
|
+
contract above.
|
|
253
|
+
- Monitor failed and warning backup jobs. Retention cannot make up for days
|
|
254
|
+
without successful archives.
|
|
255
|
+
- A fixed schedule does not ensure upstream jobs, such as Unmanic processing,
|
|
256
|
+
have completed; establish that dependency separately.
|
|
257
|
+
|
|
258
|
+
The default `auto,zstd,22` may be slow or exceed a 1 GiB memory limit on some
|
|
259
|
+
data sets. It has not been benchmarked under that limit and may be terminated
|
|
260
|
+
for memory use. Compression can be overridden per group without recreating
|
|
261
|
+
the repository.
|
|
262
|
+
|
|
263
|
+
Kubernetes Secret projection permissions, storage ownership, mount behavior,
|
|
264
|
+
node placement, registry access, and workload scheduling must be verified in
|
|
265
|
+
the target cluster. They are not verified by this Python package or its
|
|
266
|
+
container integration test.
|
|
267
|
+
|
|
268
|
+
## Testing
|
|
269
|
+
|
|
270
|
+
Run the unit tests:
|
|
271
|
+
|
|
272
|
+
```shell
|
|
273
|
+
python -m pytest -m "not integration"
|
|
274
|
+
```
|
|
275
|
+
|
|
276
|
+
The opt-in local Borg integration test uses a temporary encrypted repository:
|
|
277
|
+
|
|
278
|
+
```shell
|
|
279
|
+
RUN_BORG_LOCAL_INTEGRATION=1 python -m pytest -m integration \
|
|
280
|
+
tests/integration/test_local_borg_integration.py
|
|
281
|
+
```
|
|
282
|
+
|
|
283
|
+
The opt-in Docker SSH integration test uses disposable containers and a
|
|
284
|
+
temporary repository; it does not use production credentials or BorgBase:
|
|
285
|
+
|
|
286
|
+
```shell
|
|
287
|
+
RUN_BORG_DOCKER_INTEGRATION=1 python -m pytest -m integration \
|
|
288
|
+
tests/integration/test_docker_integration.py
|
|
289
|
+
```
|
|
290
|
+
|
|
291
|
+
Do not treat skipped integration tests as successful restore validation.
|
|
292
|
+
|
|
293
|
+
## Recovery
|
|
294
|
+
|
|
295
|
+
Recovery requires the repository and the matching external keyfile and
|
|
296
|
+
passphrase. Keep independent recovery copies of the keyfile and passphrase;
|
|
297
|
+
the repository must not be their only copy. Also retain the SSH private key
|
|
298
|
+
and verified `known_hosts` entry for a remote repository.
|
|
299
|
+
|
|
300
|
+
With the required Borg environment and credentials securely configured, list
|
|
301
|
+
archives and inspect the desired archive:
|
|
302
|
+
|
|
303
|
+
```shell
|
|
304
|
+
borg list "$BORG_REPO"
|
|
305
|
+
borg list "$BORG_REPO"::'bbkp__<source-series>__<timestamp>__<run-uuid>'
|
|
306
|
+
```
|
|
307
|
+
|
|
308
|
+
Extract into an isolated recovery directory and verify the restored data
|
|
309
|
+
before returning it to service:
|
|
310
|
+
|
|
311
|
+
```shell
|
|
312
|
+
mkdir -p /restore
|
|
313
|
+
cd /restore
|
|
314
|
+
borg extract "$BORG_REPO"::'bbkp__<source-series>__<timestamp>__<run-uuid>'
|
|
315
|
+
```
|
|
316
|
+
|
|
317
|
+
Test recovery periodically using a disposable or approved recovery
|
|
318
|
+
environment. This repository's tests do not verify access to a live remote
|
|
319
|
+
repository or actual Kubernetes Secret projection and storage behavior.
|
|
@@ -0,0 +1,306 @@
|
|
|
1
|
+
# Declarative Data Backup
|
|
2
|
+
|
|
3
|
+
Declarative Data Backup creates encrypted Borg archives for configured source
|
|
4
|
+
directories. Each source gets its own archive series in one Borg repository,
|
|
5
|
+
so Borg can deduplicate chunks across all sources. The application validates
|
|
6
|
+
the complete configuration before invoking Borg and prunes a source's archives
|
|
7
|
+
only after every create operation in the run has succeeded.
|
|
8
|
+
|
|
9
|
+
## Requirements
|
|
10
|
+
|
|
11
|
+
- Python 3.11 or newer.
|
|
12
|
+
- Borg 1.4.x installed separately and available as `borg` on `PATH`.
|
|
13
|
+
- A Borg repository that has already been initialized in keyfile encryption
|
|
14
|
+
mode, its matching keyfile, and a protected passphrase source.
|
|
15
|
+
- For remote repositories, OpenSSH and a trusted `known_hosts` file.
|
|
16
|
+
|
|
17
|
+
The application never initializes a repository. Borg 2 is not supported.
|
|
18
|
+
|
|
19
|
+
## Installation
|
|
20
|
+
|
|
21
|
+
Install the package from PyPI:
|
|
22
|
+
|
|
23
|
+
```shell
|
|
24
|
+
python -m pip install declarative-data-backup
|
|
25
|
+
```
|
|
26
|
+
|
|
27
|
+
Or install a checkout with the development dependencies:
|
|
28
|
+
|
|
29
|
+
```shell
|
|
30
|
+
python -m pip install -e ".[dev]"
|
|
31
|
+
```
|
|
32
|
+
|
|
33
|
+
The PyPI distribution is named `declarative-data-backup`; its importable
|
|
34
|
+
package is `declarative_data_backup`.
|
|
35
|
+
|
|
36
|
+
## GitLab release automation
|
|
37
|
+
|
|
38
|
+
The pipeline in [.gitlab-ci.yml](.gitlab-ci.yml) publishes releases when a Git
|
|
39
|
+
tag in `major.minor.patch` format is pushed, for example `1.2.3`. Use the same
|
|
40
|
+
version for the GitLab Release name and tag. Tags with a `v` prefix, prerelease
|
|
41
|
+
suffix, or leading zeros are not published. Branch pushes do not run this
|
|
42
|
+
release pipeline.
|
|
43
|
+
|
|
44
|
+
For each version tag, the pipeline:
|
|
45
|
+
|
|
46
|
+
1. Sets the package version in the CI checkout's `pyproject.toml`, builds a
|
|
47
|
+
wheel and source distribution, and checks them with Twine.
|
|
48
|
+
2. Uploads `declarative-data-backup` to PyPI.
|
|
49
|
+
3. Builds the existing Python 3.13 Dockerfile with the same package version,
|
|
50
|
+
verifies the installed version, and pushes
|
|
51
|
+
`${CI_REGISTRY_IMAGE}/declarative-data-backup:1.2.3-python3.13` to the
|
|
52
|
+
project's GitLab Container Registry.
|
|
53
|
+
|
|
54
|
+
The source checkout's version is not committed back to the repository.
|
|
55
|
+
The two publishing jobs run independently after the package build succeeds;
|
|
56
|
+
publishing is not atomic across PyPI and the container registry.
|
|
57
|
+
|
|
58
|
+
Before the first release:
|
|
59
|
+
|
|
60
|
+
- Enable the project's GitLab Container Registry.
|
|
61
|
+
- Add `PYPI_API_TOKEN` in GitLab **Settings > CI/CD > Variables** as a masked,
|
|
62
|
+
protected variable. Use a PyPI API token authorized to publish
|
|
63
|
+
`declarative-data-backup`; for a new PyPI project, its first upload may
|
|
64
|
+
require an account-wide token, which should then be replaced with a
|
|
65
|
+
project-scoped token.
|
|
66
|
+
- Protect release tags in GitLab (for example, `*.*.*`) and restrict who can
|
|
67
|
+
create them. Protected variables are only available on protected refs.
|
|
68
|
+
- Use a trusted Docker-executor runner configured for Docker-in-Docker with
|
|
69
|
+
TLS: `privileged = true` and `volumes = ["/certs/client", "/cache"]` in its
|
|
70
|
+
`[runners.docker]` configuration. GitLab.com instance runners support this
|
|
71
|
+
setup. Privileged runners should not execute untrusted code. If your runner
|
|
72
|
+
requires a tag, add it to `publish_container`.
|
|
73
|
+
|
|
74
|
+
Registry authentication uses GitLab's built-in `CI_REGISTRY_USER` and
|
|
75
|
+
`CI_REGISTRY_PASSWORD`; no separate registry password is needed.
|
|
76
|
+
|
|
77
|
+
Create and push a release tag from the commit you want to publish:
|
|
78
|
+
|
|
79
|
+
```shell
|
|
80
|
+
git tag 1.2.3
|
|
81
|
+
git push origin 1.2.3
|
|
82
|
+
```
|
|
83
|
+
|
|
84
|
+
Creating a GitLab Release with a new `1.2.3` tag also triggers the tag pipeline.
|
|
85
|
+
Creating or editing a Release on an already existing tag does not trigger a
|
|
86
|
+
new pipeline. PyPI versions cannot be overwritten: do not move or reuse
|
|
87
|
+
release tags. If only the container job fails, retry that job rather than
|
|
88
|
+
rerunning a successful PyPI upload.
|
|
89
|
+
|
|
90
|
+
## Configure and run
|
|
91
|
+
|
|
92
|
+
Create a YAML file accessible to the backup process:
|
|
93
|
+
|
|
94
|
+
```yaml
|
|
95
|
+
media_backup:
|
|
96
|
+
folders:
|
|
97
|
+
- /server_root/media/photos
|
|
98
|
+
- /server_root/media/music
|
|
99
|
+
compression: auto,zstd,22
|
|
100
|
+
retention_days: 30
|
|
101
|
+
required_mounts:
|
|
102
|
+
- /server_root/media
|
|
103
|
+
allow_empty: true
|
|
104
|
+
|
|
105
|
+
postgres_backup:
|
|
106
|
+
folders:
|
|
107
|
+
- /server_root/path/to/postgres-backups
|
|
108
|
+
required_mounts:
|
|
109
|
+
- /server_root/path/to/postgres-backups
|
|
110
|
+
```
|
|
111
|
+
|
|
112
|
+
Set the required environment variables before starting the process:
|
|
113
|
+
|
|
114
|
+
| Variable | Purpose |
|
|
115
|
+
| --- | --- |
|
|
116
|
+
| `BACKUP_CONFIG_PATH` | Absolute path to the YAML configuration. |
|
|
117
|
+
| `SERVER_ROOT` | Absolute path to the root containing all configured source directories. |
|
|
118
|
+
| `BORG_REPO` | Existing Borg repository location, local or remote. |
|
|
119
|
+
| `BORG_KEY_FILE` | Path to the external keyfile for the encrypted repository. |
|
|
120
|
+
| `BORG_BASE_DIR` | Writable directory for Borg state and the backup coordination lock. |
|
|
121
|
+
| `BORG_RSH` | SSH command and options used by Borg for a remote repository. |
|
|
122
|
+
| `BORG_PASSCOMMAND` | Command Borg runs to obtain the repository passphrase. |
|
|
123
|
+
|
|
124
|
+
Optional variables:
|
|
125
|
+
|
|
126
|
+
| Variable | Purpose |
|
|
127
|
+
| --- | --- |
|
|
128
|
+
| `BORG_REMOTE_PATH` | Remote Borg executable path, passed as Borg's `--remote-path`. Leave unset to use Borg's default. |
|
|
129
|
+
| `BORG_EXIT_CODES` | Borg exit-code mode: `legacy` or `modern`. |
|
|
130
|
+
|
|
131
|
+
Provide credentials through a secret manager or protected files; do not put
|
|
132
|
+
passphrases, private keys, or credentials in the YAML file, source control, or
|
|
133
|
+
shell history. For example, in a controlled environment where these variables
|
|
134
|
+
have already been securely populated:
|
|
135
|
+
|
|
136
|
+
```shell
|
|
137
|
+
python -m declarative_data_backup
|
|
138
|
+
```
|
|
139
|
+
|
|
140
|
+
The command exits with status `0` on success, nonzero on configuration or Borg
|
|
141
|
+
failure, and `130` when interrupted. Borg's exit code `1` indicates warnings:
|
|
142
|
+
the run fails and pruning is skipped.
|
|
143
|
+
|
|
144
|
+
## Configuration reference
|
|
145
|
+
|
|
146
|
+
The top level must be a nonempty YAML mapping of backup group names to group
|
|
147
|
+
mappings. Group names may contain ASCII letters, digits, `_`, and `-`, and
|
|
148
|
+
must be at most 64 characters. Duplicate YAML keys, unknown group fields,
|
|
149
|
+
invalid values, and malformed or empty configuration are rejected before any
|
|
150
|
+
create or prune operation.
|
|
151
|
+
|
|
152
|
+
Each group supports:
|
|
153
|
+
|
|
154
|
+
- `folders` (required): a nonempty list of absolute directory paths. Each
|
|
155
|
+
source must exist, be readable, and resolve inside `SERVER_ROOT`. Duplicate
|
|
156
|
+
canonical paths and parent/child overlaps across groups are rejected.
|
|
157
|
+
- `compression` (optional): defaults to `auto,zstd,22`. Supported Borg 1.4
|
|
158
|
+
codecs include `none`, `lz4`, `zstd`, `zlib`, and `lzma`; `auto` may precede
|
|
159
|
+
a codec. Codec levels are checked against supported ranges: zstd `1`-`22`,
|
|
160
|
+
zlib `0`-`9`, and lzma `0`-`9`. Borg's `obfuscate` compression specifier is
|
|
161
|
+
also accepted for specifier values `1`-`6`, `110`-`123`, or `250`; it is
|
|
162
|
+
useful only with repository encryption.
|
|
163
|
+
- `retention_days` (optional): positive integer, default `30`. This controls
|
|
164
|
+
Borg's `--keep-daily` policy.
|
|
165
|
+
- `required_mounts` (optional): list of exact mount points that must appear
|
|
166
|
+
in the process's `/proc/self/mountinfo`.
|
|
167
|
+
- `allow_empty` (optional): boolean, default `true`. If set to `false`, an
|
|
168
|
+
empty source is rejected.
|
|
169
|
+
|
|
170
|
+
Configure `required_mounts` explicitly for every source that depends on a
|
|
171
|
+
separately mounted filesystem. The program does not infer mount requirements
|
|
172
|
+
from source paths. In a container or Kubernetes pod, mounting `/server_root`
|
|
173
|
+
does not prove that nested host filesystems are mounted. In particular,
|
|
174
|
+
`allow_empty: true` does not prove that a disk is present: an empty source may
|
|
175
|
+
be legitimate and can be archived.
|
|
176
|
+
|
|
177
|
+
Paths are resolved before backup. A top-level source symlink is allowed only
|
|
178
|
+
when its canonical target remains inside `SERVER_ROOT`; symlinks within a
|
|
179
|
+
source are stored as links rather than followed.
|
|
180
|
+
|
|
181
|
+
## Archives, retention, and compaction
|
|
182
|
+
|
|
183
|
+
The application creates one archive per configured source in the single
|
|
184
|
+
`BORG_REPO`. Names have this form:
|
|
185
|
+
|
|
186
|
+
```text
|
|
187
|
+
bbkp__<source-series>__<UTC timestamp>__<run UUID>
|
|
188
|
+
```
|
|
189
|
+
|
|
190
|
+
The stable source series is derived from the canonical source path and includes
|
|
191
|
+
a SHA-256 suffix. Borg receives the canonical absolute path as the source, so
|
|
192
|
+
the original directory hierarchy is retained inside the archive.
|
|
193
|
+
|
|
194
|
+
The application creates all configured archives before pruning any source.
|
|
195
|
+
If a create fails or Borg reports warnings, the run stops and no pruning is
|
|
196
|
+
performed. Retries use a new run UUID and may create another archive for that
|
|
197
|
+
day; archives are never overwritten to handle a collision.
|
|
198
|
+
|
|
199
|
+
`--keep-daily N` retains the latest archive in up to `N` daily buckets. It does
|
|
200
|
+
not guarantee `N` successful calendar days of recovery if scheduled runs fail
|
|
201
|
+
or are missed. Removing a source from the YAML does not delete its old
|
|
202
|
+
archives. The source-series selector does not match archives made with the
|
|
203
|
+
previous non-namespaced naming scheme; retain those archives unless a
|
|
204
|
+
separately reviewed migration or cleanup is performed.
|
|
205
|
+
|
|
206
|
+
Pruning marks repository data for deletion; it does not reclaim the space.
|
|
207
|
+
Repository compaction is a separate maintenance operation and is not run by
|
|
208
|
+
this application. Any scheduled compact job or manual maintenance must first
|
|
209
|
+
acquire an exclusive `flock` on:
|
|
210
|
+
|
|
211
|
+
```text
|
|
212
|
+
<BORG_BASE_DIR>/backup.lock
|
|
213
|
+
```
|
|
214
|
+
|
|
215
|
+
Use the same persistent state volume and filesystem as the backup job, with
|
|
216
|
+
working advisory-lock semantics. The backup process waits up to 3600 seconds
|
|
217
|
+
for this lock. Kubernetes `concurrencyPolicy: Forbid` does not coordinate
|
|
218
|
+
separate CronJobs, and Borg's own repository lock does not replace this
|
|
219
|
+
application-level lock. Never unlink or replace the lock file while any
|
|
220
|
+
process could hold it.
|
|
221
|
+
|
|
222
|
+
## Deployment and operational notes
|
|
223
|
+
|
|
224
|
+
The Dockerfile builds an image with Borg 1.4.1 and the Python package. The
|
|
225
|
+
application itself does not set CPU or I/O priority; a scheduled deployment
|
|
226
|
+
may wrap the command with `nice -n 19` and `ionice -c 3`. For Kubernetes or
|
|
227
|
+
another container runtime:
|
|
228
|
+
|
|
229
|
+
- Run as a non-root user and confirm it can read the keyfile and credentials
|
|
230
|
+
and write the Borg state volume.
|
|
231
|
+
- Mount source data read-only. Keep Borg cache, configuration, security state,
|
|
232
|
+
keyfile, and the coordination lock on persistent writable storage.
|
|
233
|
+
- For nested source filesystems, configure and verify the corresponding
|
|
234
|
+
`required_mounts`.
|
|
235
|
+
- Use a versioned image tag or digest and confirm private-registry pull
|
|
236
|
+
credentials have only the required read access.
|
|
237
|
+
- Do not mount a service-account token unless the workload needs one.
|
|
238
|
+
- Ensure any compact job mounts the same state volume and honors the lock
|
|
239
|
+
contract above.
|
|
240
|
+
- Monitor failed and warning backup jobs. Retention cannot make up for days
|
|
241
|
+
without successful archives.
|
|
242
|
+
- A fixed schedule does not ensure upstream jobs, such as Unmanic processing,
|
|
243
|
+
have completed; establish that dependency separately.
|
|
244
|
+
|
|
245
|
+
The default `auto,zstd,22` may be slow or exceed a 1 GiB memory limit on some
|
|
246
|
+
data sets. It has not been benchmarked under that limit and may be terminated
|
|
247
|
+
for memory use. Compression can be overridden per group without recreating
|
|
248
|
+
the repository.
|
|
249
|
+
|
|
250
|
+
Kubernetes Secret projection permissions, storage ownership, mount behavior,
|
|
251
|
+
node placement, registry access, and workload scheduling must be verified in
|
|
252
|
+
the target cluster. They are not verified by this Python package or its
|
|
253
|
+
container integration test.
|
|
254
|
+
|
|
255
|
+
## Testing
|
|
256
|
+
|
|
257
|
+
Run the unit tests:
|
|
258
|
+
|
|
259
|
+
```shell
|
|
260
|
+
python -m pytest -m "not integration"
|
|
261
|
+
```
|
|
262
|
+
|
|
263
|
+
The opt-in local Borg integration test uses a temporary encrypted repository:
|
|
264
|
+
|
|
265
|
+
```shell
|
|
266
|
+
RUN_BORG_LOCAL_INTEGRATION=1 python -m pytest -m integration \
|
|
267
|
+
tests/integration/test_local_borg_integration.py
|
|
268
|
+
```
|
|
269
|
+
|
|
270
|
+
The opt-in Docker SSH integration test uses disposable containers and a
|
|
271
|
+
temporary repository; it does not use production credentials or BorgBase:
|
|
272
|
+
|
|
273
|
+
```shell
|
|
274
|
+
RUN_BORG_DOCKER_INTEGRATION=1 python -m pytest -m integration \
|
|
275
|
+
tests/integration/test_docker_integration.py
|
|
276
|
+
```
|
|
277
|
+
|
|
278
|
+
Do not treat skipped integration tests as successful restore validation.
|
|
279
|
+
|
|
280
|
+
## Recovery
|
|
281
|
+
|
|
282
|
+
Recovery requires the repository and the matching external keyfile and
|
|
283
|
+
passphrase. Keep independent recovery copies of the keyfile and passphrase;
|
|
284
|
+
the repository must not be their only copy. Also retain the SSH private key
|
|
285
|
+
and verified `known_hosts` entry for a remote repository.
|
|
286
|
+
|
|
287
|
+
With the required Borg environment and credentials securely configured, list
|
|
288
|
+
archives and inspect the desired archive:
|
|
289
|
+
|
|
290
|
+
```shell
|
|
291
|
+
borg list "$BORG_REPO"
|
|
292
|
+
borg list "$BORG_REPO"::'bbkp__<source-series>__<timestamp>__<run-uuid>'
|
|
293
|
+
```
|
|
294
|
+
|
|
295
|
+
Extract into an isolated recovery directory and verify the restored data
|
|
296
|
+
before returning it to service:
|
|
297
|
+
|
|
298
|
+
```shell
|
|
299
|
+
mkdir -p /restore
|
|
300
|
+
cd /restore
|
|
301
|
+
borg extract "$BORG_REPO"::'bbkp__<source-series>__<timestamp>__<run-uuid>'
|
|
302
|
+
```
|
|
303
|
+
|
|
304
|
+
Test recovery periodically using a disposable or approved recovery
|
|
305
|
+
environment. This repository's tests do not verify access to a live remote
|
|
306
|
+
repository or actual Kubernetes Secret projection and storage behavior.
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=68", "wheel"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "declarative-data-backup"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "Declarative Borg-based data backups"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.11"
|
|
11
|
+
dependencies = ["PyYAML>=6,<7"]
|
|
12
|
+
|
|
13
|
+
[project.optional-dependencies]
|
|
14
|
+
dev = [
|
|
15
|
+
"pre-commit>=4,<5",
|
|
16
|
+
"pyright>=1.1.400,<2",
|
|
17
|
+
"pytest>=8,<10",
|
|
18
|
+
"ruff>=0.11,<1",
|
|
19
|
+
]
|
|
20
|
+
|
|
21
|
+
[tool.setuptools.packages.find]
|
|
22
|
+
where = ["src"]
|
|
23
|
+
include = ["declarative_data_backup*"]
|
|
24
|
+
|
|
25
|
+
[tool.ruff]
|
|
26
|
+
line-length = 88
|
|
27
|
+
target-version = "py311"
|
|
28
|
+
|
|
29
|
+
[tool.ruff.lint]
|
|
30
|
+
select = ["E4", "E7", "E9", "F", "I"]
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Declarative Borg-based data backups."""
|