declarative-data-backup 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (26) hide show
  1. declarative_data_backup-0.1.0/PKG-INFO +319 -0
  2. declarative_data_backup-0.1.0/README.md +306 -0
  3. declarative_data_backup-0.1.0/pyproject.toml +30 -0
  4. declarative_data_backup-0.1.0/setup.cfg +4 -0
  5. declarative_data_backup-0.1.0/src/declarative_data_backup/__init__.py +1 -0
  6. declarative_data_backup-0.1.0/src/declarative_data_backup/__main__.py +6 -0
  7. declarative_data_backup-0.1.0/src/declarative_data_backup/backup.py +111 -0
  8. declarative_data_backup-0.1.0/src/declarative_data_backup/borg/__init__.py +13 -0
  9. declarative_data_backup-0.1.0/src/declarative_data_backup/borg/archive_naming.py +48 -0
  10. declarative_data_backup-0.1.0/src/declarative_data_backup/borg/borg.py +166 -0
  11. declarative_data_backup-0.1.0/src/declarative_data_backup/borg/borg_cli.py +69 -0
  12. declarative_data_backup-0.1.0/src/declarative_data_backup/config/__init__.py +11 -0
  13. declarative_data_backup-0.1.0/src/declarative_data_backup/config/backup_config/__init__.py +1 -0
  14. declarative_data_backup-0.1.0/src/declarative_data_backup/config/backup_config/models.py +12 -0
  15. declarative_data_backup-0.1.0/src/declarative_data_backup/config/backup_config/utils.py +141 -0
  16. declarative_data_backup-0.1.0/src/declarative_data_backup/config/backup_config/yaml_parser.py +136 -0
  17. declarative_data_backup-0.1.0/src/declarative_data_backup/config/backup_config/yaml_raw_loader.py +79 -0
  18. declarative_data_backup-0.1.0/src/declarative_data_backup/config/logging.py +8 -0
  19. declarative_data_backup-0.1.0/src/declarative_data_backup/environment.py +70 -0
  20. declarative_data_backup-0.1.0/src/declarative_data_backup/errors.py +13 -0
  21. declarative_data_backup-0.1.0/src/declarative_data_backup.egg-info/PKG-INFO +319 -0
  22. declarative_data_backup-0.1.0/src/declarative_data_backup.egg-info/SOURCES.txt +24 -0
  23. declarative_data_backup-0.1.0/src/declarative_data_backup.egg-info/dependency_links.txt +1 -0
  24. declarative_data_backup-0.1.0/src/declarative_data_backup.egg-info/requires.txt +7 -0
  25. declarative_data_backup-0.1.0/src/declarative_data_backup.egg-info/top_level.txt +1 -0
  26. declarative_data_backup-0.1.0/tests/test_backup.py +287 -0
@@ -0,0 +1,319 @@
1
+ Metadata-Version: 2.4
2
+ Name: declarative-data-backup
3
+ Version: 0.1.0
4
+ Summary: Declarative Borg-based data backups
5
+ Requires-Python: >=3.11
6
+ Description-Content-Type: text/markdown
7
+ Requires-Dist: PyYAML<7,>=6
8
+ Provides-Extra: dev
9
+ Requires-Dist: pre-commit<5,>=4; extra == "dev"
10
+ Requires-Dist: pyright<2,>=1.1.400; extra == "dev"
11
+ Requires-Dist: pytest<10,>=8; extra == "dev"
12
+ Requires-Dist: ruff<1,>=0.11; extra == "dev"
13
+
14
+ # Declarative Data Backup
15
+
16
+ Declarative Data Backup creates encrypted Borg archives for configured source
17
+ directories. Each source gets its own archive series in one Borg repository,
18
+ so Borg can deduplicate chunks across all sources. The application validates
19
+ the complete configuration before invoking Borg and prunes a source's archives
20
+ only after every create operation in the run has succeeded.
21
+
22
+ ## Requirements
23
+
24
+ - Python 3.11 or newer.
25
+ - Borg 1.4.x installed separately and available as `borg` on `PATH`.
26
+ - A Borg repository that has already been initialized in keyfile encryption
27
+ mode, its matching keyfile, and a protected passphrase source.
28
+ - For remote repositories, OpenSSH and a trusted `known_hosts` file.
29
+
30
+ The application never initializes a repository. Borg 2 is not supported.
31
+
32
+ ## Installation
33
+
34
+ Install the package from PyPI:
35
+
36
+ ```shell
37
+ python -m pip install declarative-data-backup
38
+ ```
39
+
40
+ Or install a checkout with the development dependencies:
41
+
42
+ ```shell
43
+ python -m pip install -e ".[dev]"
44
+ ```
45
+
46
+ The PyPI distribution is named `declarative-data-backup`; its importable
47
+ package is `declarative_data_backup`.
48
+
49
+ ## GitLab release automation
50
+
51
+ The pipeline in [.gitlab-ci.yml](.gitlab-ci.yml) publishes releases when a Git
52
+ tag in `major.minor.patch` format is pushed, for example `1.2.3`. Use the same
53
+ version for the GitLab Release name and tag. Tags with a `v` prefix, prerelease
54
+ suffix, or leading zeros are not published. Branch pushes do not run this
55
+ release pipeline.
56
+
57
+ For each version tag, the pipeline:
58
+
59
+ 1. Sets the package version in the CI checkout's `pyproject.toml`, builds a
60
+ wheel and source distribution, and checks them with Twine.
61
+ 2. Uploads `declarative-data-backup` to PyPI.
62
+ 3. Builds the existing Python 3.13 Dockerfile with the same package version,
63
+ verifies the installed version, and pushes
64
+ `${CI_REGISTRY_IMAGE}/declarative-data-backup:1.2.3-python3.13` to the
65
+ project's GitLab Container Registry.
66
+
67
+ The source checkout's version is not committed back to the repository.
68
+ The two publishing jobs run independently after the package build succeeds;
69
+ publishing is not atomic across PyPI and the container registry.
70
+
71
+ Before the first release:
72
+
73
+ - Enable the project's GitLab Container Registry.
74
+ - Add `PYPI_API_TOKEN` in GitLab **Settings > CI/CD > Variables** as a masked,
75
+ protected variable. Use a PyPI API token authorized to publish
76
+ `declarative-data-backup`; for a new PyPI project, its first upload may
77
+ require an account-wide token, which should then be replaced with a
78
+ project-scoped token.
79
+ - Protect release tags in GitLab (for example, `*.*.*`) and restrict who can
80
+ create them. Protected variables are only available on protected refs.
81
+ - Use a trusted Docker-executor runner configured for Docker-in-Docker with
82
+ TLS: `privileged = true` and `volumes = ["/certs/client", "/cache"]` in its
83
+ `[runners.docker]` configuration. GitLab.com instance runners support this
84
+ setup. Privileged runners should not execute untrusted code. If your runner
85
+ requires a tag, add it to `publish_container`.
86
+
87
+ Registry authentication uses GitLab's built-in `CI_REGISTRY_USER` and
88
+ `CI_REGISTRY_PASSWORD`; no separate registry password is needed.
89
+
90
+ Create and push a release tag from the commit you want to publish:
91
+
92
+ ```shell
93
+ git tag 1.2.3
94
+ git push origin 1.2.3
95
+ ```
96
+
97
+ Creating a GitLab Release with a new `1.2.3` tag also triggers the tag pipeline.
98
+ Creating or editing a Release on an already existing tag does not trigger a
99
+ new pipeline. PyPI versions cannot be overwritten: do not move or reuse
100
+ release tags. If only the container job fails, retry that job rather than
101
+ rerunning a successful PyPI upload.
102
+
103
+ ## Configure and run
104
+
105
+ Create a YAML file accessible to the backup process:
106
+
107
+ ```yaml
108
+ media_backup:
109
+ folders:
110
+ - /server_root/media/photos
111
+ - /server_root/media/music
112
+ compression: auto,zstd,22
113
+ retention_days: 30
114
+ required_mounts:
115
+ - /server_root/media
116
+ allow_empty: true
117
+
118
+ postgres_backup:
119
+ folders:
120
+ - /server_root/path/to/postgres-backups
121
+ required_mounts:
122
+ - /server_root/path/to/postgres-backups
123
+ ```
124
+
125
+ Set the required environment variables before starting the process:
126
+
127
+ | Variable | Purpose |
128
+ | --- | --- |
129
+ | `BACKUP_CONFIG_PATH` | Absolute path to the YAML configuration. |
130
+ | `SERVER_ROOT` | Absolute path to the root containing all configured source directories. |
131
+ | `BORG_REPO` | Existing Borg repository location, local or remote. |
132
+ | `BORG_KEY_FILE` | Path to the external keyfile for the encrypted repository. |
133
+ | `BORG_BASE_DIR` | Writable directory for Borg state and the backup coordination lock. |
134
+ | `BORG_RSH` | SSH command and options used by Borg for a remote repository. |
135
+ | `BORG_PASSCOMMAND` | Command Borg runs to obtain the repository passphrase. |
136
+
137
+ Optional variables:
138
+
139
+ | Variable | Purpose |
140
+ | --- | --- |
141
+ | `BORG_REMOTE_PATH` | Remote Borg executable path, passed as Borg's `--remote-path`. Leave unset to use Borg's default. |
142
+ | `BORG_EXIT_CODES` | Borg exit-code mode: `legacy` or `modern`. |
143
+
144
+ Provide credentials through a secret manager or protected files; do not put
145
+ passphrases, private keys, or credentials in the YAML file, source control, or
146
+ shell history. For example, in a controlled environment where these variables
147
+ have already been securely populated:
148
+
149
+ ```shell
150
+ python -m declarative_data_backup
151
+ ```
152
+
153
+ The command exits with status `0` on success, nonzero on configuration or Borg
154
+ failure, and `130` when interrupted. Borg's exit code `1` indicates warnings:
155
+ the run fails and pruning is skipped.
156
+
157
+ ## Configuration reference
158
+
159
+ The top level must be a nonempty YAML mapping of backup group names to group
160
+ mappings. Group names may contain ASCII letters, digits, `_`, and `-`, and
161
+ must be at most 64 characters. Duplicate YAML keys, unknown group fields,
162
+ invalid values, and malformed or empty configuration are rejected before any
163
+ create or prune operation.
164
+
165
+ Each group supports:
166
+
167
+ - `folders` (required): a nonempty list of absolute directory paths. Each
168
+ source must exist, be readable, and resolve inside `SERVER_ROOT`. Duplicate
169
+ canonical paths and parent/child overlaps across groups are rejected.
170
+ - `compression` (optional): defaults to `auto,zstd,22`. Supported Borg 1.4
171
+ codecs include `none`, `lz4`, `zstd`, `zlib`, and `lzma`; `auto` may precede
172
+ a codec. Codec levels are checked against supported ranges: zstd `1`-`22`,
173
+ zlib `0`-`9`, and lzma `0`-`9`. Borg's `obfuscate` compression specifier is
174
+ also accepted for specifier values `1`-`6`, `110`-`123`, or `250`; it is
175
+ useful only with repository encryption.
176
+ - `retention_days` (optional): positive integer, default `30`. This controls
177
+ Borg's `--keep-daily` policy.
178
+ - `required_mounts` (optional): list of exact mount points that must appear
179
+ in the process's `/proc/self/mountinfo`.
180
+ - `allow_empty` (optional): boolean, default `true`. If set to `false`, an
181
+ empty source is rejected.
182
+
183
+ Configure `required_mounts` explicitly for every source that depends on a
184
+ separately mounted filesystem. The program does not infer mount requirements
185
+ from source paths. In a container or Kubernetes pod, mounting `/server_root`
186
+ does not prove that nested host filesystems are mounted. In particular,
187
+ `allow_empty: true` does not prove that a disk is present: an empty source may
188
+ be legitimate and can be archived.
189
+
190
+ Paths are resolved before backup. A top-level source symlink is allowed only
191
+ when its canonical target remains inside `SERVER_ROOT`; symlinks within a
192
+ source are stored as links rather than followed.
193
+
194
+ ## Archives, retention, and compaction
195
+
196
+ The application creates one archive per configured source in the single
197
+ `BORG_REPO`. Names have this form:
198
+
199
+ ```text
200
+ bbkp__<source-series>__<UTC timestamp>__<run UUID>
201
+ ```
202
+
203
+ The stable source series is derived from the canonical source path and includes
204
+ a SHA-256 suffix. Borg receives the canonical absolute path as the source, so
205
+ the original directory hierarchy is retained inside the archive.
206
+
207
+ The application creates all configured archives before pruning any source.
208
+ If a create fails or Borg reports warnings, the run stops and no pruning is
209
+ performed. Retries use a new run UUID and may create another archive for that
210
+ day; archives are never overwritten to handle a collision.
211
+
212
+ `--keep-daily N` retains the latest archive in up to `N` daily buckets. It does
213
+ not guarantee `N` successful calendar days of recovery if scheduled runs fail
214
+ or are missed. Removing a source from the YAML does not delete its old
215
+ archives. The source-series selector does not match archives made with the
216
+ previous non-namespaced naming scheme; retain those archives unless a
217
+ separately reviewed migration or cleanup is performed.
218
+
219
+ Pruning marks repository data for deletion; it does not reclaim the space.
220
+ Repository compaction is a separate maintenance operation and is not run by
221
+ this application. Any scheduled compact job or manual maintenance must first
222
+ acquire an exclusive `flock` on:
223
+
224
+ ```text
225
+ <BORG_BASE_DIR>/backup.lock
226
+ ```
227
+
228
+ Use the same persistent state volume and filesystem as the backup job, with
229
+ working advisory-lock semantics. The backup process waits up to 3600 seconds
230
+ for this lock. Kubernetes `concurrencyPolicy: Forbid` does not coordinate
231
+ separate CronJobs, and Borg's own repository lock does not replace this
232
+ application-level lock. Never unlink or replace the lock file while any
233
+ process could hold it.
234
+
235
+ ## Deployment and operational notes
236
+
237
+ The Dockerfile builds an image with Borg 1.4.1 and the Python package. The
238
+ application itself does not set CPU or I/O priority; a scheduled deployment
239
+ may wrap the command with `nice -n 19` and `ionice -c 3`. For Kubernetes or
240
+ another container runtime:
241
+
242
+ - Run as a non-root user and confirm it can read the keyfile and credentials
243
+ and write the Borg state volume.
244
+ - Mount source data read-only. Keep Borg cache, configuration, security state,
245
+ keyfile, and the coordination lock on persistent writable storage.
246
+ - For nested source filesystems, configure and verify the corresponding
247
+ `required_mounts`.
248
+ - Use a versioned image tag or digest and confirm private-registry pull
249
+ credentials have only the required read access.
250
+ - Do not mount a service-account token unless the workload needs one.
251
+ - Ensure any compact job mounts the same state volume and honors the lock
252
+ contract above.
253
+ - Monitor failed and warning backup jobs. Retention cannot make up for days
254
+ without successful archives.
255
+ - A fixed schedule does not ensure upstream jobs, such as Unmanic processing,
256
+ have completed; establish that dependency separately.
257
+
258
+ The default `auto,zstd,22` may be slow or exceed a 1 GiB memory limit on some
259
+ data sets. It has not been benchmarked under that limit and may be terminated
260
+ for memory use. Compression can be overridden per group without recreating
261
+ the repository.
262
+
263
+ Kubernetes Secret projection permissions, storage ownership, mount behavior,
264
+ node placement, registry access, and workload scheduling must be verified in
265
+ the target cluster. They are not verified by this Python package or its
266
+ container integration test.
267
+
268
+ ## Testing
269
+
270
+ Run the unit tests:
271
+
272
+ ```shell
273
+ python -m pytest -m "not integration"
274
+ ```
275
+
276
+ The opt-in local Borg integration test uses a temporary encrypted repository:
277
+
278
+ ```shell
279
+ RUN_BORG_LOCAL_INTEGRATION=1 python -m pytest -m integration \
280
+ tests/integration/test_local_borg_integration.py
281
+ ```
282
+
283
+ The opt-in Docker SSH integration test uses disposable containers and a
284
+ temporary repository; it does not use production credentials or BorgBase:
285
+
286
+ ```shell
287
+ RUN_BORG_DOCKER_INTEGRATION=1 python -m pytest -m integration \
288
+ tests/integration/test_docker_integration.py
289
+ ```
290
+
291
+ Do not treat skipped integration tests as successful restore validation.
292
+
293
+ ## Recovery
294
+
295
+ Recovery requires the repository and the matching external keyfile and
296
+ passphrase. Keep independent recovery copies of the keyfile and passphrase;
297
+ the repository must not be their only copy. Also retain the SSH private key
298
+ and verified `known_hosts` entry for a remote repository.
299
+
300
+ With the required Borg environment and credentials securely configured, list
301
+ archives and inspect the desired archive:
302
+
303
+ ```shell
304
+ borg list "$BORG_REPO"
305
+ borg list "$BORG_REPO"::'bbkp__<source-series>__<timestamp>__<run-uuid>'
306
+ ```
307
+
308
+ Extract into an isolated recovery directory and verify the restored data
309
+ before returning it to service:
310
+
311
+ ```shell
312
+ mkdir -p /restore
313
+ cd /restore
314
+ borg extract "$BORG_REPO"::'bbkp__<source-series>__<timestamp>__<run-uuid>'
315
+ ```
316
+
317
+ Test recovery periodically using a disposable or approved recovery
318
+ environment. This repository's tests do not verify access to a live remote
319
+ repository or actual Kubernetes Secret projection and storage behavior.
@@ -0,0 +1,306 @@
1
+ # Declarative Data Backup
2
+
3
+ Declarative Data Backup creates encrypted Borg archives for configured source
4
+ directories. Each source gets its own archive series in one Borg repository,
5
+ so Borg can deduplicate chunks across all sources. The application validates
6
+ the complete configuration before invoking Borg and prunes a source's archives
7
+ only after every create operation in the run has succeeded.
8
+
9
+ ## Requirements
10
+
11
+ - Python 3.11 or newer.
12
+ - Borg 1.4.x installed separately and available as `borg` on `PATH`.
13
+ - A Borg repository that has already been initialized in keyfile encryption
14
+ mode, its matching keyfile, and a protected passphrase source.
15
+ - For remote repositories, OpenSSH and a trusted `known_hosts` file.
16
+
17
+ The application never initializes a repository. Borg 2 is not supported.
18
+
19
+ ## Installation
20
+
21
+ Install the package from PyPI:
22
+
23
+ ```shell
24
+ python -m pip install declarative-data-backup
25
+ ```
26
+
27
+ Or install a checkout with the development dependencies:
28
+
29
+ ```shell
30
+ python -m pip install -e ".[dev]"
31
+ ```
32
+
33
+ The PyPI distribution is named `declarative-data-backup`; its importable
34
+ package is `declarative_data_backup`.
35
+
36
+ ## GitLab release automation
37
+
38
+ The pipeline in [.gitlab-ci.yml](.gitlab-ci.yml) publishes releases when a Git
39
+ tag in `major.minor.patch` format is pushed, for example `1.2.3`. Use the same
40
+ version for the GitLab Release name and tag. Tags with a `v` prefix, prerelease
41
+ suffix, or leading zeros are not published. Branch pushes do not run this
42
+ release pipeline.
43
+
44
+ For each version tag, the pipeline:
45
+
46
+ 1. Sets the package version in the CI checkout's `pyproject.toml`, builds a
47
+ wheel and source distribution, and checks them with Twine.
48
+ 2. Uploads `declarative-data-backup` to PyPI.
49
+ 3. Builds the existing Python 3.13 Dockerfile with the same package version,
50
+ verifies the installed version, and pushes
51
+ `${CI_REGISTRY_IMAGE}/declarative-data-backup:1.2.3-python3.13` to the
52
+ project's GitLab Container Registry.
53
+
54
+ The source checkout's version is not committed back to the repository.
55
+ The two publishing jobs run independently after the package build succeeds;
56
+ publishing is not atomic across PyPI and the container registry.
57
+
58
+ Before the first release:
59
+
60
+ - Enable the project's GitLab Container Registry.
61
+ - Add `PYPI_API_TOKEN` in GitLab **Settings > CI/CD > Variables** as a masked,
62
+ protected variable. Use a PyPI API token authorized to publish
63
+ `declarative-data-backup`; for a new PyPI project, its first upload may
64
+ require an account-wide token, which should then be replaced with a
65
+ project-scoped token.
66
+ - Protect release tags in GitLab (for example, `*.*.*`) and restrict who can
67
+ create them. Protected variables are only available on protected refs.
68
+ - Use a trusted Docker-executor runner configured for Docker-in-Docker with
69
+ TLS: `privileged = true` and `volumes = ["/certs/client", "/cache"]` in its
70
+ `[runners.docker]` configuration. GitLab.com instance runners support this
71
+ setup. Privileged runners should not execute untrusted code. If your runner
72
+ requires a tag, add it to `publish_container`.
73
+
74
+ Registry authentication uses GitLab's built-in `CI_REGISTRY_USER` and
75
+ `CI_REGISTRY_PASSWORD`; no separate registry password is needed.
76
+
77
+ Create and push a release tag from the commit you want to publish:
78
+
79
+ ```shell
80
+ git tag 1.2.3
81
+ git push origin 1.2.3
82
+ ```
83
+
84
+ Creating a GitLab Release with a new `1.2.3` tag also triggers the tag pipeline.
85
+ Creating or editing a Release on an already existing tag does not trigger a
86
+ new pipeline. PyPI versions cannot be overwritten: do not move or reuse
87
+ release tags. If only the container job fails, retry that job rather than
88
+ rerunning a successful PyPI upload.
89
+
90
+ ## Configure and run
91
+
92
+ Create a YAML file accessible to the backup process:
93
+
94
+ ```yaml
95
+ media_backup:
96
+ folders:
97
+ - /server_root/media/photos
98
+ - /server_root/media/music
99
+ compression: auto,zstd,22
100
+ retention_days: 30
101
+ required_mounts:
102
+ - /server_root/media
103
+ allow_empty: true
104
+
105
+ postgres_backup:
106
+ folders:
107
+ - /server_root/path/to/postgres-backups
108
+ required_mounts:
109
+ - /server_root/path/to/postgres-backups
110
+ ```
111
+
112
+ Set the required environment variables before starting the process:
113
+
114
+ | Variable | Purpose |
115
+ | --- | --- |
116
+ | `BACKUP_CONFIG_PATH` | Absolute path to the YAML configuration. |
117
+ | `SERVER_ROOT` | Absolute path to the root containing all configured source directories. |
118
+ | `BORG_REPO` | Existing Borg repository location, local or remote. |
119
+ | `BORG_KEY_FILE` | Path to the external keyfile for the encrypted repository. |
120
+ | `BORG_BASE_DIR` | Writable directory for Borg state and the backup coordination lock. |
121
+ | `BORG_RSH` | SSH command and options used by Borg for a remote repository. |
122
+ | `BORG_PASSCOMMAND` | Command Borg runs to obtain the repository passphrase. |
123
+
124
+ Optional variables:
125
+
126
+ | Variable | Purpose |
127
+ | --- | --- |
128
+ | `BORG_REMOTE_PATH` | Remote Borg executable path, passed as Borg's `--remote-path`. Leave unset to use Borg's default. |
129
+ | `BORG_EXIT_CODES` | Borg exit-code mode: `legacy` or `modern`. |
130
+
131
+ Provide credentials through a secret manager or protected files; do not put
132
+ passphrases, private keys, or credentials in the YAML file, source control, or
133
+ shell history. For example, in a controlled environment where these variables
134
+ have already been securely populated:
135
+
136
+ ```shell
137
+ python -m declarative_data_backup
138
+ ```
139
+
140
+ The command exits with status `0` on success, nonzero on configuration or Borg
141
+ failure, and `130` when interrupted. Borg's exit code `1` indicates warnings:
142
+ the run fails and pruning is skipped.
143
+
144
+ ## Configuration reference
145
+
146
+ The top level must be a nonempty YAML mapping of backup group names to group
147
+ mappings. Group names may contain ASCII letters, digits, `_`, and `-`, and
148
+ must be at most 64 characters. Duplicate YAML keys, unknown group fields,
149
+ invalid values, and malformed or empty configuration are rejected before any
150
+ create or prune operation.
151
+
152
+ Each group supports:
153
+
154
+ - `folders` (required): a nonempty list of absolute directory paths. Each
155
+ source must exist, be readable, and resolve inside `SERVER_ROOT`. Duplicate
156
+ canonical paths and parent/child overlaps across groups are rejected.
157
+ - `compression` (optional): defaults to `auto,zstd,22`. Supported Borg 1.4
158
+ codecs include `none`, `lz4`, `zstd`, `zlib`, and `lzma`; `auto` may precede
159
+ a codec. Codec levels are checked against supported ranges: zstd `1`-`22`,
160
+ zlib `0`-`9`, and lzma `0`-`9`. Borg's `obfuscate` compression specifier is
161
+ also accepted for specifier values `1`-`6`, `110`-`123`, or `250`; it is
162
+ useful only with repository encryption.
163
+ - `retention_days` (optional): positive integer, default `30`. This controls
164
+ Borg's `--keep-daily` policy.
165
+ - `required_mounts` (optional): list of exact mount points that must appear
166
+ in the process's `/proc/self/mountinfo`.
167
+ - `allow_empty` (optional): boolean, default `true`. If set to `false`, an
168
+ empty source is rejected.
169
+
170
+ Configure `required_mounts` explicitly for every source that depends on a
171
+ separately mounted filesystem. The program does not infer mount requirements
172
+ from source paths. In a container or Kubernetes pod, mounting `/server_root`
173
+ does not prove that nested host filesystems are mounted. In particular,
174
+ `allow_empty: true` does not prove that a disk is present: an empty source may
175
+ be legitimate and can be archived.
176
+
177
+ Paths are resolved before backup. A top-level source symlink is allowed only
178
+ when its canonical target remains inside `SERVER_ROOT`; symlinks within a
179
+ source are stored as links rather than followed.
180
+
181
+ ## Archives, retention, and compaction
182
+
183
+ The application creates one archive per configured source in the single
184
+ `BORG_REPO`. Names have this form:
185
+
186
+ ```text
187
+ bbkp__<source-series>__<UTC timestamp>__<run UUID>
188
+ ```
189
+
190
+ The stable source series is derived from the canonical source path and includes
191
+ a SHA-256 suffix. Borg receives the canonical absolute path as the source, so
192
+ the original directory hierarchy is retained inside the archive.
193
+
194
+ The application creates all configured archives before pruning any source.
195
+ If a create fails or Borg reports warnings, the run stops and no pruning is
196
+ performed. Retries use a new run UUID and may create another archive for that
197
+ day; archives are never overwritten to handle a collision.
198
+
199
+ `--keep-daily N` retains the latest archive in up to `N` daily buckets. It does
200
+ not guarantee `N` successful calendar days of recovery if scheduled runs fail
201
+ or are missed. Removing a source from the YAML does not delete its old
202
+ archives. The source-series selector does not match archives made with the
203
+ previous non-namespaced naming scheme; retain those archives unless a
204
+ separately reviewed migration or cleanup is performed.
205
+
206
+ Pruning marks repository data for deletion; it does not reclaim the space.
207
+ Repository compaction is a separate maintenance operation and is not run by
208
+ this application. Any scheduled compact job or manual maintenance must first
209
+ acquire an exclusive `flock` on:
210
+
211
+ ```text
212
+ <BORG_BASE_DIR>/backup.lock
213
+ ```
214
+
215
+ Use the same persistent state volume and filesystem as the backup job, with
216
+ working advisory-lock semantics. The backup process waits up to 3600 seconds
217
+ for this lock. Kubernetes `concurrencyPolicy: Forbid` does not coordinate
218
+ separate CronJobs, and Borg's own repository lock does not replace this
219
+ application-level lock. Never unlink or replace the lock file while any
220
+ process could hold it.
221
+
222
+ ## Deployment and operational notes
223
+
224
+ The Dockerfile builds an image with Borg 1.4.1 and the Python package. The
225
+ application itself does not set CPU or I/O priority; a scheduled deployment
226
+ may wrap the command with `nice -n 19` and `ionice -c 3`. For Kubernetes or
227
+ another container runtime:
228
+
229
+ - Run as a non-root user and confirm it can read the keyfile and credentials
230
+ and write the Borg state volume.
231
+ - Mount source data read-only. Keep Borg cache, configuration, security state,
232
+ keyfile, and the coordination lock on persistent writable storage.
233
+ - For nested source filesystems, configure and verify the corresponding
234
+ `required_mounts`.
235
+ - Use a versioned image tag or digest and confirm private-registry pull
236
+ credentials have only the required read access.
237
+ - Do not mount a service-account token unless the workload needs one.
238
+ - Ensure any compact job mounts the same state volume and honors the lock
239
+ contract above.
240
+ - Monitor failed and warning backup jobs. Retention cannot make up for days
241
+ without successful archives.
242
+ - A fixed schedule does not ensure upstream jobs, such as Unmanic processing,
243
+ have completed; establish that dependency separately.
244
+
245
+ The default `auto,zstd,22` may be slow or exceed a 1 GiB memory limit on some
246
+ data sets. It has not been benchmarked under that limit and may be terminated
247
+ for memory use. Compression can be overridden per group without recreating
248
+ the repository.
249
+
250
+ Kubernetes Secret projection permissions, storage ownership, mount behavior,
251
+ node placement, registry access, and workload scheduling must be verified in
252
+ the target cluster. They are not verified by this Python package or its
253
+ container integration test.
254
+
255
+ ## Testing
256
+
257
+ Run the unit tests:
258
+
259
+ ```shell
260
+ python -m pytest -m "not integration"
261
+ ```
262
+
263
+ The opt-in local Borg integration test uses a temporary encrypted repository:
264
+
265
+ ```shell
266
+ RUN_BORG_LOCAL_INTEGRATION=1 python -m pytest -m integration \
267
+ tests/integration/test_local_borg_integration.py
268
+ ```
269
+
270
+ The opt-in Docker SSH integration test uses disposable containers and a
271
+ temporary repository; it does not use production credentials or BorgBase:
272
+
273
+ ```shell
274
+ RUN_BORG_DOCKER_INTEGRATION=1 python -m pytest -m integration \
275
+ tests/integration/test_docker_integration.py
276
+ ```
277
+
278
+ Do not treat skipped integration tests as successful restore validation.
279
+
280
+ ## Recovery
281
+
282
+ Recovery requires the repository and the matching external keyfile and
283
+ passphrase. Keep independent recovery copies of the keyfile and passphrase;
284
+ the repository must not be their only copy. Also retain the SSH private key
285
+ and verified `known_hosts` entry for a remote repository.
286
+
287
+ With the required Borg environment and credentials securely configured, list
288
+ archives and inspect the desired archive:
289
+
290
+ ```shell
291
+ borg list "$BORG_REPO"
292
+ borg list "$BORG_REPO"::'bbkp__<source-series>__<timestamp>__<run-uuid>'
293
+ ```
294
+
295
+ Extract into an isolated recovery directory and verify the restored data
296
+ before returning it to service:
297
+
298
+ ```shell
299
+ mkdir -p /restore
300
+ cd /restore
301
+ borg extract "$BORG_REPO"::'bbkp__<source-series>__<timestamp>__<run-uuid>'
302
+ ```
303
+
304
+ Test recovery periodically using a disposable or approved recovery
305
+ environment. This repository's tests do not verify access to a live remote
306
+ repository or actual Kubernetes Secret projection and storage behavior.
@@ -0,0 +1,30 @@
1
+ [build-system]
2
+ requires = ["setuptools>=68", "wheel"]
3
+ build-backend = "setuptools.build_meta"
4
+
5
+ [project]
6
+ name = "declarative-data-backup"
7
+ version = "0.1.0"
8
+ description = "Declarative Borg-based data backups"
9
+ readme = "README.md"
10
+ requires-python = ">=3.11"
11
+ dependencies = ["PyYAML>=6,<7"]
12
+
13
+ [project.optional-dependencies]
14
+ dev = [
15
+ "pre-commit>=4,<5",
16
+ "pyright>=1.1.400,<2",
17
+ "pytest>=8,<10",
18
+ "ruff>=0.11,<1",
19
+ ]
20
+
21
+ [tool.setuptools.packages.find]
22
+ where = ["src"]
23
+ include = ["declarative_data_backup*"]
24
+
25
+ [tool.ruff]
26
+ line-length = 88
27
+ target-version = "py311"
28
+
29
+ [tool.ruff.lint]
30
+ select = ["E4", "E7", "E9", "F", "I"]
@@ -0,0 +1,4 @@
1
+ [egg_info]
2
+ tag_build =
3
+ tag_date = 0
4
+
@@ -0,0 +1 @@
1
+ """Declarative Borg-based data backups."""
@@ -0,0 +1,6 @@
1
+ import sys
2
+
3
+ from .backup import main
4
+
5
+ if __name__ == "__main__":
6
+ sys.exit(main())