declarative-data-backup 0.5.2__tar.gz → 0.7.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (31) hide show
  1. {declarative_data_backup-0.5.2 → declarative_data_backup-0.7.0}/PKG-INFO +71 -29
  2. {declarative_data_backup-0.5.2 → declarative_data_backup-0.7.0}/README.md +69 -28
  3. {declarative_data_backup-0.5.2 → declarative_data_backup-0.7.0}/pyproject.toml +2 -2
  4. declarative_data_backup-0.7.0/src/declarative_data_backup/backup.py +276 -0
  5. {declarative_data_backup-0.5.2 → declarative_data_backup-0.7.0}/src/declarative_data_backup/config/backup_config/models.py +5 -2
  6. {declarative_data_backup-0.5.2 → declarative_data_backup-0.7.0}/src/declarative_data_backup/config/backup_config/utils.py +93 -22
  7. {declarative_data_backup-0.5.2 → declarative_data_backup-0.7.0}/src/declarative_data_backup/config/backup_config/yaml_parser.py +32 -3
  8. declarative_data_backup-0.7.0/src/declarative_data_backup/dirty_groups.py +150 -0
  9. {declarative_data_backup-0.5.2 → declarative_data_backup-0.7.0}/src/declarative_data_backup/environment.py +67 -1
  10. declarative_data_backup-0.7.0/src/declarative_data_backup/exclusions.py +127 -0
  11. {declarative_data_backup-0.5.2 → declarative_data_backup-0.7.0}/src/declarative_data_backup.egg-info/PKG-INFO +71 -29
  12. {declarative_data_backup-0.5.2 → declarative_data_backup-0.7.0}/src/declarative_data_backup.egg-info/SOURCES.txt +5 -1
  13. {declarative_data_backup-0.5.2 → declarative_data_backup-0.7.0}/src/declarative_data_backup.egg-info/requires.txt +1 -0
  14. {declarative_data_backup-0.5.2 → declarative_data_backup-0.7.0}/tests/test_backup.py +251 -5
  15. declarative_data_backup-0.7.0/tests/test_dirty_groups.py +188 -0
  16. declarative_data_backup-0.7.0/tests/test_exclusions.py +72 -0
  17. declarative_data_backup-0.5.2/src/declarative_data_backup/backup.py +0 -216
  18. {declarative_data_backup-0.5.2 → declarative_data_backup-0.7.0}/setup.cfg +0 -0
  19. {declarative_data_backup-0.5.2 → declarative_data_backup-0.7.0}/src/declarative_data_backup/__init__.py +0 -0
  20. {declarative_data_backup-0.5.2 → declarative_data_backup-0.7.0}/src/declarative_data_backup/__main__.py +0 -0
  21. {declarative_data_backup-0.5.2 → declarative_data_backup-0.7.0}/src/declarative_data_backup/borg/__init__.py +0 -0
  22. {declarative_data_backup-0.5.2 → declarative_data_backup-0.7.0}/src/declarative_data_backup/borg/archive_naming.py +0 -0
  23. {declarative_data_backup-0.5.2 → declarative_data_backup-0.7.0}/src/declarative_data_backup/borg/borg.py +0 -0
  24. {declarative_data_backup-0.5.2 → declarative_data_backup-0.7.0}/src/declarative_data_backup/borg/borg_cli.py +0 -0
  25. {declarative_data_backup-0.5.2 → declarative_data_backup-0.7.0}/src/declarative_data_backup/config/__init__.py +0 -0
  26. {declarative_data_backup-0.5.2 → declarative_data_backup-0.7.0}/src/declarative_data_backup/config/backup_config/__init__.py +0 -0
  27. {declarative_data_backup-0.5.2 → declarative_data_backup-0.7.0}/src/declarative_data_backup/config/backup_config/yaml_raw_loader.py +0 -0
  28. {declarative_data_backup-0.5.2 → declarative_data_backup-0.7.0}/src/declarative_data_backup/config/logging.py +0 -0
  29. {declarative_data_backup-0.5.2 → declarative_data_backup-0.7.0}/src/declarative_data_backup/errors.py +0 -0
  30. {declarative_data_backup-0.5.2 → declarative_data_backup-0.7.0}/src/declarative_data_backup.egg-info/dependency_links.txt +0 -0
  31. {declarative_data_backup-0.5.2 → declarative_data_backup-0.7.0}/src/declarative_data_backup.egg-info/top_level.txt +0 -0
@@ -1,10 +1,11 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: declarative-data-backup
3
- Version: 0.5.2
3
+ Version: 0.7.0
4
4
  Summary: Declarative Borg-based data backups
5
5
  Requires-Python: >=3.11
6
6
  Description-Content-Type: text/markdown
7
7
  Requires-Dist: PyYAML<7,>=6
8
+ Requires-Dist: redis<7,>=5
8
9
 
9
10
  # Declarative Data Backup
10
11
 
@@ -24,6 +25,15 @@ only after every create operation in the run has succeeded.
24
25
 
25
26
  The application never initializes a repository. Borg 2 is not supported.
26
27
 
28
+ ## Optional filesystem watcher
29
+
30
+ The separate Linux Rust [watcher](watcher/README.md) records changed backup
31
+ groups in Redis using filesystem-wide fanotify marks. The Python runner accepts
32
+ the same rooted YAML schema. `BACKUP_MODE=shadow` reports which groups would
33
+ be selected while still backing up all groups; `BACKUP_MODE=dirty` consumes
34
+ selected groups. The default remains `BACKUP_MODE=full`; keep full scheduling
35
+ until the watcher and Redis consumer have been validated in shadow operation.
36
+
27
37
  ## Installation
28
38
 
29
39
  Install the package from PyPI into a `uv`-managed environment:
@@ -45,13 +55,16 @@ package is `declarative_data_backup`.
45
55
 
46
56
  ## GitLab release automation
47
57
 
48
- The pipeline in [.gitlab-ci.yml](.gitlab-ci.yml) publishes releases when a Git
49
- tag in `major.minor.patch` format is pushed, for example `1.2.3`. Use the same
58
+ The pipeline in [.gitlab-ci.yml](.gitlab-ci.yml) publishes Python releases when
59
+ a Git tag in `major.minor.patch` format is pushed, for example `1.2.3`. Use the same
50
60
  version for the GitLab Release name and tag. Tags with a `v` prefix, prerelease
51
61
  suffix, or leading zeros are not published. Branch pushes do not run this
52
62
  release pipeline.
53
63
 
54
- For each version tag, the pipeline:
64
+ Watcher releases use independent `watcher/major.minor.patch` tags and publish
65
+ only the watcher image; see [watcher releases](watcher/README.md#releases).
66
+
67
+ For each Python version tag, the pipeline:
55
68
 
56
69
  1. Sets the package version in the CI checkout's `pyproject.toml` and uses
57
70
  `uv build` to create a wheel and source distribution.
@@ -103,9 +116,10 @@ Create a YAML file accessible to the backup process:
103
116
 
104
117
  ```yaml
105
118
  media_backup:
119
+ root: /server_root/media
106
120
  folders:
107
- - /server_root/media/photos
108
- - /server_root/media/music
121
+ - photos
122
+ - music
109
123
  compression: auto,zstd,22
110
124
  retention_days: 30
111
125
  required_mounts:
@@ -113,18 +127,19 @@ media_backup:
113
127
  allow_empty: true
114
128
 
115
129
  postgres_backup:
130
+ root: /server_root/backups
116
131
  folders:
117
- - /server_root/path/to/postgres-backups
118
- required_mounts:
119
- - /server_root/path/to/postgres-backups
132
+ - postgres
120
133
 
121
134
  apprise_backup:
135
+ root: /server_root/data
122
136
  files:
123
- - /server_root/data/apprise/apprise.yml
137
+ - apprise/apprise.yml
124
138
 
125
139
  jellyfin_backup:
140
+ root: /server_root/data
126
141
  folders:
127
- - /server_root/data/jellyfin
142
+ - jellyfin
128
143
  exclude:
129
144
  - "**/cache/**"
130
145
  - "**/log/**"
@@ -152,8 +167,14 @@ Optional variables:
152
167
  | `BACKUP_FILES_CACHE_TTL_MARGIN` | Extra archive creations to retain file-cache entries beyond the number of configured sources. The app sets Borg's TTL to source count plus this margin; defaults to `20`. |
153
168
  | `BORG_FILES_CACHE_TTL` | Optional absolute Borg TTL override; it must be at least the number of configured sources. Usually leave unset to use the source-count-based TTL and configured margin. |
154
169
  | `BACKUP_FAILURES_PATH` | Optional absolute path where failed backup group names are written as JSON. |
170
+ | `BACKUP_SUCCESSES_PATH` | Optional absolute path where successfully backed-up group names are written as JSON, including groups whose subsequent pruning failed. |
155
171
  | `BACKUP_PROGRESS` | Set to `true` to log the latest Borg progress every 30 seconds during archive creation. Defaults to `false`. |
156
172
  | `BACKUP_VERBOSE` | Set to `true` to show Borg's per-file create and per-archive prune output. Defaults to `false` for concise logs. |
173
+ | `BACKUP_MODE` | `full` (default) backs up every configured group; `shadow` reports the dirty selection but backs up all groups; `dirty` consumes `ddb:v1:dirty-groups` from Redis. Shadow and dirty modes require the Redis variables below. |
174
+ | `REDIS_URL` | Required in shadow/dirty mode. A `redis://` TCP URL without credentials. |
175
+ | `REDIS_DBINDEX` | Required in shadow/dirty mode. Nonnegative logical database index shared with the watcher. |
176
+ | `REDIS_USERNAME` | Optional ACL username in shadow/dirty mode. |
177
+ | `REDIS_PASSWORD` | Required in shadow/dirty mode. Supply through a Kubernetes Secret or protected source. |
157
178
 
158
179
  Provide credentials through a secret manager or protected files; do not put
159
180
  passphrases, private keys, or credentials in the YAML file, source control, or
@@ -191,22 +212,32 @@ locks without first confirming that no repository operation is still running.
191
212
  The top level must be a nonempty YAML mapping of backup group names to group
192
213
  mappings. Group names may contain ASCII letters, digits, `_`, and `-`, and
193
214
  must be at most 64 characters. Duplicate YAML keys, unknown group fields,
194
- invalid types, unsupported pattern selectors, and empty configuration are
195
- rejected before any create or prune operation.
215
+ invalid types, unsupported patterns, and empty configuration are rejected
216
+ before any create or prune operation.
196
217
 
197
218
  Each group supports:
198
219
 
199
- - `folders` (optional): list of absolute directory paths.
200
- - `files` (optional): list of absolute regular-file paths. Folders and files
201
- share the same containment, readability, duplicate, and overlap checks;
202
- configure at least one source per group.
203
- - `exclude` (optional): list of Borg 1.4 glob patterns, scoped to this group
204
- and passed directly to Borg for folder sources. To avoid approximating
205
- Borg's pattern matching, a group with explicit `files` cannot also configure
206
- `exclude`; this is rejected before any Borg operation.
207
- Patterns are otherwise passed unchanged to Borg, which evaluates their
208
- matching semantics. If Borg rejects a pattern during create, the run stops
209
- before pruning any source.
220
+ - `root` (required): absolute primary source root, lexically and canonically
221
+ inside `SERVER_ROOT`. It must exist, be a directory, and be accessible.
222
+ - `watcher_alternative_roots` (optional): absolute roots representing the same
223
+ relative layout. These roots are watcher-only: Python does not require their
224
+ directories or mounts to exist, and they are never backed up.
225
+ - `folders` (optional): nonempty relative directory paths under `root`. A
226
+ folder `.` selects the root itself.
227
+ - `files` (optional): nonempty relative regular-file paths under `root`.
228
+ Sources reject absolute paths, NUL bytes, and `..` components. Their
229
+ canonical targets must remain inside `root`. Configure at least one source.
230
+ - `exclude` (optional): root-relative glob patterns shared with the watcher.
231
+ Patterns are anchored and case-sensitive. Version 1 allows ASCII
232
+ letters/digits, space, `_`, `-`, `.`, `/`, `*`, `?`, and `**` as a whole
233
+ component; empty, `.` and `..` components are rejected. An excluded
234
+ directory excludes descendants, and a trailing `/**` excludes the directory
235
+ itself too. Patterns are translated to root-scoped Borg regular expressions,
236
+ not passed through as Borg syntax. Patterns whose translated regex exceeds
237
+ 64,000 characters are rejected. A group with explicit `files` cannot also
238
+ configure `exclude`; this safety check remains until a policy and
239
+ archive-level tests define the behavior for an explicitly selected file
240
+ matched by an exclusion.
210
241
  - `compression` (optional): defaults to `auto,zstd,22`. Supported Borg 1.4
211
242
  codecs include `none`, `lz4`, `zstd`, `zlib`, and `lzma`; `auto` may precede
212
243
  a codec. Codec levels are checked against supported ranges: zstd `1`-`22`,
@@ -215,8 +246,10 @@ Each group supports:
215
246
  useful only with repository encryption.
216
247
  - `retention_days` (optional): positive integer, default `30`. This controls
217
248
  Borg's `--keep-daily` policy.
218
- - `required_mounts` (optional): list of exact mount points that must appear
219
- in the process's `/proc/self/mountinfo`.
249
+ - `required_mounts` (optional): absolute paths to existing exact mount points
250
+ that must appear in the process's `/proc/self/mountinfo`. These are
251
+ availability requirements, not source selectors. Alternative roots are not
252
+ automatically added.
220
253
  - `allow_empty` (optional): boolean, default `true`. If set to `false`, an
221
254
  empty source is rejected.
222
255
 
@@ -232,9 +265,18 @@ consistent database snapshot. Use an application-supported export, backup API,
232
265
  or coordinated filesystem snapshot for live database data. Exclusions do not
233
266
  make a database backup consistent.
234
267
 
235
- Paths are resolved before backup. A top-level source symlink is allowed only
236
- when its canonical target remains inside `SERVER_ROOT`; symlinks within a
237
- source are stored as links rather than followed.
268
+ Primary roots and sources are resolved before backup. A source symlink is
269
+ allowed only when its canonical target remains inside the group's `root`;
270
+ symlinks within a selected source are stored as links rather than followed.
271
+
272
+ Dirty mode acquires the same `BORG_BASE_DIR/backup.lock` as full mode before
273
+ recovering in-flight groups, clearing stale group names, or claiming work.
274
+ Claims use Redis `SMOVE`; successful claims are removed only after all creates
275
+ and retention pruning for that group succeed. Failed groups are requeued, and
276
+ claims left by a crashed process are recovered on the next run. Redis errors
277
+ fail the job; they are never treated as an empty dirty set. Keep periodic full
278
+ reconciliation backups as a safety net. The daily CronJob remains in full mode
279
+ until dirty-mode rollout is explicitly enabled.
238
280
 
239
281
  ## Archives, retention, and compaction
240
282
 
@@ -16,6 +16,15 @@ only after every create operation in the run has succeeded.
16
16
 
17
17
  The application never initializes a repository. Borg 2 is not supported.
18
18
 
19
+ ## Optional filesystem watcher
20
+
21
+ The separate Linux Rust [watcher](watcher/README.md) records changed backup
22
+ groups in Redis using filesystem-wide fanotify marks. The Python runner accepts
23
+ the same rooted YAML schema. `BACKUP_MODE=shadow` reports which groups would
24
+ be selected while still backing up all groups; `BACKUP_MODE=dirty` consumes
25
+ selected groups. The default remains `BACKUP_MODE=full`; keep full scheduling
26
+ until the watcher and Redis consumer have been validated in shadow operation.
27
+
19
28
  ## Installation
20
29
 
21
30
  Install the package from PyPI into a `uv`-managed environment:
@@ -37,13 +46,16 @@ package is `declarative_data_backup`.
37
46
 
38
47
  ## GitLab release automation
39
48
 
40
- The pipeline in [.gitlab-ci.yml](.gitlab-ci.yml) publishes releases when a Git
41
- tag in `major.minor.patch` format is pushed, for example `1.2.3`. Use the same
49
+ The pipeline in [.gitlab-ci.yml](.gitlab-ci.yml) publishes Python releases when
50
+ a Git tag in `major.minor.patch` format is pushed, for example `1.2.3`. Use the same
42
51
  version for the GitLab Release name and tag. Tags with a `v` prefix, prerelease
43
52
  suffix, or leading zeros are not published. Branch pushes do not run this
44
53
  release pipeline.
45
54
 
46
- For each version tag, the pipeline:
55
+ Watcher releases use independent `watcher/major.minor.patch` tags and publish
56
+ only the watcher image; see [watcher releases](watcher/README.md#releases).
57
+
58
+ For each Python version tag, the pipeline:
47
59
 
48
60
  1. Sets the package version in the CI checkout's `pyproject.toml` and uses
49
61
  `uv build` to create a wheel and source distribution.
@@ -95,9 +107,10 @@ Create a YAML file accessible to the backup process:
95
107
 
96
108
  ```yaml
97
109
  media_backup:
110
+ root: /server_root/media
98
111
  folders:
99
- - /server_root/media/photos
100
- - /server_root/media/music
112
+ - photos
113
+ - music
101
114
  compression: auto,zstd,22
102
115
  retention_days: 30
103
116
  required_mounts:
@@ -105,18 +118,19 @@ media_backup:
105
118
  allow_empty: true
106
119
 
107
120
  postgres_backup:
121
+ root: /server_root/backups
108
122
  folders:
109
- - /server_root/path/to/postgres-backups
110
- required_mounts:
111
- - /server_root/path/to/postgres-backups
123
+ - postgres
112
124
 
113
125
  apprise_backup:
126
+ root: /server_root/data
114
127
  files:
115
- - /server_root/data/apprise/apprise.yml
128
+ - apprise/apprise.yml
116
129
 
117
130
  jellyfin_backup:
131
+ root: /server_root/data
118
132
  folders:
119
- - /server_root/data/jellyfin
133
+ - jellyfin
120
134
  exclude:
121
135
  - "**/cache/**"
122
136
  - "**/log/**"
@@ -144,8 +158,14 @@ Optional variables:
144
158
  | `BACKUP_FILES_CACHE_TTL_MARGIN` | Extra archive creations to retain file-cache entries beyond the number of configured sources. The app sets Borg's TTL to source count plus this margin; defaults to `20`. |
145
159
  | `BORG_FILES_CACHE_TTL` | Optional absolute Borg TTL override; it must be at least the number of configured sources. Usually leave unset to use the source-count-based TTL and configured margin. |
146
160
  | `BACKUP_FAILURES_PATH` | Optional absolute path where failed backup group names are written as JSON. |
161
+ | `BACKUP_SUCCESSES_PATH` | Optional absolute path where successfully backed-up group names are written as JSON, including groups whose subsequent pruning failed. |
147
162
  | `BACKUP_PROGRESS` | Set to `true` to log the latest Borg progress every 30 seconds during archive creation. Defaults to `false`. |
148
163
  | `BACKUP_VERBOSE` | Set to `true` to show Borg's per-file create and per-archive prune output. Defaults to `false` for concise logs. |
164
+ | `BACKUP_MODE` | `full` (default) backs up every configured group; `shadow` reports the dirty selection but backs up all groups; `dirty` consumes `ddb:v1:dirty-groups` from Redis. Shadow and dirty modes require the Redis variables below. |
165
+ | `REDIS_URL` | Required in shadow/dirty mode. A `redis://` TCP URL without credentials. |
166
+ | `REDIS_DBINDEX` | Required in shadow/dirty mode. Nonnegative logical database index shared with the watcher. |
167
+ | `REDIS_USERNAME` | Optional ACL username in shadow/dirty mode. |
168
+ | `REDIS_PASSWORD` | Required in shadow/dirty mode. Supply through a Kubernetes Secret or protected source. |
149
169
 
150
170
  Provide credentials through a secret manager or protected files; do not put
151
171
  passphrases, private keys, or credentials in the YAML file, source control, or
@@ -183,22 +203,32 @@ locks without first confirming that no repository operation is still running.
183
203
  The top level must be a nonempty YAML mapping of backup group names to group
184
204
  mappings. Group names may contain ASCII letters, digits, `_`, and `-`, and
185
205
  must be at most 64 characters. Duplicate YAML keys, unknown group fields,
186
- invalid types, unsupported pattern selectors, and empty configuration are
187
- rejected before any create or prune operation.
206
+ invalid types, unsupported patterns, and empty configuration are rejected
207
+ before any create or prune operation.
188
208
 
189
209
  Each group supports:
190
210
 
191
- - `folders` (optional): list of absolute directory paths.
192
- - `files` (optional): list of absolute regular-file paths. Folders and files
193
- share the same containment, readability, duplicate, and overlap checks;
194
- configure at least one source per group.
195
- - `exclude` (optional): list of Borg 1.4 glob patterns, scoped to this group
196
- and passed directly to Borg for folder sources. To avoid approximating
197
- Borg's pattern matching, a group with explicit `files` cannot also configure
198
- `exclude`; this is rejected before any Borg operation.
199
- Patterns are otherwise passed unchanged to Borg, which evaluates their
200
- matching semantics. If Borg rejects a pattern during create, the run stops
201
- before pruning any source.
211
+ - `root` (required): absolute primary source root, lexically and canonically
212
+ inside `SERVER_ROOT`. It must exist, be a directory, and be accessible.
213
+ - `watcher_alternative_roots` (optional): absolute roots representing the same
214
+ relative layout. These roots are watcher-only: Python does not require their
215
+ directories or mounts to exist, and they are never backed up.
216
+ - `folders` (optional): nonempty relative directory paths under `root`. A
217
+ folder `.` selects the root itself.
218
+ - `files` (optional): nonempty relative regular-file paths under `root`.
219
+ Sources reject absolute paths, NUL bytes, and `..` components. Their
220
+ canonical targets must remain inside `root`. Configure at least one source.
221
+ - `exclude` (optional): root-relative glob patterns shared with the watcher.
222
+ Patterns are anchored and case-sensitive. Version 1 allows ASCII
223
+ letters/digits, space, `_`, `-`, `.`, `/`, `*`, `?`, and `**` as a whole
224
+ component; empty, `.` and `..` components are rejected. An excluded
225
+ directory excludes descendants, and a trailing `/**` excludes the directory
226
+ itself too. Patterns are translated to root-scoped Borg regular expressions,
227
+ not passed through as Borg syntax. Patterns whose translated regex exceeds
228
+ 64,000 characters are rejected. A group with explicit `files` cannot also
229
+ configure `exclude`; this safety check remains until a policy and
230
+ archive-level tests define the behavior for an explicitly selected file
231
+ matched by an exclusion.
202
232
  - `compression` (optional): defaults to `auto,zstd,22`. Supported Borg 1.4
203
233
  codecs include `none`, `lz4`, `zstd`, `zlib`, and `lzma`; `auto` may precede
204
234
  a codec. Codec levels are checked against supported ranges: zstd `1`-`22`,
@@ -207,8 +237,10 @@ Each group supports:
207
237
  useful only with repository encryption.
208
238
  - `retention_days` (optional): positive integer, default `30`. This controls
209
239
  Borg's `--keep-daily` policy.
210
- - `required_mounts` (optional): list of exact mount points that must appear
211
- in the process's `/proc/self/mountinfo`.
240
+ - `required_mounts` (optional): absolute paths to existing exact mount points
241
+ that must appear in the process's `/proc/self/mountinfo`. These are
242
+ availability requirements, not source selectors. Alternative roots are not
243
+ automatically added.
212
244
  - `allow_empty` (optional): boolean, default `true`. If set to `false`, an
213
245
  empty source is rejected.
214
246
 
@@ -224,9 +256,18 @@ consistent database snapshot. Use an application-supported export, backup API,
224
256
  or coordinated filesystem snapshot for live database data. Exclusions do not
225
257
  make a database backup consistent.
226
258
 
227
- Paths are resolved before backup. A top-level source symlink is allowed only
228
- when its canonical target remains inside `SERVER_ROOT`; symlinks within a
229
- source are stored as links rather than followed.
259
+ Primary roots and sources are resolved before backup. A source symlink is
260
+ allowed only when its canonical target remains inside the group's `root`;
261
+ symlinks within a selected source are stored as links rather than followed.
262
+
263
+ Dirty mode acquires the same `BORG_BASE_DIR/backup.lock` as full mode before
264
+ recovering in-flight groups, clearing stale group names, or claiming work.
265
+ Claims use Redis `SMOVE`; successful claims are removed only after all creates
266
+ and retention pruning for that group succeed. Failed groups are requeued, and
267
+ claims left by a crashed process are recovered on the next run. Redis errors
268
+ fail the job; they are never treated as an empty dirty set. Keep periodic full
269
+ reconciliation backups as a safety net. The daily CronJob remains in full mode
270
+ until dirty-mode rollout is explicitly enabled.
230
271
 
231
272
  ## Archives, retention, and compaction
232
273
 
@@ -4,11 +4,11 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "declarative-data-backup"
7
- version = "0.5.2"
7
+ version = "0.7.0"
8
8
  description = "Declarative Borg-based data backups"
9
9
  readme = "README.md"
10
10
  requires-python = ">=3.11"
11
- dependencies = ["PyYAML>=6,<7"]
11
+ dependencies = ["PyYAML>=6,<7", "redis>=5,<7"]
12
12
 
13
13
  [dependency-groups]
14
14
  dev = [
@@ -0,0 +1,276 @@
1
+ import json
2
+ import logging
3
+ import os
4
+ import signal
5
+ import uuid
6
+ from collections.abc import Mapping
7
+ from dataclasses import replace
8
+ from datetime import UTC, datetime
9
+ from pathlib import Path
10
+ from types import FrameType
11
+
12
+ from .borg import (
13
+ backup_folder,
14
+ check_keyfile,
15
+ prune,
16
+ repository_lock,
17
+ )
18
+ from .borg.borg_cli import ensure_borg_binary
19
+ from .config import configure_logging, load_yaml, parse_backup_definitions
20
+ from .config.backup_config.models import BackupDefinition
21
+ from .dirty_groups import DirtyGroupConsumer
22
+ from .environment import Environment
23
+ from .errors import BackupError, BackupInterrupted, BorgWarningError
24
+ from .exclusions import translate_excludes
25
+
26
+ logger = logging.getLogger(__name__)
27
+
28
+
29
+ def write_group_summary(path: Path | None, groups: set[str]) -> None:
30
+ if path is None:
31
+ return
32
+ try:
33
+ path.write_text(json.dumps(sorted(groups)), encoding="utf-8")
34
+ except OSError as exc:
35
+ logger.warning(
36
+ "Unable to write backup group summary to %s: %s",
37
+ path,
38
+ exc.strerror or "I/O error",
39
+ )
40
+
41
+
42
+ def check_settings(settings: Environment) -> None:
43
+ ensure_borg_binary()
44
+ if not settings.borg_base_dir.is_dir():
45
+ raise BackupError(
46
+ f"BORG_BASE_DIR does not exist or is not a directory: "
47
+ f"{settings.borg_base_dir}."
48
+ )
49
+ if not os.access(settings.borg_base_dir, os.R_OK | os.W_OK | os.X_OK):
50
+ raise BackupError(
51
+ f"BORG_BASE_DIR is not accessible for Borg state: {settings.borg_base_dir}."
52
+ )
53
+ check_keyfile(settings)
54
+
55
+
56
+ def load_definitions(settings: Environment) -> list[BackupDefinition]:
57
+ return parse_backup_definitions(
58
+ load_yaml(settings.config_path),
59
+ settings.server_root,
60
+ )
61
+
62
+
63
+ def run_backup(
64
+ settings: Environment,
65
+ definitions: list[BackupDefinition],
66
+ ) -> None:
67
+ total_sources = sum(len(definition.sources) for definition in definitions)
68
+ logger.info(
69
+ "Validated %d backup groups containing %d sources",
70
+ len(definitions),
71
+ total_sources,
72
+ )
73
+ if (
74
+ settings.borg_files_cache_ttl is not None
75
+ and settings.borg_files_cache_ttl < total_sources
76
+ ):
77
+ raise BackupError(
78
+ "BORG_FILES_CACHE_TTL must be at least the number of configured "
79
+ f"source creates ({total_sources})."
80
+ )
81
+ if settings.borg_files_cache_ttl is None:
82
+ files_cache_ttl = total_sources + settings.backup_files_cache_ttl_margin
83
+ settings = replace(settings, borg_files_cache_ttl=files_cache_ttl)
84
+ logger.info(
85
+ "Using Borg files-cache TTL %d (%d source creates + %d margin).",
86
+ files_cache_ttl,
87
+ total_sources,
88
+ settings.backup_files_cache_ttl_margin,
89
+ )
90
+ else:
91
+ logger.info(
92
+ "Using explicit Borg files-cache TTL %d.",
93
+ settings.borg_files_cache_ttl,
94
+ )
95
+
96
+ run_id = uuid.uuid4().hex
97
+ with repository_lock(settings.borg_base_dir / "backup.lock"):
98
+ backed_up_groups: set[str] = set()
99
+ write_group_summary(settings.backed_up_groups_path, backed_up_groups)
100
+ dirty_groups = (
101
+ DirtyGroupConsumer(settings)
102
+ if settings.backup_mode in {"shadow", "dirty"}
103
+ else None
104
+ )
105
+ try:
106
+ if settings.backup_mode in {"shadow", "dirty"} and dirty_groups is not None:
107
+ group_names = {definition.name for definition in definitions}
108
+ if settings.backup_mode == "shadow":
109
+ dirty_groups.preview(group_names)
110
+ else:
111
+ dirty_groups.prepare(group_names)
112
+
113
+ created_at = datetime.now(UTC)
114
+ successful_definitions = []
115
+ failures: list[BackupError] = []
116
+ failed_groups: set[str] = set()
117
+ for definition in definitions:
118
+ if settings.backup_mode == "dirty" and dirty_groups is not None:
119
+ claimed = dirty_groups.claim(definition.name)
120
+ else:
121
+ claimed = True
122
+ if not claimed:
123
+ logger.info(
124
+ "Skipping backup group '%s'; it is not dirty.",
125
+ definition.name,
126
+ )
127
+ continue
128
+
129
+ logger.info(
130
+ "Starting backup group '%s' (%d sources).",
131
+ definition.name,
132
+ len(definition.sources),
133
+ )
134
+ try:
135
+ excludes = translate_excludes(
136
+ definition.root,
137
+ definition.excludes,
138
+ )
139
+ for folder in definition.sources:
140
+ backup_folder(
141
+ folder,
142
+ definition.compression,
143
+ definition.name,
144
+ settings,
145
+ created_at,
146
+ run_id,
147
+ excludes,
148
+ )
149
+ except BackupError as exc:
150
+ if settings.backup_mode == "dirty" and dirty_groups is not None:
151
+ try:
152
+ dirty_groups.requeue(definition.name)
153
+ except BackupError as state_error:
154
+ failed_groups.add(definition.name)
155
+ write_group_summary(
156
+ settings.failure_groups_path, failed_groups
157
+ )
158
+ raise BackupError(f"{exc}; {state_error}") from state_error
159
+ logger.error(
160
+ "Backup group '%s' failed; retention pruning will be skipped "
161
+ "for this group: %s",
162
+ definition.name,
163
+ exc,
164
+ )
165
+ failures.append(exc)
166
+ failed_groups.add(definition.name)
167
+ write_group_summary(settings.failure_groups_path, failed_groups)
168
+ continue
169
+
170
+ logger.info("Backup group '%s' completed.", definition.name)
171
+ successful_definitions.append(definition)
172
+ backed_up_groups.add(definition.name)
173
+ write_group_summary(settings.backed_up_groups_path, backed_up_groups)
174
+
175
+ for definition in successful_definitions:
176
+ logger.info(
177
+ "Starting retention pruning for group '%s' (%d sources).",
178
+ definition.name,
179
+ len(definition.sources),
180
+ )
181
+ pruning_succeeded = True
182
+ for folder in definition.sources:
183
+ try:
184
+ prune(
185
+ folder,
186
+ definition.retention_days,
187
+ definition.name,
188
+ settings,
189
+ )
190
+ except BackupError as exc:
191
+ logger.error(
192
+ "Retention pruning failed for group '%s' source '%s': %s",
193
+ definition.name,
194
+ folder,
195
+ exc,
196
+ )
197
+ failures.append(exc)
198
+ failed_groups.add(definition.name)
199
+ write_group_summary(settings.failure_groups_path, failed_groups)
200
+ pruning_succeeded = False
201
+
202
+ if settings.backup_mode == "dirty" and dirty_groups is not None:
203
+ try:
204
+ if pruning_succeeded:
205
+ dirty_groups.acknowledge(definition.name)
206
+ else:
207
+ dirty_groups.requeue(definition.name)
208
+ except BackupError as state_error:
209
+ failed_groups.add(definition.name)
210
+ write_group_summary(settings.failure_groups_path, failed_groups)
211
+ raise state_error
212
+
213
+ if pruning_succeeded:
214
+ logger.info(
215
+ "Retention pruning for group '%s' completed.",
216
+ definition.name,
217
+ )
218
+
219
+ if failures:
220
+ write_group_summary(settings.failure_groups_path, failed_groups)
221
+ message = "; ".join(str(failure) for failure in failures)
222
+ if any(isinstance(failure, BorgWarningError) for failure in failures):
223
+ raise BorgWarningError(message)
224
+ raise BackupError(message)
225
+ finally:
226
+ if dirty_groups is not None:
227
+ dirty_groups.close()
228
+
229
+
230
+ def main(environ: Mapping[str, str] | None = None) -> int:
231
+ configure_logging()
232
+
233
+ def interrupt(signum: int, frame: FrameType | None) -> None:
234
+ # Repeated signals must not interrupt Borg cleanup or lock release.
235
+ signal.signal(signal.SIGTERM, signal.SIG_IGN)
236
+ signal.signal(signal.SIGINT, signal.SIG_IGN)
237
+ raise BackupInterrupted(signum)
238
+
239
+ previous_sigterm = signal.signal(signal.SIGTERM, interrupt)
240
+ previous_sigint = signal.signal(signal.SIGINT, interrupt)
241
+ try:
242
+ settings = Environment.from_env(environ)
243
+ logger.info("Starting backup using configuration %s", settings.config_path)
244
+ check_settings(settings)
245
+ run_backup(settings, load_definitions(settings))
246
+ logger.info("Backup completed successfully.")
247
+ return 0
248
+
249
+ except BorgWarningError as exc:
250
+ logger.error(
251
+ "Backup failed because Borg reported warnings; "
252
+ "retention pruning may have completed for other groups: %s",
253
+ exc,
254
+ )
255
+ return 1
256
+ except BackupError as exc:
257
+ logger.error("%s", exc)
258
+ return 1
259
+ except OSError as exc:
260
+ logger.error(
261
+ "Unexpected I/O failure: %s.",
262
+ exc.strerror or "I/O error",
263
+ )
264
+ return 1
265
+ except BackupInterrupted as exc:
266
+ logger.error(
267
+ "Interrupted by %s; no further backups or pruning will run.",
268
+ signal.Signals(exc.signum).name,
269
+ )
270
+ return 128 + exc.signum
271
+ except KeyboardInterrupt:
272
+ logger.error("Interrupted.")
273
+ return 130
274
+ finally:
275
+ signal.signal(signal.SIGTERM, previous_sigterm)
276
+ signal.signal(signal.SIGINT, previous_sigint)
@@ -5,14 +5,17 @@ from pathlib import Path
5
5
  @dataclass(frozen=True)
6
6
  class BackupDefinition:
7
7
  name: str
8
+ root: Path
8
9
  folders: tuple[Path, ...]
10
+ files: tuple[Path, ...]
11
+ canonical_sources: tuple[Path, ...]
9
12
  compression: str
10
13
  retention_days: int
11
14
  required_mounts: tuple[Path, ...] = ()
15
+ watcher_alternative_roots: tuple[Path, ...] = ()
12
16
  allow_empty: bool = True
13
- files: tuple[Path, ...] = ()
14
17
  excludes: tuple[str, ...] = ()
15
18
 
16
19
  @property
17
20
  def sources(self) -> tuple[Path, ...]:
18
- return self.folders + self.files
21
+ return self.canonical_sources