declarative-data-backup 0.5.2__tar.gz → 0.7.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {declarative_data_backup-0.5.2 → declarative_data_backup-0.7.0}/PKG-INFO +71 -29
- {declarative_data_backup-0.5.2 → declarative_data_backup-0.7.0}/README.md +69 -28
- {declarative_data_backup-0.5.2 → declarative_data_backup-0.7.0}/pyproject.toml +2 -2
- declarative_data_backup-0.7.0/src/declarative_data_backup/backup.py +276 -0
- {declarative_data_backup-0.5.2 → declarative_data_backup-0.7.0}/src/declarative_data_backup/config/backup_config/models.py +5 -2
- {declarative_data_backup-0.5.2 → declarative_data_backup-0.7.0}/src/declarative_data_backup/config/backup_config/utils.py +93 -22
- {declarative_data_backup-0.5.2 → declarative_data_backup-0.7.0}/src/declarative_data_backup/config/backup_config/yaml_parser.py +32 -3
- declarative_data_backup-0.7.0/src/declarative_data_backup/dirty_groups.py +150 -0
- {declarative_data_backup-0.5.2 → declarative_data_backup-0.7.0}/src/declarative_data_backup/environment.py +67 -1
- declarative_data_backup-0.7.0/src/declarative_data_backup/exclusions.py +127 -0
- {declarative_data_backup-0.5.2 → declarative_data_backup-0.7.0}/src/declarative_data_backup.egg-info/PKG-INFO +71 -29
- {declarative_data_backup-0.5.2 → declarative_data_backup-0.7.0}/src/declarative_data_backup.egg-info/SOURCES.txt +5 -1
- {declarative_data_backup-0.5.2 → declarative_data_backup-0.7.0}/src/declarative_data_backup.egg-info/requires.txt +1 -0
- {declarative_data_backup-0.5.2 → declarative_data_backup-0.7.0}/tests/test_backup.py +251 -5
- declarative_data_backup-0.7.0/tests/test_dirty_groups.py +188 -0
- declarative_data_backup-0.7.0/tests/test_exclusions.py +72 -0
- declarative_data_backup-0.5.2/src/declarative_data_backup/backup.py +0 -216
- {declarative_data_backup-0.5.2 → declarative_data_backup-0.7.0}/setup.cfg +0 -0
- {declarative_data_backup-0.5.2 → declarative_data_backup-0.7.0}/src/declarative_data_backup/__init__.py +0 -0
- {declarative_data_backup-0.5.2 → declarative_data_backup-0.7.0}/src/declarative_data_backup/__main__.py +0 -0
- {declarative_data_backup-0.5.2 → declarative_data_backup-0.7.0}/src/declarative_data_backup/borg/__init__.py +0 -0
- {declarative_data_backup-0.5.2 → declarative_data_backup-0.7.0}/src/declarative_data_backup/borg/archive_naming.py +0 -0
- {declarative_data_backup-0.5.2 → declarative_data_backup-0.7.0}/src/declarative_data_backup/borg/borg.py +0 -0
- {declarative_data_backup-0.5.2 → declarative_data_backup-0.7.0}/src/declarative_data_backup/borg/borg_cli.py +0 -0
- {declarative_data_backup-0.5.2 → declarative_data_backup-0.7.0}/src/declarative_data_backup/config/__init__.py +0 -0
- {declarative_data_backup-0.5.2 → declarative_data_backup-0.7.0}/src/declarative_data_backup/config/backup_config/__init__.py +0 -0
- {declarative_data_backup-0.5.2 → declarative_data_backup-0.7.0}/src/declarative_data_backup/config/backup_config/yaml_raw_loader.py +0 -0
- {declarative_data_backup-0.5.2 → declarative_data_backup-0.7.0}/src/declarative_data_backup/config/logging.py +0 -0
- {declarative_data_backup-0.5.2 → declarative_data_backup-0.7.0}/src/declarative_data_backup/errors.py +0 -0
- {declarative_data_backup-0.5.2 → declarative_data_backup-0.7.0}/src/declarative_data_backup.egg-info/dependency_links.txt +0 -0
- {declarative_data_backup-0.5.2 → declarative_data_backup-0.7.0}/src/declarative_data_backup.egg-info/top_level.txt +0 -0
|
@@ -1,10 +1,11 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: declarative-data-backup
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.7.0
|
|
4
4
|
Summary: Declarative Borg-based data backups
|
|
5
5
|
Requires-Python: >=3.11
|
|
6
6
|
Description-Content-Type: text/markdown
|
|
7
7
|
Requires-Dist: PyYAML<7,>=6
|
|
8
|
+
Requires-Dist: redis<7,>=5
|
|
8
9
|
|
|
9
10
|
# Declarative Data Backup
|
|
10
11
|
|
|
@@ -24,6 +25,15 @@ only after every create operation in the run has succeeded.
|
|
|
24
25
|
|
|
25
26
|
The application never initializes a repository. Borg 2 is not supported.
|
|
26
27
|
|
|
28
|
+
## Optional filesystem watcher
|
|
29
|
+
|
|
30
|
+
The separate Linux Rust [watcher](watcher/README.md) records changed backup
|
|
31
|
+
groups in Redis using filesystem-wide fanotify marks. The Python runner accepts
|
|
32
|
+
the same rooted YAML schema. `BACKUP_MODE=shadow` reports which groups would
|
|
33
|
+
be selected while still backing up all groups; `BACKUP_MODE=dirty` consumes
|
|
34
|
+
selected groups. The default remains `BACKUP_MODE=full`; keep full scheduling
|
|
35
|
+
until the watcher and Redis consumer have been validated in shadow operation.
|
|
36
|
+
|
|
27
37
|
## Installation
|
|
28
38
|
|
|
29
39
|
Install the package from PyPI into a `uv`-managed environment:
|
|
@@ -45,13 +55,16 @@ package is `declarative_data_backup`.
|
|
|
45
55
|
|
|
46
56
|
## GitLab release automation
|
|
47
57
|
|
|
48
|
-
The pipeline in [.gitlab-ci.yml](.gitlab-ci.yml) publishes releases when
|
|
49
|
-
tag in `major.minor.patch` format is pushed, for example `1.2.3`. Use the same
|
|
58
|
+
The pipeline in [.gitlab-ci.yml](.gitlab-ci.yml) publishes Python releases when
|
|
59
|
+
a Git tag in `major.minor.patch` format is pushed, for example `1.2.3`. Use the same
|
|
50
60
|
version for the GitLab Release name and tag. Tags with a `v` prefix, prerelease
|
|
51
61
|
suffix, or leading zeros are not published. Branch pushes do not run this
|
|
52
62
|
release pipeline.
|
|
53
63
|
|
|
54
|
-
|
|
64
|
+
Watcher releases use independent `watcher/major.minor.patch` tags and publish
|
|
65
|
+
only the watcher image; see [watcher releases](watcher/README.md#releases).
|
|
66
|
+
|
|
67
|
+
For each Python version tag, the pipeline:
|
|
55
68
|
|
|
56
69
|
1. Sets the package version in the CI checkout's `pyproject.toml` and uses
|
|
57
70
|
`uv build` to create a wheel and source distribution.
|
|
@@ -103,9 +116,10 @@ Create a YAML file accessible to the backup process:
|
|
|
103
116
|
|
|
104
117
|
```yaml
|
|
105
118
|
media_backup:
|
|
119
|
+
root: /server_root/media
|
|
106
120
|
folders:
|
|
107
|
-
-
|
|
108
|
-
-
|
|
121
|
+
- photos
|
|
122
|
+
- music
|
|
109
123
|
compression: auto,zstd,22
|
|
110
124
|
retention_days: 30
|
|
111
125
|
required_mounts:
|
|
@@ -113,18 +127,19 @@ media_backup:
|
|
|
113
127
|
allow_empty: true
|
|
114
128
|
|
|
115
129
|
postgres_backup:
|
|
130
|
+
root: /server_root/backups
|
|
116
131
|
folders:
|
|
117
|
-
-
|
|
118
|
-
required_mounts:
|
|
119
|
-
- /server_root/path/to/postgres-backups
|
|
132
|
+
- postgres
|
|
120
133
|
|
|
121
134
|
apprise_backup:
|
|
135
|
+
root: /server_root/data
|
|
122
136
|
files:
|
|
123
|
-
-
|
|
137
|
+
- apprise/apprise.yml
|
|
124
138
|
|
|
125
139
|
jellyfin_backup:
|
|
140
|
+
root: /server_root/data
|
|
126
141
|
folders:
|
|
127
|
-
-
|
|
142
|
+
- jellyfin
|
|
128
143
|
exclude:
|
|
129
144
|
- "**/cache/**"
|
|
130
145
|
- "**/log/**"
|
|
@@ -152,8 +167,14 @@ Optional variables:
|
|
|
152
167
|
| `BACKUP_FILES_CACHE_TTL_MARGIN` | Extra archive creations to retain file-cache entries beyond the number of configured sources. The app sets Borg's TTL to source count plus this margin; defaults to `20`. |
|
|
153
168
|
| `BORG_FILES_CACHE_TTL` | Optional absolute Borg TTL override; it must be at least the number of configured sources. Usually leave unset to use the source-count-based TTL and configured margin. |
|
|
154
169
|
| `BACKUP_FAILURES_PATH` | Optional absolute path where failed backup group names are written as JSON. |
|
|
170
|
+
| `BACKUP_SUCCESSES_PATH` | Optional absolute path where successfully backed-up group names are written as JSON, including groups whose subsequent pruning failed. |
|
|
155
171
|
| `BACKUP_PROGRESS` | Set to `true` to log the latest Borg progress every 30 seconds during archive creation. Defaults to `false`. |
|
|
156
172
|
| `BACKUP_VERBOSE` | Set to `true` to show Borg's per-file create and per-archive prune output. Defaults to `false` for concise logs. |
|
|
173
|
+
| `BACKUP_MODE` | `full` (default) backs up every configured group; `shadow` reports the dirty selection but backs up all groups; `dirty` consumes `ddb:v1:dirty-groups` from Redis. Shadow and dirty modes require the Redis variables below. |
|
|
174
|
+
| `REDIS_URL` | Required in shadow/dirty mode. A `redis://` TCP URL without credentials. |
|
|
175
|
+
| `REDIS_DBINDEX` | Required in shadow/dirty mode. Nonnegative logical database index shared with the watcher. |
|
|
176
|
+
| `REDIS_USERNAME` | Optional ACL username in shadow/dirty mode. |
|
|
177
|
+
| `REDIS_PASSWORD` | Required in shadow/dirty mode. Supply through a Kubernetes Secret or protected source. |
|
|
157
178
|
|
|
158
179
|
Provide credentials through a secret manager or protected files; do not put
|
|
159
180
|
passphrases, private keys, or credentials in the YAML file, source control, or
|
|
@@ -191,22 +212,32 @@ locks without first confirming that no repository operation is still running.
|
|
|
191
212
|
The top level must be a nonempty YAML mapping of backup group names to group
|
|
192
213
|
mappings. Group names may contain ASCII letters, digits, `_`, and `-`, and
|
|
193
214
|
must be at most 64 characters. Duplicate YAML keys, unknown group fields,
|
|
194
|
-
invalid types, unsupported
|
|
195
|
-
|
|
215
|
+
invalid types, unsupported patterns, and empty configuration are rejected
|
|
216
|
+
before any create or prune operation.
|
|
196
217
|
|
|
197
218
|
Each group supports:
|
|
198
219
|
|
|
199
|
-
- `
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
220
|
+
- `root` (required): absolute primary source root, lexically and canonically
|
|
221
|
+
inside `SERVER_ROOT`. It must exist, be a directory, and be accessible.
|
|
222
|
+
- `watcher_alternative_roots` (optional): absolute roots representing the same
|
|
223
|
+
relative layout. These roots are watcher-only: Python does not require their
|
|
224
|
+
directories or mounts to exist, and they are never backed up.
|
|
225
|
+
- `folders` (optional): nonempty relative directory paths under `root`. A
|
|
226
|
+
folder `.` selects the root itself.
|
|
227
|
+
- `files` (optional): nonempty relative regular-file paths under `root`.
|
|
228
|
+
Sources reject absolute paths, NUL bytes, and `..` components. Their
|
|
229
|
+
canonical targets must remain inside `root`. Configure at least one source.
|
|
230
|
+
- `exclude` (optional): root-relative glob patterns shared with the watcher.
|
|
231
|
+
Patterns are anchored and case-sensitive. Version 1 allows ASCII
|
|
232
|
+
letters/digits, space, `_`, `-`, `.`, `/`, `*`, `?`, and `**` as a whole
|
|
233
|
+
component; empty, `.` and `..` components are rejected. An excluded
|
|
234
|
+
directory excludes descendants, and a trailing `/**` excludes the directory
|
|
235
|
+
itself too. Patterns are translated to root-scoped Borg regular expressions,
|
|
236
|
+
not passed through as Borg syntax. Patterns whose translated regex exceeds
|
|
237
|
+
64,000 characters are rejected. A group with explicit `files` cannot also
|
|
238
|
+
configure `exclude`; this safety check remains until a policy and
|
|
239
|
+
archive-level tests define the behavior for an explicitly selected file
|
|
240
|
+
matched by an exclusion.
|
|
210
241
|
- `compression` (optional): defaults to `auto,zstd,22`. Supported Borg 1.4
|
|
211
242
|
codecs include `none`, `lz4`, `zstd`, `zlib`, and `lzma`; `auto` may precede
|
|
212
243
|
a codec. Codec levels are checked against supported ranges: zstd `1`-`22`,
|
|
@@ -215,8 +246,10 @@ Each group supports:
|
|
|
215
246
|
useful only with repository encryption.
|
|
216
247
|
- `retention_days` (optional): positive integer, default `30`. This controls
|
|
217
248
|
Borg's `--keep-daily` policy.
|
|
218
|
-
- `required_mounts` (optional):
|
|
219
|
-
in the process's `/proc/self/mountinfo`.
|
|
249
|
+
- `required_mounts` (optional): absolute paths to existing exact mount points
|
|
250
|
+
that must appear in the process's `/proc/self/mountinfo`. These are
|
|
251
|
+
availability requirements, not source selectors. Alternative roots are not
|
|
252
|
+
automatically added.
|
|
220
253
|
- `allow_empty` (optional): boolean, default `true`. If set to `false`, an
|
|
221
254
|
empty source is rejected.
|
|
222
255
|
|
|
@@ -232,9 +265,18 @@ consistent database snapshot. Use an application-supported export, backup API,
|
|
|
232
265
|
or coordinated filesystem snapshot for live database data. Exclusions do not
|
|
233
266
|
make a database backup consistent.
|
|
234
267
|
|
|
235
|
-
|
|
236
|
-
when its canonical target remains inside `
|
|
237
|
-
source are stored as links rather than followed.
|
|
268
|
+
Primary roots and sources are resolved before backup. A source symlink is
|
|
269
|
+
allowed only when its canonical target remains inside the group's `root`;
|
|
270
|
+
symlinks within a selected source are stored as links rather than followed.
|
|
271
|
+
|
|
272
|
+
Dirty mode acquires the same `BORG_BASE_DIR/backup.lock` as full mode before
|
|
273
|
+
recovering in-flight groups, clearing stale group names, or claiming work.
|
|
274
|
+
Claims use Redis `SMOVE`; successful claims are removed only after all creates
|
|
275
|
+
and retention pruning for that group succeed. Failed groups are requeued, and
|
|
276
|
+
claims left by a crashed process are recovered on the next run. Redis errors
|
|
277
|
+
fail the job; they are never treated as an empty dirty set. Keep periodic full
|
|
278
|
+
reconciliation backups as a safety net. The daily CronJob remains in full mode
|
|
279
|
+
until dirty-mode rollout is explicitly enabled.
|
|
238
280
|
|
|
239
281
|
## Archives, retention, and compaction
|
|
240
282
|
|
|
@@ -16,6 +16,15 @@ only after every create operation in the run has succeeded.
|
|
|
16
16
|
|
|
17
17
|
The application never initializes a repository. Borg 2 is not supported.
|
|
18
18
|
|
|
19
|
+
## Optional filesystem watcher
|
|
20
|
+
|
|
21
|
+
The separate Linux Rust [watcher](watcher/README.md) records changed backup
|
|
22
|
+
groups in Redis using filesystem-wide fanotify marks. The Python runner accepts
|
|
23
|
+
the same rooted YAML schema. `BACKUP_MODE=shadow` reports which groups would
|
|
24
|
+
be selected while still backing up all groups; `BACKUP_MODE=dirty` consumes
|
|
25
|
+
selected groups. The default remains `BACKUP_MODE=full`; keep full scheduling
|
|
26
|
+
until the watcher and Redis consumer have been validated in shadow operation.
|
|
27
|
+
|
|
19
28
|
## Installation
|
|
20
29
|
|
|
21
30
|
Install the package from PyPI into a `uv`-managed environment:
|
|
@@ -37,13 +46,16 @@ package is `declarative_data_backup`.
|
|
|
37
46
|
|
|
38
47
|
## GitLab release automation
|
|
39
48
|
|
|
40
|
-
The pipeline in [.gitlab-ci.yml](.gitlab-ci.yml) publishes releases when
|
|
41
|
-
tag in `major.minor.patch` format is pushed, for example `1.2.3`. Use the same
|
|
49
|
+
The pipeline in [.gitlab-ci.yml](.gitlab-ci.yml) publishes Python releases when
|
|
50
|
+
a Git tag in `major.minor.patch` format is pushed, for example `1.2.3`. Use the same
|
|
42
51
|
version for the GitLab Release name and tag. Tags with a `v` prefix, prerelease
|
|
43
52
|
suffix, or leading zeros are not published. Branch pushes do not run this
|
|
44
53
|
release pipeline.
|
|
45
54
|
|
|
46
|
-
|
|
55
|
+
Watcher releases use independent `watcher/major.minor.patch` tags and publish
|
|
56
|
+
only the watcher image; see [watcher releases](watcher/README.md#releases).
|
|
57
|
+
|
|
58
|
+
For each Python version tag, the pipeline:
|
|
47
59
|
|
|
48
60
|
1. Sets the package version in the CI checkout's `pyproject.toml` and uses
|
|
49
61
|
`uv build` to create a wheel and source distribution.
|
|
@@ -95,9 +107,10 @@ Create a YAML file accessible to the backup process:
|
|
|
95
107
|
|
|
96
108
|
```yaml
|
|
97
109
|
media_backup:
|
|
110
|
+
root: /server_root/media
|
|
98
111
|
folders:
|
|
99
|
-
-
|
|
100
|
-
-
|
|
112
|
+
- photos
|
|
113
|
+
- music
|
|
101
114
|
compression: auto,zstd,22
|
|
102
115
|
retention_days: 30
|
|
103
116
|
required_mounts:
|
|
@@ -105,18 +118,19 @@ media_backup:
|
|
|
105
118
|
allow_empty: true
|
|
106
119
|
|
|
107
120
|
postgres_backup:
|
|
121
|
+
root: /server_root/backups
|
|
108
122
|
folders:
|
|
109
|
-
-
|
|
110
|
-
required_mounts:
|
|
111
|
-
- /server_root/path/to/postgres-backups
|
|
123
|
+
- postgres
|
|
112
124
|
|
|
113
125
|
apprise_backup:
|
|
126
|
+
root: /server_root/data
|
|
114
127
|
files:
|
|
115
|
-
-
|
|
128
|
+
- apprise/apprise.yml
|
|
116
129
|
|
|
117
130
|
jellyfin_backup:
|
|
131
|
+
root: /server_root/data
|
|
118
132
|
folders:
|
|
119
|
-
-
|
|
133
|
+
- jellyfin
|
|
120
134
|
exclude:
|
|
121
135
|
- "**/cache/**"
|
|
122
136
|
- "**/log/**"
|
|
@@ -144,8 +158,14 @@ Optional variables:
|
|
|
144
158
|
| `BACKUP_FILES_CACHE_TTL_MARGIN` | Extra archive creations to retain file-cache entries beyond the number of configured sources. The app sets Borg's TTL to source count plus this margin; defaults to `20`. |
|
|
145
159
|
| `BORG_FILES_CACHE_TTL` | Optional absolute Borg TTL override; it must be at least the number of configured sources. Usually leave unset to use the source-count-based TTL and configured margin. |
|
|
146
160
|
| `BACKUP_FAILURES_PATH` | Optional absolute path where failed backup group names are written as JSON. |
|
|
161
|
+
| `BACKUP_SUCCESSES_PATH` | Optional absolute path where successfully backed-up group names are written as JSON, including groups whose subsequent pruning failed. |
|
|
147
162
|
| `BACKUP_PROGRESS` | Set to `true` to log the latest Borg progress every 30 seconds during archive creation. Defaults to `false`. |
|
|
148
163
|
| `BACKUP_VERBOSE` | Set to `true` to show Borg's per-file create and per-archive prune output. Defaults to `false` for concise logs. |
|
|
164
|
+
| `BACKUP_MODE` | `full` (default) backs up every configured group; `shadow` reports the dirty selection but backs up all groups; `dirty` consumes `ddb:v1:dirty-groups` from Redis. Shadow and dirty modes require the Redis variables below. |
|
|
165
|
+
| `REDIS_URL` | Required in shadow/dirty mode. A `redis://` TCP URL without credentials. |
|
|
166
|
+
| `REDIS_DBINDEX` | Required in shadow/dirty mode. Nonnegative logical database index shared with the watcher. |
|
|
167
|
+
| `REDIS_USERNAME` | Optional ACL username in shadow/dirty mode. |
|
|
168
|
+
| `REDIS_PASSWORD` | Required in shadow/dirty mode. Supply through a Kubernetes Secret or protected source. |
|
|
149
169
|
|
|
150
170
|
Provide credentials through a secret manager or protected files; do not put
|
|
151
171
|
passphrases, private keys, or credentials in the YAML file, source control, or
|
|
@@ -183,22 +203,32 @@ locks without first confirming that no repository operation is still running.
|
|
|
183
203
|
The top level must be a nonempty YAML mapping of backup group names to group
|
|
184
204
|
mappings. Group names may contain ASCII letters, digits, `_`, and `-`, and
|
|
185
205
|
must be at most 64 characters. Duplicate YAML keys, unknown group fields,
|
|
186
|
-
invalid types, unsupported
|
|
187
|
-
|
|
206
|
+
invalid types, unsupported patterns, and empty configuration are rejected
|
|
207
|
+
before any create or prune operation.
|
|
188
208
|
|
|
189
209
|
Each group supports:
|
|
190
210
|
|
|
191
|
-
- `
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
211
|
+
- `root` (required): absolute primary source root, lexically and canonically
|
|
212
|
+
inside `SERVER_ROOT`. It must exist, be a directory, and be accessible.
|
|
213
|
+
- `watcher_alternative_roots` (optional): absolute roots representing the same
|
|
214
|
+
relative layout. These roots are watcher-only: Python does not require their
|
|
215
|
+
directories or mounts to exist, and they are never backed up.
|
|
216
|
+
- `folders` (optional): nonempty relative directory paths under `root`. A
|
|
217
|
+
folder `.` selects the root itself.
|
|
218
|
+
- `files` (optional): nonempty relative regular-file paths under `root`.
|
|
219
|
+
Sources reject absolute paths, NUL bytes, and `..` components. Their
|
|
220
|
+
canonical targets must remain inside `root`. Configure at least one source.
|
|
221
|
+
- `exclude` (optional): root-relative glob patterns shared with the watcher.
|
|
222
|
+
Patterns are anchored and case-sensitive. Version 1 allows ASCII
|
|
223
|
+
letters/digits, space, `_`, `-`, `.`, `/`, `*`, `?`, and `**` as a whole
|
|
224
|
+
component; empty, `.` and `..` components are rejected. An excluded
|
|
225
|
+
directory excludes descendants, and a trailing `/**` excludes the directory
|
|
226
|
+
itself too. Patterns are translated to root-scoped Borg regular expressions,
|
|
227
|
+
not passed through as Borg syntax. Patterns whose translated regex exceeds
|
|
228
|
+
64,000 characters are rejected. A group with explicit `files` cannot also
|
|
229
|
+
configure `exclude`; this safety check remains until a policy and
|
|
230
|
+
archive-level tests define the behavior for an explicitly selected file
|
|
231
|
+
matched by an exclusion.
|
|
202
232
|
- `compression` (optional): defaults to `auto,zstd,22`. Supported Borg 1.4
|
|
203
233
|
codecs include `none`, `lz4`, `zstd`, `zlib`, and `lzma`; `auto` may precede
|
|
204
234
|
a codec. Codec levels are checked against supported ranges: zstd `1`-`22`,
|
|
@@ -207,8 +237,10 @@ Each group supports:
|
|
|
207
237
|
useful only with repository encryption.
|
|
208
238
|
- `retention_days` (optional): positive integer, default `30`. This controls
|
|
209
239
|
Borg's `--keep-daily` policy.
|
|
210
|
-
- `required_mounts` (optional):
|
|
211
|
-
in the process's `/proc/self/mountinfo`.
|
|
240
|
+
- `required_mounts` (optional): absolute paths to existing exact mount points
|
|
241
|
+
that must appear in the process's `/proc/self/mountinfo`. These are
|
|
242
|
+
availability requirements, not source selectors. Alternative roots are not
|
|
243
|
+
automatically added.
|
|
212
244
|
- `allow_empty` (optional): boolean, default `true`. If set to `false`, an
|
|
213
245
|
empty source is rejected.
|
|
214
246
|
|
|
@@ -224,9 +256,18 @@ consistent database snapshot. Use an application-supported export, backup API,
|
|
|
224
256
|
or coordinated filesystem snapshot for live database data. Exclusions do not
|
|
225
257
|
make a database backup consistent.
|
|
226
258
|
|
|
227
|
-
|
|
228
|
-
when its canonical target remains inside `
|
|
229
|
-
source are stored as links rather than followed.
|
|
259
|
+
Primary roots and sources are resolved before backup. A source symlink is
|
|
260
|
+
allowed only when its canonical target remains inside the group's `root`;
|
|
261
|
+
symlinks within a selected source are stored as links rather than followed.
|
|
262
|
+
|
|
263
|
+
Dirty mode acquires the same `BORG_BASE_DIR/backup.lock` as full mode before
|
|
264
|
+
recovering in-flight groups, clearing stale group names, or claiming work.
|
|
265
|
+
Claims use Redis `SMOVE`; successful claims are removed only after all creates
|
|
266
|
+
and retention pruning for that group succeed. Failed groups are requeued, and
|
|
267
|
+
claims left by a crashed process are recovered on the next run. Redis errors
|
|
268
|
+
fail the job; they are never treated as an empty dirty set. Keep periodic full
|
|
269
|
+
reconciliation backups as a safety net. The daily CronJob remains in full mode
|
|
270
|
+
until dirty-mode rollout is explicitly enabled.
|
|
230
271
|
|
|
231
272
|
## Archives, retention, and compaction
|
|
232
273
|
|
|
@@ -4,11 +4,11 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "declarative-data-backup"
|
|
7
|
-
version = "0.
|
|
7
|
+
version = "0.7.0"
|
|
8
8
|
description = "Declarative Borg-based data backups"
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.11"
|
|
11
|
-
dependencies = ["PyYAML>=6,<7"]
|
|
11
|
+
dependencies = ["PyYAML>=6,<7", "redis>=5,<7"]
|
|
12
12
|
|
|
13
13
|
[dependency-groups]
|
|
14
14
|
dev = [
|
|
@@ -0,0 +1,276 @@
|
|
|
1
|
+
import json
|
|
2
|
+
import logging
|
|
3
|
+
import os
|
|
4
|
+
import signal
|
|
5
|
+
import uuid
|
|
6
|
+
from collections.abc import Mapping
|
|
7
|
+
from dataclasses import replace
|
|
8
|
+
from datetime import UTC, datetime
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
from types import FrameType
|
|
11
|
+
|
|
12
|
+
from .borg import (
|
|
13
|
+
backup_folder,
|
|
14
|
+
check_keyfile,
|
|
15
|
+
prune,
|
|
16
|
+
repository_lock,
|
|
17
|
+
)
|
|
18
|
+
from .borg.borg_cli import ensure_borg_binary
|
|
19
|
+
from .config import configure_logging, load_yaml, parse_backup_definitions
|
|
20
|
+
from .config.backup_config.models import BackupDefinition
|
|
21
|
+
from .dirty_groups import DirtyGroupConsumer
|
|
22
|
+
from .environment import Environment
|
|
23
|
+
from .errors import BackupError, BackupInterrupted, BorgWarningError
|
|
24
|
+
from .exclusions import translate_excludes
|
|
25
|
+
|
|
26
|
+
logger = logging.getLogger(__name__)
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def write_group_summary(path: Path | None, groups: set[str]) -> None:
|
|
30
|
+
if path is None:
|
|
31
|
+
return
|
|
32
|
+
try:
|
|
33
|
+
path.write_text(json.dumps(sorted(groups)), encoding="utf-8")
|
|
34
|
+
except OSError as exc:
|
|
35
|
+
logger.warning(
|
|
36
|
+
"Unable to write backup group summary to %s: %s",
|
|
37
|
+
path,
|
|
38
|
+
exc.strerror or "I/O error",
|
|
39
|
+
)
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def check_settings(settings: Environment) -> None:
|
|
43
|
+
ensure_borg_binary()
|
|
44
|
+
if not settings.borg_base_dir.is_dir():
|
|
45
|
+
raise BackupError(
|
|
46
|
+
f"BORG_BASE_DIR does not exist or is not a directory: "
|
|
47
|
+
f"{settings.borg_base_dir}."
|
|
48
|
+
)
|
|
49
|
+
if not os.access(settings.borg_base_dir, os.R_OK | os.W_OK | os.X_OK):
|
|
50
|
+
raise BackupError(
|
|
51
|
+
f"BORG_BASE_DIR is not accessible for Borg state: {settings.borg_base_dir}."
|
|
52
|
+
)
|
|
53
|
+
check_keyfile(settings)
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def load_definitions(settings: Environment) -> list[BackupDefinition]:
|
|
57
|
+
return parse_backup_definitions(
|
|
58
|
+
load_yaml(settings.config_path),
|
|
59
|
+
settings.server_root,
|
|
60
|
+
)
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def run_backup(
|
|
64
|
+
settings: Environment,
|
|
65
|
+
definitions: list[BackupDefinition],
|
|
66
|
+
) -> None:
|
|
67
|
+
total_sources = sum(len(definition.sources) for definition in definitions)
|
|
68
|
+
logger.info(
|
|
69
|
+
"Validated %d backup groups containing %d sources",
|
|
70
|
+
len(definitions),
|
|
71
|
+
total_sources,
|
|
72
|
+
)
|
|
73
|
+
if (
|
|
74
|
+
settings.borg_files_cache_ttl is not None
|
|
75
|
+
and settings.borg_files_cache_ttl < total_sources
|
|
76
|
+
):
|
|
77
|
+
raise BackupError(
|
|
78
|
+
"BORG_FILES_CACHE_TTL must be at least the number of configured "
|
|
79
|
+
f"source creates ({total_sources})."
|
|
80
|
+
)
|
|
81
|
+
if settings.borg_files_cache_ttl is None:
|
|
82
|
+
files_cache_ttl = total_sources + settings.backup_files_cache_ttl_margin
|
|
83
|
+
settings = replace(settings, borg_files_cache_ttl=files_cache_ttl)
|
|
84
|
+
logger.info(
|
|
85
|
+
"Using Borg files-cache TTL %d (%d source creates + %d margin).",
|
|
86
|
+
files_cache_ttl,
|
|
87
|
+
total_sources,
|
|
88
|
+
settings.backup_files_cache_ttl_margin,
|
|
89
|
+
)
|
|
90
|
+
else:
|
|
91
|
+
logger.info(
|
|
92
|
+
"Using explicit Borg files-cache TTL %d.",
|
|
93
|
+
settings.borg_files_cache_ttl,
|
|
94
|
+
)
|
|
95
|
+
|
|
96
|
+
run_id = uuid.uuid4().hex
|
|
97
|
+
with repository_lock(settings.borg_base_dir / "backup.lock"):
|
|
98
|
+
backed_up_groups: set[str] = set()
|
|
99
|
+
write_group_summary(settings.backed_up_groups_path, backed_up_groups)
|
|
100
|
+
dirty_groups = (
|
|
101
|
+
DirtyGroupConsumer(settings)
|
|
102
|
+
if settings.backup_mode in {"shadow", "dirty"}
|
|
103
|
+
else None
|
|
104
|
+
)
|
|
105
|
+
try:
|
|
106
|
+
if settings.backup_mode in {"shadow", "dirty"} and dirty_groups is not None:
|
|
107
|
+
group_names = {definition.name for definition in definitions}
|
|
108
|
+
if settings.backup_mode == "shadow":
|
|
109
|
+
dirty_groups.preview(group_names)
|
|
110
|
+
else:
|
|
111
|
+
dirty_groups.prepare(group_names)
|
|
112
|
+
|
|
113
|
+
created_at = datetime.now(UTC)
|
|
114
|
+
successful_definitions = []
|
|
115
|
+
failures: list[BackupError] = []
|
|
116
|
+
failed_groups: set[str] = set()
|
|
117
|
+
for definition in definitions:
|
|
118
|
+
if settings.backup_mode == "dirty" and dirty_groups is not None:
|
|
119
|
+
claimed = dirty_groups.claim(definition.name)
|
|
120
|
+
else:
|
|
121
|
+
claimed = True
|
|
122
|
+
if not claimed:
|
|
123
|
+
logger.info(
|
|
124
|
+
"Skipping backup group '%s'; it is not dirty.",
|
|
125
|
+
definition.name,
|
|
126
|
+
)
|
|
127
|
+
continue
|
|
128
|
+
|
|
129
|
+
logger.info(
|
|
130
|
+
"Starting backup group '%s' (%d sources).",
|
|
131
|
+
definition.name,
|
|
132
|
+
len(definition.sources),
|
|
133
|
+
)
|
|
134
|
+
try:
|
|
135
|
+
excludes = translate_excludes(
|
|
136
|
+
definition.root,
|
|
137
|
+
definition.excludes,
|
|
138
|
+
)
|
|
139
|
+
for folder in definition.sources:
|
|
140
|
+
backup_folder(
|
|
141
|
+
folder,
|
|
142
|
+
definition.compression,
|
|
143
|
+
definition.name,
|
|
144
|
+
settings,
|
|
145
|
+
created_at,
|
|
146
|
+
run_id,
|
|
147
|
+
excludes,
|
|
148
|
+
)
|
|
149
|
+
except BackupError as exc:
|
|
150
|
+
if settings.backup_mode == "dirty" and dirty_groups is not None:
|
|
151
|
+
try:
|
|
152
|
+
dirty_groups.requeue(definition.name)
|
|
153
|
+
except BackupError as state_error:
|
|
154
|
+
failed_groups.add(definition.name)
|
|
155
|
+
write_group_summary(
|
|
156
|
+
settings.failure_groups_path, failed_groups
|
|
157
|
+
)
|
|
158
|
+
raise BackupError(f"{exc}; {state_error}") from state_error
|
|
159
|
+
logger.error(
|
|
160
|
+
"Backup group '%s' failed; retention pruning will be skipped "
|
|
161
|
+
"for this group: %s",
|
|
162
|
+
definition.name,
|
|
163
|
+
exc,
|
|
164
|
+
)
|
|
165
|
+
failures.append(exc)
|
|
166
|
+
failed_groups.add(definition.name)
|
|
167
|
+
write_group_summary(settings.failure_groups_path, failed_groups)
|
|
168
|
+
continue
|
|
169
|
+
|
|
170
|
+
logger.info("Backup group '%s' completed.", definition.name)
|
|
171
|
+
successful_definitions.append(definition)
|
|
172
|
+
backed_up_groups.add(definition.name)
|
|
173
|
+
write_group_summary(settings.backed_up_groups_path, backed_up_groups)
|
|
174
|
+
|
|
175
|
+
for definition in successful_definitions:
|
|
176
|
+
logger.info(
|
|
177
|
+
"Starting retention pruning for group '%s' (%d sources).",
|
|
178
|
+
definition.name,
|
|
179
|
+
len(definition.sources),
|
|
180
|
+
)
|
|
181
|
+
pruning_succeeded = True
|
|
182
|
+
for folder in definition.sources:
|
|
183
|
+
try:
|
|
184
|
+
prune(
|
|
185
|
+
folder,
|
|
186
|
+
definition.retention_days,
|
|
187
|
+
definition.name,
|
|
188
|
+
settings,
|
|
189
|
+
)
|
|
190
|
+
except BackupError as exc:
|
|
191
|
+
logger.error(
|
|
192
|
+
"Retention pruning failed for group '%s' source '%s': %s",
|
|
193
|
+
definition.name,
|
|
194
|
+
folder,
|
|
195
|
+
exc,
|
|
196
|
+
)
|
|
197
|
+
failures.append(exc)
|
|
198
|
+
failed_groups.add(definition.name)
|
|
199
|
+
write_group_summary(settings.failure_groups_path, failed_groups)
|
|
200
|
+
pruning_succeeded = False
|
|
201
|
+
|
|
202
|
+
if settings.backup_mode == "dirty" and dirty_groups is not None:
|
|
203
|
+
try:
|
|
204
|
+
if pruning_succeeded:
|
|
205
|
+
dirty_groups.acknowledge(definition.name)
|
|
206
|
+
else:
|
|
207
|
+
dirty_groups.requeue(definition.name)
|
|
208
|
+
except BackupError as state_error:
|
|
209
|
+
failed_groups.add(definition.name)
|
|
210
|
+
write_group_summary(settings.failure_groups_path, failed_groups)
|
|
211
|
+
raise state_error
|
|
212
|
+
|
|
213
|
+
if pruning_succeeded:
|
|
214
|
+
logger.info(
|
|
215
|
+
"Retention pruning for group '%s' completed.",
|
|
216
|
+
definition.name,
|
|
217
|
+
)
|
|
218
|
+
|
|
219
|
+
if failures:
|
|
220
|
+
write_group_summary(settings.failure_groups_path, failed_groups)
|
|
221
|
+
message = "; ".join(str(failure) for failure in failures)
|
|
222
|
+
if any(isinstance(failure, BorgWarningError) for failure in failures):
|
|
223
|
+
raise BorgWarningError(message)
|
|
224
|
+
raise BackupError(message)
|
|
225
|
+
finally:
|
|
226
|
+
if dirty_groups is not None:
|
|
227
|
+
dirty_groups.close()
|
|
228
|
+
|
|
229
|
+
|
|
230
|
+
def main(environ: Mapping[str, str] | None = None) -> int:
|
|
231
|
+
configure_logging()
|
|
232
|
+
|
|
233
|
+
def interrupt(signum: int, frame: FrameType | None) -> None:
|
|
234
|
+
# Repeated signals must not interrupt Borg cleanup or lock release.
|
|
235
|
+
signal.signal(signal.SIGTERM, signal.SIG_IGN)
|
|
236
|
+
signal.signal(signal.SIGINT, signal.SIG_IGN)
|
|
237
|
+
raise BackupInterrupted(signum)
|
|
238
|
+
|
|
239
|
+
previous_sigterm = signal.signal(signal.SIGTERM, interrupt)
|
|
240
|
+
previous_sigint = signal.signal(signal.SIGINT, interrupt)
|
|
241
|
+
try:
|
|
242
|
+
settings = Environment.from_env(environ)
|
|
243
|
+
logger.info("Starting backup using configuration %s", settings.config_path)
|
|
244
|
+
check_settings(settings)
|
|
245
|
+
run_backup(settings, load_definitions(settings))
|
|
246
|
+
logger.info("Backup completed successfully.")
|
|
247
|
+
return 0
|
|
248
|
+
|
|
249
|
+
except BorgWarningError as exc:
|
|
250
|
+
logger.error(
|
|
251
|
+
"Backup failed because Borg reported warnings; "
|
|
252
|
+
"retention pruning may have completed for other groups: %s",
|
|
253
|
+
exc,
|
|
254
|
+
)
|
|
255
|
+
return 1
|
|
256
|
+
except BackupError as exc:
|
|
257
|
+
logger.error("%s", exc)
|
|
258
|
+
return 1
|
|
259
|
+
except OSError as exc:
|
|
260
|
+
logger.error(
|
|
261
|
+
"Unexpected I/O failure: %s.",
|
|
262
|
+
exc.strerror or "I/O error",
|
|
263
|
+
)
|
|
264
|
+
return 1
|
|
265
|
+
except BackupInterrupted as exc:
|
|
266
|
+
logger.error(
|
|
267
|
+
"Interrupted by %s; no further backups or pruning will run.",
|
|
268
|
+
signal.Signals(exc.signum).name,
|
|
269
|
+
)
|
|
270
|
+
return 128 + exc.signum
|
|
271
|
+
except KeyboardInterrupt:
|
|
272
|
+
logger.error("Interrupted.")
|
|
273
|
+
return 130
|
|
274
|
+
finally:
|
|
275
|
+
signal.signal(signal.SIGTERM, previous_sigterm)
|
|
276
|
+
signal.signal(signal.SIGINT, previous_sigint)
|
|
@@ -5,14 +5,17 @@ from pathlib import Path
|
|
|
5
5
|
@dataclass(frozen=True)
|
|
6
6
|
class BackupDefinition:
|
|
7
7
|
name: str
|
|
8
|
+
root: Path
|
|
8
9
|
folders: tuple[Path, ...]
|
|
10
|
+
files: tuple[Path, ...]
|
|
11
|
+
canonical_sources: tuple[Path, ...]
|
|
9
12
|
compression: str
|
|
10
13
|
retention_days: int
|
|
11
14
|
required_mounts: tuple[Path, ...] = ()
|
|
15
|
+
watcher_alternative_roots: tuple[Path, ...] = ()
|
|
12
16
|
allow_empty: bool = True
|
|
13
|
-
files: tuple[Path, ...] = ()
|
|
14
17
|
excludes: tuple[str, ...] = ()
|
|
15
18
|
|
|
16
19
|
@property
|
|
17
20
|
def sources(self) -> tuple[Path, ...]:
|
|
18
|
-
return self.
|
|
21
|
+
return self.canonical_sources
|