declarative-data-backup 0.5.2__tar.gz → 0.6.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {declarative_data_backup-0.5.2 → declarative_data_backup-0.6.0}/PKG-INFO +70 -29
- {declarative_data_backup-0.5.2 → declarative_data_backup-0.6.0}/README.md +68 -28
- {declarative_data_backup-0.5.2 → declarative_data_backup-0.6.0}/pyproject.toml +2 -2
- {declarative_data_backup-0.5.2 → declarative_data_backup-0.6.0}/src/declarative_data_backup/backup.py +107 -55
- {declarative_data_backup-0.5.2 → declarative_data_backup-0.6.0}/src/declarative_data_backup/config/backup_config/models.py +5 -2
- {declarative_data_backup-0.5.2 → declarative_data_backup-0.6.0}/src/declarative_data_backup/config/backup_config/utils.py +93 -22
- {declarative_data_backup-0.5.2 → declarative_data_backup-0.6.0}/src/declarative_data_backup/config/backup_config/yaml_parser.py +32 -3
- declarative_data_backup-0.6.0/src/declarative_data_backup/dirty_groups.py +150 -0
- {declarative_data_backup-0.5.2 → declarative_data_backup-0.6.0}/src/declarative_data_backup/environment.py +65 -1
- declarative_data_backup-0.6.0/src/declarative_data_backup/exclusions.py +127 -0
- {declarative_data_backup-0.5.2 → declarative_data_backup-0.6.0}/src/declarative_data_backup.egg-info/PKG-INFO +70 -29
- {declarative_data_backup-0.5.2 → declarative_data_backup-0.6.0}/src/declarative_data_backup.egg-info/SOURCES.txt +5 -1
- {declarative_data_backup-0.5.2 → declarative_data_backup-0.6.0}/src/declarative_data_backup.egg-info/requires.txt +1 -0
- {declarative_data_backup-0.5.2 → declarative_data_backup-0.6.0}/tests/test_backup.py +192 -2
- declarative_data_backup-0.6.0/tests/test_dirty_groups.py +188 -0
- declarative_data_backup-0.6.0/tests/test_exclusions.py +72 -0
- {declarative_data_backup-0.5.2 → declarative_data_backup-0.6.0}/setup.cfg +0 -0
- {declarative_data_backup-0.5.2 → declarative_data_backup-0.6.0}/src/declarative_data_backup/__init__.py +0 -0
- {declarative_data_backup-0.5.2 → declarative_data_backup-0.6.0}/src/declarative_data_backup/__main__.py +0 -0
- {declarative_data_backup-0.5.2 → declarative_data_backup-0.6.0}/src/declarative_data_backup/borg/__init__.py +0 -0
- {declarative_data_backup-0.5.2 → declarative_data_backup-0.6.0}/src/declarative_data_backup/borg/archive_naming.py +0 -0
- {declarative_data_backup-0.5.2 → declarative_data_backup-0.6.0}/src/declarative_data_backup/borg/borg.py +0 -0
- {declarative_data_backup-0.5.2 → declarative_data_backup-0.6.0}/src/declarative_data_backup/borg/borg_cli.py +0 -0
- {declarative_data_backup-0.5.2 → declarative_data_backup-0.6.0}/src/declarative_data_backup/config/__init__.py +0 -0
- {declarative_data_backup-0.5.2 → declarative_data_backup-0.6.0}/src/declarative_data_backup/config/backup_config/__init__.py +0 -0
- {declarative_data_backup-0.5.2 → declarative_data_backup-0.6.0}/src/declarative_data_backup/config/backup_config/yaml_raw_loader.py +0 -0
- {declarative_data_backup-0.5.2 → declarative_data_backup-0.6.0}/src/declarative_data_backup/config/logging.py +0 -0
- {declarative_data_backup-0.5.2 → declarative_data_backup-0.6.0}/src/declarative_data_backup/errors.py +0 -0
- {declarative_data_backup-0.5.2 → declarative_data_backup-0.6.0}/src/declarative_data_backup.egg-info/dependency_links.txt +0 -0
- {declarative_data_backup-0.5.2 → declarative_data_backup-0.6.0}/src/declarative_data_backup.egg-info/top_level.txt +0 -0
|
@@ -1,10 +1,11 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: declarative-data-backup
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.6.0
|
|
4
4
|
Summary: Declarative Borg-based data backups
|
|
5
5
|
Requires-Python: >=3.11
|
|
6
6
|
Description-Content-Type: text/markdown
|
|
7
7
|
Requires-Dist: PyYAML<7,>=6
|
|
8
|
+
Requires-Dist: redis<7,>=5
|
|
8
9
|
|
|
9
10
|
# Declarative Data Backup
|
|
10
11
|
|
|
@@ -24,6 +25,15 @@ only after every create operation in the run has succeeded.
|
|
|
24
25
|
|
|
25
26
|
The application never initializes a repository. Borg 2 is not supported.
|
|
26
27
|
|
|
28
|
+
## Optional filesystem watcher
|
|
29
|
+
|
|
30
|
+
The separate Linux Rust [watcher](watcher/README.md) records changed backup
|
|
31
|
+
groups in Redis using filesystem-wide fanotify marks. The Python runner accepts
|
|
32
|
+
the same rooted YAML schema. `BACKUP_MODE=shadow` reports which groups would
|
|
33
|
+
be selected while still backing up all groups; `BACKUP_MODE=dirty` consumes
|
|
34
|
+
selected groups. The default remains `BACKUP_MODE=full`; keep full scheduling
|
|
35
|
+
until the watcher and Redis consumer have been validated in shadow operation.
|
|
36
|
+
|
|
27
37
|
## Installation
|
|
28
38
|
|
|
29
39
|
Install the package from PyPI into a `uv`-managed environment:
|
|
@@ -45,13 +55,16 @@ package is `declarative_data_backup`.
|
|
|
45
55
|
|
|
46
56
|
## GitLab release automation
|
|
47
57
|
|
|
48
|
-
The pipeline in [.gitlab-ci.yml](.gitlab-ci.yml) publishes releases when
|
|
49
|
-
tag in `major.minor.patch` format is pushed, for example `1.2.3`. Use the same
|
|
58
|
+
The pipeline in [.gitlab-ci.yml](.gitlab-ci.yml) publishes Python releases when
|
|
59
|
+
a Git tag in `major.minor.patch` format is pushed, for example `1.2.3`. Use the same
|
|
50
60
|
version for the GitLab Release name and tag. Tags with a `v` prefix, prerelease
|
|
51
61
|
suffix, or leading zeros are not published. Branch pushes do not run this
|
|
52
62
|
release pipeline.
|
|
53
63
|
|
|
54
|
-
|
|
64
|
+
Watcher releases use independent `watcher/major.minor.patch` tags and publish
|
|
65
|
+
only the watcher image; see [watcher releases](watcher/README.md#releases).
|
|
66
|
+
|
|
67
|
+
For each Python version tag, the pipeline:
|
|
55
68
|
|
|
56
69
|
1. Sets the package version in the CI checkout's `pyproject.toml` and uses
|
|
57
70
|
`uv build` to create a wheel and source distribution.
|
|
@@ -103,9 +116,10 @@ Create a YAML file accessible to the backup process:
|
|
|
103
116
|
|
|
104
117
|
```yaml
|
|
105
118
|
media_backup:
|
|
119
|
+
root: /server_root/media
|
|
106
120
|
folders:
|
|
107
|
-
-
|
|
108
|
-
-
|
|
121
|
+
- photos
|
|
122
|
+
- music
|
|
109
123
|
compression: auto,zstd,22
|
|
110
124
|
retention_days: 30
|
|
111
125
|
required_mounts:
|
|
@@ -113,18 +127,19 @@ media_backup:
|
|
|
113
127
|
allow_empty: true
|
|
114
128
|
|
|
115
129
|
postgres_backup:
|
|
130
|
+
root: /server_root/backups
|
|
116
131
|
folders:
|
|
117
|
-
-
|
|
118
|
-
required_mounts:
|
|
119
|
-
- /server_root/path/to/postgres-backups
|
|
132
|
+
- postgres
|
|
120
133
|
|
|
121
134
|
apprise_backup:
|
|
135
|
+
root: /server_root/data
|
|
122
136
|
files:
|
|
123
|
-
-
|
|
137
|
+
- apprise/apprise.yml
|
|
124
138
|
|
|
125
139
|
jellyfin_backup:
|
|
140
|
+
root: /server_root/data
|
|
126
141
|
folders:
|
|
127
|
-
-
|
|
142
|
+
- jellyfin
|
|
128
143
|
exclude:
|
|
129
144
|
- "**/cache/**"
|
|
130
145
|
- "**/log/**"
|
|
@@ -154,6 +169,11 @@ Optional variables:
|
|
|
154
169
|
| `BACKUP_FAILURES_PATH` | Optional absolute path where failed backup group names are written as JSON. |
|
|
155
170
|
| `BACKUP_PROGRESS` | Set to `true` to log the latest Borg progress every 30 seconds during archive creation. Defaults to `false`. |
|
|
156
171
|
| `BACKUP_VERBOSE` | Set to `true` to show Borg's per-file create and per-archive prune output. Defaults to `false` for concise logs. |
|
|
172
|
+
| `BACKUP_MODE` | `full` (default) backs up every configured group; `shadow` reports the dirty selection but backs up all groups; `dirty` consumes `ddb:v1:dirty-groups` from Redis. Shadow and dirty modes require the Redis variables below. |
|
|
173
|
+
| `REDIS_URL` | Required in shadow/dirty mode. A `redis://` TCP URL without credentials. |
|
|
174
|
+
| `REDIS_DBINDEX` | Required in shadow/dirty mode. Nonnegative logical database index shared with the watcher. |
|
|
175
|
+
| `REDIS_USERNAME` | Optional ACL username in shadow/dirty mode. |
|
|
176
|
+
| `REDIS_PASSWORD` | Required in shadow/dirty mode. Supply through a Kubernetes Secret or protected source. |
|
|
157
177
|
|
|
158
178
|
Provide credentials through a secret manager or protected files; do not put
|
|
159
179
|
passphrases, private keys, or credentials in the YAML file, source control, or
|
|
@@ -191,22 +211,32 @@ locks without first confirming that no repository operation is still running.
|
|
|
191
211
|
The top level must be a nonempty YAML mapping of backup group names to group
|
|
192
212
|
mappings. Group names may contain ASCII letters, digits, `_`, and `-`, and
|
|
193
213
|
must be at most 64 characters. Duplicate YAML keys, unknown group fields,
|
|
194
|
-
invalid types, unsupported
|
|
195
|
-
|
|
214
|
+
invalid types, unsupported patterns, and empty configuration are rejected
|
|
215
|
+
before any create or prune operation.
|
|
196
216
|
|
|
197
217
|
Each group supports:
|
|
198
218
|
|
|
199
|
-
- `
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
219
|
+
- `root` (required): absolute primary source root, lexically and canonically
|
|
220
|
+
inside `SERVER_ROOT`. It must exist, be a directory, and be accessible.
|
|
221
|
+
- `watcher_alternative_roots` (optional): absolute roots representing the same
|
|
222
|
+
relative layout. These roots are watcher-only: Python does not require their
|
|
223
|
+
directories or mounts to exist, and they are never backed up.
|
|
224
|
+
- `folders` (optional): nonempty relative directory paths under `root`. A
|
|
225
|
+
folder `.` selects the root itself.
|
|
226
|
+
- `files` (optional): nonempty relative regular-file paths under `root`.
|
|
227
|
+
Sources reject absolute paths, NUL bytes, and `..` components. Their
|
|
228
|
+
canonical targets must remain inside `root`. Configure at least one source.
|
|
229
|
+
- `exclude` (optional): root-relative glob patterns shared with the watcher.
|
|
230
|
+
Patterns are anchored and case-sensitive. Version 1 allows ASCII
|
|
231
|
+
letters/digits, space, `_`, `-`, `.`, `/`, `*`, `?`, and `**` as a whole
|
|
232
|
+
component; empty, `.` and `..` components are rejected. An excluded
|
|
233
|
+
directory excludes descendants, and a trailing `/**` excludes the directory
|
|
234
|
+
itself too. Patterns are translated to root-scoped Borg regular expressions,
|
|
235
|
+
not passed through as Borg syntax. Patterns whose translated regex exceeds
|
|
236
|
+
64,000 characters are rejected. A group with explicit `files` cannot also
|
|
237
|
+
configure `exclude`; this safety check remains until a policy and
|
|
238
|
+
archive-level tests define the behavior for an explicitly selected file
|
|
239
|
+
matched by an exclusion.
|
|
210
240
|
- `compression` (optional): defaults to `auto,zstd,22`. Supported Borg 1.4
|
|
211
241
|
codecs include `none`, `lz4`, `zstd`, `zlib`, and `lzma`; `auto` may precede
|
|
212
242
|
a codec. Codec levels are checked against supported ranges: zstd `1`-`22`,
|
|
@@ -215,8 +245,10 @@ Each group supports:
|
|
|
215
245
|
useful only with repository encryption.
|
|
216
246
|
- `retention_days` (optional): positive integer, default `30`. This controls
|
|
217
247
|
Borg's `--keep-daily` policy.
|
|
218
|
-
- `required_mounts` (optional):
|
|
219
|
-
in the process's `/proc/self/mountinfo`.
|
|
248
|
+
- `required_mounts` (optional): absolute paths to existing exact mount points
|
|
249
|
+
that must appear in the process's `/proc/self/mountinfo`. These are
|
|
250
|
+
availability requirements, not source selectors. Alternative roots are not
|
|
251
|
+
automatically added.
|
|
220
252
|
- `allow_empty` (optional): boolean, default `true`. If set to `false`, an
|
|
221
253
|
empty source is rejected.
|
|
222
254
|
|
|
@@ -232,9 +264,18 @@ consistent database snapshot. Use an application-supported export, backup API,
|
|
|
232
264
|
or coordinated filesystem snapshot for live database data. Exclusions do not
|
|
233
265
|
make a database backup consistent.
|
|
234
266
|
|
|
235
|
-
|
|
236
|
-
when its canonical target remains inside `
|
|
237
|
-
source are stored as links rather than followed.
|
|
267
|
+
Primary roots and sources are resolved before backup. A source symlink is
|
|
268
|
+
allowed only when its canonical target remains inside the group's `root`;
|
|
269
|
+
symlinks within a selected source are stored as links rather than followed.
|
|
270
|
+
|
|
271
|
+
Dirty mode acquires the same `BORG_BASE_DIR/backup.lock` as full mode before
|
|
272
|
+
recovering in-flight groups, clearing stale group names, or claiming work.
|
|
273
|
+
Claims use Redis `SMOVE`; successful claims are removed only after all creates
|
|
274
|
+
and retention pruning for that group succeed. Failed groups are requeued, and
|
|
275
|
+
claims left by a crashed process are recovered on the next run. Redis errors
|
|
276
|
+
fail the job; they are never treated as an empty dirty set. Keep periodic full
|
|
277
|
+
reconciliation backups as a safety net. The daily CronJob remains in full mode
|
|
278
|
+
until dirty-mode rollout is explicitly enabled.
|
|
238
279
|
|
|
239
280
|
## Archives, retention, and compaction
|
|
240
281
|
|
|
@@ -16,6 +16,15 @@ only after every create operation in the run has succeeded.
|
|
|
16
16
|
|
|
17
17
|
The application never initializes a repository. Borg 2 is not supported.
|
|
18
18
|
|
|
19
|
+
## Optional filesystem watcher
|
|
20
|
+
|
|
21
|
+
The separate Linux Rust [watcher](watcher/README.md) records changed backup
|
|
22
|
+
groups in Redis using filesystem-wide fanotify marks. The Python runner accepts
|
|
23
|
+
the same rooted YAML schema. `BACKUP_MODE=shadow` reports which groups would
|
|
24
|
+
be selected while still backing up all groups; `BACKUP_MODE=dirty` consumes
|
|
25
|
+
selected groups. The default remains `BACKUP_MODE=full`; keep full scheduling
|
|
26
|
+
until the watcher and Redis consumer have been validated in shadow operation.
|
|
27
|
+
|
|
19
28
|
## Installation
|
|
20
29
|
|
|
21
30
|
Install the package from PyPI into a `uv`-managed environment:
|
|
@@ -37,13 +46,16 @@ package is `declarative_data_backup`.
|
|
|
37
46
|
|
|
38
47
|
## GitLab release automation
|
|
39
48
|
|
|
40
|
-
The pipeline in [.gitlab-ci.yml](.gitlab-ci.yml) publishes releases when
|
|
41
|
-
tag in `major.minor.patch` format is pushed, for example `1.2.3`. Use the same
|
|
49
|
+
The pipeline in [.gitlab-ci.yml](.gitlab-ci.yml) publishes Python releases when
|
|
50
|
+
a Git tag in `major.minor.patch` format is pushed, for example `1.2.3`. Use the same
|
|
42
51
|
version for the GitLab Release name and tag. Tags with a `v` prefix, prerelease
|
|
43
52
|
suffix, or leading zeros are not published. Branch pushes do not run this
|
|
44
53
|
release pipeline.
|
|
45
54
|
|
|
46
|
-
|
|
55
|
+
Watcher releases use independent `watcher/major.minor.patch` tags and publish
|
|
56
|
+
only the watcher image; see [watcher releases](watcher/README.md#releases).
|
|
57
|
+
|
|
58
|
+
For each Python version tag, the pipeline:
|
|
47
59
|
|
|
48
60
|
1. Sets the package version in the CI checkout's `pyproject.toml` and uses
|
|
49
61
|
`uv build` to create a wheel and source distribution.
|
|
@@ -95,9 +107,10 @@ Create a YAML file accessible to the backup process:
|
|
|
95
107
|
|
|
96
108
|
```yaml
|
|
97
109
|
media_backup:
|
|
110
|
+
root: /server_root/media
|
|
98
111
|
folders:
|
|
99
|
-
-
|
|
100
|
-
-
|
|
112
|
+
- photos
|
|
113
|
+
- music
|
|
101
114
|
compression: auto,zstd,22
|
|
102
115
|
retention_days: 30
|
|
103
116
|
required_mounts:
|
|
@@ -105,18 +118,19 @@ media_backup:
|
|
|
105
118
|
allow_empty: true
|
|
106
119
|
|
|
107
120
|
postgres_backup:
|
|
121
|
+
root: /server_root/backups
|
|
108
122
|
folders:
|
|
109
|
-
-
|
|
110
|
-
required_mounts:
|
|
111
|
-
- /server_root/path/to/postgres-backups
|
|
123
|
+
- postgres
|
|
112
124
|
|
|
113
125
|
apprise_backup:
|
|
126
|
+
root: /server_root/data
|
|
114
127
|
files:
|
|
115
|
-
-
|
|
128
|
+
- apprise/apprise.yml
|
|
116
129
|
|
|
117
130
|
jellyfin_backup:
|
|
131
|
+
root: /server_root/data
|
|
118
132
|
folders:
|
|
119
|
-
-
|
|
133
|
+
- jellyfin
|
|
120
134
|
exclude:
|
|
121
135
|
- "**/cache/**"
|
|
122
136
|
- "**/log/**"
|
|
@@ -146,6 +160,11 @@ Optional variables:
|
|
|
146
160
|
| `BACKUP_FAILURES_PATH` | Optional absolute path where failed backup group names are written as JSON. |
|
|
147
161
|
| `BACKUP_PROGRESS` | Set to `true` to log the latest Borg progress every 30 seconds during archive creation. Defaults to `false`. |
|
|
148
162
|
| `BACKUP_VERBOSE` | Set to `true` to show Borg's per-file create and per-archive prune output. Defaults to `false` for concise logs. |
|
|
163
|
+
| `BACKUP_MODE` | `full` (default) backs up every configured group; `shadow` reports the dirty selection but backs up all groups; `dirty` consumes `ddb:v1:dirty-groups` from Redis. Shadow and dirty modes require the Redis variables below. |
|
|
164
|
+
| `REDIS_URL` | Required in shadow/dirty mode. A `redis://` TCP URL without credentials. |
|
|
165
|
+
| `REDIS_DBINDEX` | Required in shadow/dirty mode. Nonnegative logical database index shared with the watcher. |
|
|
166
|
+
| `REDIS_USERNAME` | Optional ACL username in shadow/dirty mode. |
|
|
167
|
+
| `REDIS_PASSWORD` | Required in shadow/dirty mode. Supply through a Kubernetes Secret or protected source. |
|
|
149
168
|
|
|
150
169
|
Provide credentials through a secret manager or protected files; do not put
|
|
151
170
|
passphrases, private keys, or credentials in the YAML file, source control, or
|
|
@@ -183,22 +202,32 @@ locks without first confirming that no repository operation is still running.
|
|
|
183
202
|
The top level must be a nonempty YAML mapping of backup group names to group
|
|
184
203
|
mappings. Group names may contain ASCII letters, digits, `_`, and `-`, and
|
|
185
204
|
must be at most 64 characters. Duplicate YAML keys, unknown group fields,
|
|
186
|
-
invalid types, unsupported
|
|
187
|
-
|
|
205
|
+
invalid types, unsupported patterns, and empty configuration are rejected
|
|
206
|
+
before any create or prune operation.
|
|
188
207
|
|
|
189
208
|
Each group supports:
|
|
190
209
|
|
|
191
|
-
- `
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
210
|
+
- `root` (required): absolute primary source root, lexically and canonically
|
|
211
|
+
inside `SERVER_ROOT`. It must exist, be a directory, and be accessible.
|
|
212
|
+
- `watcher_alternative_roots` (optional): absolute roots representing the same
|
|
213
|
+
relative layout. These roots are watcher-only: Python does not require their
|
|
214
|
+
directories or mounts to exist, and they are never backed up.
|
|
215
|
+
- `folders` (optional): nonempty relative directory paths under `root`. A
|
|
216
|
+
folder `.` selects the root itself.
|
|
217
|
+
- `files` (optional): nonempty relative regular-file paths under `root`.
|
|
218
|
+
Sources reject absolute paths, NUL bytes, and `..` components. Their
|
|
219
|
+
canonical targets must remain inside `root`. Configure at least one source.
|
|
220
|
+
- `exclude` (optional): root-relative glob patterns shared with the watcher.
|
|
221
|
+
Patterns are anchored and case-sensitive. Version 1 allows ASCII
|
|
222
|
+
letters/digits, space, `_`, `-`, `.`, `/`, `*`, `?`, and `**` as a whole
|
|
223
|
+
component; empty, `.` and `..` components are rejected. An excluded
|
|
224
|
+
directory excludes descendants, and a trailing `/**` excludes the directory
|
|
225
|
+
itself too. Patterns are translated to root-scoped Borg regular expressions,
|
|
226
|
+
not passed through as Borg syntax. Patterns whose translated regex exceeds
|
|
227
|
+
64,000 characters are rejected. A group with explicit `files` cannot also
|
|
228
|
+
configure `exclude`; this safety check remains until a policy and
|
|
229
|
+
archive-level tests define the behavior for an explicitly selected file
|
|
230
|
+
matched by an exclusion.
|
|
202
231
|
- `compression` (optional): defaults to `auto,zstd,22`. Supported Borg 1.4
|
|
203
232
|
codecs include `none`, `lz4`, `zstd`, `zlib`, and `lzma`; `auto` may precede
|
|
204
233
|
a codec. Codec levels are checked against supported ranges: zstd `1`-`22`,
|
|
@@ -207,8 +236,10 @@ Each group supports:
|
|
|
207
236
|
useful only with repository encryption.
|
|
208
237
|
- `retention_days` (optional): positive integer, default `30`. This controls
|
|
209
238
|
Borg's `--keep-daily` policy.
|
|
210
|
-
- `required_mounts` (optional):
|
|
211
|
-
in the process's `/proc/self/mountinfo`.
|
|
239
|
+
- `required_mounts` (optional): absolute paths to existing exact mount points
|
|
240
|
+
that must appear in the process's `/proc/self/mountinfo`. These are
|
|
241
|
+
availability requirements, not source selectors. Alternative roots are not
|
|
242
|
+
automatically added.
|
|
212
243
|
- `allow_empty` (optional): boolean, default `true`. If set to `false`, an
|
|
213
244
|
empty source is rejected.
|
|
214
245
|
|
|
@@ -224,9 +255,18 @@ consistent database snapshot. Use an application-supported export, backup API,
|
|
|
224
255
|
or coordinated filesystem snapshot for live database data. Exclusions do not
|
|
225
256
|
make a database backup consistent.
|
|
226
257
|
|
|
227
|
-
|
|
228
|
-
when its canonical target remains inside `
|
|
229
|
-
source are stored as links rather than followed.
|
|
258
|
+
Primary roots and sources are resolved before backup. A source symlink is
|
|
259
|
+
allowed only when its canonical target remains inside the group's `root`;
|
|
260
|
+
symlinks within a selected source are stored as links rather than followed.
|
|
261
|
+
|
|
262
|
+
Dirty mode acquires the same `BORG_BASE_DIR/backup.lock` as full mode before
|
|
263
|
+
recovering in-flight groups, clearing stale group names, or claiming work.
|
|
264
|
+
Claims use Redis `SMOVE`; successful claims are removed only after all creates
|
|
265
|
+
and retention pruning for that group succeed. Failed groups are requeued, and
|
|
266
|
+
claims left by a crashed process are recovered on the next run. Redis errors
|
|
267
|
+
fail the job; they are never treated as an empty dirty set. Keep periodic full
|
|
268
|
+
reconciliation backups as a safety net. The daily CronJob remains in full mode
|
|
269
|
+
until dirty-mode rollout is explicitly enabled.
|
|
230
270
|
|
|
231
271
|
## Archives, retention, and compaction
|
|
232
272
|
|
|
@@ -4,11 +4,11 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "declarative-data-backup"
|
|
7
|
-
version = "0.
|
|
7
|
+
version = "0.6.0"
|
|
8
8
|
description = "Declarative Borg-based data backups"
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.11"
|
|
11
|
-
dependencies = ["PyYAML>=6,<7"]
|
|
11
|
+
dependencies = ["PyYAML>=6,<7", "redis>=5,<7"]
|
|
12
12
|
|
|
13
13
|
[dependency-groups]
|
|
14
14
|
dev = [
|
|
@@ -17,8 +17,10 @@ from .borg import (
|
|
|
17
17
|
from .borg.borg_cli import ensure_borg_binary
|
|
18
18
|
from .config import configure_logging, load_yaml, parse_backup_definitions
|
|
19
19
|
from .config.backup_config.models import BackupDefinition
|
|
20
|
+
from .dirty_groups import DirtyGroupConsumer
|
|
20
21
|
from .environment import Environment
|
|
21
22
|
from .errors import BackupError, BackupInterrupted, BorgWarningError
|
|
23
|
+
from .exclusions import translate_excludes
|
|
22
24
|
|
|
23
25
|
logger = logging.getLogger(__name__)
|
|
24
26
|
|
|
@@ -93,78 +95,128 @@ def run_backup(
|
|
|
93
95
|
|
|
94
96
|
run_id = uuid.uuid4().hex
|
|
95
97
|
with repository_lock(settings.borg_base_dir / "backup.lock"):
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
definition.name
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
98
|
+
dirty_groups = (
|
|
99
|
+
DirtyGroupConsumer(settings)
|
|
100
|
+
if settings.backup_mode in {"shadow", "dirty"}
|
|
101
|
+
else None
|
|
102
|
+
)
|
|
103
|
+
try:
|
|
104
|
+
if settings.backup_mode in {"shadow", "dirty"} and dirty_groups is not None:
|
|
105
|
+
group_names = {definition.name for definition in definitions}
|
|
106
|
+
if settings.backup_mode == "shadow":
|
|
107
|
+
dirty_groups.preview(group_names)
|
|
108
|
+
else:
|
|
109
|
+
dirty_groups.prepare(group_names)
|
|
110
|
+
|
|
111
|
+
created_at = datetime.now(UTC)
|
|
112
|
+
successful_definitions = []
|
|
113
|
+
failures: list[BackupError] = []
|
|
114
|
+
failed_groups: set[str] = set()
|
|
115
|
+
for definition in definitions:
|
|
116
|
+
if settings.backup_mode == "dirty" and dirty_groups is not None:
|
|
117
|
+
claimed = dirty_groups.claim(definition.name)
|
|
118
|
+
else:
|
|
119
|
+
claimed = True
|
|
120
|
+
if not claimed:
|
|
121
|
+
logger.info(
|
|
122
|
+
"Skipping backup group '%s'; it is not dirty.",
|
|
111
123
|
definition.name,
|
|
112
|
-
settings,
|
|
113
|
-
created_at,
|
|
114
|
-
run_id,
|
|
115
|
-
definition.excludes,
|
|
116
124
|
)
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
"
|
|
125
|
+
continue
|
|
126
|
+
|
|
127
|
+
logger.info(
|
|
128
|
+
"Starting backup group '%s' (%d sources).",
|
|
121
129
|
definition.name,
|
|
122
|
-
|
|
130
|
+
len(definition.sources),
|
|
123
131
|
)
|
|
124
|
-
failures.append(exc)
|
|
125
|
-
failed_groups.add(definition.name)
|
|
126
|
-
continue
|
|
127
|
-
|
|
128
|
-
logger.info("Backup group '%s' completed.", definition.name)
|
|
129
|
-
successful_definitions.append(definition)
|
|
130
|
-
|
|
131
|
-
for definition in successful_definitions:
|
|
132
|
-
logger.info(
|
|
133
|
-
"Starting retention pruning for group '%s' (%d sources).",
|
|
134
|
-
definition.name,
|
|
135
|
-
len(definition.sources),
|
|
136
|
-
)
|
|
137
|
-
pruning_succeeded = True
|
|
138
|
-
for folder in definition.sources:
|
|
139
132
|
try:
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
definition.
|
|
143
|
-
definition.name,
|
|
144
|
-
settings,
|
|
133
|
+
excludes = translate_excludes(
|
|
134
|
+
definition.root,
|
|
135
|
+
definition.excludes,
|
|
145
136
|
)
|
|
137
|
+
for folder in definition.sources:
|
|
138
|
+
backup_folder(
|
|
139
|
+
folder,
|
|
140
|
+
definition.compression,
|
|
141
|
+
definition.name,
|
|
142
|
+
settings,
|
|
143
|
+
created_at,
|
|
144
|
+
run_id,
|
|
145
|
+
excludes,
|
|
146
|
+
)
|
|
146
147
|
except BackupError as exc:
|
|
148
|
+
if settings.backup_mode == "dirty" and dirty_groups is not None:
|
|
149
|
+
try:
|
|
150
|
+
dirty_groups.requeue(definition.name)
|
|
151
|
+
except BackupError as state_error:
|
|
152
|
+
failed_groups.add(definition.name)
|
|
153
|
+
write_failed_groups(settings, failed_groups)
|
|
154
|
+
raise BackupError(f"{exc}; {state_error}") from state_error
|
|
147
155
|
logger.error(
|
|
148
|
-
"
|
|
156
|
+
"Backup group '%s' failed; retention pruning will be skipped "
|
|
157
|
+
"for this group: %s",
|
|
149
158
|
definition.name,
|
|
150
|
-
folder,
|
|
151
159
|
exc,
|
|
152
160
|
)
|
|
153
161
|
failures.append(exc)
|
|
154
162
|
failed_groups.add(definition.name)
|
|
155
|
-
|
|
163
|
+
continue
|
|
156
164
|
|
|
157
|
-
|
|
165
|
+
logger.info("Backup group '%s' completed.", definition.name)
|
|
166
|
+
successful_definitions.append(definition)
|
|
167
|
+
|
|
168
|
+
for definition in successful_definitions:
|
|
158
169
|
logger.info(
|
|
159
|
-
"
|
|
170
|
+
"Starting retention pruning for group '%s' (%d sources).",
|
|
171
|
+
definition.name,
|
|
172
|
+
len(definition.sources),
|
|
160
173
|
)
|
|
174
|
+
pruning_succeeded = True
|
|
175
|
+
for folder in definition.sources:
|
|
176
|
+
try:
|
|
177
|
+
prune(
|
|
178
|
+
folder,
|
|
179
|
+
definition.retention_days,
|
|
180
|
+
definition.name,
|
|
181
|
+
settings,
|
|
182
|
+
)
|
|
183
|
+
except BackupError as exc:
|
|
184
|
+
logger.error(
|
|
185
|
+
"Retention pruning failed for group '%s' source '%s': %s",
|
|
186
|
+
definition.name,
|
|
187
|
+
folder,
|
|
188
|
+
exc,
|
|
189
|
+
)
|
|
190
|
+
failures.append(exc)
|
|
191
|
+
failed_groups.add(definition.name)
|
|
192
|
+
pruning_succeeded = False
|
|
193
|
+
|
|
194
|
+
if settings.backup_mode == "dirty" and dirty_groups is not None:
|
|
195
|
+
try:
|
|
196
|
+
if pruning_succeeded:
|
|
197
|
+
dirty_groups.acknowledge(definition.name)
|
|
198
|
+
else:
|
|
199
|
+
dirty_groups.requeue(definition.name)
|
|
200
|
+
except BackupError as state_error:
|
|
201
|
+
failed_groups.add(definition.name)
|
|
202
|
+
write_failed_groups(settings, failed_groups)
|
|
203
|
+
raise state_error
|
|
204
|
+
|
|
205
|
+
if pruning_succeeded:
|
|
206
|
+
logger.info(
|
|
207
|
+
"Retention pruning for group '%s' completed.",
|
|
208
|
+
definition.name,
|
|
209
|
+
)
|
|
161
210
|
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
211
|
+
if failures:
|
|
212
|
+
write_failed_groups(settings, failed_groups)
|
|
213
|
+
message = "; ".join(str(failure) for failure in failures)
|
|
214
|
+
if any(isinstance(failure, BorgWarningError) for failure in failures):
|
|
215
|
+
raise BorgWarningError(message)
|
|
216
|
+
raise BackupError(message)
|
|
217
|
+
finally:
|
|
218
|
+
if dirty_groups is not None:
|
|
219
|
+
dirty_groups.close()
|
|
168
220
|
|
|
169
221
|
|
|
170
222
|
def main(environ: Mapping[str, str] | None = None) -> int:
|
|
@@ -5,14 +5,17 @@ from pathlib import Path
|
|
|
5
5
|
@dataclass(frozen=True)
|
|
6
6
|
class BackupDefinition:
|
|
7
7
|
name: str
|
|
8
|
+
root: Path
|
|
8
9
|
folders: tuple[Path, ...]
|
|
10
|
+
files: tuple[Path, ...]
|
|
11
|
+
canonical_sources: tuple[Path, ...]
|
|
9
12
|
compression: str
|
|
10
13
|
retention_days: int
|
|
11
14
|
required_mounts: tuple[Path, ...] = ()
|
|
15
|
+
watcher_alternative_roots: tuple[Path, ...] = ()
|
|
12
16
|
allow_empty: bool = True
|
|
13
|
-
files: tuple[Path, ...] = ()
|
|
14
17
|
excludes: tuple[str, ...] = ()
|
|
15
18
|
|
|
16
19
|
@property
|
|
17
20
|
def sources(self) -> tuple[Path, ...]:
|
|
18
|
-
return self.
|
|
21
|
+
return self.canonical_sources
|