declarative-data-backup 0.5.1__tar.gz → 0.6.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (30) hide show
  1. {declarative_data_backup-0.5.1 → declarative_data_backup-0.6.0}/PKG-INFO +70 -29
  2. {declarative_data_backup-0.5.1 → declarative_data_backup-0.6.0}/README.md +68 -28
  3. {declarative_data_backup-0.5.1 → declarative_data_backup-0.6.0}/pyproject.toml +2 -2
  4. {declarative_data_backup-0.5.1 → declarative_data_backup-0.6.0}/src/declarative_data_backup/backup.py +107 -55
  5. {declarative_data_backup-0.5.1 → declarative_data_backup-0.6.0}/src/declarative_data_backup/borg/borg_cli.py +4 -2
  6. {declarative_data_backup-0.5.1 → declarative_data_backup-0.6.0}/src/declarative_data_backup/config/backup_config/models.py +5 -2
  7. {declarative_data_backup-0.5.1 → declarative_data_backup-0.6.0}/src/declarative_data_backup/config/backup_config/utils.py +93 -22
  8. {declarative_data_backup-0.5.1 → declarative_data_backup-0.6.0}/src/declarative_data_backup/config/backup_config/yaml_parser.py +32 -3
  9. declarative_data_backup-0.6.0/src/declarative_data_backup/dirty_groups.py +150 -0
  10. {declarative_data_backup-0.5.1 → declarative_data_backup-0.6.0}/src/declarative_data_backup/environment.py +65 -1
  11. declarative_data_backup-0.6.0/src/declarative_data_backup/exclusions.py +127 -0
  12. {declarative_data_backup-0.5.1 → declarative_data_backup-0.6.0}/src/declarative_data_backup.egg-info/PKG-INFO +70 -29
  13. {declarative_data_backup-0.5.1 → declarative_data_backup-0.6.0}/src/declarative_data_backup.egg-info/SOURCES.txt +5 -1
  14. {declarative_data_backup-0.5.1 → declarative_data_backup-0.6.0}/src/declarative_data_backup.egg-info/requires.txt +1 -0
  15. {declarative_data_backup-0.5.1 → declarative_data_backup-0.6.0}/tests/test_backup.py +192 -2
  16. declarative_data_backup-0.6.0/tests/test_dirty_groups.py +188 -0
  17. declarative_data_backup-0.6.0/tests/test_exclusions.py +72 -0
  18. {declarative_data_backup-0.5.1 → declarative_data_backup-0.6.0}/setup.cfg +0 -0
  19. {declarative_data_backup-0.5.1 → declarative_data_backup-0.6.0}/src/declarative_data_backup/__init__.py +0 -0
  20. {declarative_data_backup-0.5.1 → declarative_data_backup-0.6.0}/src/declarative_data_backup/__main__.py +0 -0
  21. {declarative_data_backup-0.5.1 → declarative_data_backup-0.6.0}/src/declarative_data_backup/borg/__init__.py +0 -0
  22. {declarative_data_backup-0.5.1 → declarative_data_backup-0.6.0}/src/declarative_data_backup/borg/archive_naming.py +0 -0
  23. {declarative_data_backup-0.5.1 → declarative_data_backup-0.6.0}/src/declarative_data_backup/borg/borg.py +0 -0
  24. {declarative_data_backup-0.5.1 → declarative_data_backup-0.6.0}/src/declarative_data_backup/config/__init__.py +0 -0
  25. {declarative_data_backup-0.5.1 → declarative_data_backup-0.6.0}/src/declarative_data_backup/config/backup_config/__init__.py +0 -0
  26. {declarative_data_backup-0.5.1 → declarative_data_backup-0.6.0}/src/declarative_data_backup/config/backup_config/yaml_raw_loader.py +0 -0
  27. {declarative_data_backup-0.5.1 → declarative_data_backup-0.6.0}/src/declarative_data_backup/config/logging.py +0 -0
  28. {declarative_data_backup-0.5.1 → declarative_data_backup-0.6.0}/src/declarative_data_backup/errors.py +0 -0
  29. {declarative_data_backup-0.5.1 → declarative_data_backup-0.6.0}/src/declarative_data_backup.egg-info/dependency_links.txt +0 -0
  30. {declarative_data_backup-0.5.1 → declarative_data_backup-0.6.0}/src/declarative_data_backup.egg-info/top_level.txt +0 -0
@@ -1,10 +1,11 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: declarative-data-backup
3
- Version: 0.5.1
3
+ Version: 0.6.0
4
4
  Summary: Declarative Borg-based data backups
5
5
  Requires-Python: >=3.11
6
6
  Description-Content-Type: text/markdown
7
7
  Requires-Dist: PyYAML<7,>=6
8
+ Requires-Dist: redis<7,>=5
8
9
 
9
10
  # Declarative Data Backup
10
11
 
@@ -24,6 +25,15 @@ only after every create operation in the run has succeeded.
24
25
 
25
26
  The application never initializes a repository. Borg 2 is not supported.
26
27
 
28
+ ## Optional filesystem watcher
29
+
30
+ The separate Linux Rust [watcher](watcher/README.md) records changed backup
31
+ groups in Redis using filesystem-wide fanotify marks. The Python runner accepts
32
+ the same rooted YAML schema. `BACKUP_MODE=shadow` reports which groups would
33
+ be selected while still backing up all groups; `BACKUP_MODE=dirty` consumes
34
+ selected groups. The default remains `BACKUP_MODE=full`; keep full scheduling
35
+ until the watcher and Redis consumer have been validated in shadow operation.
36
+
27
37
  ## Installation
28
38
 
29
39
  Install the package from PyPI into a `uv`-managed environment:
@@ -45,13 +55,16 @@ package is `declarative_data_backup`.
45
55
 
46
56
  ## GitLab release automation
47
57
 
48
- The pipeline in [.gitlab-ci.yml](.gitlab-ci.yml) publishes releases when a Git
49
- tag in `major.minor.patch` format is pushed, for example `1.2.3`. Use the same
58
+ The pipeline in [.gitlab-ci.yml](.gitlab-ci.yml) publishes Python releases when
59
+ a Git tag in `major.minor.patch` format is pushed, for example `1.2.3`. Use the same
50
60
  version for the GitLab Release name and tag. Tags with a `v` prefix, prerelease
51
61
  suffix, or leading zeros are not published. Branch pushes do not run this
52
62
  release pipeline.
53
63
 
54
- For each version tag, the pipeline:
64
+ Watcher releases use independent `watcher/major.minor.patch` tags and publish
65
+ only the watcher image; see [watcher releases](watcher/README.md#releases).
66
+
67
+ For each Python version tag, the pipeline:
55
68
 
56
69
  1. Sets the package version in the CI checkout's `pyproject.toml` and uses
57
70
  `uv build` to create a wheel and source distribution.
@@ -103,9 +116,10 @@ Create a YAML file accessible to the backup process:
103
116
 
104
117
  ```yaml
105
118
  media_backup:
119
+ root: /server_root/media
106
120
  folders:
107
- - /server_root/media/photos
108
- - /server_root/media/music
121
+ - photos
122
+ - music
109
123
  compression: auto,zstd,22
110
124
  retention_days: 30
111
125
  required_mounts:
@@ -113,18 +127,19 @@ media_backup:
113
127
  allow_empty: true
114
128
 
115
129
  postgres_backup:
130
+ root: /server_root/backups
116
131
  folders:
117
- - /server_root/path/to/postgres-backups
118
- required_mounts:
119
- - /server_root/path/to/postgres-backups
132
+ - postgres
120
133
 
121
134
  apprise_backup:
135
+ root: /server_root/data
122
136
  files:
123
- - /server_root/data/apprise/apprise.yml
137
+ - apprise/apprise.yml
124
138
 
125
139
  jellyfin_backup:
140
+ root: /server_root/data
126
141
  folders:
127
- - /server_root/data/jellyfin
142
+ - jellyfin
128
143
  exclude:
129
144
  - "**/cache/**"
130
145
  - "**/log/**"
@@ -154,6 +169,11 @@ Optional variables:
154
169
  | `BACKUP_FAILURES_PATH` | Optional absolute path where failed backup group names are written as JSON. |
155
170
  | `BACKUP_PROGRESS` | Set to `true` to log the latest Borg progress every 30 seconds during archive creation. Defaults to `false`. |
156
171
  | `BACKUP_VERBOSE` | Set to `true` to show Borg's per-file create and per-archive prune output. Defaults to `false` for concise logs. |
172
+ | `BACKUP_MODE` | `full` (default) backs up every configured group; `shadow` reports the dirty selection but backs up all groups; `dirty` consumes `ddb:v1:dirty-groups` from Redis. Shadow and dirty modes require the Redis variables below. |
173
+ | `REDIS_URL` | Required in shadow/dirty mode. A `redis://` TCP URL without credentials. |
174
+ | `REDIS_DBINDEX` | Required in shadow/dirty mode. Nonnegative logical database index shared with the watcher. |
175
+ | `REDIS_USERNAME` | Optional ACL username in shadow/dirty mode. |
176
+ | `REDIS_PASSWORD` | Required in shadow/dirty mode. Supply through a Kubernetes Secret or protected source. |
157
177
 
158
178
  Provide credentials through a secret manager or protected files; do not put
159
179
  passphrases, private keys, or credentials in the YAML file, source control, or
@@ -191,22 +211,32 @@ locks without first confirming that no repository operation is still running.
191
211
  The top level must be a nonempty YAML mapping of backup group names to group
192
212
  mappings. Group names may contain ASCII letters, digits, `_`, and `-`, and
193
213
  must be at most 64 characters. Duplicate YAML keys, unknown group fields,
194
- invalid types, unsupported pattern selectors, and empty configuration are
195
- rejected before any create or prune operation.
214
+ invalid types, unsupported patterns, and empty configuration are rejected
215
+ before any create or prune operation.
196
216
 
197
217
  Each group supports:
198
218
 
199
- - `folders` (optional): list of absolute directory paths.
200
- - `files` (optional): list of absolute regular-file paths. Folders and files
201
- share the same containment, readability, duplicate, and overlap checks;
202
- configure at least one source per group.
203
- - `exclude` (optional): list of Borg 1.4 glob patterns, scoped to this group
204
- and passed directly to Borg for folder sources. To avoid approximating
205
- Borg's pattern matching, a group with explicit `files` cannot also configure
206
- `exclude`; this is rejected before any Borg operation.
207
- Patterns are otherwise passed unchanged to Borg, which evaluates their
208
- matching semantics. If Borg rejects a pattern during create, the run stops
209
- before pruning any source.
219
+ - `root` (required): absolute primary source root, lexically and canonically
220
+ inside `SERVER_ROOT`. It must exist, be a directory, and be accessible.
221
+ - `watcher_alternative_roots` (optional): absolute roots representing the same
222
+ relative layout. These roots are watcher-only: Python does not require their
223
+ directories or mounts to exist, and they are never backed up.
224
+ - `folders` (optional): nonempty relative directory paths under `root`. A
225
+ folder `.` selects the root itself.
226
+ - `files` (optional): nonempty relative regular-file paths under `root`.
227
+ Sources reject absolute paths, NUL bytes, and `..` components. Their
228
+ canonical targets must remain inside `root`. Configure at least one source.
229
+ - `exclude` (optional): root-relative glob patterns shared with the watcher.
230
+ Patterns are anchored and case-sensitive. Version 1 allows ASCII
231
+ letters/digits, space, `_`, `-`, `.`, `/`, `*`, `?`, and `**` as a whole
232
+ component; empty, `.` and `..` components are rejected. An excluded
233
+ directory excludes descendants, and a trailing `/**` excludes the directory
234
+ itself too. Patterns are translated to root-scoped Borg regular expressions,
235
+ not passed through as Borg syntax. Patterns whose translated regex exceeds
236
+ 64,000 characters are rejected. A group with explicit `files` cannot also
237
+ configure `exclude`; this safety check remains until a policy and
238
+ archive-level tests define the behavior for an explicitly selected file
239
+ matched by an exclusion.
210
240
  - `compression` (optional): defaults to `auto,zstd,22`. Supported Borg 1.4
211
241
  codecs include `none`, `lz4`, `zstd`, `zlib`, and `lzma`; `auto` may precede
212
242
  a codec. Codec levels are checked against supported ranges: zstd `1`-`22`,
@@ -215,8 +245,10 @@ Each group supports:
215
245
  useful only with repository encryption.
216
246
  - `retention_days` (optional): positive integer, default `30`. This controls
217
247
  Borg's `--keep-daily` policy.
218
- - `required_mounts` (optional): list of exact mount points that must appear
219
- in the process's `/proc/self/mountinfo`.
248
+ - `required_mounts` (optional): absolute paths to existing exact mount points
249
+ that must appear in the process's `/proc/self/mountinfo`. These are
250
+ availability requirements, not source selectors. Alternative roots are not
251
+ automatically added.
220
252
  - `allow_empty` (optional): boolean, default `true`. If set to `false`, an
221
253
  empty source is rejected.
222
254
 
@@ -232,9 +264,18 @@ consistent database snapshot. Use an application-supported export, backup API,
232
264
  or coordinated filesystem snapshot for live database data. Exclusions do not
233
265
  make a database backup consistent.
234
266
 
235
- Paths are resolved before backup. A top-level source symlink is allowed only
236
- when its canonical target remains inside `SERVER_ROOT`; symlinks within a
237
- source are stored as links rather than followed.
267
+ Primary roots and sources are resolved before backup. A source symlink is
268
+ allowed only when its canonical target remains inside the group's `root`;
269
+ symlinks within a selected source are stored as links rather than followed.
270
+
271
+ Dirty mode acquires the same `BORG_BASE_DIR/backup.lock` as full mode before
272
+ recovering in-flight groups, clearing stale group names, or claiming work.
273
+ Claims use Redis `SMOVE`; successful claims are removed only after all creates
274
+ and retention pruning for that group succeed. Failed groups are requeued, and
275
+ claims left by a crashed process are recovered on the next run. Redis errors
276
+ fail the job; they are never treated as an empty dirty set. Keep periodic full
277
+ reconciliation backups as a safety net. The daily CronJob remains in full mode
278
+ until dirty-mode rollout is explicitly enabled.
238
279
 
239
280
  ## Archives, retention, and compaction
240
281
 
@@ -16,6 +16,15 @@ only after every create operation in the run has succeeded.
16
16
 
17
17
  The application never initializes a repository. Borg 2 is not supported.
18
18
 
19
+ ## Optional filesystem watcher
20
+
21
+ The separate Linux Rust [watcher](watcher/README.md) records changed backup
22
+ groups in Redis using filesystem-wide fanotify marks. The Python runner accepts
23
+ the same rooted YAML schema. `BACKUP_MODE=shadow` reports which groups would
24
+ be selected while still backing up all groups; `BACKUP_MODE=dirty` consumes
25
+ selected groups. The default remains `BACKUP_MODE=full`; keep full scheduling
26
+ until the watcher and Redis consumer have been validated in shadow operation.
27
+
19
28
  ## Installation
20
29
 
21
30
  Install the package from PyPI into a `uv`-managed environment:
@@ -37,13 +46,16 @@ package is `declarative_data_backup`.
37
46
 
38
47
  ## GitLab release automation
39
48
 
40
- The pipeline in [.gitlab-ci.yml](.gitlab-ci.yml) publishes releases when a Git
41
- tag in `major.minor.patch` format is pushed, for example `1.2.3`. Use the same
49
+ The pipeline in [.gitlab-ci.yml](.gitlab-ci.yml) publishes Python releases when
50
+ a Git tag in `major.minor.patch` format is pushed, for example `1.2.3`. Use the same
42
51
  version for the GitLab Release name and tag. Tags with a `v` prefix, prerelease
43
52
  suffix, or leading zeros are not published. Branch pushes do not run this
44
53
  release pipeline.
45
54
 
46
- For each version tag, the pipeline:
55
+ Watcher releases use independent `watcher/major.minor.patch` tags and publish
56
+ only the watcher image; see [watcher releases](watcher/README.md#releases).
57
+
58
+ For each Python version tag, the pipeline:
47
59
 
48
60
  1. Sets the package version in the CI checkout's `pyproject.toml` and uses
49
61
  `uv build` to create a wheel and source distribution.
@@ -95,9 +107,10 @@ Create a YAML file accessible to the backup process:
95
107
 
96
108
  ```yaml
97
109
  media_backup:
110
+ root: /server_root/media
98
111
  folders:
99
- - /server_root/media/photos
100
- - /server_root/media/music
112
+ - photos
113
+ - music
101
114
  compression: auto,zstd,22
102
115
  retention_days: 30
103
116
  required_mounts:
@@ -105,18 +118,19 @@ media_backup:
105
118
  allow_empty: true
106
119
 
107
120
  postgres_backup:
121
+ root: /server_root/backups
108
122
  folders:
109
- - /server_root/path/to/postgres-backups
110
- required_mounts:
111
- - /server_root/path/to/postgres-backups
123
+ - postgres
112
124
 
113
125
  apprise_backup:
126
+ root: /server_root/data
114
127
  files:
115
- - /server_root/data/apprise/apprise.yml
128
+ - apprise/apprise.yml
116
129
 
117
130
  jellyfin_backup:
131
+ root: /server_root/data
118
132
  folders:
119
- - /server_root/data/jellyfin
133
+ - jellyfin
120
134
  exclude:
121
135
  - "**/cache/**"
122
136
  - "**/log/**"
@@ -146,6 +160,11 @@ Optional variables:
146
160
  | `BACKUP_FAILURES_PATH` | Optional absolute path where failed backup group names are written as JSON. |
147
161
  | `BACKUP_PROGRESS` | Set to `true` to log the latest Borg progress every 30 seconds during archive creation. Defaults to `false`. |
148
162
  | `BACKUP_VERBOSE` | Set to `true` to show Borg's per-file create and per-archive prune output. Defaults to `false` for concise logs. |
163
+ | `BACKUP_MODE` | `full` (default) backs up every configured group; `shadow` reports the dirty selection but backs up all groups; `dirty` consumes `ddb:v1:dirty-groups` from Redis. Shadow and dirty modes require the Redis variables below. |
164
+ | `REDIS_URL` | Required in shadow/dirty mode. A `redis://` TCP URL without credentials. |
165
+ | `REDIS_DBINDEX` | Required in shadow/dirty mode. Nonnegative logical database index shared with the watcher. |
166
+ | `REDIS_USERNAME` | Optional ACL username in shadow/dirty mode. |
167
+ | `REDIS_PASSWORD` | Required in shadow/dirty mode. Supply through a Kubernetes Secret or protected source. |
149
168
 
150
169
  Provide credentials through a secret manager or protected files; do not put
151
170
  passphrases, private keys, or credentials in the YAML file, source control, or
@@ -183,22 +202,32 @@ locks without first confirming that no repository operation is still running.
183
202
  The top level must be a nonempty YAML mapping of backup group names to group
184
203
  mappings. Group names may contain ASCII letters, digits, `_`, and `-`, and
185
204
  must be at most 64 characters. Duplicate YAML keys, unknown group fields,
186
- invalid types, unsupported pattern selectors, and empty configuration are
187
- rejected before any create or prune operation.
205
+ invalid types, unsupported patterns, and empty configuration are rejected
206
+ before any create or prune operation.
188
207
 
189
208
  Each group supports:
190
209
 
191
- - `folders` (optional): list of absolute directory paths.
192
- - `files` (optional): list of absolute regular-file paths. Folders and files
193
- share the same containment, readability, duplicate, and overlap checks;
194
- configure at least one source per group.
195
- - `exclude` (optional): list of Borg 1.4 glob patterns, scoped to this group
196
- and passed directly to Borg for folder sources. To avoid approximating
197
- Borg's pattern matching, a group with explicit `files` cannot also configure
198
- `exclude`; this is rejected before any Borg operation.
199
- Patterns are otherwise passed unchanged to Borg, which evaluates their
200
- matching semantics. If Borg rejects a pattern during create, the run stops
201
- before pruning any source.
210
+ - `root` (required): absolute primary source root, lexically and canonically
211
+ inside `SERVER_ROOT`. It must exist, be a directory, and be accessible.
212
+ - `watcher_alternative_roots` (optional): absolute roots representing the same
213
+ relative layout. These roots are watcher-only: Python does not require their
214
+ directories or mounts to exist, and they are never backed up.
215
+ - `folders` (optional): nonempty relative directory paths under `root`. A
216
+ folder `.` selects the root itself.
217
+ - `files` (optional): nonempty relative regular-file paths under `root`.
218
+ Sources reject absolute paths, NUL bytes, and `..` components. Their
219
+ canonical targets must remain inside `root`. Configure at least one source.
220
+ - `exclude` (optional): root-relative glob patterns shared with the watcher.
221
+ Patterns are anchored and case-sensitive. Version 1 allows ASCII
222
+ letters/digits, space, `_`, `-`, `.`, `/`, `*`, `?`, and `**` as a whole
223
+ component; empty, `.` and `..` components are rejected. An excluded
224
+ directory excludes descendants, and a trailing `/**` excludes the directory
225
+ itself too. Patterns are translated to root-scoped Borg regular expressions,
226
+ not passed through as Borg syntax. Patterns whose translated regex exceeds
227
+ 64,000 characters are rejected. A group with explicit `files` cannot also
228
+ configure `exclude`; this safety check remains until a policy and
229
+ archive-level tests define the behavior for an explicitly selected file
230
+ matched by an exclusion.
202
231
  - `compression` (optional): defaults to `auto,zstd,22`. Supported Borg 1.4
203
232
  codecs include `none`, `lz4`, `zstd`, `zlib`, and `lzma`; `auto` may precede
204
233
  a codec. Codec levels are checked against supported ranges: zstd `1`-`22`,
@@ -207,8 +236,10 @@ Each group supports:
207
236
  useful only with repository encryption.
208
237
  - `retention_days` (optional): positive integer, default `30`. This controls
209
238
  Borg's `--keep-daily` policy.
210
- - `required_mounts` (optional): list of exact mount points that must appear
211
- in the process's `/proc/self/mountinfo`.
239
+ - `required_mounts` (optional): absolute paths to existing exact mount points
240
+ that must appear in the process's `/proc/self/mountinfo`. These are
241
+ availability requirements, not source selectors. Alternative roots are not
242
+ automatically added.
212
243
  - `allow_empty` (optional): boolean, default `true`. If set to `false`, an
213
244
  empty source is rejected.
214
245
 
@@ -224,9 +255,18 @@ consistent database snapshot. Use an application-supported export, backup API,
224
255
  or coordinated filesystem snapshot for live database data. Exclusions do not
225
256
  make a database backup consistent.
226
257
 
227
- Paths are resolved before backup. A top-level source symlink is allowed only
228
- when its canonical target remains inside `SERVER_ROOT`; symlinks within a
229
- source are stored as links rather than followed.
258
+ Primary roots and sources are resolved before backup. A source symlink is
259
+ allowed only when its canonical target remains inside the group's `root`;
260
+ symlinks within a selected source are stored as links rather than followed.
261
+
262
+ Dirty mode acquires the same `BORG_BASE_DIR/backup.lock` as full mode before
263
+ recovering in-flight groups, clearing stale group names, or claiming work.
264
+ Claims use Redis `SMOVE`; successful claims are removed only after all creates
265
+ and retention pruning for that group succeed. Failed groups are requeued, and
266
+ claims left by a crashed process are recovered on the next run. Redis errors
267
+ fail the job; they are never treated as an empty dirty set. Keep periodic full
268
+ reconciliation backups as a safety net. The daily CronJob remains in full mode
269
+ until dirty-mode rollout is explicitly enabled.
230
270
 
231
271
  ## Archives, retention, and compaction
232
272
 
@@ -4,11 +4,11 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "declarative-data-backup"
7
- version = "0.5.1"
7
+ version = "0.6.0"
8
8
  description = "Declarative Borg-based data backups"
9
9
  readme = "README.md"
10
10
  requires-python = ">=3.11"
11
- dependencies = ["PyYAML>=6,<7"]
11
+ dependencies = ["PyYAML>=6,<7", "redis>=5,<7"]
12
12
 
13
13
  [dependency-groups]
14
14
  dev = [
@@ -17,8 +17,10 @@ from .borg import (
17
17
  from .borg.borg_cli import ensure_borg_binary
18
18
  from .config import configure_logging, load_yaml, parse_backup_definitions
19
19
  from .config.backup_config.models import BackupDefinition
20
+ from .dirty_groups import DirtyGroupConsumer
20
21
  from .environment import Environment
21
22
  from .errors import BackupError, BackupInterrupted, BorgWarningError
23
+ from .exclusions import translate_excludes
22
24
 
23
25
  logger = logging.getLogger(__name__)
24
26
 
@@ -93,78 +95,128 @@ def run_backup(
93
95
 
94
96
  run_id = uuid.uuid4().hex
95
97
  with repository_lock(settings.borg_base_dir / "backup.lock"):
96
- created_at = datetime.now(UTC)
97
- successful_definitions = []
98
- failures: list[BackupError] = []
99
- failed_groups: set[str] = set()
100
- for definition in definitions:
101
- logger.info(
102
- "Starting backup group '%s' (%d sources).",
103
- definition.name,
104
- len(definition.sources),
105
- )
106
- try:
107
- for folder in definition.sources:
108
- backup_folder(
109
- folder,
110
- definition.compression,
98
+ dirty_groups = (
99
+ DirtyGroupConsumer(settings)
100
+ if settings.backup_mode in {"shadow", "dirty"}
101
+ else None
102
+ )
103
+ try:
104
+ if settings.backup_mode in {"shadow", "dirty"} and dirty_groups is not None:
105
+ group_names = {definition.name for definition in definitions}
106
+ if settings.backup_mode == "shadow":
107
+ dirty_groups.preview(group_names)
108
+ else:
109
+ dirty_groups.prepare(group_names)
110
+
111
+ created_at = datetime.now(UTC)
112
+ successful_definitions = []
113
+ failures: list[BackupError] = []
114
+ failed_groups: set[str] = set()
115
+ for definition in definitions:
116
+ if settings.backup_mode == "dirty" and dirty_groups is not None:
117
+ claimed = dirty_groups.claim(definition.name)
118
+ else:
119
+ claimed = True
120
+ if not claimed:
121
+ logger.info(
122
+ "Skipping backup group '%s'; it is not dirty.",
111
123
  definition.name,
112
- settings,
113
- created_at,
114
- run_id,
115
- definition.excludes,
116
124
  )
117
- except BackupError as exc:
118
- logger.error(
119
- "Backup group '%s' failed; retention pruning will be skipped "
120
- "for this group: %s",
125
+ continue
126
+
127
+ logger.info(
128
+ "Starting backup group '%s' (%d sources).",
121
129
  definition.name,
122
- exc,
130
+ len(definition.sources),
123
131
  )
124
- failures.append(exc)
125
- failed_groups.add(definition.name)
126
- continue
127
-
128
- logger.info("Backup group '%s' completed.", definition.name)
129
- successful_definitions.append(definition)
130
-
131
- for definition in successful_definitions:
132
- logger.info(
133
- "Starting retention pruning for group '%s' (%d sources).",
134
- definition.name,
135
- len(definition.sources),
136
- )
137
- pruning_succeeded = True
138
- for folder in definition.sources:
139
132
  try:
140
- prune(
141
- folder,
142
- definition.retention_days,
143
- definition.name,
144
- settings,
133
+ excludes = translate_excludes(
134
+ definition.root,
135
+ definition.excludes,
145
136
  )
137
+ for folder in definition.sources:
138
+ backup_folder(
139
+ folder,
140
+ definition.compression,
141
+ definition.name,
142
+ settings,
143
+ created_at,
144
+ run_id,
145
+ excludes,
146
+ )
146
147
  except BackupError as exc:
148
+ if settings.backup_mode == "dirty" and dirty_groups is not None:
149
+ try:
150
+ dirty_groups.requeue(definition.name)
151
+ except BackupError as state_error:
152
+ failed_groups.add(definition.name)
153
+ write_failed_groups(settings, failed_groups)
154
+ raise BackupError(f"{exc}; {state_error}") from state_error
147
155
  logger.error(
148
- "Retention pruning failed for group '%s' source '%s': %s",
156
+ "Backup group '%s' failed; retention pruning will be skipped "
157
+ "for this group: %s",
149
158
  definition.name,
150
- folder,
151
159
  exc,
152
160
  )
153
161
  failures.append(exc)
154
162
  failed_groups.add(definition.name)
155
- pruning_succeeded = False
163
+ continue
156
164
 
157
- if pruning_succeeded:
165
+ logger.info("Backup group '%s' completed.", definition.name)
166
+ successful_definitions.append(definition)
167
+
168
+ for definition in successful_definitions:
158
169
  logger.info(
159
- "Retention pruning for group '%s' completed.", definition.name
170
+ "Starting retention pruning for group '%s' (%d sources).",
171
+ definition.name,
172
+ len(definition.sources),
160
173
  )
174
+ pruning_succeeded = True
175
+ for folder in definition.sources:
176
+ try:
177
+ prune(
178
+ folder,
179
+ definition.retention_days,
180
+ definition.name,
181
+ settings,
182
+ )
183
+ except BackupError as exc:
184
+ logger.error(
185
+ "Retention pruning failed for group '%s' source '%s': %s",
186
+ definition.name,
187
+ folder,
188
+ exc,
189
+ )
190
+ failures.append(exc)
191
+ failed_groups.add(definition.name)
192
+ pruning_succeeded = False
193
+
194
+ if settings.backup_mode == "dirty" and dirty_groups is not None:
195
+ try:
196
+ if pruning_succeeded:
197
+ dirty_groups.acknowledge(definition.name)
198
+ else:
199
+ dirty_groups.requeue(definition.name)
200
+ except BackupError as state_error:
201
+ failed_groups.add(definition.name)
202
+ write_failed_groups(settings, failed_groups)
203
+ raise state_error
204
+
205
+ if pruning_succeeded:
206
+ logger.info(
207
+ "Retention pruning for group '%s' completed.",
208
+ definition.name,
209
+ )
161
210
 
162
- if failures:
163
- write_failed_groups(settings, failed_groups)
164
- message = "; ".join(str(failure) for failure in failures)
165
- if any(isinstance(failure, BorgWarningError) for failure in failures):
166
- raise BorgWarningError(message)
167
- raise BackupError(message)
211
+ if failures:
212
+ write_failed_groups(settings, failed_groups)
213
+ message = "; ".join(str(failure) for failure in failures)
214
+ if any(isinstance(failure, BorgWarningError) for failure in failures):
215
+ raise BorgWarningError(message)
216
+ raise BackupError(message)
217
+ finally:
218
+ if dirty_groups is not None:
219
+ dirty_groups.close()
168
220
 
169
221
 
170
222
  def main(environ: Mapping[str, str] | None = None) -> int:
@@ -100,9 +100,11 @@ def run_with_progress(
100
100
  print(text, file=sys.stderr, flush=True)
101
101
 
102
102
  with borg_process(command, environment, capture_stderr=True) as process:
103
- assert process.stderr is not None
103
+ stderr = process.stderr
104
+ if stderr is None:
105
+ fail("Borg stderr was not captured for progress reporting.")
104
106
  with selectors.DefaultSelector() as selector:
105
- selector.register(process.stderr, selectors.EVENT_READ)
107
+ selector.register(stderr, selectors.EVENT_READ)
106
108
  pending = b""
107
109
  while selector.get_map():
108
110
  for key, _ in selector.select(max(0, next_report - time.monotonic())):
@@ -5,14 +5,17 @@ from pathlib import Path
5
5
  @dataclass(frozen=True)
6
6
  class BackupDefinition:
7
7
  name: str
8
+ root: Path
8
9
  folders: tuple[Path, ...]
10
+ files: tuple[Path, ...]
11
+ canonical_sources: tuple[Path, ...]
9
12
  compression: str
10
13
  retention_days: int
11
14
  required_mounts: tuple[Path, ...] = ()
15
+ watcher_alternative_roots: tuple[Path, ...] = ()
12
16
  allow_empty: bool = True
13
- files: tuple[Path, ...] = ()
14
17
  excludes: tuple[str, ...] = ()
15
18
 
16
19
  @property
17
20
  def sources(self) -> tuple[Path, ...]:
18
- return self.folders + self.files
21
+ return self.canonical_sources