safekeep 0.5.2__tar.gz → 0.6.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {safekeep-0.5.2/safekeep.egg-info → safekeep-0.6.0}/PKG-INFO +59 -2
- {safekeep-0.5.2 → safekeep-0.6.0}/README.md +56 -1
- {safekeep-0.5.2 → safekeep-0.6.0}/pyproject.toml +5 -1
- {safekeep-0.5.2 → safekeep-0.6.0}/safekeep/__init__.py +1 -1
- {safekeep-0.5.2 → safekeep-0.6.0}/safekeep/cli.py +55 -6
- {safekeep-0.5.2 → safekeep-0.6.0}/safekeep/config.py +115 -5
- {safekeep-0.5.2 → safekeep-0.6.0}/safekeep/copier.py +15 -4
- {safekeep-0.5.2 → safekeep-0.6.0}/safekeep/daemon.py +89 -4
- safekeep-0.6.0/safekeep/s3dest.py +278 -0
- {safekeep-0.5.2 → safekeep-0.6.0/safekeep.egg-info}/PKG-INFO +59 -2
- {safekeep-0.5.2 → safekeep-0.6.0}/safekeep.egg-info/SOURCES.txt +4 -0
- safekeep-0.6.0/safekeep.egg-info/requires.txt +3 -0
- {safekeep-0.5.2 → safekeep-0.6.0}/tests/test_cli.py +96 -0
- {safekeep-0.5.2 → safekeep-0.6.0}/tests/test_config.py +122 -0
- {safekeep-0.5.2 → safekeep-0.6.0}/tests/test_daemon.py +164 -0
- safekeep-0.6.0/tests/test_s3_itest.py +157 -0
- safekeep-0.6.0/tests/test_s3dest.py +343 -0
- {safekeep-0.5.2 → safekeep-0.6.0}/LICENSE +0 -0
- {safekeep-0.5.2 → safekeep-0.6.0}/safekeep/matcher.py +0 -0
- {safekeep-0.5.2 → safekeep-0.6.0}/safekeep/platform.py +0 -0
- {safekeep-0.5.2 → safekeep-0.6.0}/safekeep/volumes.py +0 -0
- {safekeep-0.5.2 → safekeep-0.6.0}/safekeep.egg-info/dependency_links.txt +0 -0
- {safekeep-0.5.2 → safekeep-0.6.0}/safekeep.egg-info/entry_points.txt +0 -0
- {safekeep-0.5.2 → safekeep-0.6.0}/safekeep.egg-info/top_level.txt +0 -0
- {safekeep-0.5.2 → safekeep-0.6.0}/setup.cfg +0 -0
- {safekeep-0.5.2 → safekeep-0.6.0}/tests/test_copier.py +0 -0
- {safekeep-0.5.2 → safekeep-0.6.0}/tests/test_matcher.py +0 -0
- {safekeep-0.5.2 → safekeep-0.6.0}/tests/test_platform.py +0 -0
- {safekeep-0.5.2 → safekeep-0.6.0}/tests/test_trend.py +0 -0
- {safekeep-0.5.2 → safekeep-0.6.0}/tests/test_volumes.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: safekeep
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.6.0
|
|
4
4
|
Summary: Selective always-on backups for macOS and Linux: fswatch-driven copy of allow-listed files, never deletes
|
|
5
5
|
License-Expression: MIT
|
|
6
6
|
Project-URL: Repository, https://github.com/foxhound87/safekeep
|
|
@@ -21,6 +21,8 @@ Classifier: Topic :: System :: Archiving :: Backup
|
|
|
21
21
|
Requires-Python: >=3.9
|
|
22
22
|
Description-Content-Type: text/markdown
|
|
23
23
|
License-File: LICENSE
|
|
24
|
+
Provides-Extra: s3
|
|
25
|
+
Requires-Dist: botocore<2,>=1.31; extra == "s3"
|
|
24
26
|
Dynamic: license-file
|
|
25
27
|
|
|
26
28
|
# safekeep
|
|
@@ -56,13 +58,19 @@ Documentation: https://foxhound87.github.io/safekeep/
|
|
|
56
58
|
`.sync` file can neither add nor remove destinations (a `dest:` line is an
|
|
57
59
|
invalid line), so a repo cloned from a third party can't redirect the backup
|
|
58
60
|
somewhere else.
|
|
61
|
+
- **S3-compatible destination** (optional `safekeep[s3]` extra): one
|
|
62
|
+
S3-compatible bucket alongside the filesystem destinations — same
|
|
63
|
+
size/mtime idempotency, metadata-based, and **never deletes anything**
|
|
64
|
+
(`--prune` only touches filesystem destinations).
|
|
59
65
|
|
|
60
66
|
## Installation
|
|
61
67
|
|
|
62
68
|
```bash
|
|
63
|
-
pipx install safekeep
|
|
69
|
+
pipx install safekeep # recommended for a CLI
|
|
70
|
+
pipx install "safekeep[s3]" # + S3-compatible destination (botocore)
|
|
64
71
|
# or
|
|
65
72
|
pip install safekeep
|
|
73
|
+
pip install "safekeep[s3]"
|
|
66
74
|
```
|
|
67
75
|
|
|
68
76
|
**Requirements**: macOS or Linux (POSIX with systemd), Python >= 3.9 and
|
|
@@ -171,6 +179,55 @@ safekeep sync-once --dry-run # prints what would be copied, copies nothing
|
|
|
171
179
|
safekeep run # daemon: watch + copy (foreground)
|
|
172
180
|
```
|
|
173
181
|
|
|
182
|
+
## S3-compatible destinations
|
|
183
|
+
|
|
184
|
+
safekeep can push every project to **one S3-compatible bucket**
|
|
185
|
+
(S3 = Simple Storage Service) in addition to (or instead of) the filesystem
|
|
186
|
+
destinations. The client is [botocore](https://botocore.amazonaws.com/),
|
|
187
|
+
installed via the optional extra — the core package stays stdlib-only:
|
|
188
|
+
|
|
189
|
+
```bash
|
|
190
|
+
pipx install "safekeep[s3]"
|
|
191
|
+
```
|
|
192
|
+
|
|
193
|
+
`~/.safekeep` (the `s3_*` keys are valid **only** in this file — a `.sync`
|
|
194
|
+
can't add them; keep the file at `chmod 600`):
|
|
195
|
+
|
|
196
|
+
```ini
|
|
197
|
+
s3_endpoint: https://s3.example.com:443 # http:// → cleartext warning
|
|
198
|
+
s3_bucket: my-backup-bucket # at least one dest: OR s3_bucket:
|
|
199
|
+
s3_prefix: safekeep # optional, no '..'
|
|
200
|
+
s3_region: us-east-1 # default
|
|
201
|
+
s3_access_key: REPLACEME_ACCESS_KEY # required when s3_bucket is set
|
|
202
|
+
s3_secret_key: REPLACEME_SECRET_KEY # chmod 600 — or use s3_secret_key_env:
|
|
203
|
+
# s3_secret_key_env: SAFEKEEP_S3_SECRET # secret from env, never in the file
|
|
204
|
+
s3_addressing: path # path (default) | virtual
|
|
205
|
+
s3_tls_check: true # false = self-signed certificates
|
|
206
|
+
```
|
|
207
|
+
|
|
208
|
+
Semantics (see [SPEC.md](SPEC.md) §18): upload-only with `needs_copy` based on
|
|
209
|
+
object metadata (`x-amz-meta-mtime` in nanoseconds, `x-amz-meta-mode`,
|
|
210
|
+
symlinks = zero-byte objects with `x-amz-meta-symlink`), same 1s mtime
|
|
211
|
+
tolerance as filesystem destinations, network errors → `pending` with
|
|
212
|
+
backoff, and **no deletions ever** — `--prune` never touches S3. A slow PUT
|
|
213
|
+
blocks the single-process daemon until it times out (connect 10s / read 60s);
|
|
214
|
+
that's an accepted v1 limitation.
|
|
215
|
+
|
|
216
|
+
Compatible services:
|
|
217
|
+
|
|
218
|
+
| Service | Example `s3_endpoint` | `s3_addressing` | Notes |
|
|
219
|
+
|---|---|---|---|
|
|
220
|
+
| MinIO (self-hosted) | `https://minio.example.com:9000` | `path` | SigV4 (Signature Version 4), any region |
|
|
221
|
+
| Silo | `https://s3.example.com:443` | `path` | fork of MinIO — same settings |
|
|
222
|
+
| Ceph RGW | `https://rgw.example.com` | `path` | S3-compatible API |
|
|
223
|
+
| Wasabi | `https://s3.wasabisys.com` | `path` | region required, must match the bucket |
|
|
224
|
+
| Backblaze B2 | `https://s3.<region>.backblazeb2.com` | `virtual` | use an Application Key as access/secret |
|
|
225
|
+
| AWS S3 | *(omit `s3_endpoint`)* | `virtual` | default endpoint, region must match |
|
|
226
|
+
|
|
227
|
+
`safekeep status` shows the bucket as `[ok]`/`[offline]` (cached HeadBucket),
|
|
228
|
+
and `safekeep doctor` reports credentials, file permissions and bucket
|
|
229
|
+
reachability — all as **non-fatal** warnings.
|
|
230
|
+
|
|
174
231
|
## CLI commands
|
|
175
232
|
|
|
176
233
|
```
|
|
@@ -31,13 +31,19 @@ Documentation: https://foxhound87.github.io/safekeep/
|
|
|
31
31
|
`.sync` file can neither add nor remove destinations (a `dest:` line is an
|
|
32
32
|
invalid line), so a repo cloned from a third party can't redirect the backup
|
|
33
33
|
somewhere else.
|
|
34
|
+
- **S3-compatible destination** (optional `safekeep[s3]` extra): one
|
|
35
|
+
S3-compatible bucket alongside the filesystem destinations — same
|
|
36
|
+
size/mtime idempotency, metadata-based, and **never deletes anything**
|
|
37
|
+
(`--prune` only touches filesystem destinations).
|
|
34
38
|
|
|
35
39
|
## Installation
|
|
36
40
|
|
|
37
41
|
```bash
|
|
38
|
-
pipx install safekeep
|
|
42
|
+
pipx install safekeep # recommended for a CLI
|
|
43
|
+
pipx install "safekeep[s3]" # + S3-compatible destination (botocore)
|
|
39
44
|
# or
|
|
40
45
|
pip install safekeep
|
|
46
|
+
pip install "safekeep[s3]"
|
|
41
47
|
```
|
|
42
48
|
|
|
43
49
|
**Requirements**: macOS or Linux (POSIX with systemd), Python >= 3.9 and
|
|
@@ -146,6 +152,55 @@ safekeep sync-once --dry-run # prints what would be copied, copies nothing
|
|
|
146
152
|
safekeep run # daemon: watch + copy (foreground)
|
|
147
153
|
```
|
|
148
154
|
|
|
155
|
+
## S3-compatible destinations
|
|
156
|
+
|
|
157
|
+
safekeep can push every project to **one S3-compatible bucket**
|
|
158
|
+
(S3 = Simple Storage Service) in addition to (or instead of) the filesystem
|
|
159
|
+
destinations. The client is [botocore](https://botocore.amazonaws.com/),
|
|
160
|
+
installed via the optional extra — the core package stays stdlib-only:
|
|
161
|
+
|
|
162
|
+
```bash
|
|
163
|
+
pipx install "safekeep[s3]"
|
|
164
|
+
```
|
|
165
|
+
|
|
166
|
+
`~/.safekeep` (the `s3_*` keys are valid **only** in this file — a `.sync`
|
|
167
|
+
can't add them; keep the file at `chmod 600`):
|
|
168
|
+
|
|
169
|
+
```ini
|
|
170
|
+
s3_endpoint: https://s3.example.com:443 # http:// → cleartext warning
|
|
171
|
+
s3_bucket: my-backup-bucket # at least one dest: OR s3_bucket:
|
|
172
|
+
s3_prefix: safekeep # optional, no '..'
|
|
173
|
+
s3_region: us-east-1 # default
|
|
174
|
+
s3_access_key: REPLACEME_ACCESS_KEY # required when s3_bucket is set
|
|
175
|
+
s3_secret_key: REPLACEME_SECRET_KEY # chmod 600 — or use s3_secret_key_env:
|
|
176
|
+
# s3_secret_key_env: SAFEKEEP_S3_SECRET # secret from env, never in the file
|
|
177
|
+
s3_addressing: path # path (default) | virtual
|
|
178
|
+
s3_tls_check: true # false = self-signed certificates
|
|
179
|
+
```
|
|
180
|
+
|
|
181
|
+
Semantics (see [SPEC.md](SPEC.md) §18): upload-only with `needs_copy` based on
|
|
182
|
+
object metadata (`x-amz-meta-mtime` in nanoseconds, `x-amz-meta-mode`,
|
|
183
|
+
symlinks = zero-byte objects with `x-amz-meta-symlink`), same 1s mtime
|
|
184
|
+
tolerance as filesystem destinations, network errors → `pending` with
|
|
185
|
+
backoff, and **no deletions ever** — `--prune` never touches S3. A slow PUT
|
|
186
|
+
blocks the single-process daemon until it times out (connect 10s / read 60s);
|
|
187
|
+
that's an accepted v1 limitation.
|
|
188
|
+
|
|
189
|
+
Compatible services:
|
|
190
|
+
|
|
191
|
+
| Service | Example `s3_endpoint` | `s3_addressing` | Notes |
|
|
192
|
+
|---|---|---|---|
|
|
193
|
+
| MinIO (self-hosted) | `https://minio.example.com:9000` | `path` | SigV4 (Signature Version 4), any region |
|
|
194
|
+
| Silo | `https://s3.example.com:443` | `path` | fork of MinIO — same settings |
|
|
195
|
+
| Ceph RGW | `https://rgw.example.com` | `path` | S3-compatible API |
|
|
196
|
+
| Wasabi | `https://s3.wasabisys.com` | `path` | region required, must match the bucket |
|
|
197
|
+
| Backblaze B2 | `https://s3.<region>.backblazeb2.com` | `virtual` | use an Application Key as access/secret |
|
|
198
|
+
| AWS S3 | *(omit `s3_endpoint`)* | `virtual` | default endpoint, region must match |
|
|
199
|
+
|
|
200
|
+
`safekeep status` shows the bucket as `[ok]`/`[offline]` (cached HeadBucket),
|
|
201
|
+
and `safekeep doctor` reports credentials, file permissions and bucket
|
|
202
|
+
reachability — all as **non-fatal** warnings.
|
|
203
|
+
|
|
149
204
|
## CLI commands
|
|
150
205
|
|
|
151
206
|
```
|
|
@@ -25,7 +25,11 @@ classifiers = [
|
|
|
25
25
|
"Programming Language :: Python :: 3.14",
|
|
26
26
|
"Topic :: System :: Archiving :: Backup",
|
|
27
27
|
]
|
|
28
|
-
# zero runtime dependencies: solo stdlib (fswatch è un binario esterno)
|
|
28
|
+
# zero runtime dependencies: solo stdlib (fswatch è un binario esterno).
|
|
29
|
+
# L'extra `s3` (botocore, import lazy in s3dest.py) è l'unica dipendenza
|
|
30
|
+
# opzionale: serve SOLO se il config usa le chiavi `s3_*` (SPEC.md §18).
|
|
31
|
+
[project.optional-dependencies]
|
|
32
|
+
s3 = ["botocore>=1.31,<2"]
|
|
29
33
|
|
|
30
34
|
[project.urls]
|
|
31
35
|
Repository = "https://github.com/foxhound87/safekeep"
|
|
@@ -31,6 +31,7 @@ from safekeep.platform import (
|
|
|
31
31
|
systemd_user_available,
|
|
32
32
|
wsl_distro,
|
|
33
33
|
)
|
|
34
|
+
from safekeep.s3dest import S3ExtraMissing, secret_perms_insecure
|
|
34
35
|
from safekeep.volumes import dest_state
|
|
35
36
|
|
|
36
37
|
# Radice del repo quando si gira da checkout; in un install pip punta altrove
|
|
@@ -60,7 +61,8 @@ def build_parser():
|
|
|
60
61
|
help='stampa cosa copierebbe, senza copiare')
|
|
61
62
|
once.add_argument('--prune', action='store_true',
|
|
62
63
|
help='rimuove dalla dest i file il cui sorgente esiste ancora ma '
|
|
63
|
-
'non è più incluso dal matcher (default: mai cancellare
|
|
64
|
+
'non è più incluso dal matcher (default: mai cancellare; '
|
|
65
|
+
'solo dest fs — la dest S3 non viene toccata, SPEC §18.2)')
|
|
64
66
|
sub.add_parser('status', parents=[common],
|
|
65
67
|
help='config, progetti e stato delle dest — sola lettura')
|
|
66
68
|
doc = sub.add_parser('doctor', parents=[common],
|
|
@@ -78,6 +80,7 @@ def cmd_sync_once(args):
|
|
|
78
80
|
daemon = Daemon(args.config)
|
|
79
81
|
daemon.load_config()
|
|
80
82
|
setup_logging('debug' if args.v else daemon.cfg.log_level)
|
|
83
|
+
daemon.log_s3_secret_perms() # §18.4: warning una-tantum, dopo gli handler
|
|
81
84
|
daemon.validate_dests() # CR-02: dest dentro source/$HOME → exit ≠ 0
|
|
82
85
|
daemon.load_projects()
|
|
83
86
|
rows = daemon.sync_once(dry_run=args.dry_run, only=args.project,
|
|
@@ -123,6 +126,11 @@ def cmd_status(args):
|
|
|
123
126
|
print('progetto nessuno (.sync non trovato)')
|
|
124
127
|
for dest in daemon.cfg.dests:
|
|
125
128
|
print(f'dest {dest} [{dest_state(dest)}]')
|
|
129
|
+
if daemon.s3 is not None:
|
|
130
|
+
# HeadBucket con cache 30s (§18.3): ok/offline, senza toccare la
|
|
131
|
+
# config — `status` resta sola lettura
|
|
132
|
+
print(f's3 {daemon.s3.root} [{daemon.s3.state()}] '
|
|
133
|
+
f'— {daemon.s3.endpoint_label} (region {daemon.s3.cfg.region})')
|
|
126
134
|
return 0
|
|
127
135
|
|
|
128
136
|
|
|
@@ -180,6 +188,7 @@ def cmd_doctor(args):
|
|
|
180
188
|
cfg_path = os.path.expanduser(args.config or '~/.safekeep')
|
|
181
189
|
cfg = None
|
|
182
190
|
daemon = None
|
|
191
|
+
s3_missing = None
|
|
183
192
|
if not os.path.exists(cfg_path):
|
|
184
193
|
if os.path.exists(EXAMPLE):
|
|
185
194
|
hint = f'creala: cp {EXAMPLE} {cfg_path}'
|
|
@@ -193,14 +202,23 @@ def cmd_doctor(args):
|
|
|
193
202
|
try:
|
|
194
203
|
daemon = Daemon(cfg_path)
|
|
195
204
|
cfg = daemon.load_config()
|
|
205
|
+
except S3ExtraMissing as e:
|
|
206
|
+
# extra `safekeep[s3]` mancante: riga NON fatale con hint (§18).
|
|
207
|
+
# `load_config` ha già validato il resto → daemon/cfg restano
|
|
208
|
+
# utilizzabili per tutti gli altri check, solo senza dest S3.
|
|
209
|
+
cfg = daemon.cfg
|
|
210
|
+
s3_missing = e
|
|
196
211
|
except ConfigError as e:
|
|
197
212
|
daemon = None
|
|
198
213
|
out(False, f'config non valida: {e}', is_fatal=True)
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
214
|
+
|
|
215
|
+
if cfg is not None:
|
|
216
|
+
# nessuna `source` → auto-discovery da $HOME: info, non warning (SPEC §6)
|
|
217
|
+
mode = (f'{len(cfg.sources)} source' if cfg.sources
|
|
218
|
+
else f'auto-discovery da {os.path.expanduser("~")}')
|
|
219
|
+
out(True, f'config valida: {cfg_path} ({mode}, {len(cfg.dests)} dest)')
|
|
220
|
+
if s3_missing is not None:
|
|
221
|
+
out(False, f'{s3_missing} — warning non fatale', is_fatal=False)
|
|
204
222
|
|
|
205
223
|
if daemon is not None:
|
|
206
224
|
# base = source, oppure $HOME in auto-mode: copre anche "dest dentro $HOME"
|
|
@@ -211,6 +229,37 @@ def cmd_doctor(args):
|
|
|
211
229
|
else:
|
|
212
230
|
out(True, 'dest non annidate nelle source/$HOME (nessun loop di copia)')
|
|
213
231
|
|
|
232
|
+
# --- dest S3 (SPEC.md §18): tutti check NON fatali ---------------------
|
|
233
|
+
if cfg is not None and cfg.s3 is not None:
|
|
234
|
+
s3 = cfg.s3
|
|
235
|
+
orig = (f'variabile d\'ambiente {s3.secret_env!r}' if s3.secret_env
|
|
236
|
+
else 'file di config')
|
|
237
|
+
out(True, f's3 credenziali risolte da {orig} (bucket {s3.bucket})')
|
|
238
|
+
if s3.secret_env:
|
|
239
|
+
out(True, f's3 secret non nel file (da {s3.secret_env}): '
|
|
240
|
+
'chmod 600 non pertinente')
|
|
241
|
+
elif secret_perms_insecure(cfg_path, s3):
|
|
242
|
+
out(False, f's3_secret_key in {cfg_path} con permessi > 0600 — '
|
|
243
|
+
f'remedio: chmod 600 {cfg_path} — warning non fatale')
|
|
244
|
+
else:
|
|
245
|
+
out(True, f's3_secret_key: permessi del config ≤ 0600')
|
|
246
|
+
if s3.endpoint and s3.endpoint.startswith('http://'):
|
|
247
|
+
out(False, f's3 endpoint cleartext {s3.endpoint}: il secret e i '
|
|
248
|
+
'dati viaggiano in chiaro — usa https:// — '
|
|
249
|
+
'warning non fatale')
|
|
250
|
+
else:
|
|
251
|
+
out(True, f's3 endpoint: {s3.endpoint or "AWS default"} '
|
|
252
|
+
f'(region {s3.region}, addressing {s3.addressing})')
|
|
253
|
+
if daemon is not None and daemon.s3 is not None:
|
|
254
|
+
# extra installato: HeadBucket con cache 30s (§18.3)
|
|
255
|
+
state = daemon.s3.state()
|
|
256
|
+
if state == 'ok':
|
|
257
|
+
out(True, f's3 HeadBucket {daemon.s3.root}: ok')
|
|
258
|
+
else:
|
|
259
|
+
out(False, f's3 HeadBucket {daemon.s3.root}: offline — endpoint '
|
|
260
|
+
f'{daemon.s3.endpoint_label} non raggiungibile — '
|
|
261
|
+
'warning non fatale')
|
|
262
|
+
|
|
214
263
|
if daemon is not None:
|
|
215
264
|
total, broken, discarded = check_sync_files(daemon)
|
|
216
265
|
if broken:
|
|
@@ -10,6 +10,7 @@ Una riga a forma di chiave non nota (`dest:`, `-dest:`, …) resta invalida: con
|
|
|
10
10
|
import logging
|
|
11
11
|
import os
|
|
12
12
|
import re
|
|
13
|
+
from dataclasses import dataclass
|
|
13
14
|
|
|
14
15
|
from .matcher import BUILTIN_RULES, Matcher, parse_rule, strip_comment
|
|
15
16
|
|
|
@@ -20,10 +21,29 @@ class ConfigError(ValueError):
|
|
|
20
21
|
"""Errore di config: il messaggio contiene sempre l'origine e la riga."""
|
|
21
22
|
|
|
22
23
|
|
|
24
|
+
@dataclass
|
|
25
|
+
class S3Config:
|
|
26
|
+
"""Blocco `s3_*` del config globale (SPEC.md §18.1). Solo una dest S3.
|
|
27
|
+
|
|
28
|
+
`secret_key` è già **risolto** a parse-time: da `s3_secret_key` oppure
|
|
29
|
+
dalla variabile `s3_secret_key_env` (assente → ConfigError fail-fast).
|
|
30
|
+
"""
|
|
31
|
+
endpoint: str = None # None → endpoint AWS default
|
|
32
|
+
bucket: str = None
|
|
33
|
+
prefix: str = '' # normalizzato: niente / iniziale/finali
|
|
34
|
+
region: str = 'us-east-1'
|
|
35
|
+
access_key: str = None
|
|
36
|
+
secret_key: str = None
|
|
37
|
+
secret_env: str = None # nome var se il secret viene da env
|
|
38
|
+
addressing: str = 'path' # path (default) | virtual
|
|
39
|
+
tls_check: bool = True
|
|
40
|
+
|
|
41
|
+
|
|
23
42
|
class GlobalConfig:
|
|
24
43
|
def __init__(self):
|
|
25
44
|
self.sources = [] # radici osservate (ripetibili, opzionali)
|
|
26
45
|
self.dests = [] # destinazioni globali (ripetibili) — le UNICHE
|
|
46
|
+
self.s3 = None # S3Config | None (una sola dest S3, §18)
|
|
27
47
|
self.layout = 'relative' # relative | full
|
|
28
48
|
self.defaults = [] # regole a bassa priorità (list[Rule])
|
|
29
49
|
self.log_level = 'info' # debug | info | warn | error
|
|
@@ -38,10 +58,19 @@ class SyncConfig:
|
|
|
38
58
|
|
|
39
59
|
|
|
40
60
|
_SYNC_KEY = re.compile(r'(name|defaults|include|exclude)\s*:\s*(.*)')
|
|
41
|
-
|
|
61
|
+
# Chiave globale: inizia con lettera/_, poi [A-Za-z0-9_] (serve a `s3_endpoint`,
|
|
62
|
+
# cifra inclusa); una riga che non matcha resta "riga non valida" (fail-fast).
|
|
63
|
+
_CONF_KEY = re.compile(r'([A-Za-z_][A-Za-z0-9_]*)\s*:\s*(.*)')
|
|
42
64
|
# Riga a forma di chiave (`dest:`, `-dest:`, `foo:`): NON è una riga nuda.
|
|
43
65
|
_KEY_SHAPE = re.compile(r'[A-Za-z_-][A-Za-z0-9_-]*\s*:')
|
|
44
66
|
|
|
67
|
+
# Chiavi del blocco S3 (SPEC.md §18.1): `s3_*` fuori da questo insieme è
|
|
68
|
+
# errore fail-fast, come ogni altra chiave ignota del config globale.
|
|
69
|
+
_S3_KEYS = frozenset({
|
|
70
|
+
's3_endpoint', 's3_bucket', 's3_prefix', 's3_region', 's3_access_key',
|
|
71
|
+
's3_secret_key', 's3_secret_key_env', 's3_addressing', 's3_tls_check',
|
|
72
|
+
})
|
|
73
|
+
|
|
45
74
|
|
|
46
75
|
def bare_rule(line):
|
|
47
76
|
"""Riga nuda stile gitignore → regola (SPEC.md §4.2): `pattern` include,
|
|
@@ -114,9 +143,10 @@ def parse_config(text, origin='config'):
|
|
|
114
143
|
"""Parse del config globale → GlobalConfig. Errori fatali sempre (fail-fast).
|
|
115
144
|
|
|
116
145
|
`source` è opzionale (nessuna riga → modalità auto-discovery da `$HOME`,
|
|
117
|
-
SPEC.md §6);
|
|
118
|
-
|
|
146
|
+
SPEC.md §6); serve almeno una `dest:` **oppure** `s3_bucket` (dest
|
|
147
|
+
S3-compatible, SPEC.md §18 — backward compat: senza `s3_*` nulla cambia)."""
|
|
119
148
|
cfg = GlobalConfig()
|
|
149
|
+
s3raw = {} # chiave → (valore, riga) del blocco s3_*
|
|
120
150
|
for lineno, raw in enumerate(text.splitlines(), 1):
|
|
121
151
|
line = strip_comment(raw).strip()
|
|
122
152
|
if not line:
|
|
@@ -137,6 +167,10 @@ def parse_config(text, origin='config'):
|
|
|
137
167
|
if not val:
|
|
138
168
|
raise ConfigError(f'{origin}:{lineno}: dest vuota')
|
|
139
169
|
cfg.dests.append(os.path.expanduser(val))
|
|
170
|
+
elif key.startswith('s3_'):
|
|
171
|
+
if key not in _S3_KEYS:
|
|
172
|
+
raise ConfigError(f'{origin}:{lineno}: chiave s3 sconosciuta: {key}')
|
|
173
|
+
s3raw[key] = (val, lineno) # ultima occorrenza vince (come layout)
|
|
140
174
|
elif key == 'layout':
|
|
141
175
|
if val not in ('relative', 'full'):
|
|
142
176
|
raise ConfigError(f'{origin}:{lineno}: layout non valido: {val!r} (relative|full)')
|
|
@@ -155,12 +189,88 @@ def parse_config(text, origin='config'):
|
|
|
155
189
|
cfg.log_level = val
|
|
156
190
|
else:
|
|
157
191
|
raise ConfigError(f'{origin}:{lineno}: chiave sconosciuta: {key}')
|
|
158
|
-
|
|
192
|
+
cfg.s3 = _build_s3(s3raw, origin)
|
|
193
|
+
if not cfg.dests and (cfg.s3 is None or not cfg.s3.bucket):
|
|
159
194
|
raise ConfigError(f'{origin}: nessuna dest: serve almeno una riga '
|
|
160
|
-
f'`dest: <path>`
|
|
195
|
+
f'`dest: <path>` oppure `s3_bucket: <nome>` '
|
|
196
|
+
f'(esempio: examples/safekeep.example)')
|
|
161
197
|
return cfg
|
|
162
198
|
|
|
163
199
|
|
|
200
|
+
def _build_s3(raw, origin):
|
|
201
|
+
"""Validazione fail-fast del blocco `s3_*` → S3Config (SPEC.md §18.1).
|
|
202
|
+
|
|
203
|
+
`raw` = {chiave: (valore, riga)}: ogni errore riporta origine e riga.
|
|
204
|
+
Ritorna None se nessuna chiave `s3_*` presente (config solo-fs invariata).
|
|
205
|
+
"""
|
|
206
|
+
if not raw:
|
|
207
|
+
return None
|
|
208
|
+
|
|
209
|
+
def err(key, detail):
|
|
210
|
+
# riga del campo mancante, altrimenti quella di s3_bucket (l'inizio
|
|
211
|
+
# del blocco incompleto), altrimenti la prima riga s3_* presente
|
|
212
|
+
entry = raw.get(key) or raw.get('s3_bucket') or next(iter(raw.values()))
|
|
213
|
+
raise ConfigError(f'{origin}:{entry[1]}: {detail}')
|
|
214
|
+
|
|
215
|
+
def get(key, default=''):
|
|
216
|
+
return raw[key][0] if key in raw else default
|
|
217
|
+
|
|
218
|
+
bucket = get('s3_bucket').strip()
|
|
219
|
+
if not bucket:
|
|
220
|
+
err('s3_bucket',
|
|
221
|
+
's3_bucket vuota o assente: serve `s3_bucket: <nome>` '
|
|
222
|
+
'(SPEC.md §18.1, almeno una dest: oppure s3_bucket)')
|
|
223
|
+
access = get('s3_access_key').strip()
|
|
224
|
+
if not access:
|
|
225
|
+
err('s3_access_key',
|
|
226
|
+
f's3_bucket senza s3_access_key: serve `s3_access_key: <chiave>` '
|
|
227
|
+
f'(bucket {bucket!r})')
|
|
228
|
+
secret, env = get('s3_secret_key'), get('s3_secret_key_env').strip()
|
|
229
|
+
if secret and env:
|
|
230
|
+
err('s3_secret_key',
|
|
231
|
+
's3_secret_key e s3_secret_key_env insieme: ne va scelta una sola '
|
|
232
|
+
'(SPEC.md §18.4)')
|
|
233
|
+
if not secret and not env:
|
|
234
|
+
err('s3_secret_key',
|
|
235
|
+
's3_bucket senza credenziali: serve `s3_secret_key: <secret>` '
|
|
236
|
+
'(chmod 600 il config) oppure `s3_secret_key_env: <NOME_VAR>`')
|
|
237
|
+
if env:
|
|
238
|
+
secret = os.environ.get(env)
|
|
239
|
+
if secret is None:
|
|
240
|
+
err('s3_secret_key_env',
|
|
241
|
+
f's3_secret_key_env: variabile d\'ambiente {env!r} assente')
|
|
242
|
+
prefix = get('s3_prefix').strip().strip('/')
|
|
243
|
+
if '..' in prefix.split('/'):
|
|
244
|
+
err('s3_prefix', f's3_prefix con "..": {get("s3_prefix")!r} non è ammesso')
|
|
245
|
+
addressing = get('s3_addressing', 'path').strip()
|
|
246
|
+
if addressing not in ('path', 'virtual'):
|
|
247
|
+
err('s3_addressing',
|
|
248
|
+
f's3_addressing non valido: {addressing!r} (path|virtual)')
|
|
249
|
+
tls = get('s3_tls_check', 'true').strip()
|
|
250
|
+
if tls not in ('true', 'false'):
|
|
251
|
+
err('s3_tls_check', f's3_tls_check non valido: {tls!r} (true|false)')
|
|
252
|
+
endpoint = get('s3_endpoint').strip()
|
|
253
|
+
if endpoint and not endpoint.startswith(('http://', 'https://')):
|
|
254
|
+
err('s3_endpoint',
|
|
255
|
+
f's3_endpoint senza scheme http(s): {endpoint!r} '
|
|
256
|
+
f'(esempio: https://s3.example.com:443)')
|
|
257
|
+
if endpoint.startswith('http://'):
|
|
258
|
+
log.warning('%s:%d: s3_endpoint cleartext %s: il secret e i dati '
|
|
259
|
+
'viaggiano in chiaro — usa https:// (SPEC.md §18.1)',
|
|
260
|
+
origin, raw['s3_endpoint'][1], endpoint)
|
|
261
|
+
return S3Config(
|
|
262
|
+
endpoint=endpoint or None,
|
|
263
|
+
bucket=bucket,
|
|
264
|
+
prefix=prefix,
|
|
265
|
+
region=get('s3_region').strip() or 'us-east-1',
|
|
266
|
+
access_key=access,
|
|
267
|
+
secret_key=secret,
|
|
268
|
+
secret_env=env or None,
|
|
269
|
+
addressing=addressing,
|
|
270
|
+
tls_check=(tls == 'true'),
|
|
271
|
+
)
|
|
272
|
+
|
|
273
|
+
|
|
164
274
|
def source_root_for(sources, path):
|
|
165
275
|
"""La `source` configurata più profonda che contiene `path`; `path` se nessuna.
|
|
166
276
|
|
|
@@ -206,7 +206,8 @@ def needs_copy(src: str, dst: str) -> bool:
|
|
|
206
206
|
|
|
207
207
|
|
|
208
208
|
def reconcile_project(source_root, dest_root, matcher, layout, dest_path_fn,
|
|
209
|
-
stats=None, dry_run=False, dest_base=None
|
|
209
|
+
stats=None, dry_run=False, dest_base=None,
|
|
210
|
+
copy_fn=None, needs_copy_fn=None) -> int:
|
|
210
211
|
"""Walk della sorgente con pruning delle dir con verdict escluso; copia ogni
|
|
211
212
|
file con verdict incluso che differisce (un file non matchato da nessuna
|
|
212
213
|
regola NON viene copiato: allow-list, SPEC.md §4.3). Ritorna il numero di copie.
|
|
@@ -223,12 +224,22 @@ def reconcile_project(source_root, dest_root, matcher, layout, dest_path_fn,
|
|
|
223
224
|
fermato la passata: il caller metta quella dest in pending (CR-05).
|
|
224
225
|
`dry_run`: conta ed elenca senza copiare.
|
|
225
226
|
|
|
227
|
+
Seam destinazioni (SPEC.md §18.2): `copy_fn(src, dst)` e
|
|
228
|
+
`needs_copy_fn(src, dst)` — default invariati (`copier.copy_one` con
|
|
229
|
+
containment + `copier.needs_copy`); una dest S3 passa i propri. Gli
|
|
230
|
+
errori di rete S3 arrivano come `OSError(ENODEV)` e cadono nel percorso
|
|
231
|
+
pending già esistente.
|
|
232
|
+
|
|
226
233
|
Nessun errore I/O di un singolo file abortisce il walk: viene loggato e
|
|
227
234
|
contato, e gli altri file vengono processati comunque.
|
|
228
235
|
"""
|
|
229
236
|
root = os.path.abspath(os.path.expanduser(source_root))
|
|
230
237
|
base_root = os.path.abspath(os.path.expanduser(dest_base)) if dest_base else root
|
|
231
238
|
prefix = os.path.relpath(root, base_root) # radice del walk rispetto a base_root
|
|
239
|
+
# default invariati: le dest fs continuano a usare copy_one/needs_copy
|
|
240
|
+
do_copy = copy_fn if copy_fn is not None else (
|
|
241
|
+
lambda src, dst: copy_one(src, dst, dest_root))
|
|
242
|
+
do_need = needs_copy_fn if needs_copy_fn is not None else needs_copy
|
|
232
243
|
copied = 0
|
|
233
244
|
excluded = 0
|
|
234
245
|
skipped = 0
|
|
@@ -266,7 +277,7 @@ def reconcile_project(source_root, dest_root, matcher, layout, dest_path_fn,
|
|
|
266
277
|
dst = dest_path_fn(dest_root, base_root,
|
|
267
278
|
os.path.join(prefix, r), layout)
|
|
268
279
|
try:
|
|
269
|
-
if dry_run or
|
|
280
|
+
if dry_run or do_copy(full, dst):
|
|
270
281
|
copied += 1
|
|
271
282
|
if plan is not None:
|
|
272
283
|
plan.append((full, dst))
|
|
@@ -288,8 +299,8 @@ def reconcile_project(source_root, dest_root, matcher, layout, dest_path_fn,
|
|
|
288
299
|
dst = dest_path_fn(dest_root, base_root,
|
|
289
300
|
os.path.join(prefix, r), layout)
|
|
290
301
|
try:
|
|
291
|
-
if
|
|
292
|
-
if dry_run or
|
|
302
|
+
if do_need(src, dst):
|
|
303
|
+
if dry_run or do_copy(src, dst):
|
|
293
304
|
copied += 1
|
|
294
305
|
if plan is not None:
|
|
295
306
|
plan.append((src, dst))
|