batchvault 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- batchvault/__init__.py +61 -0
- batchvault/__main__.py +3 -0
- batchvault/api.py +884 -0
- batchvault/blocks.py +176 -0
- batchvault/cli.py +806 -0
- batchvault/config.py +169 -0
- batchvault/crypto.py +193 -0
- batchvault/doctor.py +67 -0
- batchvault/engines/__init__.py +0 -0
- batchvault/engines/base.py +57 -0
- batchvault/engines/native.py +237 -0
- batchvault/engines/registry.py +61 -0
- batchvault/engines/sevenzip.py +303 -0
- batchvault/errors.py +38 -0
- batchvault/eta.py +39 -0
- batchvault/filters.py +51 -0
- batchvault/inputs.py +226 -0
- batchvault/levels.py +169 -0
- batchvault/locate.py +140 -0
- batchvault/logs.py +86 -0
- batchvault/manifest.py +210 -0
- batchvault/models.py +164 -0
- batchvault/names.py +36 -0
- batchvault/notify.py +73 -0
- batchvault/options.py +71 -0
- batchvault/py.typed +0 -0
- batchvault/security.py +40 -0
- batchvault/signing.py +152 -0
- batchvault/ui.py +298 -0
- batchvault/volumes.py +201 -0
- batchvault-0.1.0.dist-info/METADATA +306 -0
- batchvault-0.1.0.dist-info/RECORD +35 -0
- batchvault-0.1.0.dist-info/WHEEL +4 -0
- batchvault-0.1.0.dist-info/entry_points.txt +2 -0
- batchvault-0.1.0.dist-info/licenses/LICENSE +21 -0
batchvault/cli.py
ADDED
|
@@ -0,0 +1,806 @@
|
|
|
1
|
+
"""Command-line interface: a thin wrapper over :mod:`batchvault.api`.
|
|
2
|
+
|
|
3
|
+
Exit codes: 0 success, 1 some files failed, were skipped or the checks failed,
|
|
4
|
+
2 usage or configuration error, 3 unexpected error, 130 interrupted.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import argparse
|
|
10
|
+
import json
|
|
11
|
+
import logging
|
|
12
|
+
import os
|
|
13
|
+
import sys
|
|
14
|
+
import threading
|
|
15
|
+
from contextlib import contextmanager
|
|
16
|
+
from pathlib import Path
|
|
17
|
+
|
|
18
|
+
from rich.console import Console
|
|
19
|
+
from rich.markup import escape
|
|
20
|
+
|
|
21
|
+
from batchvault import __version__, api, notify, signing
|
|
22
|
+
from batchvault.config import (
|
|
23
|
+
defaults_for,
|
|
24
|
+
find_config,
|
|
25
|
+
load_config,
|
|
26
|
+
reject_restricted,
|
|
27
|
+
)
|
|
28
|
+
from batchvault.doctor import passed, run_checks
|
|
29
|
+
from batchvault.engines.registry import ENGINE_CHOICES
|
|
30
|
+
from batchvault.errors import BatchVaultError, ConfigError, InputError
|
|
31
|
+
from batchvault.inputs import ROOT_HEADER, looks_like_archive
|
|
32
|
+
from batchvault.levels import AUTO, level_argument
|
|
33
|
+
from batchvault.logs import LOG_LEVELS, close_logging, setup_logging
|
|
34
|
+
from batchvault.manifest import Manifest
|
|
35
|
+
from batchvault.models import RunResult, Status
|
|
36
|
+
from batchvault.options import FORMATS
|
|
37
|
+
from batchvault.security import redact, redact_arguments
|
|
38
|
+
|
|
39
|
+
log = logging.getLogger(__name__)
|
|
40
|
+
|
|
41
|
+
ENV_PASSWORD = "BATCHVAULT_PASSWORD"
|
|
42
|
+
ENV_KEY_PASSWORD = "BATCHVAULT_KEY_PASSWORD"
|
|
43
|
+
ENV_NOTIFY_URL = "BATCHVAULT_NOTIFY_URL"
|
|
44
|
+
ENV_CONFIG = "BATCHVAULT_CONFIG"
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def _parents() -> dict[str, argparse.ArgumentParser]:
|
|
48
|
+
parents = {name: argparse.ArgumentParser(add_help=False) for name in (
|
|
49
|
+
"common", "jobs", "seven", "patterns", "run", "retry", "public",
|
|
50
|
+
)} # fmt: skip
|
|
51
|
+
|
|
52
|
+
common = parents["common"]
|
|
53
|
+
common.add_argument(
|
|
54
|
+
"--json", action="store_true", help="Print the result as JSON on stdout."
|
|
55
|
+
)
|
|
56
|
+
common.add_argument(
|
|
57
|
+
"-q", "--quiet", action="store_true", help="No progress or summary output."
|
|
58
|
+
)
|
|
59
|
+
common.add_argument("--log-file", help="Write the log here instead of the default.")
|
|
60
|
+
common.add_argument(
|
|
61
|
+
"--log-level", choices=LOG_LEVELS, default="INFO", help="Log level."
|
|
62
|
+
)
|
|
63
|
+
common.add_argument(
|
|
64
|
+
"--no-log", action="store_true", help="Do not write a log file."
|
|
65
|
+
)
|
|
66
|
+
common.add_argument(
|
|
67
|
+
"--config", metavar="FILE", help="Config file (default: batchvault.toml)."
|
|
68
|
+
)
|
|
69
|
+
|
|
70
|
+
parents["jobs"].add_argument(
|
|
71
|
+
"-j",
|
|
72
|
+
"--jobs",
|
|
73
|
+
type=int,
|
|
74
|
+
default=0,
|
|
75
|
+
metavar="N",
|
|
76
|
+
help="Worker threads to use; 0 means none, one file at a time (default: 0).",
|
|
77
|
+
)
|
|
78
|
+
|
|
79
|
+
parents["seven"].add_argument(
|
|
80
|
+
"--7z", dest="seven_zip", metavar="PATH", help="Path to the 7-Zip executable."
|
|
81
|
+
)
|
|
82
|
+
|
|
83
|
+
patterns = parents["patterns"]
|
|
84
|
+
patterns.add_argument(
|
|
85
|
+
"--include",
|
|
86
|
+
action="append",
|
|
87
|
+
default=[],
|
|
88
|
+
metavar="GLOB",
|
|
89
|
+
help="Only keep matching paths (repeatable; @FILE reads patterns from a file).",
|
|
90
|
+
)
|
|
91
|
+
patterns.add_argument(
|
|
92
|
+
"--exclude",
|
|
93
|
+
action="append",
|
|
94
|
+
default=[],
|
|
95
|
+
metavar="GLOB",
|
|
96
|
+
help="Drop matching paths (repeatable; wins over --include).",
|
|
97
|
+
)
|
|
98
|
+
|
|
99
|
+
run = parents["run"]
|
|
100
|
+
run.add_argument(
|
|
101
|
+
"--json-events",
|
|
102
|
+
action="store_true",
|
|
103
|
+
help="Print one JSON event per line on stdout while running.",
|
|
104
|
+
)
|
|
105
|
+
run.add_argument(
|
|
106
|
+
"--notify",
|
|
107
|
+
metavar="URL",
|
|
108
|
+
help=f"POST a JSON summary to this webhook (or set {ENV_NOTIFY_URL}).",
|
|
109
|
+
)
|
|
110
|
+
run.add_argument(
|
|
111
|
+
"--notify-on",
|
|
112
|
+
choices=notify.NOTIFY_ON,
|
|
113
|
+
default="failure",
|
|
114
|
+
help="When to notify (default: failure).",
|
|
115
|
+
)
|
|
116
|
+
run.add_argument(
|
|
117
|
+
"--password-file",
|
|
118
|
+
metavar="FILE",
|
|
119
|
+
help=f"Read the archive password from this file (or set {ENV_PASSWORD}).",
|
|
120
|
+
)
|
|
121
|
+
|
|
122
|
+
parents["retry"].add_argument(
|
|
123
|
+
"--retries",
|
|
124
|
+
type=int,
|
|
125
|
+
default=3,
|
|
126
|
+
metavar="N",
|
|
127
|
+
help="Retry a file this many times after an I/O or engine error (default: 3).",
|
|
128
|
+
)
|
|
129
|
+
|
|
130
|
+
parents["public"].add_argument(
|
|
131
|
+
"--public-key",
|
|
132
|
+
metavar="FILE",
|
|
133
|
+
help="Require a valid manifest signature made with the matching private key.",
|
|
134
|
+
)
|
|
135
|
+
return parents
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
def _configure(subparser, name: str, config: dict | None) -> None:
|
|
139
|
+
"""Apply defaults from the config file to one command."""
|
|
140
|
+
if not config:
|
|
141
|
+
return
|
|
142
|
+
actions = {action.dest: action for action in subparser._actions}
|
|
143
|
+
defaults = defaults_for(config, name, actions)
|
|
144
|
+
if defaults:
|
|
145
|
+
subparser.set_defaults(**defaults)
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
def build_parser(config: dict | None = None) -> argparse.ArgumentParser:
|
|
149
|
+
p = _parents()
|
|
150
|
+
common, jobs, seven, patterns, run, retry, public = (
|
|
151
|
+
p["common"], p["jobs"], p["seven"], p["patterns"], p["run"], p["retry"],
|
|
152
|
+
p["public"],
|
|
153
|
+
) # fmt: skip
|
|
154
|
+
|
|
155
|
+
parser = argparse.ArgumentParser(
|
|
156
|
+
prog="batchvault",
|
|
157
|
+
description="Batch archive, compress, split and restore files.",
|
|
158
|
+
)
|
|
159
|
+
parser.add_argument(
|
|
160
|
+
"--version", action="version", version=f"%(prog)s {__version__}"
|
|
161
|
+
)
|
|
162
|
+
commands = parser.add_subparsers(dest="command", required=True, metavar="COMMAND")
|
|
163
|
+
|
|
164
|
+
pack = commands.add_parser(
|
|
165
|
+
"pack",
|
|
166
|
+
parents=[common, jobs, seven, patterns, run, retry],
|
|
167
|
+
help="Compress files into split volumes.",
|
|
168
|
+
description="Compress files into split volumes and write a manifest.",
|
|
169
|
+
epilog="Example: batchvault pack --list archive_list.txt -o archives",
|
|
170
|
+
)
|
|
171
|
+
pack.add_argument("files", nargs="*", metavar="FILE", help="Files to archive.")
|
|
172
|
+
pack.add_argument(
|
|
173
|
+
"--list",
|
|
174
|
+
metavar="FILE",
|
|
175
|
+
help="Text or JSON list of files to archive ('-' reads standard input).",
|
|
176
|
+
)
|
|
177
|
+
pack.add_argument("-o", "--output", default="archives", help="Output folder.")
|
|
178
|
+
pack.add_argument("--engine", choices=ENGINE_CHOICES, default="auto")
|
|
179
|
+
pack.add_argument(
|
|
180
|
+
"--format",
|
|
181
|
+
choices=FORMATS,
|
|
182
|
+
help="Native engine compression (default: xz). 7-Zip always writes 7z.",
|
|
183
|
+
)
|
|
184
|
+
pack.add_argument(
|
|
185
|
+
"--level",
|
|
186
|
+
type=level_argument,
|
|
187
|
+
choices=[*range(10), AUTO],
|
|
188
|
+
default=AUTO,
|
|
189
|
+
metavar="0-9|auto",
|
|
190
|
+
help="Compression level, or auto to pick one per file type (default: auto).",
|
|
191
|
+
)
|
|
192
|
+
pack.add_argument(
|
|
193
|
+
"--volume-size",
|
|
194
|
+
default="10m",
|
|
195
|
+
metavar="SIZE",
|
|
196
|
+
help="Volume size, e.g. 500k, 10m, 2g (default: 10m).",
|
|
197
|
+
)
|
|
198
|
+
pack.add_argument("--manifest", metavar="PATH", help="Manifest file location.")
|
|
199
|
+
pack.add_argument(
|
|
200
|
+
"--root",
|
|
201
|
+
metavar="DIR",
|
|
202
|
+
help="Advanced: store paths relative to DIR. By default folders are kept "
|
|
203
|
+
"automatically (relative to the folder you listed, or the folder your "
|
|
204
|
+
"files share).",
|
|
205
|
+
)
|
|
206
|
+
pack.add_argument("--force", action="store_true", help="Replace existing archives.")
|
|
207
|
+
pack.add_argument(
|
|
208
|
+
"--update",
|
|
209
|
+
action="store_true",
|
|
210
|
+
help="Skip unchanged files and re-archive changed ones.",
|
|
211
|
+
)
|
|
212
|
+
pack.add_argument(
|
|
213
|
+
"--verify", action="store_true", help="Test each archive after creating it."
|
|
214
|
+
)
|
|
215
|
+
pack.add_argument(
|
|
216
|
+
"--encrypt",
|
|
217
|
+
action="store_true",
|
|
218
|
+
help=f"Encrypt with the password from {ENV_PASSWORD} or --password-file.",
|
|
219
|
+
)
|
|
220
|
+
pack.add_argument(
|
|
221
|
+
"--sign-key",
|
|
222
|
+
metavar="FILE",
|
|
223
|
+
help=f"Sign the manifest with this private key (password: {ENV_KEY_PASSWORD}).",
|
|
224
|
+
)
|
|
225
|
+
pack.add_argument(
|
|
226
|
+
"--delete-source",
|
|
227
|
+
action="store_true",
|
|
228
|
+
help="Delete each original after proving the archive restores identically.",
|
|
229
|
+
)
|
|
230
|
+
pack.add_argument(
|
|
231
|
+
"--yes", action="store_true", help="Do not ask before --delete-source deletes."
|
|
232
|
+
)
|
|
233
|
+
pack.add_argument(
|
|
234
|
+
"--allow-archives",
|
|
235
|
+
action="store_true",
|
|
236
|
+
help="Allow archiving BatchVault/7-Zip volumes and manifests.",
|
|
237
|
+
)
|
|
238
|
+
pack.add_argument(
|
|
239
|
+
"--strict",
|
|
240
|
+
action="store_true",
|
|
241
|
+
help="Treat settings the engine cannot honour as errors.",
|
|
242
|
+
)
|
|
243
|
+
pack.add_argument(
|
|
244
|
+
"--dry-run",
|
|
245
|
+
action="store_true",
|
|
246
|
+
help="Show what would be archived without writing anything.",
|
|
247
|
+
)
|
|
248
|
+
pack.set_defaults(handler=cmd_pack)
|
|
249
|
+
_configure(pack, "pack", config)
|
|
250
|
+
|
|
251
|
+
unpack = commands.add_parser(
|
|
252
|
+
"unpack",
|
|
253
|
+
parents=[common, jobs, seven, run, retry, public],
|
|
254
|
+
help="Restore files from archives.",
|
|
255
|
+
description="Restore files from a manifest, folder, volume or list file.",
|
|
256
|
+
epilog="Example: batchvault unpack archives -o restored",
|
|
257
|
+
)
|
|
258
|
+
unpack.add_argument(
|
|
259
|
+
"source", metavar="SOURCE", help="Manifest, folder, volume or list file."
|
|
260
|
+
)
|
|
261
|
+
unpack.add_argument("-o", "--output", default="restored", help="Output folder.")
|
|
262
|
+
unpack.add_argument(
|
|
263
|
+
"--force", action="store_true", help="Overwrite existing files."
|
|
264
|
+
)
|
|
265
|
+
unpack.add_argument(
|
|
266
|
+
"--update",
|
|
267
|
+
action="store_true",
|
|
268
|
+
help="Skip files that already match; replace the rest.",
|
|
269
|
+
)
|
|
270
|
+
unpack.add_argument(
|
|
271
|
+
"--max-output-size",
|
|
272
|
+
metavar="SIZE",
|
|
273
|
+
help="Refuse an archive that would expand beyond this size, per file "
|
|
274
|
+
"(e.g. 50g). Use it when restoring archives you do not trust.",
|
|
275
|
+
)
|
|
276
|
+
unpack.set_defaults(handler=cmd_unpack)
|
|
277
|
+
_configure(unpack, "unpack", config)
|
|
278
|
+
|
|
279
|
+
verify = commands.add_parser(
|
|
280
|
+
"verify",
|
|
281
|
+
parents=[common, jobs, seven, run, public],
|
|
282
|
+
help="Check that archives are complete and intact.",
|
|
283
|
+
description="Check archives without restoring them.",
|
|
284
|
+
)
|
|
285
|
+
verify.add_argument(
|
|
286
|
+
"source", metavar="SOURCE", help="Manifest, folder, volume or list file."
|
|
287
|
+
)
|
|
288
|
+
verify.add_argument(
|
|
289
|
+
"--against",
|
|
290
|
+
metavar="FOLDER",
|
|
291
|
+
help="Also compare with the original files in this folder (and below).",
|
|
292
|
+
)
|
|
293
|
+
verify.set_defaults(handler=cmd_verify)
|
|
294
|
+
_configure(verify, "verify", config)
|
|
295
|
+
|
|
296
|
+
inspect = commands.add_parser(
|
|
297
|
+
"inspect",
|
|
298
|
+
parents=[common],
|
|
299
|
+
help="Show what is in an archive set.",
|
|
300
|
+
description="Show sizes, ratios, engine, format and completeness.",
|
|
301
|
+
)
|
|
302
|
+
inspect.add_argument(
|
|
303
|
+
"source", metavar="SOURCE", help="Manifest, folder, volume or list file."
|
|
304
|
+
)
|
|
305
|
+
inspect.set_defaults(handler=cmd_inspect)
|
|
306
|
+
_configure(inspect, "inspect", config)
|
|
307
|
+
|
|
308
|
+
listing = commands.add_parser("list", help="List files or archives.")
|
|
309
|
+
kinds = listing.add_subparsers(dest="kind", required=True, metavar="KIND")
|
|
310
|
+
|
|
311
|
+
files = kinds.add_parser(
|
|
312
|
+
"files",
|
|
313
|
+
parents=[common, patterns],
|
|
314
|
+
help="List the files in a folder.",
|
|
315
|
+
description="List files (JSON on stdout unless -o is given).",
|
|
316
|
+
)
|
|
317
|
+
files.add_argument("folder")
|
|
318
|
+
files.add_argument("-o", "--output", metavar="FILE", help="Write .json or .txt.")
|
|
319
|
+
files.set_defaults(handler=cmd_list_files)
|
|
320
|
+
_configure(files, "list-files", config)
|
|
321
|
+
|
|
322
|
+
archives = kinds.add_parser(
|
|
323
|
+
"archives",
|
|
324
|
+
parents=[common],
|
|
325
|
+
help="List the archives in a folder.",
|
|
326
|
+
description="List archives (JSON on stdout unless -o is given).",
|
|
327
|
+
)
|
|
328
|
+
archives.add_argument("folder")
|
|
329
|
+
archives.add_argument("-o", "--output", metavar="FILE", help="Write .json or .txt.")
|
|
330
|
+
archives.set_defaults(handler=cmd_list_archives)
|
|
331
|
+
_configure(archives, "list-archives", config)
|
|
332
|
+
|
|
333
|
+
keygen = commands.add_parser(
|
|
334
|
+
"keygen",
|
|
335
|
+
parents=[common],
|
|
336
|
+
help="Create a key pair for signing manifests.",
|
|
337
|
+
description=(
|
|
338
|
+
"Create an Ed25519 key pair (needs batchvault[crypto]). The private "
|
|
339
|
+
f"key is encrypted if {ENV_KEY_PASSWORD} is set."
|
|
340
|
+
),
|
|
341
|
+
)
|
|
342
|
+
keygen.add_argument("--private", default="batchvault.key", metavar="FILE")
|
|
343
|
+
keygen.add_argument("--public", default="batchvault.pub", metavar="FILE")
|
|
344
|
+
keygen.add_argument("--force", action="store_true", help="Replace existing keys.")
|
|
345
|
+
keygen.set_defaults(handler=cmd_keygen)
|
|
346
|
+
_configure(keygen, "keygen", config)
|
|
347
|
+
|
|
348
|
+
sign = commands.add_parser(
|
|
349
|
+
"sign",
|
|
350
|
+
parents=[common],
|
|
351
|
+
help="Sign a manifest.",
|
|
352
|
+
description="Write a detached signature next to a manifest.",
|
|
353
|
+
)
|
|
354
|
+
sign.add_argument("manifest", metavar="MANIFEST")
|
|
355
|
+
sign.add_argument("--key", required=True, metavar="FILE", help="Private key.")
|
|
356
|
+
sign.set_defaults(handler=cmd_sign)
|
|
357
|
+
_configure(sign, "sign", config)
|
|
358
|
+
|
|
359
|
+
doctor = commands.add_parser(
|
|
360
|
+
"doctor",
|
|
361
|
+
parents=[common, seven],
|
|
362
|
+
help="Check this machine's setup.",
|
|
363
|
+
description="Check Python modules, 7-Zip and the log folder.",
|
|
364
|
+
)
|
|
365
|
+
doctor.set_defaults(handler=cmd_doctor)
|
|
366
|
+
_configure(doctor, "doctor", config)
|
|
367
|
+
|
|
368
|
+
return parser
|
|
369
|
+
|
|
370
|
+
|
|
371
|
+
def _consoles() -> tuple[Console, Console]:
|
|
372
|
+
return Console(), Console(stderr=True)
|
|
373
|
+
|
|
374
|
+
|
|
375
|
+
class EventStream:
|
|
376
|
+
"""Progress callback that prints one JSON object per line on stdout."""
|
|
377
|
+
|
|
378
|
+
def __init__(self):
|
|
379
|
+
self._lock = threading.Lock()
|
|
380
|
+
|
|
381
|
+
def __call__(self, event) -> None:
|
|
382
|
+
self._write(event.to_dict())
|
|
383
|
+
|
|
384
|
+
def result(self, result: RunResult) -> None:
|
|
385
|
+
self._write({"event": "result", **result.to_dict()})
|
|
386
|
+
|
|
387
|
+
def _write(self, data: dict) -> None:
|
|
388
|
+
with self._lock:
|
|
389
|
+
print(json.dumps(data, ensure_ascii=False), flush=True)
|
|
390
|
+
|
|
391
|
+
|
|
392
|
+
@contextmanager
|
|
393
|
+
def _progress(args, label: str, err: Console):
|
|
394
|
+
from batchvault import ui
|
|
395
|
+
|
|
396
|
+
if args.json_events:
|
|
397
|
+
yield EventStream()
|
|
398
|
+
else:
|
|
399
|
+
with ui.RunProgress(err, label, enabled=not args.quiet) as progress:
|
|
400
|
+
yield progress
|
|
401
|
+
|
|
402
|
+
|
|
403
|
+
def _check_output_flags(args) -> None:
|
|
404
|
+
if args.json and args.json_events:
|
|
405
|
+
raise InputError("Use either --json or --json-events, not both.")
|
|
406
|
+
|
|
407
|
+
|
|
408
|
+
def _read_password(args) -> str | None:
|
|
409
|
+
path = getattr(args, "password_file", None)
|
|
410
|
+
if path:
|
|
411
|
+
try:
|
|
412
|
+
lines = Path(path).expanduser().read_text(encoding="utf-8").splitlines()
|
|
413
|
+
except OSError as exc:
|
|
414
|
+
raise InputError(f"Cannot read password file {path}: {exc}") from exc
|
|
415
|
+
if not lines or not lines[0]:
|
|
416
|
+
raise InputError(f"Password file {path} is empty.")
|
|
417
|
+
return lines[0]
|
|
418
|
+
return os.environ.get(ENV_PASSWORD) or None
|
|
419
|
+
|
|
420
|
+
|
|
421
|
+
def _notify_url(args) -> str | None:
|
|
422
|
+
url = args.notify or os.environ.get(ENV_NOTIFY_URL)
|
|
423
|
+
if not url:
|
|
424
|
+
return None
|
|
425
|
+
|
|
426
|
+
notify.validate_url(url)
|
|
427
|
+
if notify.is_insecure(url):
|
|
428
|
+
_, err = _consoles()
|
|
429
|
+
err.print(
|
|
430
|
+
"[yellow]Warning: the notification URL uses http://, so the summary "
|
|
431
|
+
"(file names and paths) is sent unencrypted. Prefer https://.[/yellow]"
|
|
432
|
+
)
|
|
433
|
+
return url
|
|
434
|
+
|
|
435
|
+
|
|
436
|
+
def _finish(
|
|
437
|
+
args, command: str, result: RunResult, title: str | None, progress, url
|
|
438
|
+
) -> int:
|
|
439
|
+
from batchvault import ui
|
|
440
|
+
|
|
441
|
+
out, err = _consoles()
|
|
442
|
+
if args.json_events:
|
|
443
|
+
progress.result(result)
|
|
444
|
+
elif args.json:
|
|
445
|
+
print(json.dumps(result.to_dict(), indent=2, ensure_ascii=False))
|
|
446
|
+
elif not args.quiet:
|
|
447
|
+
ui.print_summary(out, result, title, command)
|
|
448
|
+
|
|
449
|
+
if url and notify.should_notify(args.notify_on, result):
|
|
450
|
+
problem = notify.send(url, notify.build_payload(command, result))
|
|
451
|
+
if problem:
|
|
452
|
+
log.warning("Notification failed: %s", redact(problem))
|
|
453
|
+
err.print(
|
|
454
|
+
f"[yellow]Warning: notification failed ({escape(problem)}).[/yellow]"
|
|
455
|
+
)
|
|
456
|
+
|
|
457
|
+
return 0 if result.status == Status.SUCCESS else 1
|
|
458
|
+
|
|
459
|
+
|
|
460
|
+
def _confirm_deletion(args, progress):
|
|
461
|
+
def confirm(paths: list[Path]) -> bool:
|
|
462
|
+
if args.yes:
|
|
463
|
+
return True
|
|
464
|
+
quiet = args.json or args.json_events or args.quiet
|
|
465
|
+
if quiet or not sys.stdin.isatty():
|
|
466
|
+
return False # never delete without an explicit yes
|
|
467
|
+
|
|
468
|
+
if hasattr(progress, "pause"):
|
|
469
|
+
progress.pause()
|
|
470
|
+
print(f"{len(paths)} original file(s) were archived and verified:")
|
|
471
|
+
for path in paths[:10]:
|
|
472
|
+
print(f" {path}")
|
|
473
|
+
if len(paths) > 10:
|
|
474
|
+
print(f" ... and {len(paths) - 10} more")
|
|
475
|
+
answer = input("Delete them? [y/N] ")
|
|
476
|
+
return answer.strip().lower() in ("y", "yes")
|
|
477
|
+
|
|
478
|
+
return confirm
|
|
479
|
+
|
|
480
|
+
|
|
481
|
+
def _describe_level(level) -> str:
|
|
482
|
+
return "auto (chosen for each file's type)" if level == AUTO else str(level)
|
|
483
|
+
|
|
484
|
+
|
|
485
|
+
def _describe_sources(args) -> str:
|
|
486
|
+
parts = []
|
|
487
|
+
if args.files:
|
|
488
|
+
parts.append(f"{len(args.files)} given")
|
|
489
|
+
if args.list:
|
|
490
|
+
parts.append("list: " + ("standard input" if args.list == "-" else args.list))
|
|
491
|
+
return ", ".join(parts) or "-"
|
|
492
|
+
|
|
493
|
+
|
|
494
|
+
def cmd_pack(args) -> int:
|
|
495
|
+
from batchvault import ui
|
|
496
|
+
|
|
497
|
+
_check_output_flags(args)
|
|
498
|
+
if not args.files and not args.list:
|
|
499
|
+
raise InputError("Give at least one FILE or use --list.")
|
|
500
|
+
|
|
501
|
+
password = None
|
|
502
|
+
if args.encrypt:
|
|
503
|
+
password = _read_password(args)
|
|
504
|
+
if not password:
|
|
505
|
+
raise InputError(
|
|
506
|
+
f"--encrypt needs a password: set {ENV_PASSWORD} "
|
|
507
|
+
"or use --password-file."
|
|
508
|
+
)
|
|
509
|
+
url = _notify_url(args)
|
|
510
|
+
|
|
511
|
+
out, err = _consoles()
|
|
512
|
+
if not (args.quiet or args.json or args.json_events):
|
|
513
|
+
ui.print_run_info(
|
|
514
|
+
out,
|
|
515
|
+
"BatchVault pack",
|
|
516
|
+
[
|
|
517
|
+
("Engine", args.engine),
|
|
518
|
+
("Format", args.format or "xz (native engine)"),
|
|
519
|
+
("Level", _describe_level(args.level)),
|
|
520
|
+
("Volume size", args.volume_size),
|
|
521
|
+
("Jobs", str(args.jobs) if args.jobs else "0 (sequential)"),
|
|
522
|
+
("Source files", _describe_sources(args)),
|
|
523
|
+
(
|
|
524
|
+
"Folders",
|
|
525
|
+
f"kept, relative to {Path(args.root).resolve()}"
|
|
526
|
+
if args.root
|
|
527
|
+
else "kept",
|
|
528
|
+
),
|
|
529
|
+
("Archives folder", str(Path(args.output).resolve())),
|
|
530
|
+
],
|
|
531
|
+
)
|
|
532
|
+
|
|
533
|
+
with _progress(args, "Archiving", err) as progress:
|
|
534
|
+
result = api.pack(
|
|
535
|
+
args.files,
|
|
536
|
+
args.output,
|
|
537
|
+
list_file=args.list,
|
|
538
|
+
root=args.root,
|
|
539
|
+
engine=args.engine,
|
|
540
|
+
seven_zip=args.seven_zip,
|
|
541
|
+
format=args.format,
|
|
542
|
+
level=args.level,
|
|
543
|
+
volume_size=args.volume_size,
|
|
544
|
+
jobs=args.jobs,
|
|
545
|
+
include=args.include,
|
|
546
|
+
exclude=args.exclude,
|
|
547
|
+
force=args.force,
|
|
548
|
+
update=args.update,
|
|
549
|
+
verify=args.verify,
|
|
550
|
+
manifest=args.manifest,
|
|
551
|
+
allow_archives=args.allow_archives,
|
|
552
|
+
strict=args.strict,
|
|
553
|
+
dry_run=args.dry_run,
|
|
554
|
+
password=password,
|
|
555
|
+
retries=args.retries,
|
|
556
|
+
sign_key=args.sign_key,
|
|
557
|
+
sign_key_password=os.environ.get(ENV_KEY_PASSWORD) or None,
|
|
558
|
+
delete_source=args.delete_source and not args.dry_run,
|
|
559
|
+
confirm=_confirm_deletion(args, progress),
|
|
560
|
+
progress=progress,
|
|
561
|
+
)
|
|
562
|
+
|
|
563
|
+
title = None # the default title names the source files and archives
|
|
564
|
+
if args.dry_run:
|
|
565
|
+
title = "Pack plan (dry run): source files -> archives"
|
|
566
|
+
if not (args.quiet or args.json or args.json_events):
|
|
567
|
+
out.print("[bold]Dry run:[/bold] nothing was written.")
|
|
568
|
+
return _finish(args, "pack", result, title, progress, url)
|
|
569
|
+
|
|
570
|
+
|
|
571
|
+
def cmd_unpack(args) -> int:
|
|
572
|
+
from batchvault import ui
|
|
573
|
+
|
|
574
|
+
_check_output_flags(args)
|
|
575
|
+
password = _read_password(args)
|
|
576
|
+
url = _notify_url(args)
|
|
577
|
+
|
|
578
|
+
out, err = _consoles()
|
|
579
|
+
if not (args.quiet or args.json or args.json_events):
|
|
580
|
+
ui.print_run_info(
|
|
581
|
+
out,
|
|
582
|
+
"BatchVault unpack",
|
|
583
|
+
[
|
|
584
|
+
("Archives", args.source),
|
|
585
|
+
("Restore files to", str(Path(args.output).resolve())),
|
|
586
|
+
("Existing files", "overwrite" if args.force else "skip"),
|
|
587
|
+
("Jobs", str(args.jobs) if args.jobs else "0 (sequential)"),
|
|
588
|
+
],
|
|
589
|
+
)
|
|
590
|
+
with _progress(args, "Restoring", err) as progress:
|
|
591
|
+
result = api.unpack(
|
|
592
|
+
args.source,
|
|
593
|
+
args.output,
|
|
594
|
+
force=args.force,
|
|
595
|
+
update=args.update,
|
|
596
|
+
jobs=args.jobs,
|
|
597
|
+
seven_zip=args.seven_zip,
|
|
598
|
+
password=password,
|
|
599
|
+
public_key=args.public_key,
|
|
600
|
+
retries=args.retries,
|
|
601
|
+
max_output_size=args.max_output_size,
|
|
602
|
+
progress=progress,
|
|
603
|
+
)
|
|
604
|
+
return _finish(args, "unpack", result, None, progress, url)
|
|
605
|
+
|
|
606
|
+
|
|
607
|
+
def cmd_verify(args) -> int:
|
|
608
|
+
from batchvault import ui
|
|
609
|
+
|
|
610
|
+
_check_output_flags(args)
|
|
611
|
+
password = _read_password(args)
|
|
612
|
+
url = _notify_url(args)
|
|
613
|
+
|
|
614
|
+
out, err = _consoles()
|
|
615
|
+
if not (args.quiet or args.json or args.json_events):
|
|
616
|
+
ui.print_run_info(
|
|
617
|
+
out,
|
|
618
|
+
"BatchVault verify",
|
|
619
|
+
[
|
|
620
|
+
("Archives", args.source),
|
|
621
|
+
("Original files", args.against or "not compared"),
|
|
622
|
+
("Jobs", str(args.jobs) if args.jobs else "0 (sequential)"),
|
|
623
|
+
],
|
|
624
|
+
)
|
|
625
|
+
with _progress(args, "Verifying", err) as progress:
|
|
626
|
+
result = api.verify(
|
|
627
|
+
args.source,
|
|
628
|
+
against=args.against,
|
|
629
|
+
jobs=args.jobs,
|
|
630
|
+
seven_zip=args.seven_zip,
|
|
631
|
+
password=password,
|
|
632
|
+
public_key=args.public_key,
|
|
633
|
+
progress=progress,
|
|
634
|
+
)
|
|
635
|
+
return _finish(args, "verify", result, None, progress, url)
|
|
636
|
+
|
|
637
|
+
|
|
638
|
+
def cmd_inspect(args) -> int:
|
|
639
|
+
from batchvault import ui
|
|
640
|
+
|
|
641
|
+
entries = api.inspect(args.source)
|
|
642
|
+
if args.json:
|
|
643
|
+
print(json.dumps([e.to_dict() for e in entries], indent=2, ensure_ascii=False))
|
|
644
|
+
elif not args.quiet:
|
|
645
|
+
ui.print_inspect(Console(), entries, args.source)
|
|
646
|
+
return 0
|
|
647
|
+
|
|
648
|
+
|
|
649
|
+
def cmd_keygen(args) -> int:
|
|
650
|
+
private, public = Path(args.private).expanduser(), Path(args.public).expanduser()
|
|
651
|
+
signing.generate_keypair(
|
|
652
|
+
private,
|
|
653
|
+
public,
|
|
654
|
+
password=os.environ.get(ENV_KEY_PASSWORD) or None,
|
|
655
|
+
overwrite=args.force,
|
|
656
|
+
)
|
|
657
|
+
if args.json:
|
|
658
|
+
print(json.dumps({"private": str(private), "public": str(public)}))
|
|
659
|
+
elif not args.quiet:
|
|
660
|
+
out, _ = _consoles()
|
|
661
|
+
out.print(f"Private key: {escape(str(private))} (keep it secret)")
|
|
662
|
+
out.print(f"Public key: {escape(str(public))} (share it)")
|
|
663
|
+
return 0
|
|
664
|
+
|
|
665
|
+
|
|
666
|
+
def cmd_sign(args) -> int:
|
|
667
|
+
manifest_path = Path(args.manifest).expanduser()
|
|
668
|
+
Manifest.load(manifest_path) # refuse to sign something that is not a manifest
|
|
669
|
+
key = signing.load_private_key(
|
|
670
|
+
Path(args.key).expanduser(), os.environ.get(ENV_KEY_PASSWORD) or None
|
|
671
|
+
)
|
|
672
|
+
signature = signing.sign_manifest(manifest_path, key)
|
|
673
|
+
if args.json:
|
|
674
|
+
print(json.dumps({"signature": str(signature)}))
|
|
675
|
+
elif not args.quiet:
|
|
676
|
+
_consoles()[0].print(f"Signature written to {escape(str(signature))}")
|
|
677
|
+
return 0
|
|
678
|
+
|
|
679
|
+
|
|
680
|
+
def _write_listing(args, entries, noun: str) -> None:
|
|
681
|
+
_, err = _consoles()
|
|
682
|
+
if not args.output:
|
|
683
|
+
print(json.dumps([e.to_dict() for e in entries], indent=2, ensure_ascii=False))
|
|
684
|
+
return
|
|
685
|
+
|
|
686
|
+
path = Path(args.output).expanduser()
|
|
687
|
+
if path.suffix.lower() == ".json":
|
|
688
|
+
text = json.dumps([e.to_dict() for e in entries], indent=2, ensure_ascii=False)
|
|
689
|
+
text += "\n"
|
|
690
|
+
else:
|
|
691
|
+
text = "".join(f"{entry.path}\n" for entry in entries)
|
|
692
|
+
roots = {entry.root for entry in entries if entry.root}
|
|
693
|
+
if len(roots) == 1:
|
|
694
|
+
# Remembered by ``pack`` so the listed folder's name is kept.
|
|
695
|
+
text = f"{ROOT_HEADER} {roots.pop()}\n" + text
|
|
696
|
+
path.write_text(text, encoding="utf-8")
|
|
697
|
+
|
|
698
|
+
if not args.quiet:
|
|
699
|
+
err.print(f"Found {len(entries)} {noun}(s). List saved to: {escape(str(path))}")
|
|
700
|
+
|
|
701
|
+
|
|
702
|
+
def cmd_list_files(args) -> int:
|
|
703
|
+
_, err = _consoles()
|
|
704
|
+
entries = api.list_files(args.folder, include=args.include, exclude=args.exclude)
|
|
705
|
+
|
|
706
|
+
if any(looks_like_archive(entry.path) for entry in entries) and not args.quiet:
|
|
707
|
+
err.print(
|
|
708
|
+
"[yellow]Warning: this folder contains archive volumes or a manifest. "
|
|
709
|
+
"Did you mean `batchvault list archives`?[/yellow]"
|
|
710
|
+
)
|
|
711
|
+
_write_listing(args, entries, "file")
|
|
712
|
+
return 0
|
|
713
|
+
|
|
714
|
+
|
|
715
|
+
def cmd_list_archives(args) -> int:
|
|
716
|
+
_, err = _consoles()
|
|
717
|
+
entries = api.list_archives(args.folder)
|
|
718
|
+
|
|
719
|
+
if not entries and not args.quiet:
|
|
720
|
+
err.print(
|
|
721
|
+
"[yellow]Warning: no archives found. "
|
|
722
|
+
"Did you mean `batchvault list files`?[/yellow]"
|
|
723
|
+
)
|
|
724
|
+
_write_listing(args, entries, "archive")
|
|
725
|
+
return 0
|
|
726
|
+
|
|
727
|
+
|
|
728
|
+
def cmd_doctor(args) -> int:
|
|
729
|
+
from batchvault import ui
|
|
730
|
+
|
|
731
|
+
checks = run_checks(args.seven_zip)
|
|
732
|
+
if args.json:
|
|
733
|
+
print(
|
|
734
|
+
json.dumps(
|
|
735
|
+
[vars(check) for check in checks], indent=2, ensure_ascii=False
|
|
736
|
+
)
|
|
737
|
+
)
|
|
738
|
+
elif not args.quiet:
|
|
739
|
+
ui.print_checks(Console(), checks)
|
|
740
|
+
return 0 if passed(checks) else 1
|
|
741
|
+
|
|
742
|
+
|
|
743
|
+
def _use_utf8() -> None:
|
|
744
|
+
# Keep file names readable when output is redirected on Windows.
|
|
745
|
+
for stream in (sys.stdout, sys.stderr):
|
|
746
|
+
try:
|
|
747
|
+
stream.reconfigure(encoding="utf-8")
|
|
748
|
+
except (AttributeError, ValueError):
|
|
749
|
+
pass
|
|
750
|
+
|
|
751
|
+
|
|
752
|
+
def _load_config(argv: list[str]) -> dict | None:
|
|
753
|
+
pre = argparse.ArgumentParser(add_help=False, allow_abbrev=False)
|
|
754
|
+
pre.add_argument("--config")
|
|
755
|
+
known, _ = pre.parse_known_args(argv)
|
|
756
|
+
|
|
757
|
+
explicit = known.config or os.environ.get(ENV_CONFIG)
|
|
758
|
+
if explicit:
|
|
759
|
+
path = Path(explicit).expanduser()
|
|
760
|
+
if not path.is_file():
|
|
761
|
+
raise ConfigError(f"Config file not found: {path}")
|
|
762
|
+
return load_config(path) # named by the user, so trusted
|
|
763
|
+
|
|
764
|
+
path = find_config()
|
|
765
|
+
if path is None:
|
|
766
|
+
return None
|
|
767
|
+
config = load_config(path)
|
|
768
|
+
reject_restricted(config, path) # found by searching: not fully trusted
|
|
769
|
+
return config
|
|
770
|
+
|
|
771
|
+
|
|
772
|
+
def main(argv: list[str] | None = None) -> int:
|
|
773
|
+
_use_utf8()
|
|
774
|
+
_, err = _consoles()
|
|
775
|
+
argv = list(sys.argv[1:] if argv is None else argv)
|
|
776
|
+
|
|
777
|
+
try:
|
|
778
|
+
parser = build_parser(_load_config(argv))
|
|
779
|
+
except BatchVaultError as exc:
|
|
780
|
+
err.print(f"[bold red]Error:[/bold red] {escape(str(exc))}")
|
|
781
|
+
return 2
|
|
782
|
+
|
|
783
|
+
args = parser.parse_args(argv)
|
|
784
|
+
|
|
785
|
+
log_path = None
|
|
786
|
+
if not args.no_log:
|
|
787
|
+
log_path = setup_logging(args.log_level, args.log_file)
|
|
788
|
+
log.info("batchvault %s: %s", __version__, " ".join(redact_arguments(argv)))
|
|
789
|
+
|
|
790
|
+
try:
|
|
791
|
+
return args.handler(args)
|
|
792
|
+
except BatchVaultError as exc:
|
|
793
|
+
log.error("%s", exc)
|
|
794
|
+
err.print(f"[bold red]Error:[/bold red] {escape(str(exc))}")
|
|
795
|
+
return 2
|
|
796
|
+
except KeyboardInterrupt:
|
|
797
|
+
log.warning("Interrupted")
|
|
798
|
+
err.print("[yellow]Interrupted.[/yellow]")
|
|
799
|
+
return 130
|
|
800
|
+
except Exception as exc:
|
|
801
|
+
log.exception("Unexpected error")
|
|
802
|
+
where = f" See the log: {log_path}" if log_path else ""
|
|
803
|
+
err.print(f"[bold red]Unexpected error:[/bold red] {escape(str(exc))}.{where}")
|
|
804
|
+
return 3
|
|
805
|
+
finally:
|
|
806
|
+
close_logging()
|