sharefetch 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sharefetch/__init__.py +11 -0
- sharefetch/__main__.py +8 -0
- sharefetch/auth.py +343 -0
- sharefetch/cli.py +782 -0
- sharefetch/config.py +374 -0
- sharefetch/engine.py +1055 -0
- sharefetch/errors.py +57 -0
- sharefetch/events.py +129 -0
- sharefetch/graph.py +474 -0
- sharefetch/gui.py +1170 -0
- sharefetch/hashes.py +123 -0
- sharefetch/listing.py +493 -0
- sharefetch/manifest.py +198 -0
- sharefetch/pacing.py +211 -0
- sharefetch/paths.py +245 -0
- sharefetch/py.typed +0 -0
- sharefetch/resolve.py +660 -0
- sharefetch/state.py +707 -0
- sharefetch/tokencache.py +214 -0
- sharefetch/transfer.py +815 -0
- sharefetch/urls.py +195 -0
- sharefetch/verify.py +378 -0
- sharefetch-1.0.0.dist-info/METADATA +193 -0
- sharefetch-1.0.0.dist-info/RECORD +28 -0
- sharefetch-1.0.0.dist-info/WHEEL +5 -0
- sharefetch-1.0.0.dist-info/entry_points.txt +5 -0
- sharefetch-1.0.0.dist-info/licenses/LICENSE +21 -0
- sharefetch-1.0.0.dist-info/top_level.txt +1 -0
sharefetch/cli.py
ADDED
|
@@ -0,0 +1,782 @@
|
|
|
1
|
+
"""Command-line front end.
|
|
2
|
+
|
|
3
|
+
The one place the operator's words become a :class:`~sharefetch.config.Settings`
|
|
4
|
+
and a run's events become lines on a terminal. Every bound this module enforces
|
|
5
|
+
is a validator in :mod:`sharefetch.config`, so the command-line front end and
|
|
6
|
+
the graphical front end cannot disagree about what an option accepts.
|
|
7
|
+
|
|
8
|
+
Two departures from the obvious construction, both deliberate:
|
|
9
|
+
|
|
10
|
+
``Engine`` in place of ``engine.run``
|
|
11
|
+
``engine.run`` is a three-line convenience that builds an
|
|
12
|
+
:class:`~sharefetch.engine.Engine` and calls its ``run``, and it returns no
|
|
13
|
+
handle on the instance. Ctrl+C has to reach ``Engine.stop``, which is the
|
|
14
|
+
method that lets the files in flight finish their chunk, flushes the state
|
|
15
|
+
file and writes the manifest, so this module builds the engine itself and
|
|
16
|
+
keeps the reference. ``engine_factory`` is the seam a test replaces.
|
|
17
|
+
|
|
18
|
+
``_check_log_file``
|
|
19
|
+
Every other boundary check is a ``config`` validator. ``--log-file`` has no
|
|
20
|
+
counterpart in the graphical interface, so its
|
|
21
|
+
check lives here and no second front end can diverge from it.
|
|
22
|
+
|
|
23
|
+
``print`` appears twice by design and nowhere else: the ``--dry-run`` listing
|
|
24
|
+
and the version banner argparse writes. Every other word this module puts in
|
|
25
|
+
front of an operator goes through the logger, which carries the redaction
|
|
26
|
+
filter.
|
|
27
|
+
"""
|
|
28
|
+
|
|
29
|
+
import argparse
|
|
30
|
+
import logging
|
|
31
|
+
import signal
|
|
32
|
+
import sys
|
|
33
|
+
import types
|
|
34
|
+
from collections.abc import Callable
|
|
35
|
+
from pathlib import Path
|
|
36
|
+
from typing import Any, Protocol, runtime_checkable
|
|
37
|
+
|
|
38
|
+
from . import __version__
|
|
39
|
+
from .config import (
|
|
40
|
+
CHUNK_KIB_BOUNDS,
|
|
41
|
+
CONNECTIONS_BOUNDS,
|
|
42
|
+
DEFAULT_CHUNK_KIB,
|
|
43
|
+
DEFAULT_CLIENT_ID,
|
|
44
|
+
DEFAULT_CONNECTIONS,
|
|
45
|
+
DEFAULT_DELAY_SECONDS,
|
|
46
|
+
DEFAULT_TENANT,
|
|
47
|
+
DELAY_BOUNDS,
|
|
48
|
+
LIMIT_RATE_MIN_KBPS,
|
|
49
|
+
MIN_DEPTH,
|
|
50
|
+
Settings,
|
|
51
|
+
check_chunk_kib,
|
|
52
|
+
check_client_id,
|
|
53
|
+
check_connections,
|
|
54
|
+
check_delay,
|
|
55
|
+
check_dest,
|
|
56
|
+
check_limit_rate,
|
|
57
|
+
check_max_depth,
|
|
58
|
+
check_tenant,
|
|
59
|
+
)
|
|
60
|
+
from .engine import Engine
|
|
61
|
+
from .errors import SharefetchError, ValidationError
|
|
62
|
+
from .events import (
|
|
63
|
+
DeviceCodeCompleted,
|
|
64
|
+
DeviceCodeRequested,
|
|
65
|
+
EnumerationComplete,
|
|
66
|
+
EnumerationProgress,
|
|
67
|
+
Event,
|
|
68
|
+
FileFinished,
|
|
69
|
+
FileProgress,
|
|
70
|
+
FileStarted,
|
|
71
|
+
LogRecord,
|
|
72
|
+
Reconciled,
|
|
73
|
+
Resolved,
|
|
74
|
+
RunFinished,
|
|
75
|
+
RunSummary,
|
|
76
|
+
Throttled,
|
|
77
|
+
)
|
|
78
|
+
from .graph import RedactingFilter
|
|
79
|
+
from .paths import sanitise_segment
|
|
80
|
+
from .state import StateStore
|
|
81
|
+
from .urls import SUPPORTED_FORMS, ShareRef, SourceRef
|
|
82
|
+
from .urls import parse as parse_url
|
|
83
|
+
|
|
84
|
+
logger = logging.getLogger(__name__)
|
|
85
|
+
|
|
86
|
+
PROGRAM_NAME = "sharefetch"
|
|
87
|
+
|
|
88
|
+
# A skipped item exits 1 alongside a failed one,
|
|
89
|
+
# because the run did not transfer everything it was asked for even though
|
|
90
|
+
# nothing malfunctioned.
|
|
91
|
+
EXIT_OK = 0
|
|
92
|
+
EXIT_INCOMPLETE = 1
|
|
93
|
+
EXIT_FATAL = 2
|
|
94
|
+
EXIT_INTERRUPTED = 130
|
|
95
|
+
|
|
96
|
+
LOG_FORMAT = "%(asctime)s %(levelname)-8s %(name)s %(message)s"
|
|
97
|
+
LOG_DATE_FORMAT = "%Y-%m-%dT%H:%M:%S"
|
|
98
|
+
|
|
99
|
+
# Column widths for the --dry-run listing. The status column fits the longest
|
|
100
|
+
# member of config.ITEM_STATUSES and the byte column fits a terabyte written
|
|
101
|
+
# with thousands separators.
|
|
102
|
+
_STATUS_COLUMN = 9
|
|
103
|
+
_BYTES_COLUMN = 15
|
|
104
|
+
|
|
105
|
+
_LOG_LEVELS = frozenset({"DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"})
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
# ----------------------------------------------------------------------
|
|
109
|
+
# Parsing
|
|
110
|
+
# ----------------------------------------------------------------------
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
class _Parser(argparse.ArgumentParser):
|
|
114
|
+
"""An ``ArgumentParser`` whose refusals are ``ValidationError``.
|
|
115
|
+
|
|
116
|
+
argparse's own ``error`` writes to stderr and raises ``SystemExit``, which
|
|
117
|
+
would put operator output outside the logger and outside the redaction
|
|
118
|
+
filter. Raising instead routes an unknown option, a missing argument, a
|
|
119
|
+
mutually exclusive pair and a rejected value through the one path
|
|
120
|
+
:func:`main` already has for a fatal setup failure. ``--help`` and
|
|
121
|
+
``--version`` go through ``exit`` rather than ``error`` and keep their
|
|
122
|
+
``SystemExit``.
|
|
123
|
+
"""
|
|
124
|
+
|
|
125
|
+
def error(self, message: str) -> Any:
|
|
126
|
+
raise ValidationError(message)
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
def _bounded_int(name: str, check: Callable[[int], Any]) -> Callable[[str], int]:
|
|
130
|
+
"""Return an argparse ``type`` that parses an integer and applies ``check``."""
|
|
131
|
+
|
|
132
|
+
def convert(value: str) -> int:
|
|
133
|
+
try:
|
|
134
|
+
number = int(value)
|
|
135
|
+
except ValueError:
|
|
136
|
+
raise argparse.ArgumentTypeError(
|
|
137
|
+
f"{name} must be an integer, received {value!r}"
|
|
138
|
+
) from None
|
|
139
|
+
try:
|
|
140
|
+
check(number)
|
|
141
|
+
except ValidationError as exc:
|
|
142
|
+
raise argparse.ArgumentTypeError(str(exc)) from None
|
|
143
|
+
return number
|
|
144
|
+
|
|
145
|
+
return convert
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
def _bounded_float(name: str, check: Callable[[float], Any]) -> Callable[[str], float]:
|
|
149
|
+
"""Return an argparse ``type`` that parses a float and applies ``check``."""
|
|
150
|
+
|
|
151
|
+
def convert(value: str) -> float:
|
|
152
|
+
try:
|
|
153
|
+
number = float(value)
|
|
154
|
+
except ValueError:
|
|
155
|
+
raise argparse.ArgumentTypeError(
|
|
156
|
+
f"{name} must be a number, received {value!r}"
|
|
157
|
+
) from None
|
|
158
|
+
try:
|
|
159
|
+
check(number)
|
|
160
|
+
except ValidationError as exc:
|
|
161
|
+
raise argparse.ArgumentTypeError(str(exc)) from None
|
|
162
|
+
return number
|
|
163
|
+
|
|
164
|
+
return convert
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
def _checked_str(check: Callable[[str], str]) -> Callable[[str], str]:
|
|
168
|
+
"""Return an argparse ``type`` that applies a ``config`` string validator."""
|
|
169
|
+
|
|
170
|
+
def convert(value: str) -> str:
|
|
171
|
+
try:
|
|
172
|
+
return check(value)
|
|
173
|
+
except ValidationError as exc:
|
|
174
|
+
raise argparse.ArgumentTypeError(str(exc)) from None
|
|
175
|
+
|
|
176
|
+
return convert
|
|
177
|
+
|
|
178
|
+
|
|
179
|
+
def build_parser() -> argparse.ArgumentParser:
|
|
180
|
+
"""Build the parser for the full option table."""
|
|
181
|
+
low_connections, high_connections = CONNECTIONS_BOUNDS
|
|
182
|
+
low_delay, high_delay = DELAY_BOUNDS
|
|
183
|
+
low_chunk, high_chunk = CHUNK_KIB_BOUNDS
|
|
184
|
+
|
|
185
|
+
parser = _Parser(
|
|
186
|
+
prog=PROGRAM_NAME,
|
|
187
|
+
description=(
|
|
188
|
+
"Download a SharePoint Online or OneDrive for Business folder tree "
|
|
189
|
+
"by Microsoft Graph, resumably and with content-level verification."
|
|
190
|
+
),
|
|
191
|
+
epilog="supported URL forms:\n " + "\n ".join(SUPPORTED_FORMS),
|
|
192
|
+
formatter_class=argparse.RawDescriptionHelpFormatter,
|
|
193
|
+
)
|
|
194
|
+
|
|
195
|
+
parser.add_argument(
|
|
196
|
+
"url",
|
|
197
|
+
metavar="url",
|
|
198
|
+
help="the SharePoint or OneDrive URL naming the folder to download",
|
|
199
|
+
)
|
|
200
|
+
parser.add_argument(
|
|
201
|
+
"--dest",
|
|
202
|
+
metavar="DIR",
|
|
203
|
+
default=None,
|
|
204
|
+
help=(
|
|
205
|
+
"destination directory; defaults to a folder named after the "
|
|
206
|
+
"addressed folder, under the working directory"
|
|
207
|
+
),
|
|
208
|
+
)
|
|
209
|
+
parser.add_argument(
|
|
210
|
+
"--connections",
|
|
211
|
+
metavar="N",
|
|
212
|
+
type=_bounded_int("connections", check_connections),
|
|
213
|
+
default=DEFAULT_CONNECTIONS,
|
|
214
|
+
help=(
|
|
215
|
+
f"number of concurrent transfers, {low_connections} or "
|
|
216
|
+
f"{high_connections} (default: {DEFAULT_CONNECTIONS})"
|
|
217
|
+
),
|
|
218
|
+
)
|
|
219
|
+
parser.add_argument(
|
|
220
|
+
"--delay",
|
|
221
|
+
metavar="SECONDS",
|
|
222
|
+
type=_bounded_float("delay", check_delay),
|
|
223
|
+
default=DEFAULT_DELAY_SECONDS,
|
|
224
|
+
help=(
|
|
225
|
+
f"pause between requests, {low_delay} to {high_delay} seconds "
|
|
226
|
+
f"(default: {DEFAULT_DELAY_SECONDS})"
|
|
227
|
+
),
|
|
228
|
+
)
|
|
229
|
+
parser.add_argument(
|
|
230
|
+
"--limit-rate",
|
|
231
|
+
metavar="KBPS",
|
|
232
|
+
dest="limit_rate_kbps",
|
|
233
|
+
type=_bounded_int("rate limit", check_limit_rate),
|
|
234
|
+
default=None,
|
|
235
|
+
help=(
|
|
236
|
+
f"ceiling on transfer rate in KB/s, at least {LIMIT_RATE_MIN_KBPS} "
|
|
237
|
+
"(default: no ceiling)"
|
|
238
|
+
),
|
|
239
|
+
)
|
|
240
|
+
parser.add_argument(
|
|
241
|
+
"--chunk-size",
|
|
242
|
+
metavar="KIB",
|
|
243
|
+
dest="chunk_kib",
|
|
244
|
+
type=_bounded_int("chunk size in KiB", check_chunk_kib),
|
|
245
|
+
default=DEFAULT_CHUNK_KIB,
|
|
246
|
+
help=(f"read size in KiB, {low_chunk} to {high_chunk} (default: {DEFAULT_CHUNK_KIB})"),
|
|
247
|
+
)
|
|
248
|
+
parser.add_argument(
|
|
249
|
+
"--max-depth",
|
|
250
|
+
metavar="N",
|
|
251
|
+
type=_bounded_int("max depth", check_max_depth),
|
|
252
|
+
default=None,
|
|
253
|
+
help=(f"deepest folder level to walk, {MIN_DEPTH} or above (default: unlimited)"),
|
|
254
|
+
)
|
|
255
|
+
parser.add_argument(
|
|
256
|
+
"--include",
|
|
257
|
+
metavar="GLOB",
|
|
258
|
+
action="append",
|
|
259
|
+
default=[],
|
|
260
|
+
help="only transfer items matching this glob; repeatable",
|
|
261
|
+
)
|
|
262
|
+
parser.add_argument(
|
|
263
|
+
"--exclude",
|
|
264
|
+
metavar="GLOB",
|
|
265
|
+
action="append",
|
|
266
|
+
default=[],
|
|
267
|
+
help="skip items matching this glob, applied after --include; repeatable",
|
|
268
|
+
)
|
|
269
|
+
|
|
270
|
+
restart_group = parser.add_mutually_exclusive_group()
|
|
271
|
+
restart_group.add_argument(
|
|
272
|
+
"--resume",
|
|
273
|
+
dest="restart",
|
|
274
|
+
action="store_false",
|
|
275
|
+
help="continue a previous run from its state file (default)",
|
|
276
|
+
)
|
|
277
|
+
restart_group.add_argument(
|
|
278
|
+
"--restart",
|
|
279
|
+
dest="restart",
|
|
280
|
+
action="store_true",
|
|
281
|
+
help="ignore recorded progress and transfer every item again",
|
|
282
|
+
)
|
|
283
|
+
parser.set_defaults(restart=False)
|
|
284
|
+
|
|
285
|
+
parser.add_argument(
|
|
286
|
+
"--dry-run",
|
|
287
|
+
action="store_true",
|
|
288
|
+
help="enumerate, reconcile and report, writing state but no file content",
|
|
289
|
+
)
|
|
290
|
+
parser.add_argument(
|
|
291
|
+
"--verify-only",
|
|
292
|
+
action="store_true",
|
|
293
|
+
help="verify the tree already on disk, making no network call",
|
|
294
|
+
)
|
|
295
|
+
parser.add_argument(
|
|
296
|
+
"--reauth",
|
|
297
|
+
action="store_true",
|
|
298
|
+
help="ignore the cached refresh token and force a device code",
|
|
299
|
+
)
|
|
300
|
+
parser.add_argument(
|
|
301
|
+
"--tenant",
|
|
302
|
+
metavar="ID",
|
|
303
|
+
type=_checked_str(check_tenant),
|
|
304
|
+
default=DEFAULT_TENANT,
|
|
305
|
+
help=f"tenant GUID or domain (default: {DEFAULT_TENANT})",
|
|
306
|
+
)
|
|
307
|
+
parser.add_argument(
|
|
308
|
+
"--client-id",
|
|
309
|
+
metavar="GUID",
|
|
310
|
+
type=_checked_str(check_client_id),
|
|
311
|
+
default=DEFAULT_CLIENT_ID,
|
|
312
|
+
help=f"application GUID to sign in with (default: {DEFAULT_CLIENT_ID})",
|
|
313
|
+
)
|
|
314
|
+
parser.add_argument(
|
|
315
|
+
"--ignore-free-space",
|
|
316
|
+
action="store_true",
|
|
317
|
+
help="skip the pre-flight capacity check",
|
|
318
|
+
)
|
|
319
|
+
parser.add_argument(
|
|
320
|
+
"--log-file",
|
|
321
|
+
metavar="PATH",
|
|
322
|
+
default=None,
|
|
323
|
+
help="write the log to this file as well as to stderr",
|
|
324
|
+
)
|
|
325
|
+
parser.add_argument(
|
|
326
|
+
"--verbose",
|
|
327
|
+
action="store_true",
|
|
328
|
+
help="log at DEBUG in place of INFO",
|
|
329
|
+
)
|
|
330
|
+
parser.add_argument(
|
|
331
|
+
"--version",
|
|
332
|
+
action="version",
|
|
333
|
+
version=f"%(prog)s {__version__}",
|
|
334
|
+
help="print the version and exit",
|
|
335
|
+
)
|
|
336
|
+
return parser
|
|
337
|
+
|
|
338
|
+
|
|
339
|
+
def _default_dest(reference: SourceRef) -> Path:
|
|
340
|
+
"""Return the destination a URL implies, under the working directory.
|
|
341
|
+
|
|
342
|
+
Raises:
|
|
343
|
+
ValidationError: the URL is a sharing link. A sharing link carries a
|
|
344
|
+
token and no folder name, and the name only becomes known once the
|
|
345
|
+
run resolves the link against the service, which is after the
|
|
346
|
+
destination has to exist. ``--dest`` is required in that case.
|
|
347
|
+
"""
|
|
348
|
+
if isinstance(reference, ShareRef):
|
|
349
|
+
raise ValidationError(
|
|
350
|
+
"a sharing link names no folder until the run resolves it against "
|
|
351
|
+
"the service, and the destination has to be chosen before that, so "
|
|
352
|
+
"--dest must be given when the URL is a sharing link"
|
|
353
|
+
)
|
|
354
|
+
name, changed = sanitise_segment(reference.path_segments[-1])
|
|
355
|
+
if changed:
|
|
356
|
+
logger.info("the default destination folder name was sanitised to %r", name)
|
|
357
|
+
return Path.cwd() / name
|
|
358
|
+
|
|
359
|
+
|
|
360
|
+
def _check_log_file(value: str | None) -> Path | None:
|
|
361
|
+
"""Return the log file as an absolute path, rejecting an unusable parent."""
|
|
362
|
+
if value is None:
|
|
363
|
+
return None
|
|
364
|
+
if not str(value).strip():
|
|
365
|
+
raise ValidationError("log file must be a file path, received an empty value")
|
|
366
|
+
path = Path(value).expanduser().resolve()
|
|
367
|
+
if path.is_dir():
|
|
368
|
+
raise ValidationError(f"log file must be a file, received {path}, which is a directory")
|
|
369
|
+
if not path.parent.is_dir():
|
|
370
|
+
raise ValidationError(
|
|
371
|
+
f"log file parent directory must exist, received {path}, "
|
|
372
|
+
f"whose parent {path.parent} does not exist"
|
|
373
|
+
)
|
|
374
|
+
return path
|
|
375
|
+
|
|
376
|
+
|
|
377
|
+
def parse_args(argv: list[str] | None = None) -> Settings:
|
|
378
|
+
"""Turn a command line into validated ``Settings``.
|
|
379
|
+
|
|
380
|
+
Every numeric bound is applied by the parser's ``type`` callables, so a
|
|
381
|
+
value outside its range never reaches this function's body. The URL, the
|
|
382
|
+
destination and the log file are checked here, because each depends on more
|
|
383
|
+
than the one string argparse hands over.
|
|
384
|
+
|
|
385
|
+
Raises:
|
|
386
|
+
ValidationError: any option, the URL, the destination or the log file
|
|
387
|
+
failed its check. The message names the value received and the
|
|
388
|
+
range or rule that rejected it.
|
|
389
|
+
SystemExit: ``--help`` or ``--version`` was given.
|
|
390
|
+
"""
|
|
391
|
+
args = build_parser().parse_args(argv)
|
|
392
|
+
|
|
393
|
+
reference = parse_url(args.url)
|
|
394
|
+
dest = check_dest(args.dest) if args.dest is not None else check_dest(_default_dest(reference))
|
|
395
|
+
|
|
396
|
+
return Settings(
|
|
397
|
+
url=args.url,
|
|
398
|
+
dest=dest,
|
|
399
|
+
connections=args.connections,
|
|
400
|
+
delay=args.delay,
|
|
401
|
+
limit_rate_kbps=args.limit_rate_kbps,
|
|
402
|
+
chunk_kib=args.chunk_kib,
|
|
403
|
+
max_depth=args.max_depth,
|
|
404
|
+
include=tuple(args.include),
|
|
405
|
+
exclude=tuple(args.exclude),
|
|
406
|
+
restart=args.restart,
|
|
407
|
+
dry_run=args.dry_run,
|
|
408
|
+
verify_only=args.verify_only,
|
|
409
|
+
reauth=args.reauth,
|
|
410
|
+
tenant=args.tenant,
|
|
411
|
+
client_id=args.client_id,
|
|
412
|
+
ignore_free_space=args.ignore_free_space,
|
|
413
|
+
log_file=_check_log_file(args.log_file),
|
|
414
|
+
verbose=args.verbose,
|
|
415
|
+
)
|
|
416
|
+
|
|
417
|
+
|
|
418
|
+
# ----------------------------------------------------------------------
|
|
419
|
+
# Logging
|
|
420
|
+
# ----------------------------------------------------------------------
|
|
421
|
+
|
|
422
|
+
|
|
423
|
+
# Marks a record that may reach the console and must not reach a log file. The
|
|
424
|
+
# only record carrying it is the device-code instruction, which holds the user
|
|
425
|
+
# code.
|
|
426
|
+
CONSOLE_ONLY = "console_only"
|
|
427
|
+
|
|
428
|
+
|
|
429
|
+
class _ConsoleOnlyFilter(logging.Filter):
|
|
430
|
+
"""Drop records marked console-only, so a log file never holds a user code."""
|
|
431
|
+
|
|
432
|
+
def filter(self, record: logging.LogRecord) -> bool:
|
|
433
|
+
return not getattr(record, CONSOLE_ONLY, False)
|
|
434
|
+
|
|
435
|
+
|
|
436
|
+
def configure_logging(verbose: bool = False, log_file: Path | None = None) -> None:
|
|
437
|
+
"""Install the handlers this front end logs through.
|
|
438
|
+
|
|
439
|
+
The library configures no handler and the front end configures them all.
|
|
440
|
+
Every handler installed here carries
|
|
441
|
+
:class:`~sharefetch.graph.RedactingFilter`, so a line written by this
|
|
442
|
+
package, by ``requests`` or by ``urllib3`` is masked alike.
|
|
443
|
+
|
|
444
|
+
Called twice in a normal run: once before parsing, so a rejected option has
|
|
445
|
+
somewhere to be reported, and once after, when ``--verbose`` and
|
|
446
|
+
``--log-file`` are known. ``force`` makes the second call replace the first
|
|
447
|
+
call's handlers rather than doubling every line.
|
|
448
|
+
"""
|
|
449
|
+
level = logging.DEBUG if verbose else logging.INFO
|
|
450
|
+
handlers: list[logging.Handler] = [logging.StreamHandler(sys.stderr)]
|
|
451
|
+
if log_file is not None:
|
|
452
|
+
file_handler = logging.FileHandler(log_file, encoding="utf-8")
|
|
453
|
+
file_handler.addFilter(_ConsoleOnlyFilter())
|
|
454
|
+
handlers.append(file_handler)
|
|
455
|
+
|
|
456
|
+
redaction = RedactingFilter()
|
|
457
|
+
for handler in handlers:
|
|
458
|
+
handler.addFilter(redaction)
|
|
459
|
+
|
|
460
|
+
logging.basicConfig(
|
|
461
|
+
level=level,
|
|
462
|
+
format=LOG_FORMAT,
|
|
463
|
+
datefmt=LOG_DATE_FORMAT,
|
|
464
|
+
handlers=handlers,
|
|
465
|
+
force=True,
|
|
466
|
+
)
|
|
467
|
+
|
|
468
|
+
|
|
469
|
+
# ----------------------------------------------------------------------
|
|
470
|
+
# The event sink
|
|
471
|
+
# ----------------------------------------------------------------------
|
|
472
|
+
|
|
473
|
+
|
|
474
|
+
class CliSink:
|
|
475
|
+
"""Writes each run event through the logger.
|
|
476
|
+
|
|
477
|
+
Levels are fixed: DEBUG for per-chunk and per-request trace, INFO for
|
|
478
|
+
milestones, WARNING for a recoverable condition and ERROR for a failed
|
|
479
|
+
item.
|
|
480
|
+
"""
|
|
481
|
+
|
|
482
|
+
def __init__(self) -> None:
|
|
483
|
+
self._handlers: dict[type, Callable[[Any], None]] = {
|
|
484
|
+
Resolved: self._on_resolved,
|
|
485
|
+
DeviceCodeRequested: self._on_device_code_requested,
|
|
486
|
+
DeviceCodeCompleted: self._on_device_code_completed,
|
|
487
|
+
EnumerationProgress: self._on_enumeration_progress,
|
|
488
|
+
EnumerationComplete: self._on_enumeration_complete,
|
|
489
|
+
Reconciled: self._on_reconciled,
|
|
490
|
+
FileStarted: self._on_file_started,
|
|
491
|
+
FileProgress: self._on_file_progress,
|
|
492
|
+
FileFinished: self._on_file_finished,
|
|
493
|
+
Throttled: self._on_throttled,
|
|
494
|
+
RunFinished: self._on_run_finished,
|
|
495
|
+
LogRecord: self._on_log_record,
|
|
496
|
+
}
|
|
497
|
+
|
|
498
|
+
def emit(self, event: Event) -> None:
|
|
499
|
+
handler = self._handlers.get(type(event))
|
|
500
|
+
if handler is None:
|
|
501
|
+
logger.debug("an event of type %s was emitted and has no sink line", type(event))
|
|
502
|
+
return
|
|
503
|
+
handler(event)
|
|
504
|
+
|
|
505
|
+
def _on_resolved(self, event: Resolved) -> None:
|
|
506
|
+
logger.info(
|
|
507
|
+
"resolved to the %s library at %s, folder %r, drive %s",
|
|
508
|
+
event.drive_name,
|
|
509
|
+
event.site_web_url,
|
|
510
|
+
event.folder_path,
|
|
511
|
+
event.drive_id,
|
|
512
|
+
)
|
|
513
|
+
|
|
514
|
+
def _on_device_code_requested(self, event: DeviceCodeRequested) -> None:
|
|
515
|
+
# The service composes ``message`` as the full instruction to the
|
|
516
|
+
# operator, including the code and the URI, and the server's own
|
|
517
|
+
# wording survives verbatim.
|
|
518
|
+
#
|
|
519
|
+
# The user code is phishing material: anyone holding it can poll the
|
|
520
|
+
# token endpoint and collect the token the moment the legitimate
|
|
521
|
+
# operator completes the sign-in. It has to reach the screen, and it
|
|
522
|
+
# must not reach a file that outlives the code's few minutes of
|
|
523
|
+
# validity, so the record is marked console-only and the file handler
|
|
524
|
+
# drops it. The expiry instant is logged on its own line, which carries
|
|
525
|
+
# no code and is safe to keep.
|
|
526
|
+
logger.info("%s", event.message, extra={CONSOLE_ONLY: True})
|
|
527
|
+
logger.info(
|
|
528
|
+
"a sign-in code was issued at %s, expiring at %s",
|
|
529
|
+
event.verification_uri,
|
|
530
|
+
event.expires_at,
|
|
531
|
+
)
|
|
532
|
+
|
|
533
|
+
def _on_device_code_completed(self, event: DeviceCodeCompleted) -> None:
|
|
534
|
+
logger.info("signed in as %s", event.account)
|
|
535
|
+
|
|
536
|
+
def _on_enumeration_progress(self, event: EnumerationProgress) -> None:
|
|
537
|
+
logger.debug(
|
|
538
|
+
"enumerated %d folder(s), %d file(s), %d byte(s) so far",
|
|
539
|
+
event.folders,
|
|
540
|
+
event.files,
|
|
541
|
+
event.total_bytes,
|
|
542
|
+
)
|
|
543
|
+
|
|
544
|
+
def _on_enumeration_complete(self, event: EnumerationComplete) -> None:
|
|
545
|
+
logger.info(
|
|
546
|
+
"enumeration complete: %d file(s) in %d folder(s), %d byte(s)",
|
|
547
|
+
event.files,
|
|
548
|
+
event.folders,
|
|
549
|
+
event.total_bytes,
|
|
550
|
+
)
|
|
551
|
+
|
|
552
|
+
def _on_reconciled(self, event: Reconciled) -> None:
|
|
553
|
+
counts = ", ".join(f"{status}={count}" for status, count in sorted(event.counts.items()))
|
|
554
|
+
logger.info("reconciled against the state file: %s", counts or "no items")
|
|
555
|
+
|
|
556
|
+
def _on_file_started(self, event: FileStarted) -> None:
|
|
557
|
+
if event.resume_from:
|
|
558
|
+
logger.info(
|
|
559
|
+
"%s: resuming at byte %d of %d", event.drive_path, event.resume_from, event.size
|
|
560
|
+
)
|
|
561
|
+
else:
|
|
562
|
+
logger.info("%s: starting, %d byte(s)", event.drive_path, event.size)
|
|
563
|
+
|
|
564
|
+
def _on_file_progress(self, event: FileProgress) -> None:
|
|
565
|
+
logger.debug(
|
|
566
|
+
"%s: %d of %d byte(s) written",
|
|
567
|
+
event.drive_path,
|
|
568
|
+
event.bytes_written,
|
|
569
|
+
event.size,
|
|
570
|
+
)
|
|
571
|
+
|
|
572
|
+
def _on_file_finished(self, event: FileFinished) -> None:
|
|
573
|
+
verification = getattr(event.verification, "status", None)
|
|
574
|
+
if event.error:
|
|
575
|
+
logger.error(
|
|
576
|
+
"%s: %s, verification %s, %s",
|
|
577
|
+
event.drive_path,
|
|
578
|
+
event.status,
|
|
579
|
+
verification or "not run",
|
|
580
|
+
event.error,
|
|
581
|
+
)
|
|
582
|
+
return
|
|
583
|
+
logger.info(
|
|
584
|
+
"%s: %s, verification %s",
|
|
585
|
+
event.drive_path,
|
|
586
|
+
event.status,
|
|
587
|
+
verification or "not run",
|
|
588
|
+
)
|
|
589
|
+
|
|
590
|
+
def _on_throttled(self, event: Throttled) -> None:
|
|
591
|
+
logger.warning("held for %.1f second(s): %s", event.seconds, event.reason)
|
|
592
|
+
|
|
593
|
+
def _on_run_finished(self, event: RunFinished) -> None:
|
|
594
|
+
summary = event.summary
|
|
595
|
+
logger.info(
|
|
596
|
+
"run finished: %d transferred, %d already present, %d skipped, %d failed, "
|
|
597
|
+
"%d verified, %d byte(s) transferred",
|
|
598
|
+
summary.transferred,
|
|
599
|
+
summary.already_present,
|
|
600
|
+
summary.skipped,
|
|
601
|
+
summary.failed,
|
|
602
|
+
summary.verified,
|
|
603
|
+
summary.bytes_transferred,
|
|
604
|
+
)
|
|
605
|
+
if summary.hash_algorithm_validated is False:
|
|
606
|
+
logger.warning(
|
|
607
|
+
"the hash implementation safeguard did not validate; content hashes "
|
|
608
|
+
"from this run carry no weight"
|
|
609
|
+
)
|
|
610
|
+
if event.manifest_path is None:
|
|
611
|
+
logger.info("no manifest was written")
|
|
612
|
+
else:
|
|
613
|
+
logger.info("manifest: %s", event.manifest_path)
|
|
614
|
+
|
|
615
|
+
def _on_log_record(self, event: LogRecord) -> None:
|
|
616
|
+
# Reaches this sink only where a caller attached the logging handler,
|
|
617
|
+
# which the command-line front end does not. Its level is a
|
|
618
|
+
# name from the record it came from, and an unknown one is carried at
|
|
619
|
+
# INFO rather than dropped.
|
|
620
|
+
name = (event.level or "").upper()
|
|
621
|
+
level = logging.getLevelName(name) if name in _LOG_LEVELS else logging.INFO
|
|
622
|
+
logger.log(level, "%s", event.message)
|
|
623
|
+
|
|
624
|
+
|
|
625
|
+
# ----------------------------------------------------------------------
|
|
626
|
+
# Running
|
|
627
|
+
# ----------------------------------------------------------------------
|
|
628
|
+
|
|
629
|
+
|
|
630
|
+
@runtime_checkable
|
|
631
|
+
class EngineLike(Protocol):
|
|
632
|
+
"""The part of :class:`~sharefetch.engine.Engine` this front end drives."""
|
|
633
|
+
|
|
634
|
+
def run(self) -> RunSummary: ...
|
|
635
|
+
|
|
636
|
+
def stop(self) -> None: ...
|
|
637
|
+
|
|
638
|
+
|
|
639
|
+
class _InterruptGuard:
|
|
640
|
+
"""Turns the operator's first Ctrl+C into ``Engine.stop``.
|
|
641
|
+
|
|
642
|
+
Letting ``KeyboardInterrupt`` unwind out of a run cuts the file in flight at
|
|
643
|
+
whatever byte the read had reached and skips the manifest. ``stop`` instead
|
|
644
|
+
ends the queue at the next chunk boundary, flushes the state file and lets
|
|
645
|
+
the run return its summary, which is what Stop means and what leaves the
|
|
646
|
+
least work for the next run to redo.
|
|
647
|
+
|
|
648
|
+
The previous handler is restored the moment the first signal arrives, so a
|
|
649
|
+
second Ctrl+C from an operator who has waited long enough aborts the process
|
|
650
|
+
the usual way.
|
|
651
|
+
|
|
652
|
+
A handler can only be installed from the main thread. Where it cannot be,
|
|
653
|
+
the guard records the reason and :func:`main` falls back to catching
|
|
654
|
+
``KeyboardInterrupt``, which is the behaviour without the guard at all.
|
|
655
|
+
"""
|
|
656
|
+
|
|
657
|
+
def __init__(self, engine: EngineLike) -> None:
|
|
658
|
+
self._engine = engine
|
|
659
|
+
self._previous: Any = None
|
|
660
|
+
self._installed = False
|
|
661
|
+
self.fired = False
|
|
662
|
+
|
|
663
|
+
def _handle(self, signum: int, frame: types.FrameType | None) -> None: # noqa: ARG002
|
|
664
|
+
self.fired = True
|
|
665
|
+
if self._previous is not None:
|
|
666
|
+
signal.signal(signal.SIGINT, self._previous)
|
|
667
|
+
self._installed = False
|
|
668
|
+
logger.warning(
|
|
669
|
+
"interrupt received: finishing the file(s) in flight and flushing state; "
|
|
670
|
+
"interrupt again to abort immediately"
|
|
671
|
+
)
|
|
672
|
+
self._engine.stop()
|
|
673
|
+
|
|
674
|
+
def __enter__(self) -> "_InterruptGuard":
|
|
675
|
+
try:
|
|
676
|
+
self._previous = signal.signal(signal.SIGINT, self._handle)
|
|
677
|
+
except ValueError as exc:
|
|
678
|
+
# signal.signal outside the main thread. Documented, not silent.
|
|
679
|
+
logger.debug("no interrupt handler was installed: %s", exc)
|
|
680
|
+
return self
|
|
681
|
+
self._installed = True
|
|
682
|
+
return self
|
|
683
|
+
|
|
684
|
+
def __exit__(self, exc_type: Any, exc: Any, tb: Any) -> bool:
|
|
685
|
+
if self._installed and self._previous is not None:
|
|
686
|
+
signal.signal(signal.SIGINT, self._previous)
|
|
687
|
+
self._installed = False
|
|
688
|
+
return False
|
|
689
|
+
|
|
690
|
+
|
|
691
|
+
def _print_dry_run_listing(dest: Path) -> None:
|
|
692
|
+
"""Print what a dry run enumerated, read back from the state file it wrote.
|
|
693
|
+
|
|
694
|
+
One of the two places this package prints. The listing is operator output
|
|
695
|
+
on stdout rather than a log line, so it can be piped into a file while the
|
|
696
|
+
log stays on stderr.
|
|
697
|
+
"""
|
|
698
|
+
store = StateStore.load(dest)
|
|
699
|
+
if store is None:
|
|
700
|
+
logger.warning(
|
|
701
|
+
"no readable state file was written under %s, so no dry-run listing can be printed",
|
|
702
|
+
dest,
|
|
703
|
+
)
|
|
704
|
+
return
|
|
705
|
+
|
|
706
|
+
items = store.all_items()
|
|
707
|
+
print(f"{len(items)} item(s) enumerated into {dest}")
|
|
708
|
+
total = 0
|
|
709
|
+
for drive_path in sorted(items):
|
|
710
|
+
item = items[drive_path]
|
|
711
|
+
total += item.size
|
|
712
|
+
print(f" {item.status:<{_STATUS_COLUMN}} {item.size:>{_BYTES_COLUMN},} {drive_path}")
|
|
713
|
+
print(f" {'total':<{_STATUS_COLUMN}} {total:>{_BYTES_COLUMN},} bytes")
|
|
714
|
+
|
|
715
|
+
|
|
716
|
+
def _exit_code(summary: RunSummary) -> int:
|
|
717
|
+
"""Map a summary onto exit code 0 or 1.
|
|
718
|
+
|
|
719
|
+
A skipped item exits 1 alongside a failed one. Nothing malfunctioned, and
|
|
720
|
+
the run still did not bring down everything it was asked for.
|
|
721
|
+
"""
|
|
722
|
+
if summary.failed or summary.skipped:
|
|
723
|
+
logger.warning(
|
|
724
|
+
"the run finished with %d failed and %d skipped item(s)",
|
|
725
|
+
summary.failed,
|
|
726
|
+
summary.skipped,
|
|
727
|
+
)
|
|
728
|
+
return EXIT_INCOMPLETE
|
|
729
|
+
return EXIT_OK
|
|
730
|
+
|
|
731
|
+
|
|
732
|
+
def main(
|
|
733
|
+
argv: list[str] | None = None, *, engine_factory: Callable[..., EngineLike] = Engine
|
|
734
|
+
) -> int:
|
|
735
|
+
"""Run one command line and return its exit code.
|
|
736
|
+
|
|
737
|
+
``engine_factory`` is called with the settings and the sink and defaults to
|
|
738
|
+
:class:`~sharefetch.engine.Engine`. A test replaces it to drive every exit
|
|
739
|
+
code without a socket, a sign-in or an elapsed clock.
|
|
740
|
+
|
|
741
|
+
Returns one of four codes: ``0`` where
|
|
742
|
+
every item is complete and verified, ``1`` where the run finished with a
|
|
743
|
+
failed or skipped item, ``2`` for a fatal condition before or during setup,
|
|
744
|
+
and ``130`` where the operator interrupted it.
|
|
745
|
+
"""
|
|
746
|
+
configure_logging()
|
|
747
|
+
|
|
748
|
+
try:
|
|
749
|
+
settings = parse_args(argv)
|
|
750
|
+
except SystemExit as exc:
|
|
751
|
+
# --help and --version. argparse has already written its output.
|
|
752
|
+
return int(exc.code or 0)
|
|
753
|
+
except SharefetchError as exc:
|
|
754
|
+
logger.error("%s", exc)
|
|
755
|
+
return EXIT_FATAL
|
|
756
|
+
|
|
757
|
+
configure_logging(verbose=settings.verbose, log_file=settings.log_file)
|
|
758
|
+
logger.debug("destination %s", settings.dest)
|
|
759
|
+
|
|
760
|
+
engine = engine_factory(settings, CliSink())
|
|
761
|
+
guard = _InterruptGuard(engine)
|
|
762
|
+
|
|
763
|
+
try:
|
|
764
|
+
with guard:
|
|
765
|
+
summary = engine.run()
|
|
766
|
+
except KeyboardInterrupt:
|
|
767
|
+
# Reached where no handler was installed, or on the second interrupt.
|
|
768
|
+
engine.stop()
|
|
769
|
+
logger.warning("interrupted by the operator; progress on disk stays resumable")
|
|
770
|
+
return EXIT_INTERRUPTED
|
|
771
|
+
except SharefetchError as exc:
|
|
772
|
+
logger.error("%s", exc)
|
|
773
|
+
return EXIT_FATAL
|
|
774
|
+
|
|
775
|
+
if settings.dry_run:
|
|
776
|
+
_print_dry_run_listing(settings.dest)
|
|
777
|
+
|
|
778
|
+
if guard.fired:
|
|
779
|
+
logger.warning("interrupted by the operator; progress on disk stays resumable")
|
|
780
|
+
return EXIT_INTERRUPTED
|
|
781
|
+
|
|
782
|
+
return _exit_code(summary)
|