acadfetch 0.6.0__tar.gz → 0.6.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {acadfetch-0.6.0/acadfetch.egg-info → acadfetch-0.6.2}/PKG-INFO +1 -1
- {acadfetch-0.6.0 → acadfetch-0.6.2}/acadfetch/client.py +13 -2
- {acadfetch-0.6.0 → acadfetch-0.6.2}/acadfetch/consts.py +1 -1
- {acadfetch-0.6.0 → acadfetch-0.6.2}/acadfetch/downloader.py +116 -25
- {acadfetch-0.6.0 → acadfetch-0.6.2}/acadfetch/errors.py +4 -0
- {acadfetch-0.6.0 → acadfetch-0.6.2}/acadfetch/manifest.py +2 -0
- {acadfetch-0.6.0 → acadfetch-0.6.2}/acadfetch/sources/base.py +2 -0
- {acadfetch-0.6.0 → acadfetch-0.6.2}/acadfetch/sources/zenodo.py +4 -0
- {acadfetch-0.6.0 → acadfetch-0.6.2}/acadfetch/verify.py +9 -3
- {acadfetch-0.6.0 → acadfetch-0.6.2/acadfetch.egg-info}/PKG-INFO +1 -1
- {acadfetch-0.6.0 → acadfetch-0.6.2}/acadfetch.egg-info/SOURCES.txt +1 -0
- {acadfetch-0.6.0 → acadfetch-0.6.2}/pyproject.toml +1 -1
- {acadfetch-0.6.0 → acadfetch-0.6.2}/tests/conftest.py +3 -1
- {acadfetch-0.6.0 → acadfetch-0.6.2}/tests/test_downloader.py +57 -0
- acadfetch-0.6.2/tests/test_hash_once.py +129 -0
- {acadfetch-0.6.0 → acadfetch-0.6.2}/tests/test_sources.py +11 -0
- {acadfetch-0.6.0 → acadfetch-0.6.2}/LICENSE +0 -0
- {acadfetch-0.6.0 → acadfetch-0.6.2}/MANIFEST.in +0 -0
- {acadfetch-0.6.0 → acadfetch-0.6.2}/README.md +0 -0
- {acadfetch-0.6.0 → acadfetch-0.6.2}/acadfetch/__init__.py +0 -0
- {acadfetch-0.6.0 → acadfetch-0.6.2}/acadfetch/__main__.py +0 -0
- {acadfetch-0.6.0 → acadfetch-0.6.2}/acadfetch/atomic.py +0 -0
- {acadfetch-0.6.0 → acadfetch-0.6.2}/acadfetch/auth.py +0 -0
- {acadfetch-0.6.0 → acadfetch-0.6.2}/acadfetch/cli.py +0 -0
- {acadfetch-0.6.0 → acadfetch-0.6.2}/acadfetch/debug.py +0 -0
- {acadfetch-0.6.0 → acadfetch-0.6.2}/acadfetch/naming.py +0 -0
- {acadfetch-0.6.0 → acadfetch-0.6.2}/acadfetch/progress.py +0 -0
- {acadfetch-0.6.0 → acadfetch-0.6.2}/acadfetch/py.typed +0 -0
- {acadfetch-0.6.0 → acadfetch-0.6.2}/acadfetch/ratelimit.py +0 -0
- {acadfetch-0.6.0 → acadfetch-0.6.2}/acadfetch/remote.py +0 -0
- {acadfetch-0.6.0 → acadfetch-0.6.2}/acadfetch/sources/__init__.py +0 -0
- {acadfetch-0.6.0 → acadfetch-0.6.2}/acadfetch/sources/arxiv.py +0 -0
- {acadfetch-0.6.0 → acadfetch-0.6.2}/acadfetch/sources/direct.py +0 -0
- {acadfetch-0.6.0 → acadfetch-0.6.2}/acadfetch/sources/doi.py +0 -0
- {acadfetch-0.6.0 → acadfetch-0.6.2}/acadfetch/sources/registry.py +0 -0
- {acadfetch-0.6.0 → acadfetch-0.6.2}/acadfetch/sources/removed.py +0 -0
- {acadfetch-0.6.0 → acadfetch-0.6.2}/acadfetch/units.py +0 -0
- {acadfetch-0.6.0 → acadfetch-0.6.2}/acadfetch/urls.py +0 -0
- {acadfetch-0.6.0 → acadfetch-0.6.2}/acadfetch.egg-info/dependency_links.txt +0 -0
- {acadfetch-0.6.0 → acadfetch-0.6.2}/acadfetch.egg-info/entry_points.txt +0 -0
- {acadfetch-0.6.0 → acadfetch-0.6.2}/acadfetch.egg-info/top_level.txt +0 -0
- {acadfetch-0.6.0 → acadfetch-0.6.2}/setup.cfg +0 -0
- {acadfetch-0.6.0 → acadfetch-0.6.2}/tests/fixtures/README.md +0 -0
- {acadfetch-0.6.0 → acadfetch-0.6.2}/tests/fixtures/ca.pem +0 -0
- {acadfetch-0.6.0 → acadfetch-0.6.2}/tests/fixtures/figshare_403_block.html +0 -0
- {acadfetch-0.6.0 → acadfetch-0.6.2}/tests/fixtures/figshare_403_forbidden.json +0 -0
- {acadfetch-0.6.0 → acadfetch-0.6.2}/tests/fixtures/figshare_article.json +0 -0
- {acadfetch-0.6.0 → acadfetch-0.6.2}/tests/fixtures/figshare_file.pdf +0 -0
- {acadfetch-0.6.0 → acadfetch-0.6.2}/tests/test_atomic.py +0 -0
- {acadfetch-0.6.0 → acadfetch-0.6.2}/tests/test_auth.py +0 -0
- {acadfetch-0.6.0 → acadfetch-0.6.2}/tests/test_checksum.py +0 -0
- {acadfetch-0.6.0 → acadfetch-0.6.2}/tests/test_citation.py +0 -0
- {acadfetch-0.6.0 → acadfetch-0.6.2}/tests/test_cli.py +0 -0
- {acadfetch-0.6.0 → acadfetch-0.6.2}/tests/test_client.py +0 -0
- {acadfetch-0.6.0 → acadfetch-0.6.2}/tests/test_control_names.py +0 -0
- {acadfetch-0.6.0 → acadfetch-0.6.2}/tests/test_debug.py +0 -0
- {acadfetch-0.6.0 → acadfetch-0.6.2}/tests/test_disk.py +0 -0
- {acadfetch-0.6.0 → acadfetch-0.6.2}/tests/test_identity.py +0 -0
- {acadfetch-0.6.0 → acadfetch-0.6.2}/tests/test_isolation.py +0 -0
- {acadfetch-0.6.0 → acadfetch-0.6.2}/tests/test_json.py +0 -0
- {acadfetch-0.6.0 → acadfetch-0.6.2}/tests/test_manifest.py +0 -0
- {acadfetch-0.6.0 → acadfetch-0.6.2}/tests/test_naming.py +0 -0
- {acadfetch-0.6.0 → acadfetch-0.6.2}/tests/test_naming_race.py +0 -0
- {acadfetch-0.6.0 → acadfetch-0.6.2}/tests/test_network.py +0 -0
- {acadfetch-0.6.0 → acadfetch-0.6.2}/tests/test_packaging.py +0 -0
- {acadfetch-0.6.0 → acadfetch-0.6.2}/tests/test_progress.py +0 -0
- {acadfetch-0.6.0 → acadfetch-0.6.2}/tests/test_ratelimit.py +0 -0
- {acadfetch-0.6.0 → acadfetch-0.6.2}/tests/test_redirect.py +0 -0
- {acadfetch-0.6.0 → acadfetch-0.6.2}/tests/test_remote.py +0 -0
- {acadfetch-0.6.0 → acadfetch-0.6.2}/tests/test_removed.py +0 -0
- {acadfetch-0.6.0 → acadfetch-0.6.2}/tests/test_urls.py +0 -0
- {acadfetch-0.6.0 → acadfetch-0.6.2}/tests/test_urls_glob.py +0 -0
- {acadfetch-0.6.0 → acadfetch-0.6.2}/tests/test_verify.py +0 -0
|
@@ -236,7 +236,12 @@ class Client:
|
|
|
236
236
|
if exc.code in RETRY_STATUS:
|
|
237
237
|
pass
|
|
238
238
|
elif not last_blocked:
|
|
239
|
-
raise HttpError(
|
|
239
|
+
raise HttpError(
|
|
240
|
+
exc.code,
|
|
241
|
+
url,
|
|
242
|
+
detail,
|
|
243
|
+
content_range=exc.headers.get("Content-Range") if exc.headers else None,
|
|
244
|
+
) from exc
|
|
240
245
|
elif waf >= block_retries:
|
|
241
246
|
raise HttpError(
|
|
242
247
|
403,
|
|
@@ -270,7 +275,13 @@ class Client:
|
|
|
270
275
|
if last_blocked
|
|
271
276
|
else last_detail
|
|
272
277
|
)
|
|
273
|
-
raise HttpError(
|
|
278
|
+
raise HttpError(
|
|
279
|
+
last.code,
|
|
280
|
+
url,
|
|
281
|
+
detail,
|
|
282
|
+
transient=True,
|
|
283
|
+
content_range=last.headers.get("Content-Range") if last.headers else None,
|
|
284
|
+
) from last
|
|
274
285
|
raise HttpError(0, url, last_detail or str(last)) from last
|
|
275
286
|
|
|
276
287
|
# -- public ------------------------------------------------------------
|
|
@@ -14,6 +14,7 @@ from __future__ import annotations
|
|
|
14
14
|
|
|
15
15
|
import contextlib
|
|
16
16
|
import fnmatch
|
|
17
|
+
import hashlib
|
|
17
18
|
import json
|
|
18
19
|
import os
|
|
19
20
|
import random
|
|
@@ -83,6 +84,9 @@ class Transfer:
|
|
|
83
84
|
etag: str | None = None
|
|
84
85
|
http_status: int | None = None
|
|
85
86
|
retrieved_at: str | None = None
|
|
87
|
+
#: Algorithm -> hex digest, computed while the bytes went by. ``None`` when
|
|
88
|
+
#: no body was read (a 416 on a complete part).
|
|
89
|
+
digests: dict[str, str] | None = None
|
|
86
90
|
|
|
87
91
|
|
|
88
92
|
@dataclass
|
|
@@ -236,6 +240,46 @@ def _chunks(resp, idle_timeout: float, url: str): # noqa: ANN001, ANN202
|
|
|
236
240
|
yield chunk
|
|
237
241
|
|
|
238
242
|
|
|
243
|
+
def _star_total(content_range: str | None) -> int | None:
|
|
244
|
+
"""``bytes */N`` → N, for the 416 that carries it."""
|
|
245
|
+
if not content_range:
|
|
246
|
+
return None
|
|
247
|
+
_, _, value = content_range.strip().partition("*/")
|
|
248
|
+
try:
|
|
249
|
+
return int(value.strip())
|
|
250
|
+
except ValueError:
|
|
251
|
+
return None
|
|
252
|
+
|
|
253
|
+
|
|
254
|
+
def _restart_or_finish(
|
|
255
|
+
file: File, part: Path, meta_path: Path, have: int, exc: HttpError
|
|
256
|
+
) -> Transfer:
|
|
257
|
+
"""What a 416 means: the part is the file, or it is stale and must go.
|
|
258
|
+
|
|
259
|
+
"Already complete" used to be assumed whenever the part existed, so a stale
|
|
260
|
+
part (a different, older object) was renamed into place with nothing
|
|
261
|
+
checked. A matching checksum is proof; otherwise the size the server
|
|
262
|
+
reports for the whole object has to equal the part exactly.
|
|
263
|
+
"""
|
|
264
|
+
parsed = verify.parse(file.checksum)
|
|
265
|
+
digests: dict[str, str] | None = None
|
|
266
|
+
if parsed is not None:
|
|
267
|
+
digests = verify.hashes(part, {"sha256", parsed[0]})
|
|
268
|
+
if digests[parsed[0]] == parsed[1]:
|
|
269
|
+
return Transfer(0, digests=digests)
|
|
270
|
+
total = _star_total(exc.content_range)
|
|
271
|
+
if total is not None and total == have:
|
|
272
|
+
if digests is None:
|
|
273
|
+
digests = verify.hashes(part, {"sha256"})
|
|
274
|
+
return Transfer(0, digests=digests)
|
|
275
|
+
_drop(part, meta_path)
|
|
276
|
+
if total is None:
|
|
277
|
+
raised, compared = "416 without a size to compare against", "no evidence"
|
|
278
|
+
else:
|
|
279
|
+
raised, compared = f"server says {total} bytes", f"the .part has {have}"
|
|
280
|
+
raise HttpError(416, file.url, f"{raised}, {compared}; restarting", transient=True)
|
|
281
|
+
|
|
282
|
+
|
|
239
283
|
def _transfer(client: Client, file: File, part: Path, meta_path: Path, opts: Options) -> Transfer:
|
|
240
284
|
"""Move bytes into ``part``, resuming if possible. Returns what happened."""
|
|
241
285
|
meta = _read_meta(meta_path) if part.exists() else {}
|
|
@@ -262,7 +306,7 @@ def _transfer(client: Client, file: File, part: Path, meta_path: Path, opts: Opt
|
|
|
262
306
|
resp = client.open(file.url, headers, attempts=1)
|
|
263
307
|
except HttpError as exc:
|
|
264
308
|
if exc.status == 416 and have:
|
|
265
|
-
return
|
|
309
|
+
return _restart_or_finish(file, part, meta_path, have, exc)
|
|
266
310
|
raise
|
|
267
311
|
|
|
268
312
|
with resp:
|
|
@@ -294,6 +338,19 @@ def _transfer(client: Client, file: File, part: Path, meta_path: Path, opts: Opt
|
|
|
294
338
|
mode, offset = "wb", 0
|
|
295
339
|
else:
|
|
296
340
|
mode, offset = "ab", have
|
|
341
|
+
|
|
342
|
+
# Hash while the bytes go by instead of reading the finished file again.
|
|
343
|
+
# A resumed transfer has to be fed what is already on disk first.
|
|
344
|
+
parsed = verify.parse(file.checksum)
|
|
345
|
+
wanted_algo = parsed[0] if parsed else None
|
|
346
|
+
hashers = {"sha256": hashlib.sha256()}
|
|
347
|
+
if wanted_algo and wanted_algo != "sha256":
|
|
348
|
+
hashers[wanted_algo] = hashlib.new(wanted_algo)
|
|
349
|
+
if mode == "ab":
|
|
350
|
+
with part.open("rb") as existing:
|
|
351
|
+
for old in iter(lambda: existing.read(1 << 20), b""):
|
|
352
|
+
for hasher in hashers.values():
|
|
353
|
+
hasher.update(old)
|
|
297
354
|
_write_meta(
|
|
298
355
|
meta_path,
|
|
299
356
|
{
|
|
@@ -323,6 +380,8 @@ def _transfer(client: Client, file: File, part: Path, meta_path: Path, opts: Opt
|
|
|
323
380
|
with part.open(mode) as fh:
|
|
324
381
|
for chunk in _chunks(resp, client.timeout, file.url):
|
|
325
382
|
fh.write(chunk)
|
|
383
|
+
for hasher in hashers.values():
|
|
384
|
+
hasher.update(chunk)
|
|
326
385
|
written += len(chunk)
|
|
327
386
|
if opts.progress:
|
|
328
387
|
opts.progress.advance(len(chunk))
|
|
@@ -350,6 +409,7 @@ def _transfer(client: Client, file: File, part: Path, meta_path: Path, opts: Opt
|
|
|
350
409
|
etag=resp.headers.get("ETag"),
|
|
351
410
|
http_status=status,
|
|
352
411
|
retrieved_at=_now(),
|
|
412
|
+
digests={algo: digest.hexdigest() for algo, digest in hashers.items()},
|
|
353
413
|
)
|
|
354
414
|
|
|
355
415
|
|
|
@@ -427,24 +487,38 @@ def _reserved(files) -> set[str]: # noqa: ANN001
|
|
|
427
487
|
return reserved
|
|
428
488
|
|
|
429
489
|
|
|
430
|
-
def
|
|
431
|
-
|
|
490
|
+
def _verdict(file: File, digests: dict[str, str] | None) -> bool | None:
|
|
491
|
+
parsed = verify.parse(file.checksum)
|
|
492
|
+
if parsed is None:
|
|
493
|
+
return None
|
|
494
|
+
if not digests:
|
|
495
|
+
return None
|
|
496
|
+
return digests.get(parsed[0]) == parsed[1]
|
|
497
|
+
|
|
498
|
+
|
|
499
|
+
def _already_there(
|
|
500
|
+
final: Path, name: str, file: File, opts: Options
|
|
501
|
+
) -> tuple[bool, dict[str, str] | None]:
|
|
502
|
+
"""Whether an existing file may stand in for this one, and its digests.
|
|
432
503
|
|
|
433
504
|
A checksum match is proof. Without a checksum, a size match only means the
|
|
434
505
|
bytes *could* be the same, so one more thing must tie the file to this URL:
|
|
435
|
-
an explicit ``-o`` (the user chose the directory)
|
|
436
|
-
recorded the same URL
|
|
506
|
+
an explicit ``-o`` (the user chose the directory) or a manifest in it that
|
|
507
|
+
recorded the same URL. Both hashes are computed in that one read, so the
|
|
508
|
+
manifest does not have to read the file again.
|
|
437
509
|
"""
|
|
438
510
|
if opts.force or not final.exists():
|
|
439
|
-
return None
|
|
440
|
-
|
|
441
|
-
if
|
|
442
|
-
|
|
511
|
+
return False, None
|
|
512
|
+
parsed = verify.parse(file.checksum)
|
|
513
|
+
algos = {"sha256"} | ({parsed[0]} if parsed else set())
|
|
514
|
+
digests = verify.hashes(final, algos)
|
|
515
|
+
if parsed is not None:
|
|
516
|
+
return digests[parsed[0]] == parsed[1], digests
|
|
443
517
|
if file.size is not None and final.stat().st_size != file.size:
|
|
444
|
-
return False
|
|
518
|
+
return False, digests
|
|
445
519
|
if opts.explicit_out:
|
|
446
|
-
return True
|
|
447
|
-
return opts.trusted.get(name) == file.url
|
|
520
|
+
return True, digests
|
|
521
|
+
return opts.trusted.get(name) == file.url, digests
|
|
448
522
|
|
|
449
523
|
|
|
450
524
|
def fetch_file(client: Client, file: File, opts: Options, taken: set[str]) -> Result:
|
|
@@ -453,14 +527,16 @@ def fetch_file(client: Client, file: File, opts: Options, taken: set[str]) -> Re
|
|
|
453
527
|
part = opts.out / (name + ".part")
|
|
454
528
|
meta_path = opts.out / (name + META_SUFFIX)
|
|
455
529
|
|
|
456
|
-
|
|
530
|
+
trusted, digests = _already_there(final, name, file, opts)
|
|
531
|
+
if trusted:
|
|
457
532
|
return Result(
|
|
458
533
|
file=file,
|
|
459
534
|
state="skipped",
|
|
460
535
|
path=str(final),
|
|
461
536
|
bytes=final.stat().st_size,
|
|
462
|
-
verified=
|
|
537
|
+
verified=_verdict(file, digests),
|
|
463
538
|
name=name,
|
|
539
|
+
sha256=(digests or {}).get("sha256"),
|
|
464
540
|
)
|
|
465
541
|
|
|
466
542
|
if opts.dry_run:
|
|
@@ -574,12 +650,12 @@ def _attempt(
|
|
|
574
650
|
retrieved_at=_now(),
|
|
575
651
|
)
|
|
576
652
|
|
|
577
|
-
verdict =
|
|
653
|
+
verdict = _verdict(file, info.digests if info else None)
|
|
578
654
|
if verdict is False:
|
|
579
655
|
_drop(part, meta_path)
|
|
580
656
|
try:
|
|
581
657
|
info = _transfer(client, file, part, meta_path, opts)
|
|
582
|
-
verdict =
|
|
658
|
+
verdict = _verdict(file, info.digests if info else None)
|
|
583
659
|
except Exception as exc: # noqa: BLE001
|
|
584
660
|
return Result(
|
|
585
661
|
file=file,
|
|
@@ -603,6 +679,7 @@ def _attempt(
|
|
|
603
679
|
bytes=final.stat().st_size,
|
|
604
680
|
verified=verdict,
|
|
605
681
|
resumed_from=resumed_from,
|
|
682
|
+
sha256=(info.digests or {}).get("sha256") if info else None,
|
|
606
683
|
final_url=info.final_url if info else None,
|
|
607
684
|
etag=info.etag if info else None,
|
|
608
685
|
http_status=info.http_status if info else None,
|
|
@@ -685,17 +762,31 @@ def _fetch_record(
|
|
|
685
762
|
return _in_record_order(results, record)
|
|
686
763
|
|
|
687
764
|
pool = ThreadPoolExecutor(max_workers=opts.concurrency)
|
|
765
|
+
# Keep the in-flight set bounded: a 5000-file record submitted in one go
|
|
766
|
+
# holds 5000 futures and progress keys at once.
|
|
767
|
+
window = max(opts.concurrency * 2, 1)
|
|
768
|
+
queue = iter(files)
|
|
688
769
|
futures = {}
|
|
770
|
+
|
|
771
|
+
def submit(file: File) -> None: # noqa: ANN001
|
|
772
|
+
with lock:
|
|
773
|
+
futures[pool.submit(fetch_file, client, file, opts, taken)] = file
|
|
774
|
+
|
|
689
775
|
try:
|
|
690
|
-
for file in
|
|
691
|
-
|
|
692
|
-
|
|
693
|
-
|
|
694
|
-
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
|
|
776
|
+
for file in queue:
|
|
777
|
+
submit(file)
|
|
778
|
+
if len(futures) >= window:
|
|
779
|
+
break
|
|
780
|
+
while futures:
|
|
781
|
+
for future in as_completed(list(futures)):
|
|
782
|
+
file = futures.pop(future)
|
|
783
|
+
try:
|
|
784
|
+
results.append(future.result())
|
|
785
|
+
except Exception as exc: # noqa: BLE001 - one worker must not end the batch
|
|
786
|
+
results.append(_crashed(file, exc))
|
|
787
|
+
nxt = next(queue, None)
|
|
788
|
+
if nxt is not None:
|
|
789
|
+
submit(nxt)
|
|
699
790
|
except KeyboardInterrupt:
|
|
700
791
|
# Abort the reads that are blocked, then let the workers unwind: they
|
|
701
792
|
# see the stop flag and keep their .part instead of retrying.
|
|
@@ -30,6 +30,7 @@ class HttpError(AcFError):
|
|
|
30
30
|
detail: str = "",
|
|
31
31
|
transient: bool = False,
|
|
32
32
|
reason: str = "",
|
|
33
|
+
content_range: str | None = None,
|
|
33
34
|
) -> None:
|
|
34
35
|
self.status = status
|
|
35
36
|
self.url = url
|
|
@@ -37,6 +38,9 @@ class HttpError(AcFError):
|
|
|
37
38
|
#: Machine-readable cause for callers that must branch on it, e.g.
|
|
38
39
|
#: ``html-page``: the body is a block/login page, not the file asked for.
|
|
39
40
|
self.reason = reason
|
|
41
|
+
#: ``Content-Range`` of the answer, so a 416 can say how big the object
|
|
42
|
+
#: is supposed to be.
|
|
43
|
+
self.content_range = content_range
|
|
40
44
|
if status:
|
|
41
45
|
text = f"HTTP {status} for {url}{': ' + detail if detail else ''}"
|
|
42
46
|
elif reason == "max-size":
|
|
@@ -135,6 +135,8 @@ def _entry( # noqa: ANN001
|
|
|
135
135
|
def _sha256(result: Result | None) -> str | None:
|
|
136
136
|
if result is None or not result.path:
|
|
137
137
|
return None
|
|
138
|
+
if result.sha256:
|
|
139
|
+
return result.sha256
|
|
138
140
|
path = Path(result.path)
|
|
139
141
|
if not path.is_file():
|
|
140
142
|
return None
|
|
@@ -58,6 +58,8 @@ class Result:
|
|
|
58
58
|
http_status: int | None = None
|
|
59
59
|
#: RFC 3339 UTC, when the last byte (or the failure) arrived.
|
|
60
60
|
retrieved_at: str | None = None
|
|
61
|
+
#: SHA-256 of the bytes, computed during the transfer or the reuse check.
|
|
62
|
+
sha256: str | None = None
|
|
61
63
|
|
|
62
64
|
@property
|
|
63
65
|
def ok(self) -> bool:
|
|
@@ -3,6 +3,10 @@
|
|
|
3
3
|
The public REST API exposes ``files[].links.self`` as a redirecting download
|
|
4
4
|
URL and publishes ``files[].checksum`` as ``md5:<hex>``, which is what makes
|
|
5
5
|
verification possible here.
|
|
6
|
+
|
|
7
|
+
``is_latest`` / ``links.latest`` are honoured for an older version DOI. The
|
|
8
|
+
current API does not send ``is_latest`` (checked 2026-11), so the branch is
|
|
9
|
+
dormant; a test pins what it would do if it returned.
|
|
6
10
|
"""
|
|
7
11
|
|
|
8
12
|
from __future__ import annotations
|
|
@@ -41,11 +41,17 @@ def _guess(candidate: str) -> str:
|
|
|
41
41
|
|
|
42
42
|
|
|
43
43
|
def hash_file(path: Path, algo: str) -> str:
|
|
44
|
-
|
|
44
|
+
return hashes(path, {algo})[algo]
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def hashes(path: Path, algos: set[str]) -> dict[str, str]:
|
|
48
|
+
"""Every requested digest in one pass over the file."""
|
|
49
|
+
digests = {algo: hashlib.new(algo) for algo in algos}
|
|
45
50
|
with path.open("rb") as fh:
|
|
46
51
|
for chunk in iter(lambda: fh.read(1 << 20), b""):
|
|
47
|
-
digest.
|
|
48
|
-
|
|
52
|
+
for digest in digests.values():
|
|
53
|
+
digest.update(chunk)
|
|
54
|
+
return {algo: digest.hexdigest() for algo, digest in digests.items()}
|
|
49
55
|
|
|
50
56
|
|
|
51
57
|
def matches(path: Path, checksum: str | None) -> bool | None:
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "acadfetch"
|
|
7
|
-
version = "0.6.
|
|
7
|
+
version = "0.6.2"
|
|
8
8
|
description = "Resumable, rate-limited, checksum-verified bulk downloads from academic data repositories"
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.10"
|
|
@@ -55,7 +55,8 @@ class Handler(BaseHTTPRequestHandler):
|
|
|
55
55
|
return
|
|
56
56
|
if cfg["answers_416"] and self.headers.get("Range"):
|
|
57
57
|
self.send_response(416)
|
|
58
|
-
|
|
58
|
+
if not cfg.get("416_omits_range"):
|
|
59
|
+
self.send_header("Content-Range", f"bytes */{len(cfg['files'].get(path, b''))}")
|
|
59
60
|
self.send_header("Content-Length", "0")
|
|
60
61
|
self.end_headers()
|
|
61
62
|
return
|
|
@@ -147,6 +148,7 @@ def _new_server():
|
|
|
147
148
|
"declares": {},
|
|
148
149
|
"types": {},
|
|
149
150
|
"answers_416": False,
|
|
151
|
+
"416_omits_range": False,
|
|
150
152
|
"ranges": [],
|
|
151
153
|
"stall_after": None,
|
|
152
154
|
"stall_for": 2.0,
|
|
@@ -588,3 +588,60 @@ def test_a_failed_transfer_records_the_status(server, tmp_path):
|
|
|
588
588
|
assert res.state == "failed"
|
|
589
589
|
assert res.http_status == 500
|
|
590
590
|
assert res.retrieved_at
|
|
591
|
+
|
|
592
|
+
|
|
593
|
+
# -- 416: only evidence makes the kept part the answer ---------------------
|
|
594
|
+
|
|
595
|
+
|
|
596
|
+
def _stale_part(tmp_path, url, body: bytes):
|
|
597
|
+
(tmp_path / "a.bin.part").write_bytes(body)
|
|
598
|
+
(tmp_path / "a.bin.part.meta.json").write_text(f'{{"url": "{url}", "etag": ""v1""}}', "utf-8")
|
|
599
|
+
|
|
600
|
+
|
|
601
|
+
def test_a_stale_part_is_restarted_not_renamed(server, tmp_path):
|
|
602
|
+
"""A part from an older object is bigger than the file; 416 must not win."""
|
|
603
|
+
payload = b"new" * 1000
|
|
604
|
+
server.cfg["files"]["/a.bin"] = payload
|
|
605
|
+
url = server.base + "/a.bin"
|
|
606
|
+
_stale_part(tmp_path, url, b"old" * 2000) # 6000 bytes vs 3000
|
|
607
|
+
server.cfg["answers_416"] = True
|
|
608
|
+
|
|
609
|
+
res = downloader.fetch_file(
|
|
610
|
+
Client(rate=1000, retries=1), File(url, "a.bin"), opts(tmp_path, retries=1), set()
|
|
611
|
+
)
|
|
612
|
+
|
|
613
|
+
assert res.state == "downloaded", res.error
|
|
614
|
+
assert (tmp_path / "a.bin").read_bytes() == payload
|
|
615
|
+
|
|
616
|
+
|
|
617
|
+
def test_a_416_without_a_size_and_without_checksum_restarts(server, tmp_path):
|
|
618
|
+
payload = b"fresh bytes"
|
|
619
|
+
server.cfg["files"]["/a.bin"] = payload
|
|
620
|
+
url = server.base + "/a.bin"
|
|
621
|
+
_stale_part(tmp_path, url, b"stale")
|
|
622
|
+
server.cfg["answers_416"] = True
|
|
623
|
+
server.cfg["416_omits_range"] = True
|
|
624
|
+
|
|
625
|
+
res = downloader.fetch_file(
|
|
626
|
+
Client(rate=1000, retries=1), File(url, "a.bin"), opts(tmp_path, retries=1), set()
|
|
627
|
+
)
|
|
628
|
+
|
|
629
|
+
assert res.state == "downloaded", res.error
|
|
630
|
+
assert (tmp_path / "a.bin").read_bytes() == payload
|
|
631
|
+
|
|
632
|
+
|
|
633
|
+
def test_a_416_with_a_matching_checksum_accepts_the_part(server, tmp_path):
|
|
634
|
+
"""Proof beats a missing Content-Range."""
|
|
635
|
+
payload = b"proven bytes"
|
|
636
|
+
server.cfg["files"]["/a.bin"] = payload
|
|
637
|
+
url = server.base + "/a.bin"
|
|
638
|
+
_stale_part(tmp_path, url, payload)
|
|
639
|
+
server.cfg["answers_416"] = True
|
|
640
|
+
server.cfg["416_omits_range"] = True
|
|
641
|
+
f = File(url, "a.bin", checksum="sha256:" + hashlib.sha256(payload).hexdigest())
|
|
642
|
+
|
|
643
|
+
res = downloader.fetch_file(Client(rate=1000, retries=0), f, opts(tmp_path, retries=0), set())
|
|
644
|
+
|
|
645
|
+
assert res.state == "downloaded", res.error
|
|
646
|
+
assert res.verified is True
|
|
647
|
+
assert server.cfg["hits"]["/a.bin"] == 1, "no refetch when the checksum proves it"
|
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import hashlib
|
|
4
|
+
import json
|
|
5
|
+
import threading
|
|
6
|
+
import time
|
|
7
|
+
|
|
8
|
+
from acadfetch import downloader, verify
|
|
9
|
+
from acadfetch.client import Client
|
|
10
|
+
from acadfetch.sources.base import File, Record
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def _opts(tmp_path, **kw):
|
|
14
|
+
kw.setdefault("concurrency", 1)
|
|
15
|
+
return downloader.Options(out=tmp_path, **kw)
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def test_hashes_computes_every_algo_in_one_pass(tmp_path):
|
|
19
|
+
path = tmp_path / "f.bin"
|
|
20
|
+
payload = b"one pass"
|
|
21
|
+
path.write_bytes(payload)
|
|
22
|
+
|
|
23
|
+
got = verify.hashes(path, {"sha256", "md5"})
|
|
24
|
+
|
|
25
|
+
assert got["sha256"] == hashlib.sha256(payload).hexdigest()
|
|
26
|
+
assert got["md5"] == hashlib.md5(payload).hexdigest()
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def test_a_fetch_never_rehashes_the_finished_file(server, tmp_path, monkeypatch):
|
|
30
|
+
payload = b"payload" * 500
|
|
31
|
+
server.cfg["files"]["/a.bin"] = payload
|
|
32
|
+
f = File(
|
|
33
|
+
server.base + "/a.bin",
|
|
34
|
+
"a.bin",
|
|
35
|
+
len(payload),
|
|
36
|
+
"sha256:" + hashlib.sha256(payload).hexdigest(),
|
|
37
|
+
)
|
|
38
|
+
|
|
39
|
+
def boom(*_a, **_k): # pragma: no cover - proves it is not reached
|
|
40
|
+
raise AssertionError("hash_file was called")
|
|
41
|
+
|
|
42
|
+
monkeypatch.setattr(verify, "hash_file", boom)
|
|
43
|
+
res = downloader.fetch_file(Client(rate=1000, retries=0), f, _opts(tmp_path), set())
|
|
44
|
+
|
|
45
|
+
assert res.state == "downloaded"
|
|
46
|
+
assert res.verified is True
|
|
47
|
+
assert res.sha256 == hashlib.sha256(payload).hexdigest()
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def test_a_resumed_transfer_hashes_the_bytes_already_on_disk(server, tmp_path):
|
|
51
|
+
payload = bytes(range(256)) * 40
|
|
52
|
+
server.cfg["files"]["/a.bin"] = payload
|
|
53
|
+
url = server.base + "/a.bin"
|
|
54
|
+
(tmp_path / "a.bin.part").write_bytes(payload[:2000])
|
|
55
|
+
(tmp_path / "a.bin.part.meta.json").write_text(
|
|
56
|
+
json.dumps({"url": url, "etag": '"v1"', "size": len(payload)}), "utf-8"
|
|
57
|
+
)
|
|
58
|
+
f = File(url, "a.bin", len(payload), "sha256:" + hashlib.sha256(payload).hexdigest())
|
|
59
|
+
|
|
60
|
+
res = downloader.fetch_file(Client(rate=1000, retries=0), f, _opts(tmp_path), set())
|
|
61
|
+
|
|
62
|
+
assert res.state == "downloaded", res.error
|
|
63
|
+
assert res.resumed_from == 2000
|
|
64
|
+
assert res.verified is True
|
|
65
|
+
assert res.sha256 == hashlib.sha256(payload).hexdigest()
|
|
66
|
+
assert (tmp_path / "a.bin").read_bytes() == payload
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def test_a_reused_file_is_read_once(server, tmp_path, monkeypatch):
|
|
70
|
+
payload = b"reuse me"
|
|
71
|
+
server.cfg["files"]["/a.bin"] = payload
|
|
72
|
+
url = server.base + "/a.bin"
|
|
73
|
+
(tmp_path / "a.bin").write_bytes(payload)
|
|
74
|
+
(tmp_path / "manifest.json").write_text(
|
|
75
|
+
json.dumps({"schema": 3, "files": [{"name": "a.bin", "url": url}]}), "utf-8"
|
|
76
|
+
)
|
|
77
|
+
calls = []
|
|
78
|
+
real = verify.hashes
|
|
79
|
+
|
|
80
|
+
def counting(path, algos):
|
|
81
|
+
calls.append(algos)
|
|
82
|
+
return real(path, algos)
|
|
83
|
+
|
|
84
|
+
monkeypatch.setattr(verify, "hashes", counting)
|
|
85
|
+
monkeypatch.setattr(
|
|
86
|
+
verify, "hash_file", lambda *_a, **_k: (_ for _ in ()).throw(AssertionError("rehash"))
|
|
87
|
+
)
|
|
88
|
+
|
|
89
|
+
opts = _opts(tmp_path)
|
|
90
|
+
opts.trusted = {"a.bin": url} # what fetch_record loads from the manifest
|
|
91
|
+
res = downloader.fetch_file(
|
|
92
|
+
Client(rate=1000, retries=0), File(url, "a.bin", len(payload)), opts, set()
|
|
93
|
+
)
|
|
94
|
+
|
|
95
|
+
assert res.state == "skipped"
|
|
96
|
+
assert len(calls) == 1, "one read produces the verdict and the manifest hash"
|
|
97
|
+
assert res.sha256 == hashlib.sha256(payload).hexdigest()
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def test_the_in_flight_window_is_bounded(server, tmp_path, monkeypatch):
|
|
101
|
+
files = []
|
|
102
|
+
for i in range(30):
|
|
103
|
+
server.cfg["files"][f"/{i}.bin"] = b"x"
|
|
104
|
+
files.append(File(server.base + f"/{i}.bin", f"{i}.bin", 1))
|
|
105
|
+
record = Record(source="direct", target="t", files=files)
|
|
106
|
+
|
|
107
|
+
state = {"now": 0, "max": 0}
|
|
108
|
+
lock = threading.Lock()
|
|
109
|
+
real = downloader.fetch_file
|
|
110
|
+
|
|
111
|
+
def watched(client, file, opts, taken):
|
|
112
|
+
with lock:
|
|
113
|
+
state["now"] += 1
|
|
114
|
+
state["max"] = max(state["max"], state["now"])
|
|
115
|
+
try:
|
|
116
|
+
time.sleep(0.01)
|
|
117
|
+
return real(client, file, opts, taken)
|
|
118
|
+
finally:
|
|
119
|
+
with lock:
|
|
120
|
+
state["now"] -= 1
|
|
121
|
+
|
|
122
|
+
monkeypatch.setattr(downloader, "fetch_file", watched)
|
|
123
|
+
results = downloader.fetch_record(
|
|
124
|
+
record, downloader.Options(out=tmp_path, concurrency=3), Client(rate=10000)
|
|
125
|
+
)
|
|
126
|
+
|
|
127
|
+
assert len(results) == 30
|
|
128
|
+
assert state["max"] <= 6, state["max"]
|
|
129
|
+
assert state["max"] >= 2
|
|
@@ -320,3 +320,14 @@ def test_two_legacy_papers_no_longer_share_a_name():
|
|
|
320
320
|
def test_direct_resolve_rejects_a_non_url():
|
|
321
321
|
with pytest.raises(TargetError, match="not a URL"):
|
|
322
322
|
direct.resolve(HeadClient({}), "zenodo:1")
|
|
323
|
+
|
|
324
|
+
|
|
325
|
+
def test_an_older_zenodo_version_follows_latest():
|
|
326
|
+
"""The API no longer sends ``is_latest``, but if a source does, follow it."""
|
|
327
|
+
old = dict(ZENODO, is_latest=False, links={"latest": "https://zenodo.org/api/records/99"})
|
|
328
|
+
new = dict(ZENODO, id=99, doi="10.5281/zenodo.99")
|
|
329
|
+
record = zenodo.resolve(FakeClient({"records/1": old, "records/99": new}), "10.5281/zenodo.1")
|
|
330
|
+
|
|
331
|
+
assert record.doi == "10.5281/zenodo.99"
|
|
332
|
+
assert record.files[0].name == "a.csv"
|
|
333
|
+
assert record.target == "10.5281/zenodo.1", "the manifest keeps what was asked for"
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|