acadfetch 0.6.0__tar.gz → 0.6.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (73) hide show
  1. {acadfetch-0.6.0/acadfetch.egg-info → acadfetch-0.6.2}/PKG-INFO +1 -1
  2. {acadfetch-0.6.0 → acadfetch-0.6.2}/acadfetch/client.py +13 -2
  3. {acadfetch-0.6.0 → acadfetch-0.6.2}/acadfetch/consts.py +1 -1
  4. {acadfetch-0.6.0 → acadfetch-0.6.2}/acadfetch/downloader.py +116 -25
  5. {acadfetch-0.6.0 → acadfetch-0.6.2}/acadfetch/errors.py +4 -0
  6. {acadfetch-0.6.0 → acadfetch-0.6.2}/acadfetch/manifest.py +2 -0
  7. {acadfetch-0.6.0 → acadfetch-0.6.2}/acadfetch/sources/base.py +2 -0
  8. {acadfetch-0.6.0 → acadfetch-0.6.2}/acadfetch/sources/zenodo.py +4 -0
  9. {acadfetch-0.6.0 → acadfetch-0.6.2}/acadfetch/verify.py +9 -3
  10. {acadfetch-0.6.0 → acadfetch-0.6.2/acadfetch.egg-info}/PKG-INFO +1 -1
  11. {acadfetch-0.6.0 → acadfetch-0.6.2}/acadfetch.egg-info/SOURCES.txt +1 -0
  12. {acadfetch-0.6.0 → acadfetch-0.6.2}/pyproject.toml +1 -1
  13. {acadfetch-0.6.0 → acadfetch-0.6.2}/tests/conftest.py +3 -1
  14. {acadfetch-0.6.0 → acadfetch-0.6.2}/tests/test_downloader.py +57 -0
  15. acadfetch-0.6.2/tests/test_hash_once.py +129 -0
  16. {acadfetch-0.6.0 → acadfetch-0.6.2}/tests/test_sources.py +11 -0
  17. {acadfetch-0.6.0 → acadfetch-0.6.2}/LICENSE +0 -0
  18. {acadfetch-0.6.0 → acadfetch-0.6.2}/MANIFEST.in +0 -0
  19. {acadfetch-0.6.0 → acadfetch-0.6.2}/README.md +0 -0
  20. {acadfetch-0.6.0 → acadfetch-0.6.2}/acadfetch/__init__.py +0 -0
  21. {acadfetch-0.6.0 → acadfetch-0.6.2}/acadfetch/__main__.py +0 -0
  22. {acadfetch-0.6.0 → acadfetch-0.6.2}/acadfetch/atomic.py +0 -0
  23. {acadfetch-0.6.0 → acadfetch-0.6.2}/acadfetch/auth.py +0 -0
  24. {acadfetch-0.6.0 → acadfetch-0.6.2}/acadfetch/cli.py +0 -0
  25. {acadfetch-0.6.0 → acadfetch-0.6.2}/acadfetch/debug.py +0 -0
  26. {acadfetch-0.6.0 → acadfetch-0.6.2}/acadfetch/naming.py +0 -0
  27. {acadfetch-0.6.0 → acadfetch-0.6.2}/acadfetch/progress.py +0 -0
  28. {acadfetch-0.6.0 → acadfetch-0.6.2}/acadfetch/py.typed +0 -0
  29. {acadfetch-0.6.0 → acadfetch-0.6.2}/acadfetch/ratelimit.py +0 -0
  30. {acadfetch-0.6.0 → acadfetch-0.6.2}/acadfetch/remote.py +0 -0
  31. {acadfetch-0.6.0 → acadfetch-0.6.2}/acadfetch/sources/__init__.py +0 -0
  32. {acadfetch-0.6.0 → acadfetch-0.6.2}/acadfetch/sources/arxiv.py +0 -0
  33. {acadfetch-0.6.0 → acadfetch-0.6.2}/acadfetch/sources/direct.py +0 -0
  34. {acadfetch-0.6.0 → acadfetch-0.6.2}/acadfetch/sources/doi.py +0 -0
  35. {acadfetch-0.6.0 → acadfetch-0.6.2}/acadfetch/sources/registry.py +0 -0
  36. {acadfetch-0.6.0 → acadfetch-0.6.2}/acadfetch/sources/removed.py +0 -0
  37. {acadfetch-0.6.0 → acadfetch-0.6.2}/acadfetch/units.py +0 -0
  38. {acadfetch-0.6.0 → acadfetch-0.6.2}/acadfetch/urls.py +0 -0
  39. {acadfetch-0.6.0 → acadfetch-0.6.2}/acadfetch.egg-info/dependency_links.txt +0 -0
  40. {acadfetch-0.6.0 → acadfetch-0.6.2}/acadfetch.egg-info/entry_points.txt +0 -0
  41. {acadfetch-0.6.0 → acadfetch-0.6.2}/acadfetch.egg-info/top_level.txt +0 -0
  42. {acadfetch-0.6.0 → acadfetch-0.6.2}/setup.cfg +0 -0
  43. {acadfetch-0.6.0 → acadfetch-0.6.2}/tests/fixtures/README.md +0 -0
  44. {acadfetch-0.6.0 → acadfetch-0.6.2}/tests/fixtures/ca.pem +0 -0
  45. {acadfetch-0.6.0 → acadfetch-0.6.2}/tests/fixtures/figshare_403_block.html +0 -0
  46. {acadfetch-0.6.0 → acadfetch-0.6.2}/tests/fixtures/figshare_403_forbidden.json +0 -0
  47. {acadfetch-0.6.0 → acadfetch-0.6.2}/tests/fixtures/figshare_article.json +0 -0
  48. {acadfetch-0.6.0 → acadfetch-0.6.2}/tests/fixtures/figshare_file.pdf +0 -0
  49. {acadfetch-0.6.0 → acadfetch-0.6.2}/tests/test_atomic.py +0 -0
  50. {acadfetch-0.6.0 → acadfetch-0.6.2}/tests/test_auth.py +0 -0
  51. {acadfetch-0.6.0 → acadfetch-0.6.2}/tests/test_checksum.py +0 -0
  52. {acadfetch-0.6.0 → acadfetch-0.6.2}/tests/test_citation.py +0 -0
  53. {acadfetch-0.6.0 → acadfetch-0.6.2}/tests/test_cli.py +0 -0
  54. {acadfetch-0.6.0 → acadfetch-0.6.2}/tests/test_client.py +0 -0
  55. {acadfetch-0.6.0 → acadfetch-0.6.2}/tests/test_control_names.py +0 -0
  56. {acadfetch-0.6.0 → acadfetch-0.6.2}/tests/test_debug.py +0 -0
  57. {acadfetch-0.6.0 → acadfetch-0.6.2}/tests/test_disk.py +0 -0
  58. {acadfetch-0.6.0 → acadfetch-0.6.2}/tests/test_identity.py +0 -0
  59. {acadfetch-0.6.0 → acadfetch-0.6.2}/tests/test_isolation.py +0 -0
  60. {acadfetch-0.6.0 → acadfetch-0.6.2}/tests/test_json.py +0 -0
  61. {acadfetch-0.6.0 → acadfetch-0.6.2}/tests/test_manifest.py +0 -0
  62. {acadfetch-0.6.0 → acadfetch-0.6.2}/tests/test_naming.py +0 -0
  63. {acadfetch-0.6.0 → acadfetch-0.6.2}/tests/test_naming_race.py +0 -0
  64. {acadfetch-0.6.0 → acadfetch-0.6.2}/tests/test_network.py +0 -0
  65. {acadfetch-0.6.0 → acadfetch-0.6.2}/tests/test_packaging.py +0 -0
  66. {acadfetch-0.6.0 → acadfetch-0.6.2}/tests/test_progress.py +0 -0
  67. {acadfetch-0.6.0 → acadfetch-0.6.2}/tests/test_ratelimit.py +0 -0
  68. {acadfetch-0.6.0 → acadfetch-0.6.2}/tests/test_redirect.py +0 -0
  69. {acadfetch-0.6.0 → acadfetch-0.6.2}/tests/test_remote.py +0 -0
  70. {acadfetch-0.6.0 → acadfetch-0.6.2}/tests/test_removed.py +0 -0
  71. {acadfetch-0.6.0 → acadfetch-0.6.2}/tests/test_urls.py +0 -0
  72. {acadfetch-0.6.0 → acadfetch-0.6.2}/tests/test_urls_glob.py +0 -0
  73. {acadfetch-0.6.0 → acadfetch-0.6.2}/tests/test_verify.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: acadfetch
3
- Version: 0.6.0
3
+ Version: 0.6.2
4
4
  Summary: Resumable, rate-limited, checksum-verified bulk downloads from academic data repositories
5
5
  Author: young-ski
6
6
  License-Expression: MIT
@@ -236,7 +236,12 @@ class Client:
236
236
  if exc.code in RETRY_STATUS:
237
237
  pass
238
238
  elif not last_blocked:
239
- raise HttpError(exc.code, url, detail) from exc
239
+ raise HttpError(
240
+ exc.code,
241
+ url,
242
+ detail,
243
+ content_range=exc.headers.get("Content-Range") if exc.headers else None,
244
+ ) from exc
240
245
  elif waf >= block_retries:
241
246
  raise HttpError(
242
247
  403,
@@ -270,7 +275,13 @@ class Client:
270
275
  if last_blocked
271
276
  else last_detail
272
277
  )
273
- raise HttpError(last.code, url, detail, transient=True) from last
278
+ raise HttpError(
279
+ last.code,
280
+ url,
281
+ detail,
282
+ transient=True,
283
+ content_range=last.headers.get("Content-Range") if last.headers else None,
284
+ ) from last
274
285
  raise HttpError(0, url, last_detail or str(last)) from last
275
286
 
276
287
  # -- public ------------------------------------------------------------
@@ -1,6 +1,6 @@
1
1
  """Package-wide constants and version."""
2
2
 
3
- __version__ = "0.6.0"
3
+ __version__ = "0.6.2"
4
4
 
5
5
  USER_AGENT = f"acadfetch/{__version__} (+https://github.com/young-ski/acadfetch)"
6
6
 
@@ -14,6 +14,7 @@ from __future__ import annotations
14
14
 
15
15
  import contextlib
16
16
  import fnmatch
17
+ import hashlib
17
18
  import json
18
19
  import os
19
20
  import random
@@ -83,6 +84,9 @@ class Transfer:
83
84
  etag: str | None = None
84
85
  http_status: int | None = None
85
86
  retrieved_at: str | None = None
87
+ #: Algorithm -> hex digest, computed while the bytes went by. ``None`` when
88
+ #: no body was read (a 416 on a complete part).
89
+ digests: dict[str, str] | None = None
86
90
 
87
91
 
88
92
  @dataclass
@@ -236,6 +240,46 @@ def _chunks(resp, idle_timeout: float, url: str): # noqa: ANN001, ANN202
236
240
  yield chunk
237
241
 
238
242
 
243
+ def _star_total(content_range: str | None) -> int | None:
244
+ """``bytes */N`` → N, for the 416 that carries it."""
245
+ if not content_range:
246
+ return None
247
+ _, _, value = content_range.strip().partition("*/")
248
+ try:
249
+ return int(value.strip())
250
+ except ValueError:
251
+ return None
252
+
253
+
254
+ def _restart_or_finish(
255
+ file: File, part: Path, meta_path: Path, have: int, exc: HttpError
256
+ ) -> Transfer:
257
+ """What a 416 means: the part is the file, or it is stale and must go.
258
+
259
+ "Already complete" used to be assumed whenever the part existed, so a stale
260
+ part (a different, older object) was renamed into place with nothing
261
+ checked. A matching checksum is proof; otherwise the size the server
262
+ reports for the whole object has to equal the part exactly.
263
+ """
264
+ parsed = verify.parse(file.checksum)
265
+ digests: dict[str, str] | None = None
266
+ if parsed is not None:
267
+ digests = verify.hashes(part, {"sha256", parsed[0]})
268
+ if digests[parsed[0]] == parsed[1]:
269
+ return Transfer(0, digests=digests)
270
+ total = _star_total(exc.content_range)
271
+ if total is not None and total == have:
272
+ if digests is None:
273
+ digests = verify.hashes(part, {"sha256"})
274
+ return Transfer(0, digests=digests)
275
+ _drop(part, meta_path)
276
+ if total is None:
277
+ raised, compared = "416 without a size to compare against", "no evidence"
278
+ else:
279
+ raised, compared = f"server says {total} bytes", f"the .part has {have}"
280
+ raise HttpError(416, file.url, f"{raised}, {compared}; restarting", transient=True)
281
+
282
+
239
283
  def _transfer(client: Client, file: File, part: Path, meta_path: Path, opts: Options) -> Transfer:
240
284
  """Move bytes into ``part``, resuming if possible. Returns what happened."""
241
285
  meta = _read_meta(meta_path) if part.exists() else {}
@@ -262,7 +306,7 @@ def _transfer(client: Client, file: File, part: Path, meta_path: Path, opts: Opt
262
306
  resp = client.open(file.url, headers, attempts=1)
263
307
  except HttpError as exc:
264
308
  if exc.status == 416 and have:
265
- return Transfer(0) # already complete
309
+ return _restart_or_finish(file, part, meta_path, have, exc)
266
310
  raise
267
311
 
268
312
  with resp:
@@ -294,6 +338,19 @@ def _transfer(client: Client, file: File, part: Path, meta_path: Path, opts: Opt
294
338
  mode, offset = "wb", 0
295
339
  else:
296
340
  mode, offset = "ab", have
341
+
342
+ # Hash while the bytes go by instead of reading the finished file again.
343
+ # A resumed transfer has to be fed what is already on disk first.
344
+ parsed = verify.parse(file.checksum)
345
+ wanted_algo = parsed[0] if parsed else None
346
+ hashers = {"sha256": hashlib.sha256()}
347
+ if wanted_algo and wanted_algo != "sha256":
348
+ hashers[wanted_algo] = hashlib.new(wanted_algo)
349
+ if mode == "ab":
350
+ with part.open("rb") as existing:
351
+ for old in iter(lambda: existing.read(1 << 20), b""):
352
+ for hasher in hashers.values():
353
+ hasher.update(old)
297
354
  _write_meta(
298
355
  meta_path,
299
356
  {
@@ -323,6 +380,8 @@ def _transfer(client: Client, file: File, part: Path, meta_path: Path, opts: Opt
323
380
  with part.open(mode) as fh:
324
381
  for chunk in _chunks(resp, client.timeout, file.url):
325
382
  fh.write(chunk)
383
+ for hasher in hashers.values():
384
+ hasher.update(chunk)
326
385
  written += len(chunk)
327
386
  if opts.progress:
328
387
  opts.progress.advance(len(chunk))
@@ -350,6 +409,7 @@ def _transfer(client: Client, file: File, part: Path, meta_path: Path, opts: Opt
350
409
  etag=resp.headers.get("ETag"),
351
410
  http_status=status,
352
411
  retrieved_at=_now(),
412
+ digests={algo: digest.hexdigest() for algo, digest in hashers.items()},
353
413
  )
354
414
 
355
415
 
@@ -427,24 +487,38 @@ def _reserved(files) -> set[str]: # noqa: ANN001
427
487
  return reserved
428
488
 
429
489
 
430
- def _already_there(final: Path, name: str, file: File, opts: Options) -> bool | None:
431
- """Whether an existing file may stand in for this one.
490
+ def _verdict(file: File, digests: dict[str, str] | None) -> bool | None:
491
+ parsed = verify.parse(file.checksum)
492
+ if parsed is None:
493
+ return None
494
+ if not digests:
495
+ return None
496
+ return digests.get(parsed[0]) == parsed[1]
497
+
498
+
499
+ def _already_there(
500
+ final: Path, name: str, file: File, opts: Options
501
+ ) -> tuple[bool, dict[str, str] | None]:
502
+ """Whether an existing file may stand in for this one, and its digests.
432
503
 
433
504
  A checksum match is proof. Without a checksum, a size match only means the
434
505
  bytes *could* be the same, so one more thing must tie the file to this URL:
435
- an explicit ``-o`` (the user chose the directory), a manifest in it that
436
- recorded the same URL, or nothing else at all.
506
+ an explicit ``-o`` (the user chose the directory) or a manifest in it that
507
+ recorded the same URL. Both hashes are computed in that one read, so the
508
+ manifest does not have to read the file again.
437
509
  """
438
510
  if opts.force or not final.exists():
439
- return None
440
- verdict = verify.matches(final, file.checksum)
441
- if verdict is not None:
442
- return verdict
511
+ return False, None
512
+ parsed = verify.parse(file.checksum)
513
+ algos = {"sha256"} | ({parsed[0]} if parsed else set())
514
+ digests = verify.hashes(final, algos)
515
+ if parsed is not None:
516
+ return digests[parsed[0]] == parsed[1], digests
443
517
  if file.size is not None and final.stat().st_size != file.size:
444
- return False
518
+ return False, digests
445
519
  if opts.explicit_out:
446
- return True
447
- return opts.trusted.get(name) == file.url
520
+ return True, digests
521
+ return opts.trusted.get(name) == file.url, digests
448
522
 
449
523
 
450
524
  def fetch_file(client: Client, file: File, opts: Options, taken: set[str]) -> Result:
@@ -453,14 +527,16 @@ def fetch_file(client: Client, file: File, opts: Options, taken: set[str]) -> Re
453
527
  part = opts.out / (name + ".part")
454
528
  meta_path = opts.out / (name + META_SUFFIX)
455
529
 
456
- if _already_there(final, name, file, opts):
530
+ trusted, digests = _already_there(final, name, file, opts)
531
+ if trusted:
457
532
  return Result(
458
533
  file=file,
459
534
  state="skipped",
460
535
  path=str(final),
461
536
  bytes=final.stat().st_size,
462
- verified=verify.matches(final, file.checksum),
537
+ verified=_verdict(file, digests),
463
538
  name=name,
539
+ sha256=(digests or {}).get("sha256"),
464
540
  )
465
541
 
466
542
  if opts.dry_run:
@@ -574,12 +650,12 @@ def _attempt(
574
650
  retrieved_at=_now(),
575
651
  )
576
652
 
577
- verdict = verify.matches(part, file.checksum)
653
+ verdict = _verdict(file, info.digests if info else None)
578
654
  if verdict is False:
579
655
  _drop(part, meta_path)
580
656
  try:
581
657
  info = _transfer(client, file, part, meta_path, opts)
582
- verdict = verify.matches(part, file.checksum)
658
+ verdict = _verdict(file, info.digests if info else None)
583
659
  except Exception as exc: # noqa: BLE001
584
660
  return Result(
585
661
  file=file,
@@ -603,6 +679,7 @@ def _attempt(
603
679
  bytes=final.stat().st_size,
604
680
  verified=verdict,
605
681
  resumed_from=resumed_from,
682
+ sha256=(info.digests or {}).get("sha256") if info else None,
606
683
  final_url=info.final_url if info else None,
607
684
  etag=info.etag if info else None,
608
685
  http_status=info.http_status if info else None,
@@ -685,17 +762,31 @@ def _fetch_record(
685
762
  return _in_record_order(results, record)
686
763
 
687
764
  pool = ThreadPoolExecutor(max_workers=opts.concurrency)
765
+ # Keep the in-flight set bounded: a 5000-file record submitted in one go
766
+ # holds 5000 futures and progress keys at once.
767
+ window = max(opts.concurrency * 2, 1)
768
+ queue = iter(files)
688
769
  futures = {}
770
+
771
+ def submit(file: File) -> None: # noqa: ANN001
772
+ with lock:
773
+ futures[pool.submit(fetch_file, client, file, opts, taken)] = file
774
+
689
775
  try:
690
- for file in files:
691
- with lock:
692
- futures[pool.submit(fetch_file, client, file, opts, taken)] = file
693
- for future in as_completed(futures):
694
- file = futures[future]
695
- try:
696
- results.append(future.result())
697
- except Exception as exc: # noqa: BLE001 - one worker must not end the batch
698
- results.append(_crashed(file, exc))
776
+ for file in queue:
777
+ submit(file)
778
+ if len(futures) >= window:
779
+ break
780
+ while futures:
781
+ for future in as_completed(list(futures)):
782
+ file = futures.pop(future)
783
+ try:
784
+ results.append(future.result())
785
+ except Exception as exc: # noqa: BLE001 - one worker must not end the batch
786
+ results.append(_crashed(file, exc))
787
+ nxt = next(queue, None)
788
+ if nxt is not None:
789
+ submit(nxt)
699
790
  except KeyboardInterrupt:
700
791
  # Abort the reads that are blocked, then let the workers unwind: they
701
792
  # see the stop flag and keep their .part instead of retrying.
@@ -30,6 +30,7 @@ class HttpError(AcFError):
30
30
  detail: str = "",
31
31
  transient: bool = False,
32
32
  reason: str = "",
33
+ content_range: str | None = None,
33
34
  ) -> None:
34
35
  self.status = status
35
36
  self.url = url
@@ -37,6 +38,9 @@ class HttpError(AcFError):
37
38
  #: Machine-readable cause for callers that must branch on it, e.g.
38
39
  #: ``html-page``: the body is a block/login page, not the file asked for.
39
40
  self.reason = reason
41
+ #: ``Content-Range`` of the answer, so a 416 can say how big the object
42
+ #: is supposed to be.
43
+ self.content_range = content_range
40
44
  if status:
41
45
  text = f"HTTP {status} for {url}{': ' + detail if detail else ''}"
42
46
  elif reason == "max-size":
@@ -135,6 +135,8 @@ def _entry( # noqa: ANN001
135
135
  def _sha256(result: Result | None) -> str | None:
136
136
  if result is None or not result.path:
137
137
  return None
138
+ if result.sha256:
139
+ return result.sha256
138
140
  path = Path(result.path)
139
141
  if not path.is_file():
140
142
  return None
@@ -58,6 +58,8 @@ class Result:
58
58
  http_status: int | None = None
59
59
  #: RFC 3339 UTC, when the last byte (or the failure) arrived.
60
60
  retrieved_at: str | None = None
61
+ #: SHA-256 of the bytes, computed during the transfer or the reuse check.
62
+ sha256: str | None = None
61
63
 
62
64
  @property
63
65
  def ok(self) -> bool:
@@ -3,6 +3,10 @@
3
3
  The public REST API exposes ``files[].links.self`` as a redirecting download
4
4
  URL and publishes ``files[].checksum`` as ``md5:<hex>``, which is what makes
5
5
  verification possible here.
6
+
7
+ ``is_latest`` / ``links.latest`` are honoured for an older version DOI. The
8
+ current API does not send ``is_latest`` (checked 2026-11), so the branch is
9
+ dormant; a test pins what it would do if it returned.
6
10
  """
7
11
 
8
12
  from __future__ import annotations
@@ -41,11 +41,17 @@ def _guess(candidate: str) -> str:
41
41
 
42
42
 
43
43
  def hash_file(path: Path, algo: str) -> str:
44
- digest = hashlib.new(algo)
44
+ return hashes(path, {algo})[algo]
45
+
46
+
47
+ def hashes(path: Path, algos: set[str]) -> dict[str, str]:
48
+ """Every requested digest in one pass over the file."""
49
+ digests = {algo: hashlib.new(algo) for algo in algos}
45
50
  with path.open("rb") as fh:
46
51
  for chunk in iter(lambda: fh.read(1 << 20), b""):
47
- digest.update(chunk)
48
- return digest.hexdigest()
52
+ for digest in digests.values():
53
+ digest.update(chunk)
54
+ return {algo: digest.hexdigest() for algo, digest in digests.items()}
49
55
 
50
56
 
51
57
  def matches(path: Path, checksum: str | None) -> bool | None:
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: acadfetch
3
- Version: 0.6.0
3
+ Version: 0.6.2
4
4
  Summary: Resumable, rate-limited, checksum-verified bulk downloads from academic data repositories
5
5
  Author: young-ski
6
6
  License-Expression: MIT
@@ -45,6 +45,7 @@ tests/test_control_names.py
45
45
  tests/test_debug.py
46
46
  tests/test_disk.py
47
47
  tests/test_downloader.py
48
+ tests/test_hash_once.py
48
49
  tests/test_identity.py
49
50
  tests/test_isolation.py
50
51
  tests/test_json.py
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "acadfetch"
7
- version = "0.6.0"
7
+ version = "0.6.2"
8
8
  description = "Resumable, rate-limited, checksum-verified bulk downloads from academic data repositories"
9
9
  readme = "README.md"
10
10
  requires-python = ">=3.10"
@@ -55,7 +55,8 @@ class Handler(BaseHTTPRequestHandler):
55
55
  return
56
56
  if cfg["answers_416"] and self.headers.get("Range"):
57
57
  self.send_response(416)
58
- self.send_header("Content-Range", f"bytes */{len(cfg['files'].get(path, b''))}")
58
+ if not cfg.get("416_omits_range"):
59
+ self.send_header("Content-Range", f"bytes */{len(cfg['files'].get(path, b''))}")
59
60
  self.send_header("Content-Length", "0")
60
61
  self.end_headers()
61
62
  return
@@ -147,6 +148,7 @@ def _new_server():
147
148
  "declares": {},
148
149
  "types": {},
149
150
  "answers_416": False,
151
+ "416_omits_range": False,
150
152
  "ranges": [],
151
153
  "stall_after": None,
152
154
  "stall_for": 2.0,
@@ -588,3 +588,60 @@ def test_a_failed_transfer_records_the_status(server, tmp_path):
588
588
  assert res.state == "failed"
589
589
  assert res.http_status == 500
590
590
  assert res.retrieved_at
591
+
592
+
593
+ # -- 416: only evidence makes the kept part the answer ---------------------
594
+
595
+
596
+ def _stale_part(tmp_path, url, body: bytes):
597
+ (tmp_path / "a.bin.part").write_bytes(body)
598
+ (tmp_path / "a.bin.part.meta.json").write_text(f'{{"url": "{url}", "etag": ""v1""}}', "utf-8")
599
+
600
+
601
+ def test_a_stale_part_is_restarted_not_renamed(server, tmp_path):
602
+ """A part from an older object is bigger than the file; 416 must not win."""
603
+ payload = b"new" * 1000
604
+ server.cfg["files"]["/a.bin"] = payload
605
+ url = server.base + "/a.bin"
606
+ _stale_part(tmp_path, url, b"old" * 2000) # 6000 bytes vs 3000
607
+ server.cfg["answers_416"] = True
608
+
609
+ res = downloader.fetch_file(
610
+ Client(rate=1000, retries=1), File(url, "a.bin"), opts(tmp_path, retries=1), set()
611
+ )
612
+
613
+ assert res.state == "downloaded", res.error
614
+ assert (tmp_path / "a.bin").read_bytes() == payload
615
+
616
+
617
+ def test_a_416_without_a_size_and_without_checksum_restarts(server, tmp_path):
618
+ payload = b"fresh bytes"
619
+ server.cfg["files"]["/a.bin"] = payload
620
+ url = server.base + "/a.bin"
621
+ _stale_part(tmp_path, url, b"stale")
622
+ server.cfg["answers_416"] = True
623
+ server.cfg["416_omits_range"] = True
624
+
625
+ res = downloader.fetch_file(
626
+ Client(rate=1000, retries=1), File(url, "a.bin"), opts(tmp_path, retries=1), set()
627
+ )
628
+
629
+ assert res.state == "downloaded", res.error
630
+ assert (tmp_path / "a.bin").read_bytes() == payload
631
+
632
+
633
+ def test_a_416_with_a_matching_checksum_accepts_the_part(server, tmp_path):
634
+ """Proof beats a missing Content-Range."""
635
+ payload = b"proven bytes"
636
+ server.cfg["files"]["/a.bin"] = payload
637
+ url = server.base + "/a.bin"
638
+ _stale_part(tmp_path, url, payload)
639
+ server.cfg["answers_416"] = True
640
+ server.cfg["416_omits_range"] = True
641
+ f = File(url, "a.bin", checksum="sha256:" + hashlib.sha256(payload).hexdigest())
642
+
643
+ res = downloader.fetch_file(Client(rate=1000, retries=0), f, opts(tmp_path, retries=0), set())
644
+
645
+ assert res.state == "downloaded", res.error
646
+ assert res.verified is True
647
+ assert server.cfg["hits"]["/a.bin"] == 1, "no refetch when the checksum proves it"
@@ -0,0 +1,129 @@
1
+ from __future__ import annotations
2
+
3
+ import hashlib
4
+ import json
5
+ import threading
6
+ import time
7
+
8
+ from acadfetch import downloader, verify
9
+ from acadfetch.client import Client
10
+ from acadfetch.sources.base import File, Record
11
+
12
+
13
+ def _opts(tmp_path, **kw):
14
+ kw.setdefault("concurrency", 1)
15
+ return downloader.Options(out=tmp_path, **kw)
16
+
17
+
18
+ def test_hashes_computes_every_algo_in_one_pass(tmp_path):
19
+ path = tmp_path / "f.bin"
20
+ payload = b"one pass"
21
+ path.write_bytes(payload)
22
+
23
+ got = verify.hashes(path, {"sha256", "md5"})
24
+
25
+ assert got["sha256"] == hashlib.sha256(payload).hexdigest()
26
+ assert got["md5"] == hashlib.md5(payload).hexdigest()
27
+
28
+
29
+ def test_a_fetch_never_rehashes_the_finished_file(server, tmp_path, monkeypatch):
30
+ payload = b"payload" * 500
31
+ server.cfg["files"]["/a.bin"] = payload
32
+ f = File(
33
+ server.base + "/a.bin",
34
+ "a.bin",
35
+ len(payload),
36
+ "sha256:" + hashlib.sha256(payload).hexdigest(),
37
+ )
38
+
39
+ def boom(*_a, **_k): # pragma: no cover - proves it is not reached
40
+ raise AssertionError("hash_file was called")
41
+
42
+ monkeypatch.setattr(verify, "hash_file", boom)
43
+ res = downloader.fetch_file(Client(rate=1000, retries=0), f, _opts(tmp_path), set())
44
+
45
+ assert res.state == "downloaded"
46
+ assert res.verified is True
47
+ assert res.sha256 == hashlib.sha256(payload).hexdigest()
48
+
49
+
50
+ def test_a_resumed_transfer_hashes_the_bytes_already_on_disk(server, tmp_path):
51
+ payload = bytes(range(256)) * 40
52
+ server.cfg["files"]["/a.bin"] = payload
53
+ url = server.base + "/a.bin"
54
+ (tmp_path / "a.bin.part").write_bytes(payload[:2000])
55
+ (tmp_path / "a.bin.part.meta.json").write_text(
56
+ json.dumps({"url": url, "etag": '"v1"', "size": len(payload)}), "utf-8"
57
+ )
58
+ f = File(url, "a.bin", len(payload), "sha256:" + hashlib.sha256(payload).hexdigest())
59
+
60
+ res = downloader.fetch_file(Client(rate=1000, retries=0), f, _opts(tmp_path), set())
61
+
62
+ assert res.state == "downloaded", res.error
63
+ assert res.resumed_from == 2000
64
+ assert res.verified is True
65
+ assert res.sha256 == hashlib.sha256(payload).hexdigest()
66
+ assert (tmp_path / "a.bin").read_bytes() == payload
67
+
68
+
69
+ def test_a_reused_file_is_read_once(server, tmp_path, monkeypatch):
70
+ payload = b"reuse me"
71
+ server.cfg["files"]["/a.bin"] = payload
72
+ url = server.base + "/a.bin"
73
+ (tmp_path / "a.bin").write_bytes(payload)
74
+ (tmp_path / "manifest.json").write_text(
75
+ json.dumps({"schema": 3, "files": [{"name": "a.bin", "url": url}]}), "utf-8"
76
+ )
77
+ calls = []
78
+ real = verify.hashes
79
+
80
+ def counting(path, algos):
81
+ calls.append(algos)
82
+ return real(path, algos)
83
+
84
+ monkeypatch.setattr(verify, "hashes", counting)
85
+ monkeypatch.setattr(
86
+ verify, "hash_file", lambda *_a, **_k: (_ for _ in ()).throw(AssertionError("rehash"))
87
+ )
88
+
89
+ opts = _opts(tmp_path)
90
+ opts.trusted = {"a.bin": url} # what fetch_record loads from the manifest
91
+ res = downloader.fetch_file(
92
+ Client(rate=1000, retries=0), File(url, "a.bin", len(payload)), opts, set()
93
+ )
94
+
95
+ assert res.state == "skipped"
96
+ assert len(calls) == 1, "one read produces the verdict and the manifest hash"
97
+ assert res.sha256 == hashlib.sha256(payload).hexdigest()
98
+
99
+
100
+ def test_the_in_flight_window_is_bounded(server, tmp_path, monkeypatch):
101
+ files = []
102
+ for i in range(30):
103
+ server.cfg["files"][f"/{i}.bin"] = b"x"
104
+ files.append(File(server.base + f"/{i}.bin", f"{i}.bin", 1))
105
+ record = Record(source="direct", target="t", files=files)
106
+
107
+ state = {"now": 0, "max": 0}
108
+ lock = threading.Lock()
109
+ real = downloader.fetch_file
110
+
111
+ def watched(client, file, opts, taken):
112
+ with lock:
113
+ state["now"] += 1
114
+ state["max"] = max(state["max"], state["now"])
115
+ try:
116
+ time.sleep(0.01)
117
+ return real(client, file, opts, taken)
118
+ finally:
119
+ with lock:
120
+ state["now"] -= 1
121
+
122
+ monkeypatch.setattr(downloader, "fetch_file", watched)
123
+ results = downloader.fetch_record(
124
+ record, downloader.Options(out=tmp_path, concurrency=3), Client(rate=10000)
125
+ )
126
+
127
+ assert len(results) == 30
128
+ assert state["max"] <= 6, state["max"]
129
+ assert state["max"] >= 2
@@ -320,3 +320,14 @@ def test_two_legacy_papers_no_longer_share_a_name():
320
320
  def test_direct_resolve_rejects_a_non_url():
321
321
  with pytest.raises(TargetError, match="not a URL"):
322
322
  direct.resolve(HeadClient({}), "zenodo:1")
323
+
324
+
325
+ def test_an_older_zenodo_version_follows_latest():
326
+ """The API no longer sends ``is_latest``, but if a source does, follow it."""
327
+ old = dict(ZENODO, is_latest=False, links={"latest": "https://zenodo.org/api/records/99"})
328
+ new = dict(ZENODO, id=99, doi="10.5281/zenodo.99")
329
+ record = zenodo.resolve(FakeClient({"records/1": old, "records/99": new}), "10.5281/zenodo.1")
330
+
331
+ assert record.doi == "10.5281/zenodo.99"
332
+ assert record.files[0].name == "a.csv"
333
+ assert record.target == "10.5281/zenodo.1", "the manifest keeps what was asked for"
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes