acadfetch 0.2.0__tar.gz → 0.3.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (66) hide show
  1. {acadfetch-0.2.0/acadfetch.egg-info → acadfetch-0.3.0}/PKG-INFO +4 -1
  2. {acadfetch-0.2.0 → acadfetch-0.3.0}/README.md +3 -0
  3. {acadfetch-0.2.0 → acadfetch-0.3.0}/acadfetch/cli.py +70 -3
  4. {acadfetch-0.2.0 → acadfetch-0.3.0}/acadfetch/consts.py +1 -1
  5. {acadfetch-0.2.0 → acadfetch-0.3.0}/acadfetch/manifest.py +4 -2
  6. {acadfetch-0.2.0 → acadfetch-0.3.0}/acadfetch/naming.py +11 -0
  7. acadfetch-0.3.0/acadfetch/remote.py +82 -0
  8. {acadfetch-0.2.0 → acadfetch-0.3.0/acadfetch.egg-info}/PKG-INFO +4 -1
  9. {acadfetch-0.2.0 → acadfetch-0.3.0}/acadfetch.egg-info/SOURCES.txt +3 -0
  10. {acadfetch-0.2.0 → acadfetch-0.3.0}/pyproject.toml +1 -1
  11. {acadfetch-0.2.0 → acadfetch-0.3.0}/tests/conftest.py +18 -0
  12. acadfetch-0.3.0/tests/test_naming_race.py +35 -0
  13. acadfetch-0.3.0/tests/test_remote.py +187 -0
  14. {acadfetch-0.2.0 → acadfetch-0.3.0}/LICENSE +0 -0
  15. {acadfetch-0.2.0 → acadfetch-0.3.0}/MANIFEST.in +0 -0
  16. {acadfetch-0.2.0 → acadfetch-0.3.0}/acadfetch/__init__.py +0 -0
  17. {acadfetch-0.2.0 → acadfetch-0.3.0}/acadfetch/__main__.py +0 -0
  18. {acadfetch-0.2.0 → acadfetch-0.3.0}/acadfetch/atomic.py +0 -0
  19. {acadfetch-0.2.0 → acadfetch-0.3.0}/acadfetch/auth.py +0 -0
  20. {acadfetch-0.2.0 → acadfetch-0.3.0}/acadfetch/client.py +0 -0
  21. {acadfetch-0.2.0 → acadfetch-0.3.0}/acadfetch/downloader.py +0 -0
  22. {acadfetch-0.2.0 → acadfetch-0.3.0}/acadfetch/errors.py +0 -0
  23. {acadfetch-0.2.0 → acadfetch-0.3.0}/acadfetch/progress.py +0 -0
  24. {acadfetch-0.2.0 → acadfetch-0.3.0}/acadfetch/py.typed +0 -0
  25. {acadfetch-0.2.0 → acadfetch-0.3.0}/acadfetch/ratelimit.py +0 -0
  26. {acadfetch-0.2.0 → acadfetch-0.3.0}/acadfetch/sources/__init__.py +0 -0
  27. {acadfetch-0.2.0 → acadfetch-0.3.0}/acadfetch/sources/arxiv.py +0 -0
  28. {acadfetch-0.2.0 → acadfetch-0.3.0}/acadfetch/sources/base.py +0 -0
  29. {acadfetch-0.2.0 → acadfetch-0.3.0}/acadfetch/sources/direct.py +0 -0
  30. {acadfetch-0.2.0 → acadfetch-0.3.0}/acadfetch/sources/doi.py +0 -0
  31. {acadfetch-0.2.0 → acadfetch-0.3.0}/acadfetch/sources/registry.py +0 -0
  32. {acadfetch-0.2.0 → acadfetch-0.3.0}/acadfetch/sources/removed.py +0 -0
  33. {acadfetch-0.2.0 → acadfetch-0.3.0}/acadfetch/sources/zenodo.py +0 -0
  34. {acadfetch-0.2.0 → acadfetch-0.3.0}/acadfetch/units.py +0 -0
  35. {acadfetch-0.2.0 → acadfetch-0.3.0}/acadfetch/urls.py +0 -0
  36. {acadfetch-0.2.0 → acadfetch-0.3.0}/acadfetch/verify.py +0 -0
  37. {acadfetch-0.2.0 → acadfetch-0.3.0}/acadfetch.egg-info/dependency_links.txt +0 -0
  38. {acadfetch-0.2.0 → acadfetch-0.3.0}/acadfetch.egg-info/entry_points.txt +0 -0
  39. {acadfetch-0.2.0 → acadfetch-0.3.0}/acadfetch.egg-info/top_level.txt +0 -0
  40. {acadfetch-0.2.0 → acadfetch-0.3.0}/setup.cfg +0 -0
  41. {acadfetch-0.2.0 → acadfetch-0.3.0}/tests/fixtures/README.md +0 -0
  42. {acadfetch-0.2.0 → acadfetch-0.3.0}/tests/fixtures/figshare_403_block.html +0 -0
  43. {acadfetch-0.2.0 → acadfetch-0.3.0}/tests/fixtures/figshare_403_forbidden.json +0 -0
  44. {acadfetch-0.2.0 → acadfetch-0.3.0}/tests/fixtures/figshare_article.json +0 -0
  45. {acadfetch-0.2.0 → acadfetch-0.3.0}/tests/fixtures/figshare_file.pdf +0 -0
  46. {acadfetch-0.2.0 → acadfetch-0.3.0}/tests/test_atomic.py +0 -0
  47. {acadfetch-0.2.0 → acadfetch-0.3.0}/tests/test_auth.py +0 -0
  48. {acadfetch-0.2.0 → acadfetch-0.3.0}/tests/test_checksum.py +0 -0
  49. {acadfetch-0.2.0 → acadfetch-0.3.0}/tests/test_cli.py +0 -0
  50. {acadfetch-0.2.0 → acadfetch-0.3.0}/tests/test_client.py +0 -0
  51. {acadfetch-0.2.0 → acadfetch-0.3.0}/tests/test_control_names.py +0 -0
  52. {acadfetch-0.2.0 → acadfetch-0.3.0}/tests/test_disk.py +0 -0
  53. {acadfetch-0.2.0 → acadfetch-0.3.0}/tests/test_downloader.py +0 -0
  54. {acadfetch-0.2.0 → acadfetch-0.3.0}/tests/test_identity.py +0 -0
  55. {acadfetch-0.2.0 → acadfetch-0.3.0}/tests/test_isolation.py +0 -0
  56. {acadfetch-0.2.0 → acadfetch-0.3.0}/tests/test_json.py +0 -0
  57. {acadfetch-0.2.0 → acadfetch-0.3.0}/tests/test_manifest.py +0 -0
  58. {acadfetch-0.2.0 → acadfetch-0.3.0}/tests/test_naming.py +0 -0
  59. {acadfetch-0.2.0 → acadfetch-0.3.0}/tests/test_packaging.py +0 -0
  60. {acadfetch-0.2.0 → acadfetch-0.3.0}/tests/test_progress.py +0 -0
  61. {acadfetch-0.2.0 → acadfetch-0.3.0}/tests/test_ratelimit.py +0 -0
  62. {acadfetch-0.2.0 → acadfetch-0.3.0}/tests/test_redirect.py +0 -0
  63. {acadfetch-0.2.0 → acadfetch-0.3.0}/tests/test_removed.py +0 -0
  64. {acadfetch-0.2.0 → acadfetch-0.3.0}/tests/test_sources.py +0 -0
  65. {acadfetch-0.2.0 → acadfetch-0.3.0}/tests/test_urls.py +0 -0
  66. {acadfetch-0.2.0 → acadfetch-0.3.0}/tests/test_verify.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: acadfetch
3
- Version: 0.2.0
3
+ Version: 0.3.0
4
4
  Summary: Resumable, rate-limited, checksum-verified bulk downloads from academic data repositories
5
5
  Author: young-ski
6
6
  License-Expression: MIT
@@ -236,6 +236,9 @@ before duplicates were resolved.
236
236
 
237
237
  `acadfetch verify` recomputes SHA-256 for each entry and exits non-zero on any
238
238
  mismatch or missing file; rows the run never selected are reported `skipped`.
239
+ Add `--against-remote` to ask the source again and compare what it publishes
240
+ now: a manifest someone rewrote stops agreeing even though the bytes still match
241
+ it. `--require-remote` makes an unreachable source a failure.
239
242
  Commit the manifest next to your analysis; it is what makes the download
240
243
  reproducible a year later.
241
244
 
@@ -205,6 +205,9 @@ before duplicates were resolved.
205
205
 
206
206
  `acadfetch verify` recomputes SHA-256 for each entry and exits non-zero on any
207
207
  mismatch or missing file; rows the run never selected are reported `skipped`.
208
+ Add `--against-remote` to ask the source again and compare what it publishes
209
+ now: a manifest someone rewrote stops agreeing even though the bytes still match
210
+ it. `--require-remote` makes an unreachable source a failure.
208
211
  Commit the manifest next to your analysis; it is what makes the download
209
212
  reproducible a year later.
210
213
 
@@ -18,6 +18,7 @@ from pathlib import Path
18
18
  from urllib.parse import urlsplit
19
19
 
20
20
  from . import auth, consts, manifest, naming, urls, verify
21
+ from . import remote as remote_check
21
22
  from .client import Client
22
23
  from .downloader import Options, check_disk, check_total, fetch_record, human
23
24
  from .errors import AcFError, PartialError, TargetError
@@ -440,7 +441,20 @@ def cmd_verify(args: argparse.Namespace) -> int:
440
441
  if manifest.peek_schema(manifest_path) == 1:
441
442
  print(manifest.SCHEMA_1_NOTE, file=sys.stderr)
442
443
  doc["root"] = str(root)
443
- rows = manifest.verify_manifest(manifest_path, root, allow_escape=args.allow_escape)
444
+ data = manifest.load(manifest_path)
445
+ rows = manifest.verify_manifest(manifest_path, root, allow_escape=args.allow_escape, data=data)
446
+ remote: dict[str, tuple[str, str]] = {}
447
+ if args.against_remote:
448
+ published_now, reason = remote_check.published(_client(args), data)
449
+ entries = [e for e in data.get("files") or [] if isinstance(e, dict)]
450
+ # verify_manifest walks the same list in order; the rows do not carry
451
+ # the URL or checksum themselves.
452
+ remote = {
453
+ row["name"]: remote_check.judge(
454
+ None if not published_now and reason else published_now, entry, reason
455
+ )
456
+ for row, entry in zip(rows, entries, strict=False)
457
+ }
444
458
  skipped = sum(1 for row in rows if row["status"] == "skipped")
445
459
  hard = sum(1 for row in rows if row["status"] not in ("ok", "skipped"))
446
460
  # --require-all turns "you never asked for it" into a failure; the plain
@@ -450,7 +464,11 @@ def cmd_verify(args: argparse.Namespace) -> int:
450
464
  ok = len(rows) - hard - skipped
451
465
  if not args.json:
452
466
  for row in rows:
453
- print(f"{row['status']:>9} {row['name']}")
467
+ status, detail = remote.get(row["name"], ("", ""))
468
+ tail = f" remote: {status}" if status else ""
469
+ if detail:
470
+ tail += f" ({detail})"
471
+ print(f"{row['status']:>9} {row['name']}{tail}")
454
472
  if rows:
455
473
  print()
456
474
  tally = f"{ok}/{len(rows)} ok"
@@ -459,8 +477,46 @@ def cmd_verify(args: argparse.Namespace) -> int:
459
477
  print(tally)
460
478
  if not args.allow_escape and any(row["status"] == "escaped" for row in rows):
461
479
  print(f"some names leave {root} - --allow-escape follows them", file=sys.stderr)
480
+ remote_bad = 0
481
+ if remote:
482
+ total = remote_check.counts([{"remote": status} for status, _ in remote.values()])
483
+ if total[remote_check.MISMATCH]:
484
+ remote_bad = total[remote_check.MISMATCH]
485
+ if args.require_remote and total[remote_check.UNAVAILABLE]:
486
+ remote_bad += total[remote_check.UNAVAILABLE]
487
+ bad += remote_bad
488
+ data["remote_checked_at"] = datetime.now(timezone.utc).replace(microsecond=0).isoformat()
489
+ try:
490
+ manifest.write(manifest_path, data)
491
+ except OSError as exc:
492
+ print(f"acadfetch: could not record remote_checked_at: {exc}", file=sys.stderr)
493
+ doc["remote_checked_at"] = data["remote_checked_at"]
494
+ doc["remote"] = total
495
+ if total[remote_check.MISMATCH]:
496
+ print(
497
+ f"{total[remote_check.MISMATCH]} file(s) no longer match what the source "
498
+ f"publishes now",
499
+ file=sys.stderr,
500
+ )
501
+ if total[remote_check.UNAVAILABLE] and not args.require_remote:
502
+ print(
503
+ f"{total[remote_check.UNAVAILABLE]} file(s) could not be re-checked; "
504
+ f"--require-remote counts them",
505
+ file=sys.stderr,
506
+ )
462
507
  doc["rows"] = [
463
- {k: row[k] for k in ("name", "status", "state", "sha256", "expected", "actual") if k in row}
508
+ {
509
+ **{
510
+ k: row[k]
511
+ for k in ("name", "status", "state", "sha256", "expected", "actual")
512
+ if k in row
513
+ },
514
+ **(
515
+ {"remote": remote[row["name"]][0], "remote_detail": remote[row["name"]][1]}
516
+ if row["name"] in remote
517
+ else {}
518
+ ),
519
+ }
464
520
  for row in rows
465
521
  ]
466
522
  doc["summary"] = {
@@ -647,11 +703,22 @@ def build_parser() -> argparse.ArgumentParser:
647
703
  help="count a file that was never selected (--only, or a manifest-only "
648
704
  "run) as a failure instead of skipping it",
649
705
  )
706
+ ver.add_argument(
707
+ "--against-remote",
708
+ action="store_true",
709
+ help="ask the source again and compare what it publishes now (network)",
710
+ )
711
+ ver.add_argument(
712
+ "--require-remote",
713
+ action="store_true",
714
+ help="count a file that could not be re-checked as a failure",
715
+ )
650
716
  ver.add_argument(
651
717
  "--json",
652
718
  action="store_true",
653
719
  help="print one JSON report to stdout instead of rows",
654
720
  )
721
+ network_flags(ver)
655
722
  ver.set_defaults(func=cmd_verify)
656
723
 
657
724
  src = sub.add_parser("sources", help="list supported sources")
@@ -1,6 +1,6 @@
1
1
  """Package-wide constants and version."""
2
2
 
3
- __version__ = "0.2.0"
3
+ __version__ = "0.3.0"
4
4
 
5
5
  USER_AGENT = f"acadfetch/{__version__} (+https://github.com/young-ski/acadfetch)"
6
6
 
@@ -206,7 +206,9 @@ def peek_schema(path: Path) -> int | None:
206
206
  return value if isinstance(value, int) else None
207
207
 
208
208
 
209
- def verify_manifest(path: Path, root: Path, allow_escape: bool = False) -> list[dict]:
209
+ def verify_manifest(
210
+ path: Path, root: Path, allow_escape: bool = False, data: dict | None = None
211
+ ) -> list[dict]:
210
212
  """Re-hash every recorded file. Returns one row per file.
211
213
 
212
214
  A name is only allowed to point inside ``root`` unless ``allow_escape`` is
@@ -214,7 +216,7 @@ def verify_manifest(path: Path, root: Path, allow_escape: bool = False) -> list[
214
216
  outside the directory while reporting ``ok`` is how a green result stops
215
217
  meaning anything.
216
218
  """
217
- data = load(path)
219
+ data = load(path) if data is None else data
218
220
  rows = []
219
221
  for entry in data.get("files") or []:
220
222
  name = entry.get("name") if isinstance(entry, dict) else None
@@ -9,6 +9,7 @@ from __future__ import annotations
9
9
 
10
10
  import os
11
11
  import re
12
+ import threading
12
13
  import unicodedata
13
14
  from urllib.parse import unquote, urlsplit
14
15
 
@@ -74,8 +75,18 @@ def sanitize(name: str, fallback: str = "file") -> str:
74
75
  return out or fallback
75
76
 
76
77
 
78
+ #: The check-then-add below is one step only if callers cannot interleave.
79
+ #: Workers share one ``taken`` set, so the window is real and small.
80
+ _LOCK = threading.Lock()
81
+
82
+
77
83
  def unique(name: str, taken: set[str]) -> str:
78
84
  """Return a name not already in ``taken`` (case-insensitively)."""
85
+ with _LOCK:
86
+ return _unique_locked(name, taken)
87
+
88
+
89
+ def _unique_locked(name: str, taken: set[str]) -> str:
79
90
  if name.lower() not in taken:
80
91
  taken.add(name.lower())
81
92
  return name
@@ -0,0 +1,82 @@
1
+ """Re-check a manifest against what the source publishes today.
2
+
3
+ ``verify`` alone proves the bytes match the manifest. Nobody signs the manifest,
4
+ so someone who rewrites both the file and its row passes. Asking the source
5
+ again is the only thing that closes that: the row says what was published, the
6
+ source says what is published now, and a tampered row stops agreeing.
7
+
8
+ The result is a line per file, never a rewrite of the local check: a remote
9
+ mismatch is *added* to whatever the local verification said.
10
+ """
11
+
12
+ from __future__ import annotations
13
+
14
+ from collections.abc import Mapping
15
+
16
+ from . import verify
17
+ from .client import Client
18
+ from .errors import AcFError
19
+ from .sources import registry, removed
20
+
21
+ MATCH = "remote_match"
22
+ MISMATCH = "remote_mismatch"
23
+ NO_CHECKSUM = "remote_no_checksum"
24
+ UNAVAILABLE = "remote_unavailable"
25
+ STATUSES = (MATCH, MISMATCH, NO_CHECKSUM, UNAVAILABLE)
26
+
27
+
28
+ def published(client: Client, data: Mapping) -> tuple[dict[str, str | None], str]:
29
+ """URL -> checksum the source publishes now, or a reason there is none.
30
+
31
+ One record resolution per manifest: the API returns every file at once, and
32
+ a ``direct`` target resolves to a single HEAD.
33
+ """
34
+ target = data.get("target") or ""
35
+ if not isinstance(target, str) or not target:
36
+ return {}, "the manifest records no target"
37
+ if removed.explain(target):
38
+ return {}, "the source was removed; re-fetch from its file URL"
39
+ try:
40
+ record = registry.resolve(client, target)
41
+ except AcFError as exc:
42
+ return {}, str(exc)
43
+ except OSError as exc: # name resolution, a proxy that is not there
44
+ return {}, str(exc)
45
+ return {file.url: file.checksum for file in record.files}, ""
46
+
47
+
48
+ def _normalise(checksum: str | None) -> tuple[str, str] | None:
49
+ parsed = verify.parse(checksum)
50
+ return parsed
51
+
52
+
53
+ def judge(
54
+ published_now: Mapping[str, str | None] | None, row: Mapping, reason: str
55
+ ) -> tuple[str, str]:
56
+ """``(status, detail)`` for one manifest row."""
57
+ if published_now is None:
58
+ return UNAVAILABLE, reason
59
+ url = row.get("url")
60
+ if not isinstance(url, str) or url not in published_now:
61
+ return UNAVAILABLE, "no longer published at the recorded URL"
62
+ remote = published_now[url]
63
+ local = row.get("checksum")
64
+ if not remote:
65
+ return NO_CHECKSUM, "the source publishes no checksum now"
66
+ if not local:
67
+ return NO_CHECKSUM, "this row recorded no checksum"
68
+ here, there = _normalise(local), _normalise(remote)
69
+ if here is None or there is None:
70
+ return NO_CHECKSUM, "a checksum is not parseable"
71
+ return (
72
+ (MATCH, "") if here == there else (MISMATCH, f"published {there[0]}:{there[1][:12]}… now")
73
+ )
74
+
75
+
76
+ def counts(rows: list[dict]) -> dict[str, int]:
77
+ totals = dict.fromkeys(STATUSES, 0)
78
+ for row in rows:
79
+ status = row.get("remote")
80
+ if status in totals:
81
+ totals[status] += 1
82
+ return totals
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: acadfetch
3
- Version: 0.2.0
3
+ Version: 0.3.0
4
4
  Summary: Resumable, rate-limited, checksum-verified bulk downloads from academic data repositories
5
5
  Author: young-ski
6
6
  License-Expression: MIT
@@ -236,6 +236,9 @@ before duplicates were resolved.
236
236
 
237
237
  `acadfetch verify` recomputes SHA-256 for each entry and exits non-zero on any
238
238
  mismatch or missing file; rows the run never selected are reported `skipped`.
239
+ Add `--against-remote` to ask the source again and compare what it publishes
240
+ now: a manifest someone rewrote stops agreeing even though the bytes still match
241
+ it. `--require-remote` makes an unreachable source a failure.
239
242
  Commit the manifest next to your analysis; it is what makes the download
240
243
  reproducible a year later.
241
244
 
@@ -16,6 +16,7 @@ acadfetch/naming.py
16
16
  acadfetch/progress.py
17
17
  acadfetch/py.typed
18
18
  acadfetch/ratelimit.py
19
+ acadfetch/remote.py
19
20
  acadfetch/units.py
20
21
  acadfetch/urls.py
21
22
  acadfetch/verify.py
@@ -46,10 +47,12 @@ tests/test_isolation.py
46
47
  tests/test_json.py
47
48
  tests/test_manifest.py
48
49
  tests/test_naming.py
50
+ tests/test_naming_race.py
49
51
  tests/test_packaging.py
50
52
  tests/test_progress.py
51
53
  tests/test_ratelimit.py
52
54
  tests/test_redirect.py
55
+ tests/test_remote.py
53
56
  tests/test_removed.py
54
57
  tests/test_sources.py
55
58
  tests/test_urls.py
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "acadfetch"
7
- version = "0.2.0"
7
+ version = "0.3.0"
8
8
  description = "Resumable, rate-limited, checksum-verified bulk downloads from academic data repositories"
9
9
  readme = "README.md"
10
10
  requires-python = ">=3.10"
@@ -26,6 +26,21 @@ class Handler(BaseHTTPRequestHandler):
26
26
  self.wfile.write(body)
27
27
  self.wfile.flush()
28
28
 
29
+ def do_HEAD(self): # noqa: N802
30
+ cfg = self.server.cfg
31
+ path = self.path.split("?")[0]
32
+ cfg["hits"][path] = cfg["hits"].get(path, 0) + 1
33
+ if path not in cfg["files"]:
34
+ self.send_error(404)
35
+ return
36
+ self.send_response(200)
37
+ for name, value in cfg["headers"].get(path, {}).items():
38
+ self.send_header(name, value)
39
+ self.send_header("Content-Type", cfg["types"].get(path, "application/octet-stream"))
40
+ self.send_header("Content-Length", str(len(cfg["files"][path])))
41
+ self.send_header("ETag", cfg["etag"])
42
+ self.end_headers()
43
+
29
44
  def do_GET(self): # noqa: N802
30
45
  cfg = self.server.cfg
31
46
  path = self.path.split("?")[0]
@@ -89,6 +104,8 @@ class Handler(BaseHTTPRequestHandler):
89
104
  self.send_header(
90
105
  "Content-Range", f"bytes {start}-{len(payload) - 1}/{len(payload)}"
91
106
  )
107
+ for name, value in cfg["headers"].get(path, {}).items():
108
+ self.send_header(name, value)
92
109
  self.send_header("Content-Type", cfg["types"].get(path, "application/octet-stream"))
93
110
  declared = cfg["declares"].get(path, len(body))
94
111
  self.send_header("Content-Length", str(declared))
@@ -135,6 +152,7 @@ def _new_server():
135
152
  "stall_for": 2.0,
136
153
  "auth_seen": [],
137
154
  "redirects": {},
155
+ "headers": {},
138
156
  }
139
157
 
140
158
  thread = threading.Thread(target=srv.serve_forever, daemon=True)
@@ -0,0 +1,35 @@
1
+ from __future__ import annotations
2
+
3
+ import sys
4
+ import threading
5
+
6
+ from acadfetch import naming
7
+
8
+
9
+ def test_unique_hands_out_distinct_names_under_contention():
10
+ """The check-then-add needs to be one step when threads share the set.
11
+
12
+ Probabilistic by nature, so the switch interval is turned down and the
13
+ barrier repeated: without a lock this loses a name almost every round.
14
+ """
15
+ old = sys.getswitchinterval()
16
+ sys.setswitchinterval(1e-6)
17
+ try:
18
+ for _ in range(200):
19
+ taken: set[str] = set()
20
+ barrier = threading.Barrier(8)
21
+ results: list[str] = []
22
+
23
+ def worker(barrier=barrier, results=results, taken=taken) -> None: # noqa: B008
24
+ barrier.wait()
25
+ results.append(naming.unique("a.bin", taken))
26
+
27
+ threads = [threading.Thread(target=worker) for _ in range(8)]
28
+ for thread in threads:
29
+ thread.start()
30
+ for thread in threads:
31
+ thread.join()
32
+ assert len(set(results)) == 8, results
33
+ assert len(taken) == 8
34
+ finally:
35
+ sys.setswitchinterval(old)
@@ -0,0 +1,187 @@
1
+ from __future__ import annotations
2
+
3
+ import base64
4
+ import hashlib
5
+ import json
6
+
7
+ from acadfetch import cli
8
+ from acadfetch.sources.base import File, Record
9
+
10
+ BODY = b"remote check payload" * 40
11
+ MD5 = hashlib.md5(BODY)
12
+ GOOD = "md5:" + MD5.hexdigest()
13
+
14
+
15
+ def _manifest(tmp_path, url, checksum=GOOD, body=BODY, file_url=None):
16
+ (tmp_path / "a.bin").write_bytes(body)
17
+ data = {
18
+ "schema": 3,
19
+ "source": "direct",
20
+ "target": url,
21
+ "files": [
22
+ {
23
+ "name": "a.bin",
24
+ "url": file_url or url,
25
+ "checksum": checksum,
26
+ "sha256": hashlib.sha256(body).hexdigest(),
27
+ }
28
+ ],
29
+ }
30
+ path = tmp_path / "manifest.json"
31
+ path.write_text(json.dumps(data), "utf-8")
32
+ return path
33
+
34
+
35
+ def _run(capsys, *argv):
36
+ code = cli.main([str(a) for a in argv])
37
+ out, err = capsys.readouterr()
38
+ return code, out, err
39
+
40
+
41
+ def _served(server, headers=None, body=BODY):
42
+ server.cfg["files"]["/a.bin"] = body
43
+ if headers:
44
+ server.cfg["headers"]["/a.bin"] = headers
45
+ return server.base + "/a.bin"
46
+
47
+
48
+ def test_a_matching_published_checksum_is_a_match(server, capsys, tmp_path):
49
+ url = _served(server, {"Content-MD5": base64.b64encode(MD5.digest()).decode()})
50
+
51
+ code, out, _ = _run(capsys, "verify", _manifest(tmp_path, url), "--against-remote", "--json")
52
+
53
+ doc = json.loads(out)
54
+ assert code == cli.EXIT_OK
55
+ assert doc["rows"][0]["remote"] == "remote_match"
56
+ assert doc["remote"]["remote_match"] == 1
57
+
58
+
59
+ def test_a_changed_published_checksum_fails_even_when_local_is_ok(server, capsys, tmp_path):
60
+ """The manifest says one thing, the source another: local hashes still agree."""
61
+ wrong = "md5:" + "0" * 32
62
+ url = _served(server, {"Content-MD5": base64.b64encode(MD5.digest()).decode()})
63
+
64
+ code, out, err = _run(
65
+ capsys, "verify", _manifest(tmp_path, url, wrong), "--against-remote", "--json"
66
+ )
67
+
68
+ doc = json.loads(out)
69
+ assert doc["rows"][0]["status"] == "ok", "the bytes are fine locally"
70
+ assert doc["rows"][0]["remote"] == "remote_mismatch"
71
+ assert code == cli.EXIT_FAIL
72
+ assert "no longer match" in err
73
+
74
+
75
+ def test_no_published_checksum_is_reported_not_passed(server, capsys, tmp_path):
76
+ url = _served(server)
77
+
78
+ code, out, _ = _run(capsys, "verify", _manifest(tmp_path, url), "--against-remote", "--json")
79
+
80
+ doc = json.loads(out)
81
+ assert code == cli.EXIT_OK
82
+ assert doc["rows"][0]["remote"] == "remote_no_checksum"
83
+
84
+
85
+ def test_an_unreachable_target_is_unavailable_and_not_fatal(server, capsys, tmp_path):
86
+ url = server.base + "/gone.bin" # never registered
87
+
88
+ code, out, err = _run(capsys, "verify", _manifest(tmp_path, url), "--against-remote", "--json")
89
+
90
+ doc = json.loads(out)
91
+ assert code == cli.EXIT_OK
92
+ assert doc["rows"][0]["remote"] == "remote_unavailable"
93
+ assert "--require-remote" in err
94
+
95
+
96
+ def test_require_remote_makes_unavailable_fatal(server, capsys, tmp_path):
97
+ url = server.base + "/gone.bin"
98
+
99
+ code, _, _ = _run(
100
+ capsys, "verify", _manifest(tmp_path, url), "--against-remote", "--require-remote"
101
+ )
102
+
103
+ assert code == cli.EXIT_FAIL
104
+
105
+
106
+ def test_a_removed_source_is_unavailable_with_the_reason(capsys, tmp_path):
107
+ path = _manifest(tmp_path, "figshare:32519553")
108
+
109
+ code, out, _ = _run(capsys, "verify", path, "--against-remote", "--json")
110
+
111
+ assert code == cli.EXIT_OK
112
+ assert json.loads(out)["rows"][0]["remote"] == "remote_unavailable"
113
+
114
+
115
+ def test_the_check_time_is_recorded(capsys, tmp_path, monkeypatch):
116
+ from acadfetch import remote
117
+
118
+ monkeypatch.setattr(
119
+ remote.registry,
120
+ "resolve",
121
+ lambda _c, _t: Record(
122
+ source="zenodo",
123
+ target="zenodo:1",
124
+ files=[File("https://example/a.bin", "a.bin", 1, "md5:" + MD5.hexdigest())],
125
+ ),
126
+ )
127
+ path = _manifest(
128
+ tmp_path, "zenodo:1", "md5:" + MD5.hexdigest(), file_url="https://example/a.bin"
129
+ )
130
+
131
+ code, _, _ = _run(capsys, "verify", path, "--against-remote")
132
+
133
+ assert code == cli.EXIT_OK
134
+ written = json.loads(path.read_text("utf-8"))
135
+ assert written["remote_checked_at"]
136
+
137
+
138
+ def test_without_the_flag_nothing_is_written_or_asked(server, capsys, tmp_path):
139
+ url = _served(server)
140
+ path = _manifest(tmp_path, url)
141
+ before = path.read_text("utf-8")
142
+
143
+ code, out, _ = _run(capsys, "verify", path, "--json")
144
+
145
+ assert code == cli.EXIT_OK
146
+ assert "remote" not in json.loads(out)
147
+ assert path.read_text("utf-8") == before
148
+ assert server.cfg["hits"] == {}, "no request without the flag"
149
+
150
+
151
+ def test_a_zenodo_record_that_changed_is_a_mismatch(monkeypatch, capsys, tmp_path):
152
+ from acadfetch import remote
153
+
154
+ monkeypatch.setattr(
155
+ remote.registry,
156
+ "resolve",
157
+ lambda _c, _t: Record(
158
+ source="zenodo",
159
+ target="zenodo:1",
160
+ files=[File("https://example/a.bin", "a.bin", 1, "md5:" + "1" * 32)],
161
+ ),
162
+ )
163
+ path = _manifest(
164
+ tmp_path, "zenodo:1", "md5:" + MD5.hexdigest(), file_url="https://example/a.bin"
165
+ )
166
+
167
+ code, out, _ = _run(capsys, "verify", path, "--against-remote", "--json")
168
+
169
+ assert json.loads(out)["rows"][0]["remote"] == "remote_mismatch"
170
+ assert code == cli.EXIT_FAIL
171
+
172
+
173
+ def test_judge_normalises_case_and_algorithm():
174
+ from acadfetch import remote
175
+
176
+ row = {"url": "u", "checksum": "MD5:" + MD5.hexdigest().upper()}
177
+ assert remote.judge({"u": "md5:" + MD5.hexdigest()}, row, "")[0] == "remote_match"
178
+ assert remote.judge({"u": "md5:" + "2" * 32}, row, "")[0] == "remote_mismatch"
179
+
180
+
181
+ def test_judge_refuses_an_unparseable_side():
182
+ from acadfetch import remote
183
+
184
+ status, detail = remote.judge(
185
+ {"u": "md5:" + MD5.hexdigest()}, {"url": "u", "checksum": "nope"}, ""
186
+ )
187
+ assert status == "remote_no_checksum" and "parseable" in detail
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes