acadfetch 0.2.1__tar.gz → 0.3.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {acadfetch-0.2.1/acadfetch.egg-info → acadfetch-0.3.1}/PKG-INFO +10 -2
- {acadfetch-0.2.1 → acadfetch-0.3.1}/README.md +9 -1
- {acadfetch-0.2.1 → acadfetch-0.3.1}/acadfetch/cli.py +70 -3
- {acadfetch-0.2.1 → acadfetch-0.3.1}/acadfetch/consts.py +1 -1
- {acadfetch-0.2.1 → acadfetch-0.3.1}/acadfetch/manifest.py +4 -2
- acadfetch-0.3.1/acadfetch/remote.py +82 -0
- {acadfetch-0.2.1 → acadfetch-0.3.1/acadfetch.egg-info}/PKG-INFO +10 -2
- {acadfetch-0.2.1 → acadfetch-0.3.1}/acadfetch.egg-info/SOURCES.txt +2 -0
- {acadfetch-0.2.1 → acadfetch-0.3.1}/pyproject.toml +1 -1
- {acadfetch-0.2.1 → acadfetch-0.3.1}/tests/conftest.py +18 -0
- {acadfetch-0.2.1 → acadfetch-0.3.1}/tests/test_json.py +20 -0
- acadfetch-0.3.1/tests/test_remote.py +187 -0
- {acadfetch-0.2.1 → acadfetch-0.3.1}/LICENSE +0 -0
- {acadfetch-0.2.1 → acadfetch-0.3.1}/MANIFEST.in +0 -0
- {acadfetch-0.2.1 → acadfetch-0.3.1}/acadfetch/__init__.py +0 -0
- {acadfetch-0.2.1 → acadfetch-0.3.1}/acadfetch/__main__.py +0 -0
- {acadfetch-0.2.1 → acadfetch-0.3.1}/acadfetch/atomic.py +0 -0
- {acadfetch-0.2.1 → acadfetch-0.3.1}/acadfetch/auth.py +0 -0
- {acadfetch-0.2.1 → acadfetch-0.3.1}/acadfetch/client.py +0 -0
- {acadfetch-0.2.1 → acadfetch-0.3.1}/acadfetch/downloader.py +0 -0
- {acadfetch-0.2.1 → acadfetch-0.3.1}/acadfetch/errors.py +0 -0
- {acadfetch-0.2.1 → acadfetch-0.3.1}/acadfetch/naming.py +0 -0
- {acadfetch-0.2.1 → acadfetch-0.3.1}/acadfetch/progress.py +0 -0
- {acadfetch-0.2.1 → acadfetch-0.3.1}/acadfetch/py.typed +0 -0
- {acadfetch-0.2.1 → acadfetch-0.3.1}/acadfetch/ratelimit.py +0 -0
- {acadfetch-0.2.1 → acadfetch-0.3.1}/acadfetch/sources/__init__.py +0 -0
- {acadfetch-0.2.1 → acadfetch-0.3.1}/acadfetch/sources/arxiv.py +0 -0
- {acadfetch-0.2.1 → acadfetch-0.3.1}/acadfetch/sources/base.py +0 -0
- {acadfetch-0.2.1 → acadfetch-0.3.1}/acadfetch/sources/direct.py +0 -0
- {acadfetch-0.2.1 → acadfetch-0.3.1}/acadfetch/sources/doi.py +0 -0
- {acadfetch-0.2.1 → acadfetch-0.3.1}/acadfetch/sources/registry.py +0 -0
- {acadfetch-0.2.1 → acadfetch-0.3.1}/acadfetch/sources/removed.py +0 -0
- {acadfetch-0.2.1 → acadfetch-0.3.1}/acadfetch/sources/zenodo.py +0 -0
- {acadfetch-0.2.1 → acadfetch-0.3.1}/acadfetch/units.py +0 -0
- {acadfetch-0.2.1 → acadfetch-0.3.1}/acadfetch/urls.py +0 -0
- {acadfetch-0.2.1 → acadfetch-0.3.1}/acadfetch/verify.py +0 -0
- {acadfetch-0.2.1 → acadfetch-0.3.1}/acadfetch.egg-info/dependency_links.txt +0 -0
- {acadfetch-0.2.1 → acadfetch-0.3.1}/acadfetch.egg-info/entry_points.txt +0 -0
- {acadfetch-0.2.1 → acadfetch-0.3.1}/acadfetch.egg-info/top_level.txt +0 -0
- {acadfetch-0.2.1 → acadfetch-0.3.1}/setup.cfg +0 -0
- {acadfetch-0.2.1 → acadfetch-0.3.1}/tests/fixtures/README.md +0 -0
- {acadfetch-0.2.1 → acadfetch-0.3.1}/tests/fixtures/figshare_403_block.html +0 -0
- {acadfetch-0.2.1 → acadfetch-0.3.1}/tests/fixtures/figshare_403_forbidden.json +0 -0
- {acadfetch-0.2.1 → acadfetch-0.3.1}/tests/fixtures/figshare_article.json +0 -0
- {acadfetch-0.2.1 → acadfetch-0.3.1}/tests/fixtures/figshare_file.pdf +0 -0
- {acadfetch-0.2.1 → acadfetch-0.3.1}/tests/test_atomic.py +0 -0
- {acadfetch-0.2.1 → acadfetch-0.3.1}/tests/test_auth.py +0 -0
- {acadfetch-0.2.1 → acadfetch-0.3.1}/tests/test_checksum.py +0 -0
- {acadfetch-0.2.1 → acadfetch-0.3.1}/tests/test_cli.py +0 -0
- {acadfetch-0.2.1 → acadfetch-0.3.1}/tests/test_client.py +0 -0
- {acadfetch-0.2.1 → acadfetch-0.3.1}/tests/test_control_names.py +0 -0
- {acadfetch-0.2.1 → acadfetch-0.3.1}/tests/test_disk.py +0 -0
- {acadfetch-0.2.1 → acadfetch-0.3.1}/tests/test_downloader.py +0 -0
- {acadfetch-0.2.1 → acadfetch-0.3.1}/tests/test_identity.py +0 -0
- {acadfetch-0.2.1 → acadfetch-0.3.1}/tests/test_isolation.py +0 -0
- {acadfetch-0.2.1 → acadfetch-0.3.1}/tests/test_manifest.py +0 -0
- {acadfetch-0.2.1 → acadfetch-0.3.1}/tests/test_naming.py +0 -0
- {acadfetch-0.2.1 → acadfetch-0.3.1}/tests/test_naming_race.py +0 -0
- {acadfetch-0.2.1 → acadfetch-0.3.1}/tests/test_packaging.py +0 -0
- {acadfetch-0.2.1 → acadfetch-0.3.1}/tests/test_progress.py +0 -0
- {acadfetch-0.2.1 → acadfetch-0.3.1}/tests/test_ratelimit.py +0 -0
- {acadfetch-0.2.1 → acadfetch-0.3.1}/tests/test_redirect.py +0 -0
- {acadfetch-0.2.1 → acadfetch-0.3.1}/tests/test_removed.py +0 -0
- {acadfetch-0.2.1 → acadfetch-0.3.1}/tests/test_sources.py +0 -0
- {acadfetch-0.2.1 → acadfetch-0.3.1}/tests/test_urls.py +0 -0
- {acadfetch-0.2.1 → acadfetch-0.3.1}/tests/test_verify.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: acadfetch
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.3.1
|
|
4
4
|
Summary: Resumable, rate-limited, checksum-verified bulk downloads from academic data repositories
|
|
5
5
|
Author: young-ski
|
|
6
6
|
License-Expression: MIT
|
|
@@ -143,7 +143,8 @@ acadfetch fetch zenodo:123456 --json | jq -r '.files[] | select(.state=="failed"
|
|
|
143
143
|
```
|
|
144
144
|
|
|
145
145
|
Exit codes are unchanged, and a failure before any transfer still produces a
|
|
146
|
-
document with `error` set. `report` is the document's schema version (now `2
|
|
146
|
+
document with `error` set. `report` is the document's schema version (now `2`, field reference in
|
|
147
|
+
[docs/json-report.md](docs/json-report.md)):
|
|
147
148
|
fields are added, never removed or given a new meaning, and `state` may gain
|
|
148
149
|
values — treat an unknown one as non-terminal. `verify --json` reports per-file
|
|
149
150
|
`MISMATCH` rows the same way, and its `summary` counts `skipped` rows
|
|
@@ -236,6 +237,9 @@ before duplicates were resolved.
|
|
|
236
237
|
|
|
237
238
|
`acadfetch verify` recomputes SHA-256 for each entry and exits non-zero on any
|
|
238
239
|
mismatch or missing file; rows the run never selected are reported `skipped`.
|
|
240
|
+
Add `--against-remote` to ask the source again and compare what it publishes
|
|
241
|
+
now: a manifest someone rewrote stops agreeing even though the bytes still match
|
|
242
|
+
it. `--require-remote` makes an unreachable source a failure.
|
|
239
243
|
Commit the manifest next to your analysis; it is what makes the download
|
|
240
244
|
reproducible a year later.
|
|
241
245
|
|
|
@@ -288,6 +292,10 @@ honest way to test this code path.
|
|
|
288
292
|
|
|
289
293
|
## Getting involved
|
|
290
294
|
|
|
295
|
+
A nightly job downloads three fixed public records and verifies them, so a
|
|
296
|
+
rotted adapter opens an issue before a user reports one. It never blocks a
|
|
297
|
+
merge; the next good night closes the issue.
|
|
298
|
+
|
|
291
299
|
- **Stuck on something** → [docs/troubleshooting.md](docs/troubleshooting.md) first; it is short and indexed by symptom
|
|
292
300
|
- **Usage questions** → [Discussions](https://github.com/young-ski/acadfetch/discussions)
|
|
293
301
|
- **Bugs** → the bug template; say whether `verify` reports `MISMATCH`
|
|
@@ -112,7 +112,8 @@ acadfetch fetch zenodo:123456 --json | jq -r '.files[] | select(.state=="failed"
|
|
|
112
112
|
```
|
|
113
113
|
|
|
114
114
|
Exit codes are unchanged, and a failure before any transfer still produces a
|
|
115
|
-
document with `error` set. `report` is the document's schema version (now `2
|
|
115
|
+
document with `error` set. `report` is the document's schema version (now `2`, field reference in
|
|
116
|
+
[docs/json-report.md](docs/json-report.md)):
|
|
116
117
|
fields are added, never removed or given a new meaning, and `state` may gain
|
|
117
118
|
values — treat an unknown one as non-terminal. `verify --json` reports per-file
|
|
118
119
|
`MISMATCH` rows the same way, and its `summary` counts `skipped` rows
|
|
@@ -205,6 +206,9 @@ before duplicates were resolved.
|
|
|
205
206
|
|
|
206
207
|
`acadfetch verify` recomputes SHA-256 for each entry and exits non-zero on any
|
|
207
208
|
mismatch or missing file; rows the run never selected are reported `skipped`.
|
|
209
|
+
Add `--against-remote` to ask the source again and compare what it publishes
|
|
210
|
+
now: a manifest someone rewrote stops agreeing even though the bytes still match
|
|
211
|
+
it. `--require-remote` makes an unreachable source a failure.
|
|
208
212
|
Commit the manifest next to your analysis; it is what makes the download
|
|
209
213
|
reproducible a year later.
|
|
210
214
|
|
|
@@ -257,6 +261,10 @@ honest way to test this code path.
|
|
|
257
261
|
|
|
258
262
|
## Getting involved
|
|
259
263
|
|
|
264
|
+
A nightly job downloads three fixed public records and verifies them, so a
|
|
265
|
+
rotted adapter opens an issue before a user reports one. It never blocks a
|
|
266
|
+
merge; the next good night closes the issue.
|
|
267
|
+
|
|
260
268
|
- **Stuck on something** → [docs/troubleshooting.md](docs/troubleshooting.md) first; it is short and indexed by symptom
|
|
261
269
|
- **Usage questions** → [Discussions](https://github.com/young-ski/acadfetch/discussions)
|
|
262
270
|
- **Bugs** → the bug template; say whether `verify` reports `MISMATCH`
|
|
@@ -18,6 +18,7 @@ from pathlib import Path
|
|
|
18
18
|
from urllib.parse import urlsplit
|
|
19
19
|
|
|
20
20
|
from . import auth, consts, manifest, naming, urls, verify
|
|
21
|
+
from . import remote as remote_check
|
|
21
22
|
from .client import Client
|
|
22
23
|
from .downloader import Options, check_disk, check_total, fetch_record, human
|
|
23
24
|
from .errors import AcFError, PartialError, TargetError
|
|
@@ -440,7 +441,20 @@ def cmd_verify(args: argparse.Namespace) -> int:
|
|
|
440
441
|
if manifest.peek_schema(manifest_path) == 1:
|
|
441
442
|
print(manifest.SCHEMA_1_NOTE, file=sys.stderr)
|
|
442
443
|
doc["root"] = str(root)
|
|
443
|
-
|
|
444
|
+
data = manifest.load(manifest_path)
|
|
445
|
+
rows = manifest.verify_manifest(manifest_path, root, allow_escape=args.allow_escape, data=data)
|
|
446
|
+
remote: dict[str, tuple[str, str]] = {}
|
|
447
|
+
if args.against_remote:
|
|
448
|
+
published_now, reason = remote_check.published(_client(args), data)
|
|
449
|
+
entries = [e for e in data.get("files") or [] if isinstance(e, dict)]
|
|
450
|
+
# verify_manifest walks the same list in order; the rows do not carry
|
|
451
|
+
# the URL or checksum themselves.
|
|
452
|
+
remote = {
|
|
453
|
+
row["name"]: remote_check.judge(
|
|
454
|
+
None if not published_now and reason else published_now, entry, reason
|
|
455
|
+
)
|
|
456
|
+
for row, entry in zip(rows, entries, strict=False)
|
|
457
|
+
}
|
|
444
458
|
skipped = sum(1 for row in rows if row["status"] == "skipped")
|
|
445
459
|
hard = sum(1 for row in rows if row["status"] not in ("ok", "skipped"))
|
|
446
460
|
# --require-all turns "you never asked for it" into a failure; the plain
|
|
@@ -450,7 +464,11 @@ def cmd_verify(args: argparse.Namespace) -> int:
|
|
|
450
464
|
ok = len(rows) - hard - skipped
|
|
451
465
|
if not args.json:
|
|
452
466
|
for row in rows:
|
|
453
|
-
|
|
467
|
+
status, detail = remote.get(row["name"], ("", ""))
|
|
468
|
+
tail = f" remote: {status}" if status else ""
|
|
469
|
+
if detail:
|
|
470
|
+
tail += f" ({detail})"
|
|
471
|
+
print(f"{row['status']:>9} {row['name']}{tail}")
|
|
454
472
|
if rows:
|
|
455
473
|
print()
|
|
456
474
|
tally = f"{ok}/{len(rows)} ok"
|
|
@@ -459,8 +477,46 @@ def cmd_verify(args: argparse.Namespace) -> int:
|
|
|
459
477
|
print(tally)
|
|
460
478
|
if not args.allow_escape and any(row["status"] == "escaped" for row in rows):
|
|
461
479
|
print(f"some names leave {root} - --allow-escape follows them", file=sys.stderr)
|
|
480
|
+
remote_bad = 0
|
|
481
|
+
if remote:
|
|
482
|
+
total = remote_check.counts([{"remote": status} for status, _ in remote.values()])
|
|
483
|
+
if total[remote_check.MISMATCH]:
|
|
484
|
+
remote_bad = total[remote_check.MISMATCH]
|
|
485
|
+
if args.require_remote and total[remote_check.UNAVAILABLE]:
|
|
486
|
+
remote_bad += total[remote_check.UNAVAILABLE]
|
|
487
|
+
bad += remote_bad
|
|
488
|
+
data["remote_checked_at"] = datetime.now(timezone.utc).replace(microsecond=0).isoformat()
|
|
489
|
+
try:
|
|
490
|
+
manifest.write(manifest_path, data)
|
|
491
|
+
except OSError as exc:
|
|
492
|
+
print(f"acadfetch: could not record remote_checked_at: {exc}", file=sys.stderr)
|
|
493
|
+
doc["remote_checked_at"] = data["remote_checked_at"]
|
|
494
|
+
doc["remote"] = total
|
|
495
|
+
if total[remote_check.MISMATCH]:
|
|
496
|
+
print(
|
|
497
|
+
f"{total[remote_check.MISMATCH]} file(s) no longer match what the source "
|
|
498
|
+
f"publishes now",
|
|
499
|
+
file=sys.stderr,
|
|
500
|
+
)
|
|
501
|
+
if total[remote_check.UNAVAILABLE] and not args.require_remote:
|
|
502
|
+
print(
|
|
503
|
+
f"{total[remote_check.UNAVAILABLE]} file(s) could not be re-checked; "
|
|
504
|
+
f"--require-remote counts them",
|
|
505
|
+
file=sys.stderr,
|
|
506
|
+
)
|
|
462
507
|
doc["rows"] = [
|
|
463
|
-
{
|
|
508
|
+
{
|
|
509
|
+
**{
|
|
510
|
+
k: row[k]
|
|
511
|
+
for k in ("name", "status", "state", "sha256", "expected", "actual")
|
|
512
|
+
if k in row
|
|
513
|
+
},
|
|
514
|
+
**(
|
|
515
|
+
{"remote": remote[row["name"]][0], "remote_detail": remote[row["name"]][1]}
|
|
516
|
+
if row["name"] in remote
|
|
517
|
+
else {}
|
|
518
|
+
),
|
|
519
|
+
}
|
|
464
520
|
for row in rows
|
|
465
521
|
]
|
|
466
522
|
doc["summary"] = {
|
|
@@ -647,11 +703,22 @@ def build_parser() -> argparse.ArgumentParser:
|
|
|
647
703
|
help="count a file that was never selected (--only, or a manifest-only "
|
|
648
704
|
"run) as a failure instead of skipping it",
|
|
649
705
|
)
|
|
706
|
+
ver.add_argument(
|
|
707
|
+
"--against-remote",
|
|
708
|
+
action="store_true",
|
|
709
|
+
help="ask the source again and compare what it publishes now (network)",
|
|
710
|
+
)
|
|
711
|
+
ver.add_argument(
|
|
712
|
+
"--require-remote",
|
|
713
|
+
action="store_true",
|
|
714
|
+
help="count a file that could not be re-checked as a failure",
|
|
715
|
+
)
|
|
650
716
|
ver.add_argument(
|
|
651
717
|
"--json",
|
|
652
718
|
action="store_true",
|
|
653
719
|
help="print one JSON report to stdout instead of rows",
|
|
654
720
|
)
|
|
721
|
+
network_flags(ver)
|
|
655
722
|
ver.set_defaults(func=cmd_verify)
|
|
656
723
|
|
|
657
724
|
src = sub.add_parser("sources", help="list supported sources")
|
|
@@ -206,7 +206,9 @@ def peek_schema(path: Path) -> int | None:
|
|
|
206
206
|
return value if isinstance(value, int) else None
|
|
207
207
|
|
|
208
208
|
|
|
209
|
-
def verify_manifest(
|
|
209
|
+
def verify_manifest(
|
|
210
|
+
path: Path, root: Path, allow_escape: bool = False, data: dict | None = None
|
|
211
|
+
) -> list[dict]:
|
|
210
212
|
"""Re-hash every recorded file. Returns one row per file.
|
|
211
213
|
|
|
212
214
|
A name is only allowed to point inside ``root`` unless ``allow_escape`` is
|
|
@@ -214,7 +216,7 @@ def verify_manifest(path: Path, root: Path, allow_escape: bool = False) -> list[
|
|
|
214
216
|
outside the directory while reporting ``ok`` is how a green result stops
|
|
215
217
|
meaning anything.
|
|
216
218
|
"""
|
|
217
|
-
data = load(path)
|
|
219
|
+
data = load(path) if data is None else data
|
|
218
220
|
rows = []
|
|
219
221
|
for entry in data.get("files") or []:
|
|
220
222
|
name = entry.get("name") if isinstance(entry, dict) else None
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
"""Re-check a manifest against what the source publishes today.
|
|
2
|
+
|
|
3
|
+
``verify`` alone proves the bytes match the manifest. Nobody signs the manifest,
|
|
4
|
+
so someone who rewrites both the file and its row passes. Asking the source
|
|
5
|
+
again is the only thing that closes that: the row says what was published, the
|
|
6
|
+
source says what is published now, and a tampered row stops agreeing.
|
|
7
|
+
|
|
8
|
+
The result is a line per file, never a rewrite of the local check: a remote
|
|
9
|
+
mismatch is *added* to whatever the local verification said.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
from collections.abc import Mapping
|
|
15
|
+
|
|
16
|
+
from . import verify
|
|
17
|
+
from .client import Client
|
|
18
|
+
from .errors import AcFError
|
|
19
|
+
from .sources import registry, removed
|
|
20
|
+
|
|
21
|
+
MATCH = "remote_match"
|
|
22
|
+
MISMATCH = "remote_mismatch"
|
|
23
|
+
NO_CHECKSUM = "remote_no_checksum"
|
|
24
|
+
UNAVAILABLE = "remote_unavailable"
|
|
25
|
+
STATUSES = (MATCH, MISMATCH, NO_CHECKSUM, UNAVAILABLE)
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def published(client: Client, data: Mapping) -> tuple[dict[str, str | None], str]:
|
|
29
|
+
"""URL -> checksum the source publishes now, or a reason there is none.
|
|
30
|
+
|
|
31
|
+
One record resolution per manifest: the API returns every file at once, and
|
|
32
|
+
a ``direct`` target resolves to a single HEAD.
|
|
33
|
+
"""
|
|
34
|
+
target = data.get("target") or ""
|
|
35
|
+
if not isinstance(target, str) or not target:
|
|
36
|
+
return {}, "the manifest records no target"
|
|
37
|
+
if removed.explain(target):
|
|
38
|
+
return {}, "the source was removed; re-fetch from its file URL"
|
|
39
|
+
try:
|
|
40
|
+
record = registry.resolve(client, target)
|
|
41
|
+
except AcFError as exc:
|
|
42
|
+
return {}, str(exc)
|
|
43
|
+
except OSError as exc: # name resolution, a proxy that is not there
|
|
44
|
+
return {}, str(exc)
|
|
45
|
+
return {file.url: file.checksum for file in record.files}, ""
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def _normalise(checksum: str | None) -> tuple[str, str] | None:
|
|
49
|
+
parsed = verify.parse(checksum)
|
|
50
|
+
return parsed
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def judge(
|
|
54
|
+
published_now: Mapping[str, str | None] | None, row: Mapping, reason: str
|
|
55
|
+
) -> tuple[str, str]:
|
|
56
|
+
"""``(status, detail)`` for one manifest row."""
|
|
57
|
+
if published_now is None:
|
|
58
|
+
return UNAVAILABLE, reason
|
|
59
|
+
url = row.get("url")
|
|
60
|
+
if not isinstance(url, str) or url not in published_now:
|
|
61
|
+
return UNAVAILABLE, "no longer published at the recorded URL"
|
|
62
|
+
remote = published_now[url]
|
|
63
|
+
local = row.get("checksum")
|
|
64
|
+
if not remote:
|
|
65
|
+
return NO_CHECKSUM, "the source publishes no checksum now"
|
|
66
|
+
if not local:
|
|
67
|
+
return NO_CHECKSUM, "this row recorded no checksum"
|
|
68
|
+
here, there = _normalise(local), _normalise(remote)
|
|
69
|
+
if here is None or there is None:
|
|
70
|
+
return NO_CHECKSUM, "a checksum is not parseable"
|
|
71
|
+
return (
|
|
72
|
+
(MATCH, "") if here == there else (MISMATCH, f"published {there[0]}:{there[1][:12]}… now")
|
|
73
|
+
)
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def counts(rows: list[dict]) -> dict[str, int]:
|
|
77
|
+
totals = dict.fromkeys(STATUSES, 0)
|
|
78
|
+
for row in rows:
|
|
79
|
+
status = row.get("remote")
|
|
80
|
+
if status in totals:
|
|
81
|
+
totals[status] += 1
|
|
82
|
+
return totals
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: acadfetch
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.3.1
|
|
4
4
|
Summary: Resumable, rate-limited, checksum-verified bulk downloads from academic data repositories
|
|
5
5
|
Author: young-ski
|
|
6
6
|
License-Expression: MIT
|
|
@@ -143,7 +143,8 @@ acadfetch fetch zenodo:123456 --json | jq -r '.files[] | select(.state=="failed"
|
|
|
143
143
|
```
|
|
144
144
|
|
|
145
145
|
Exit codes are unchanged, and a failure before any transfer still produces a
|
|
146
|
-
document with `error` set. `report` is the document's schema version (now `2
|
|
146
|
+
document with `error` set. `report` is the document's schema version (now `2`, field reference in
|
|
147
|
+
[docs/json-report.md](docs/json-report.md)):
|
|
147
148
|
fields are added, never removed or given a new meaning, and `state` may gain
|
|
148
149
|
values — treat an unknown one as non-terminal. `verify --json` reports per-file
|
|
149
150
|
`MISMATCH` rows the same way, and its `summary` counts `skipped` rows
|
|
@@ -236,6 +237,9 @@ before duplicates were resolved.
|
|
|
236
237
|
|
|
237
238
|
`acadfetch verify` recomputes SHA-256 for each entry and exits non-zero on any
|
|
238
239
|
mismatch or missing file; rows the run never selected are reported `skipped`.
|
|
240
|
+
Add `--against-remote` to ask the source again and compare what it publishes
|
|
241
|
+
now: a manifest someone rewrote stops agreeing even though the bytes still match
|
|
242
|
+
it. `--require-remote` makes an unreachable source a failure.
|
|
239
243
|
Commit the manifest next to your analysis; it is what makes the download
|
|
240
244
|
reproducible a year later.
|
|
241
245
|
|
|
@@ -288,6 +292,10 @@ honest way to test this code path.
|
|
|
288
292
|
|
|
289
293
|
## Getting involved
|
|
290
294
|
|
|
295
|
+
A nightly job downloads three fixed public records and verifies them, so a
|
|
296
|
+
rotted adapter opens an issue before a user reports one. It never blocks a
|
|
297
|
+
merge; the next good night closes the issue.
|
|
298
|
+
|
|
291
299
|
- **Stuck on something** → [docs/troubleshooting.md](docs/troubleshooting.md) first; it is short and indexed by symptom
|
|
292
300
|
- **Usage questions** → [Discussions](https://github.com/young-ski/acadfetch/discussions)
|
|
293
301
|
- **Bugs** → the bug template; say whether `verify` reports `MISMATCH`
|
|
@@ -16,6 +16,7 @@ acadfetch/naming.py
|
|
|
16
16
|
acadfetch/progress.py
|
|
17
17
|
acadfetch/py.typed
|
|
18
18
|
acadfetch/ratelimit.py
|
|
19
|
+
acadfetch/remote.py
|
|
19
20
|
acadfetch/units.py
|
|
20
21
|
acadfetch/urls.py
|
|
21
22
|
acadfetch/verify.py
|
|
@@ -51,6 +52,7 @@ tests/test_packaging.py
|
|
|
51
52
|
tests/test_progress.py
|
|
52
53
|
tests/test_ratelimit.py
|
|
53
54
|
tests/test_redirect.py
|
|
55
|
+
tests/test_remote.py
|
|
54
56
|
tests/test_removed.py
|
|
55
57
|
tests/test_sources.py
|
|
56
58
|
tests/test_urls.py
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "acadfetch"
|
|
7
|
-
version = "0.
|
|
7
|
+
version = "0.3.1"
|
|
8
8
|
description = "Resumable, rate-limited, checksum-verified bulk downloads from academic data repositories"
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.10"
|
|
@@ -26,6 +26,21 @@ class Handler(BaseHTTPRequestHandler):
|
|
|
26
26
|
self.wfile.write(body)
|
|
27
27
|
self.wfile.flush()
|
|
28
28
|
|
|
29
|
+
def do_HEAD(self): # noqa: N802
|
|
30
|
+
cfg = self.server.cfg
|
|
31
|
+
path = self.path.split("?")[0]
|
|
32
|
+
cfg["hits"][path] = cfg["hits"].get(path, 0) + 1
|
|
33
|
+
if path not in cfg["files"]:
|
|
34
|
+
self.send_error(404)
|
|
35
|
+
return
|
|
36
|
+
self.send_response(200)
|
|
37
|
+
for name, value in cfg["headers"].get(path, {}).items():
|
|
38
|
+
self.send_header(name, value)
|
|
39
|
+
self.send_header("Content-Type", cfg["types"].get(path, "application/octet-stream"))
|
|
40
|
+
self.send_header("Content-Length", str(len(cfg["files"][path])))
|
|
41
|
+
self.send_header("ETag", cfg["etag"])
|
|
42
|
+
self.end_headers()
|
|
43
|
+
|
|
29
44
|
def do_GET(self): # noqa: N802
|
|
30
45
|
cfg = self.server.cfg
|
|
31
46
|
path = self.path.split("?")[0]
|
|
@@ -89,6 +104,8 @@ class Handler(BaseHTTPRequestHandler):
|
|
|
89
104
|
self.send_header(
|
|
90
105
|
"Content-Range", f"bytes {start}-{len(payload) - 1}/{len(payload)}"
|
|
91
106
|
)
|
|
107
|
+
for name, value in cfg["headers"].get(path, {}).items():
|
|
108
|
+
self.send_header(name, value)
|
|
92
109
|
self.send_header("Content-Type", cfg["types"].get(path, "application/octet-stream"))
|
|
93
110
|
declared = cfg["declares"].get(path, len(body))
|
|
94
111
|
self.send_header("Content-Length", str(declared))
|
|
@@ -135,6 +152,7 @@ def _new_server():
|
|
|
135
152
|
"stall_for": 2.0,
|
|
136
153
|
"auth_seen": [],
|
|
137
154
|
"redirects": {},
|
|
155
|
+
"headers": {},
|
|
138
156
|
}
|
|
139
157
|
|
|
140
158
|
thread = threading.Thread(target=srv.serve_forever, daemon=True)
|
|
@@ -273,3 +273,23 @@ def test_the_default_directory_drops_the_query(server, monkeypatch, capsys, tmp_
|
|
|
273
273
|
slug = Path(json.loads(out)["out"]).name
|
|
274
274
|
assert slug.startswith("direct-127.0.0.1-") and slug.endswith("-a.bin")
|
|
275
275
|
assert "SECRET" not in slug
|
|
276
|
+
|
|
277
|
+
|
|
278
|
+
def test_every_report_field_is_in_the_contract(served, capsys, tmp_path):
|
|
279
|
+
"""A field added to the document without a line in the contract fails here."""
|
|
280
|
+
docs = Path("docs/json-report.md").read_text("utf-8")
|
|
281
|
+
|
|
282
|
+
_, out, _ = _run(capsys, "fetch", "zenodo:1", "--json", "-o", tmp_path)
|
|
283
|
+
fetch_doc = json.loads(out)
|
|
284
|
+
for key in fetch_doc:
|
|
285
|
+
assert f"`{key}`" in docs, f"fetch field {key!r} is undocumented"
|
|
286
|
+
for key in fetch_doc["files"][0]:
|
|
287
|
+
assert f"`{key}`" in docs, f"file field {key!r} is undocumented"
|
|
288
|
+
|
|
289
|
+
manifest_path = tmp_path / "manifest.json"
|
|
290
|
+
_, vout, _ = _run(capsys, "verify", manifest_path, "--json")
|
|
291
|
+
verify_doc = json.loads(vout)
|
|
292
|
+
for key in verify_doc:
|
|
293
|
+
assert f"`{key}`" in docs, f"verify field {key!r} is undocumented"
|
|
294
|
+
for value in ("escaped", "malformed", "remote_match", "remote_unavailable"):
|
|
295
|
+
assert value in docs, f"{value!r} is undocumented"
|
|
@@ -0,0 +1,187 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import base64
|
|
4
|
+
import hashlib
|
|
5
|
+
import json
|
|
6
|
+
|
|
7
|
+
from acadfetch import cli
|
|
8
|
+
from acadfetch.sources.base import File, Record
|
|
9
|
+
|
|
10
|
+
BODY = b"remote check payload" * 40
|
|
11
|
+
MD5 = hashlib.md5(BODY)
|
|
12
|
+
GOOD = "md5:" + MD5.hexdigest()
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def _manifest(tmp_path, url, checksum=GOOD, body=BODY, file_url=None):
|
|
16
|
+
(tmp_path / "a.bin").write_bytes(body)
|
|
17
|
+
data = {
|
|
18
|
+
"schema": 3,
|
|
19
|
+
"source": "direct",
|
|
20
|
+
"target": url,
|
|
21
|
+
"files": [
|
|
22
|
+
{
|
|
23
|
+
"name": "a.bin",
|
|
24
|
+
"url": file_url or url,
|
|
25
|
+
"checksum": checksum,
|
|
26
|
+
"sha256": hashlib.sha256(body).hexdigest(),
|
|
27
|
+
}
|
|
28
|
+
],
|
|
29
|
+
}
|
|
30
|
+
path = tmp_path / "manifest.json"
|
|
31
|
+
path.write_text(json.dumps(data), "utf-8")
|
|
32
|
+
return path
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def _run(capsys, *argv):
|
|
36
|
+
code = cli.main([str(a) for a in argv])
|
|
37
|
+
out, err = capsys.readouterr()
|
|
38
|
+
return code, out, err
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def _served(server, headers=None, body=BODY):
|
|
42
|
+
server.cfg["files"]["/a.bin"] = body
|
|
43
|
+
if headers:
|
|
44
|
+
server.cfg["headers"]["/a.bin"] = headers
|
|
45
|
+
return server.base + "/a.bin"
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def test_a_matching_published_checksum_is_a_match(server, capsys, tmp_path):
|
|
49
|
+
url = _served(server, {"Content-MD5": base64.b64encode(MD5.digest()).decode()})
|
|
50
|
+
|
|
51
|
+
code, out, _ = _run(capsys, "verify", _manifest(tmp_path, url), "--against-remote", "--json")
|
|
52
|
+
|
|
53
|
+
doc = json.loads(out)
|
|
54
|
+
assert code == cli.EXIT_OK
|
|
55
|
+
assert doc["rows"][0]["remote"] == "remote_match"
|
|
56
|
+
assert doc["remote"]["remote_match"] == 1
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def test_a_changed_published_checksum_fails_even_when_local_is_ok(server, capsys, tmp_path):
|
|
60
|
+
"""The manifest says one thing, the source another: local hashes still agree."""
|
|
61
|
+
wrong = "md5:" + "0" * 32
|
|
62
|
+
url = _served(server, {"Content-MD5": base64.b64encode(MD5.digest()).decode()})
|
|
63
|
+
|
|
64
|
+
code, out, err = _run(
|
|
65
|
+
capsys, "verify", _manifest(tmp_path, url, wrong), "--against-remote", "--json"
|
|
66
|
+
)
|
|
67
|
+
|
|
68
|
+
doc = json.loads(out)
|
|
69
|
+
assert doc["rows"][0]["status"] == "ok", "the bytes are fine locally"
|
|
70
|
+
assert doc["rows"][0]["remote"] == "remote_mismatch"
|
|
71
|
+
assert code == cli.EXIT_FAIL
|
|
72
|
+
assert "no longer match" in err
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def test_no_published_checksum_is_reported_not_passed(server, capsys, tmp_path):
|
|
76
|
+
url = _served(server)
|
|
77
|
+
|
|
78
|
+
code, out, _ = _run(capsys, "verify", _manifest(tmp_path, url), "--against-remote", "--json")
|
|
79
|
+
|
|
80
|
+
doc = json.loads(out)
|
|
81
|
+
assert code == cli.EXIT_OK
|
|
82
|
+
assert doc["rows"][0]["remote"] == "remote_no_checksum"
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def test_an_unreachable_target_is_unavailable_and_not_fatal(server, capsys, tmp_path):
|
|
86
|
+
url = server.base + "/gone.bin" # never registered
|
|
87
|
+
|
|
88
|
+
code, out, err = _run(capsys, "verify", _manifest(tmp_path, url), "--against-remote", "--json")
|
|
89
|
+
|
|
90
|
+
doc = json.loads(out)
|
|
91
|
+
assert code == cli.EXIT_OK
|
|
92
|
+
assert doc["rows"][0]["remote"] == "remote_unavailable"
|
|
93
|
+
assert "--require-remote" in err
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def test_require_remote_makes_unavailable_fatal(server, capsys, tmp_path):
|
|
97
|
+
url = server.base + "/gone.bin"
|
|
98
|
+
|
|
99
|
+
code, _, _ = _run(
|
|
100
|
+
capsys, "verify", _manifest(tmp_path, url), "--against-remote", "--require-remote"
|
|
101
|
+
)
|
|
102
|
+
|
|
103
|
+
assert code == cli.EXIT_FAIL
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def test_a_removed_source_is_unavailable_with_the_reason(capsys, tmp_path):
|
|
107
|
+
path = _manifest(tmp_path, "figshare:32519553")
|
|
108
|
+
|
|
109
|
+
code, out, _ = _run(capsys, "verify", path, "--against-remote", "--json")
|
|
110
|
+
|
|
111
|
+
assert code == cli.EXIT_OK
|
|
112
|
+
assert json.loads(out)["rows"][0]["remote"] == "remote_unavailable"
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def test_the_check_time_is_recorded(capsys, tmp_path, monkeypatch):
|
|
116
|
+
from acadfetch import remote
|
|
117
|
+
|
|
118
|
+
monkeypatch.setattr(
|
|
119
|
+
remote.registry,
|
|
120
|
+
"resolve",
|
|
121
|
+
lambda _c, _t: Record(
|
|
122
|
+
source="zenodo",
|
|
123
|
+
target="zenodo:1",
|
|
124
|
+
files=[File("https://example/a.bin", "a.bin", 1, "md5:" + MD5.hexdigest())],
|
|
125
|
+
),
|
|
126
|
+
)
|
|
127
|
+
path = _manifest(
|
|
128
|
+
tmp_path, "zenodo:1", "md5:" + MD5.hexdigest(), file_url="https://example/a.bin"
|
|
129
|
+
)
|
|
130
|
+
|
|
131
|
+
code, _, _ = _run(capsys, "verify", path, "--against-remote")
|
|
132
|
+
|
|
133
|
+
assert code == cli.EXIT_OK
|
|
134
|
+
written = json.loads(path.read_text("utf-8"))
|
|
135
|
+
assert written["remote_checked_at"]
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
def test_without_the_flag_nothing_is_written_or_asked(server, capsys, tmp_path):
|
|
139
|
+
url = _served(server)
|
|
140
|
+
path = _manifest(tmp_path, url)
|
|
141
|
+
before = path.read_text("utf-8")
|
|
142
|
+
|
|
143
|
+
code, out, _ = _run(capsys, "verify", path, "--json")
|
|
144
|
+
|
|
145
|
+
assert code == cli.EXIT_OK
|
|
146
|
+
assert "remote" not in json.loads(out)
|
|
147
|
+
assert path.read_text("utf-8") == before
|
|
148
|
+
assert server.cfg["hits"] == {}, "no request without the flag"
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
def test_a_zenodo_record_that_changed_is_a_mismatch(monkeypatch, capsys, tmp_path):
|
|
152
|
+
from acadfetch import remote
|
|
153
|
+
|
|
154
|
+
monkeypatch.setattr(
|
|
155
|
+
remote.registry,
|
|
156
|
+
"resolve",
|
|
157
|
+
lambda _c, _t: Record(
|
|
158
|
+
source="zenodo",
|
|
159
|
+
target="zenodo:1",
|
|
160
|
+
files=[File("https://example/a.bin", "a.bin", 1, "md5:" + "1" * 32)],
|
|
161
|
+
),
|
|
162
|
+
)
|
|
163
|
+
path = _manifest(
|
|
164
|
+
tmp_path, "zenodo:1", "md5:" + MD5.hexdigest(), file_url="https://example/a.bin"
|
|
165
|
+
)
|
|
166
|
+
|
|
167
|
+
code, out, _ = _run(capsys, "verify", path, "--against-remote", "--json")
|
|
168
|
+
|
|
169
|
+
assert json.loads(out)["rows"][0]["remote"] == "remote_mismatch"
|
|
170
|
+
assert code == cli.EXIT_FAIL
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
def test_judge_normalises_case_and_algorithm():
|
|
174
|
+
from acadfetch import remote
|
|
175
|
+
|
|
176
|
+
row = {"url": "u", "checksum": "MD5:" + MD5.hexdigest().upper()}
|
|
177
|
+
assert remote.judge({"u": "md5:" + MD5.hexdigest()}, row, "")[0] == "remote_match"
|
|
178
|
+
assert remote.judge({"u": "md5:" + "2" * 32}, row, "")[0] == "remote_mismatch"
|
|
179
|
+
|
|
180
|
+
|
|
181
|
+
def test_judge_refuses_an_unparseable_side():
|
|
182
|
+
from acadfetch import remote
|
|
183
|
+
|
|
184
|
+
status, detail = remote.judge(
|
|
185
|
+
{"u": "md5:" + MD5.hexdigest()}, {"url": "u", "checksum": "nope"}, ""
|
|
186
|
+
)
|
|
187
|
+
assert status == "remote_no_checksum" and "parseable" in detail
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|