acadfetch 0.6.1__tar.gz → 0.6.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {acadfetch-0.6.1/acadfetch.egg-info → acadfetch-0.6.2}/PKG-INFO +1 -1
- {acadfetch-0.6.1 → acadfetch-0.6.2}/acadfetch/client.py +13 -2
- {acadfetch-0.6.1 → acadfetch-0.6.2}/acadfetch/consts.py +1 -1
- {acadfetch-0.6.1 → acadfetch-0.6.2}/acadfetch/downloader.py +41 -1
- {acadfetch-0.6.1 → acadfetch-0.6.2}/acadfetch/errors.py +4 -0
- {acadfetch-0.6.1 → acadfetch-0.6.2}/acadfetch/sources/zenodo.py +4 -0
- {acadfetch-0.6.1 → acadfetch-0.6.2/acadfetch.egg-info}/PKG-INFO +1 -1
- {acadfetch-0.6.1 → acadfetch-0.6.2}/pyproject.toml +1 -1
- {acadfetch-0.6.1 → acadfetch-0.6.2}/tests/conftest.py +3 -1
- {acadfetch-0.6.1 → acadfetch-0.6.2}/tests/test_downloader.py +57 -0
- {acadfetch-0.6.1 → acadfetch-0.6.2}/tests/test_sources.py +11 -0
- {acadfetch-0.6.1 → acadfetch-0.6.2}/LICENSE +0 -0
- {acadfetch-0.6.1 → acadfetch-0.6.2}/MANIFEST.in +0 -0
- {acadfetch-0.6.1 → acadfetch-0.6.2}/README.md +0 -0
- {acadfetch-0.6.1 → acadfetch-0.6.2}/acadfetch/__init__.py +0 -0
- {acadfetch-0.6.1 → acadfetch-0.6.2}/acadfetch/__main__.py +0 -0
- {acadfetch-0.6.1 → acadfetch-0.6.2}/acadfetch/atomic.py +0 -0
- {acadfetch-0.6.1 → acadfetch-0.6.2}/acadfetch/auth.py +0 -0
- {acadfetch-0.6.1 → acadfetch-0.6.2}/acadfetch/cli.py +0 -0
- {acadfetch-0.6.1 → acadfetch-0.6.2}/acadfetch/debug.py +0 -0
- {acadfetch-0.6.1 → acadfetch-0.6.2}/acadfetch/manifest.py +0 -0
- {acadfetch-0.6.1 → acadfetch-0.6.2}/acadfetch/naming.py +0 -0
- {acadfetch-0.6.1 → acadfetch-0.6.2}/acadfetch/progress.py +0 -0
- {acadfetch-0.6.1 → acadfetch-0.6.2}/acadfetch/py.typed +0 -0
- {acadfetch-0.6.1 → acadfetch-0.6.2}/acadfetch/ratelimit.py +0 -0
- {acadfetch-0.6.1 → acadfetch-0.6.2}/acadfetch/remote.py +0 -0
- {acadfetch-0.6.1 → acadfetch-0.6.2}/acadfetch/sources/__init__.py +0 -0
- {acadfetch-0.6.1 → acadfetch-0.6.2}/acadfetch/sources/arxiv.py +0 -0
- {acadfetch-0.6.1 → acadfetch-0.6.2}/acadfetch/sources/base.py +0 -0
- {acadfetch-0.6.1 → acadfetch-0.6.2}/acadfetch/sources/direct.py +0 -0
- {acadfetch-0.6.1 → acadfetch-0.6.2}/acadfetch/sources/doi.py +0 -0
- {acadfetch-0.6.1 → acadfetch-0.6.2}/acadfetch/sources/registry.py +0 -0
- {acadfetch-0.6.1 → acadfetch-0.6.2}/acadfetch/sources/removed.py +0 -0
- {acadfetch-0.6.1 → acadfetch-0.6.2}/acadfetch/units.py +0 -0
- {acadfetch-0.6.1 → acadfetch-0.6.2}/acadfetch/urls.py +0 -0
- {acadfetch-0.6.1 → acadfetch-0.6.2}/acadfetch/verify.py +0 -0
- {acadfetch-0.6.1 → acadfetch-0.6.2}/acadfetch.egg-info/SOURCES.txt +0 -0
- {acadfetch-0.6.1 → acadfetch-0.6.2}/acadfetch.egg-info/dependency_links.txt +0 -0
- {acadfetch-0.6.1 → acadfetch-0.6.2}/acadfetch.egg-info/entry_points.txt +0 -0
- {acadfetch-0.6.1 → acadfetch-0.6.2}/acadfetch.egg-info/top_level.txt +0 -0
- {acadfetch-0.6.1 → acadfetch-0.6.2}/setup.cfg +0 -0
- {acadfetch-0.6.1 → acadfetch-0.6.2}/tests/fixtures/README.md +0 -0
- {acadfetch-0.6.1 → acadfetch-0.6.2}/tests/fixtures/ca.pem +0 -0
- {acadfetch-0.6.1 → acadfetch-0.6.2}/tests/fixtures/figshare_403_block.html +0 -0
- {acadfetch-0.6.1 → acadfetch-0.6.2}/tests/fixtures/figshare_403_forbidden.json +0 -0
- {acadfetch-0.6.1 → acadfetch-0.6.2}/tests/fixtures/figshare_article.json +0 -0
- {acadfetch-0.6.1 → acadfetch-0.6.2}/tests/fixtures/figshare_file.pdf +0 -0
- {acadfetch-0.6.1 → acadfetch-0.6.2}/tests/test_atomic.py +0 -0
- {acadfetch-0.6.1 → acadfetch-0.6.2}/tests/test_auth.py +0 -0
- {acadfetch-0.6.1 → acadfetch-0.6.2}/tests/test_checksum.py +0 -0
- {acadfetch-0.6.1 → acadfetch-0.6.2}/tests/test_citation.py +0 -0
- {acadfetch-0.6.1 → acadfetch-0.6.2}/tests/test_cli.py +0 -0
- {acadfetch-0.6.1 → acadfetch-0.6.2}/tests/test_client.py +0 -0
- {acadfetch-0.6.1 → acadfetch-0.6.2}/tests/test_control_names.py +0 -0
- {acadfetch-0.6.1 → acadfetch-0.6.2}/tests/test_debug.py +0 -0
- {acadfetch-0.6.1 → acadfetch-0.6.2}/tests/test_disk.py +0 -0
- {acadfetch-0.6.1 → acadfetch-0.6.2}/tests/test_hash_once.py +0 -0
- {acadfetch-0.6.1 → acadfetch-0.6.2}/tests/test_identity.py +0 -0
- {acadfetch-0.6.1 → acadfetch-0.6.2}/tests/test_isolation.py +0 -0
- {acadfetch-0.6.1 → acadfetch-0.6.2}/tests/test_json.py +0 -0
- {acadfetch-0.6.1 → acadfetch-0.6.2}/tests/test_manifest.py +0 -0
- {acadfetch-0.6.1 → acadfetch-0.6.2}/tests/test_naming.py +0 -0
- {acadfetch-0.6.1 → acadfetch-0.6.2}/tests/test_naming_race.py +0 -0
- {acadfetch-0.6.1 → acadfetch-0.6.2}/tests/test_network.py +0 -0
- {acadfetch-0.6.1 → acadfetch-0.6.2}/tests/test_packaging.py +0 -0
- {acadfetch-0.6.1 → acadfetch-0.6.2}/tests/test_progress.py +0 -0
- {acadfetch-0.6.1 → acadfetch-0.6.2}/tests/test_ratelimit.py +0 -0
- {acadfetch-0.6.1 → acadfetch-0.6.2}/tests/test_redirect.py +0 -0
- {acadfetch-0.6.1 → acadfetch-0.6.2}/tests/test_remote.py +0 -0
- {acadfetch-0.6.1 → acadfetch-0.6.2}/tests/test_removed.py +0 -0
- {acadfetch-0.6.1 → acadfetch-0.6.2}/tests/test_urls.py +0 -0
- {acadfetch-0.6.1 → acadfetch-0.6.2}/tests/test_urls_glob.py +0 -0
- {acadfetch-0.6.1 → acadfetch-0.6.2}/tests/test_verify.py +0 -0
|
@@ -236,7 +236,12 @@ class Client:
|
|
|
236
236
|
if exc.code in RETRY_STATUS:
|
|
237
237
|
pass
|
|
238
238
|
elif not last_blocked:
|
|
239
|
-
raise HttpError(
|
|
239
|
+
raise HttpError(
|
|
240
|
+
exc.code,
|
|
241
|
+
url,
|
|
242
|
+
detail,
|
|
243
|
+
content_range=exc.headers.get("Content-Range") if exc.headers else None,
|
|
244
|
+
) from exc
|
|
240
245
|
elif waf >= block_retries:
|
|
241
246
|
raise HttpError(
|
|
242
247
|
403,
|
|
@@ -270,7 +275,13 @@ class Client:
|
|
|
270
275
|
if last_blocked
|
|
271
276
|
else last_detail
|
|
272
277
|
)
|
|
273
|
-
raise HttpError(
|
|
278
|
+
raise HttpError(
|
|
279
|
+
last.code,
|
|
280
|
+
url,
|
|
281
|
+
detail,
|
|
282
|
+
transient=True,
|
|
283
|
+
content_range=last.headers.get("Content-Range") if last.headers else None,
|
|
284
|
+
) from last
|
|
274
285
|
raise HttpError(0, url, last_detail or str(last)) from last
|
|
275
286
|
|
|
276
287
|
# -- public ------------------------------------------------------------
|
|
@@ -240,6 +240,46 @@ def _chunks(resp, idle_timeout: float, url: str): # noqa: ANN001, ANN202
|
|
|
240
240
|
yield chunk
|
|
241
241
|
|
|
242
242
|
|
|
243
|
+
def _star_total(content_range: str | None) -> int | None:
|
|
244
|
+
"""``bytes */N`` → N, for the 416 that carries it."""
|
|
245
|
+
if not content_range:
|
|
246
|
+
return None
|
|
247
|
+
_, _, value = content_range.strip().partition("*/")
|
|
248
|
+
try:
|
|
249
|
+
return int(value.strip())
|
|
250
|
+
except ValueError:
|
|
251
|
+
return None
|
|
252
|
+
|
|
253
|
+
|
|
254
|
+
def _restart_or_finish(
|
|
255
|
+
file: File, part: Path, meta_path: Path, have: int, exc: HttpError
|
|
256
|
+
) -> Transfer:
|
|
257
|
+
"""What a 416 means: the part is the file, or it is stale and must go.
|
|
258
|
+
|
|
259
|
+
"Already complete" used to be assumed whenever the part existed, so a stale
|
|
260
|
+
part (a different, older object) was renamed into place with nothing
|
|
261
|
+
checked. A matching checksum is proof; otherwise the size the server
|
|
262
|
+
reports for the whole object has to equal the part exactly.
|
|
263
|
+
"""
|
|
264
|
+
parsed = verify.parse(file.checksum)
|
|
265
|
+
digests: dict[str, str] | None = None
|
|
266
|
+
if parsed is not None:
|
|
267
|
+
digests = verify.hashes(part, {"sha256", parsed[0]})
|
|
268
|
+
if digests[parsed[0]] == parsed[1]:
|
|
269
|
+
return Transfer(0, digests=digests)
|
|
270
|
+
total = _star_total(exc.content_range)
|
|
271
|
+
if total is not None and total == have:
|
|
272
|
+
if digests is None:
|
|
273
|
+
digests = verify.hashes(part, {"sha256"})
|
|
274
|
+
return Transfer(0, digests=digests)
|
|
275
|
+
_drop(part, meta_path)
|
|
276
|
+
if total is None:
|
|
277
|
+
raised, compared = "416 without a size to compare against", "no evidence"
|
|
278
|
+
else:
|
|
279
|
+
raised, compared = f"server says {total} bytes", f"the .part has {have}"
|
|
280
|
+
raise HttpError(416, file.url, f"{raised}, {compared}; restarting", transient=True)
|
|
281
|
+
|
|
282
|
+
|
|
243
283
|
def _transfer(client: Client, file: File, part: Path, meta_path: Path, opts: Options) -> Transfer:
|
|
244
284
|
"""Move bytes into ``part``, resuming if possible. Returns what happened."""
|
|
245
285
|
meta = _read_meta(meta_path) if part.exists() else {}
|
|
@@ -266,7 +306,7 @@ def _transfer(client: Client, file: File, part: Path, meta_path: Path, opts: Opt
|
|
|
266
306
|
resp = client.open(file.url, headers, attempts=1)
|
|
267
307
|
except HttpError as exc:
|
|
268
308
|
if exc.status == 416 and have:
|
|
269
|
-
return
|
|
309
|
+
return _restart_or_finish(file, part, meta_path, have, exc)
|
|
270
310
|
raise
|
|
271
311
|
|
|
272
312
|
with resp:
|
|
@@ -30,6 +30,7 @@ class HttpError(AcFError):
|
|
|
30
30
|
detail: str = "",
|
|
31
31
|
transient: bool = False,
|
|
32
32
|
reason: str = "",
|
|
33
|
+
content_range: str | None = None,
|
|
33
34
|
) -> None:
|
|
34
35
|
self.status = status
|
|
35
36
|
self.url = url
|
|
@@ -37,6 +38,9 @@ class HttpError(AcFError):
|
|
|
37
38
|
#: Machine-readable cause for callers that must branch on it, e.g.
|
|
38
39
|
#: ``html-page``: the body is a block/login page, not the file asked for.
|
|
39
40
|
self.reason = reason
|
|
41
|
+
#: ``Content-Range`` of the answer, so a 416 can say how big the object
|
|
42
|
+
#: is supposed to be.
|
|
43
|
+
self.content_range = content_range
|
|
40
44
|
if status:
|
|
41
45
|
text = f"HTTP {status} for {url}{': ' + detail if detail else ''}"
|
|
42
46
|
elif reason == "max-size":
|
|
@@ -3,6 +3,10 @@
|
|
|
3
3
|
The public REST API exposes ``files[].links.self`` as a redirecting download
|
|
4
4
|
URL and publishes ``files[].checksum`` as ``md5:<hex>``, which is what makes
|
|
5
5
|
verification possible here.
|
|
6
|
+
|
|
7
|
+
``is_latest`` / ``links.latest`` are honoured for an older version DOI. The
|
|
8
|
+
current API does not send ``is_latest`` (checked 2026-11), so the branch is
|
|
9
|
+
dormant; a test pins what it would do if it returned.
|
|
6
10
|
"""
|
|
7
11
|
|
|
8
12
|
from __future__ import annotations
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "acadfetch"
|
|
7
|
-
version = "0.6.
|
|
7
|
+
version = "0.6.2"
|
|
8
8
|
description = "Resumable, rate-limited, checksum-verified bulk downloads from academic data repositories"
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.10"
|
|
@@ -55,7 +55,8 @@ class Handler(BaseHTTPRequestHandler):
|
|
|
55
55
|
return
|
|
56
56
|
if cfg["answers_416"] and self.headers.get("Range"):
|
|
57
57
|
self.send_response(416)
|
|
58
|
-
|
|
58
|
+
if not cfg.get("416_omits_range"):
|
|
59
|
+
self.send_header("Content-Range", f"bytes */{len(cfg['files'].get(path, b''))}")
|
|
59
60
|
self.send_header("Content-Length", "0")
|
|
60
61
|
self.end_headers()
|
|
61
62
|
return
|
|
@@ -147,6 +148,7 @@ def _new_server():
|
|
|
147
148
|
"declares": {},
|
|
148
149
|
"types": {},
|
|
149
150
|
"answers_416": False,
|
|
151
|
+
"416_omits_range": False,
|
|
150
152
|
"ranges": [],
|
|
151
153
|
"stall_after": None,
|
|
152
154
|
"stall_for": 2.0,
|
|
@@ -588,3 +588,60 @@ def test_a_failed_transfer_records_the_status(server, tmp_path):
|
|
|
588
588
|
assert res.state == "failed"
|
|
589
589
|
assert res.http_status == 500
|
|
590
590
|
assert res.retrieved_at
|
|
591
|
+
|
|
592
|
+
|
|
593
|
+
# -- 416: only evidence makes the kept part the answer ---------------------
|
|
594
|
+
|
|
595
|
+
|
|
596
|
+
def _stale_part(tmp_path, url, body: bytes):
|
|
597
|
+
(tmp_path / "a.bin.part").write_bytes(body)
|
|
598
|
+
(tmp_path / "a.bin.part.meta.json").write_text(f'{{"url": "{url}", "etag": ""v1""}}', "utf-8")
|
|
599
|
+
|
|
600
|
+
|
|
601
|
+
def test_a_stale_part_is_restarted_not_renamed(server, tmp_path):
|
|
602
|
+
"""A part from an older object is bigger than the file; 416 must not win."""
|
|
603
|
+
payload = b"new" * 1000
|
|
604
|
+
server.cfg["files"]["/a.bin"] = payload
|
|
605
|
+
url = server.base + "/a.bin"
|
|
606
|
+
_stale_part(tmp_path, url, b"old" * 2000) # 6000 bytes vs 3000
|
|
607
|
+
server.cfg["answers_416"] = True
|
|
608
|
+
|
|
609
|
+
res = downloader.fetch_file(
|
|
610
|
+
Client(rate=1000, retries=1), File(url, "a.bin"), opts(tmp_path, retries=1), set()
|
|
611
|
+
)
|
|
612
|
+
|
|
613
|
+
assert res.state == "downloaded", res.error
|
|
614
|
+
assert (tmp_path / "a.bin").read_bytes() == payload
|
|
615
|
+
|
|
616
|
+
|
|
617
|
+
def test_a_416_without_a_size_and_without_checksum_restarts(server, tmp_path):
|
|
618
|
+
payload = b"fresh bytes"
|
|
619
|
+
server.cfg["files"]["/a.bin"] = payload
|
|
620
|
+
url = server.base + "/a.bin"
|
|
621
|
+
_stale_part(tmp_path, url, b"stale")
|
|
622
|
+
server.cfg["answers_416"] = True
|
|
623
|
+
server.cfg["416_omits_range"] = True
|
|
624
|
+
|
|
625
|
+
res = downloader.fetch_file(
|
|
626
|
+
Client(rate=1000, retries=1), File(url, "a.bin"), opts(tmp_path, retries=1), set()
|
|
627
|
+
)
|
|
628
|
+
|
|
629
|
+
assert res.state == "downloaded", res.error
|
|
630
|
+
assert (tmp_path / "a.bin").read_bytes() == payload
|
|
631
|
+
|
|
632
|
+
|
|
633
|
+
def test_a_416_with_a_matching_checksum_accepts_the_part(server, tmp_path):
|
|
634
|
+
"""Proof beats a missing Content-Range."""
|
|
635
|
+
payload = b"proven bytes"
|
|
636
|
+
server.cfg["files"]["/a.bin"] = payload
|
|
637
|
+
url = server.base + "/a.bin"
|
|
638
|
+
_stale_part(tmp_path, url, payload)
|
|
639
|
+
server.cfg["answers_416"] = True
|
|
640
|
+
server.cfg["416_omits_range"] = True
|
|
641
|
+
f = File(url, "a.bin", checksum="sha256:" + hashlib.sha256(payload).hexdigest())
|
|
642
|
+
|
|
643
|
+
res = downloader.fetch_file(Client(rate=1000, retries=0), f, opts(tmp_path, retries=0), set())
|
|
644
|
+
|
|
645
|
+
assert res.state == "downloaded", res.error
|
|
646
|
+
assert res.verified is True
|
|
647
|
+
assert server.cfg["hits"]["/a.bin"] == 1, "no refetch when the checksum proves it"
|
|
@@ -320,3 +320,14 @@ def test_two_legacy_papers_no_longer_share_a_name():
|
|
|
320
320
|
def test_direct_resolve_rejects_a_non_url():
|
|
321
321
|
with pytest.raises(TargetError, match="not a URL"):
|
|
322
322
|
direct.resolve(HeadClient({}), "zenodo:1")
|
|
323
|
+
|
|
324
|
+
|
|
325
|
+
def test_an_older_zenodo_version_follows_latest():
|
|
326
|
+
"""The API no longer sends ``is_latest``, but if a source does, follow it."""
|
|
327
|
+
old = dict(ZENODO, is_latest=False, links={"latest": "https://zenodo.org/api/records/99"})
|
|
328
|
+
new = dict(ZENODO, id=99, doi="10.5281/zenodo.99")
|
|
329
|
+
record = zenodo.resolve(FakeClient({"records/1": old, "records/99": new}), "10.5281/zenodo.1")
|
|
330
|
+
|
|
331
|
+
assert record.doi == "10.5281/zenodo.99"
|
|
332
|
+
assert record.files[0].name == "a.csv"
|
|
333
|
+
assert record.target == "10.5281/zenodo.1", "the manifest keeps what was asked for"
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|