acadfetch 0.6.1__tar.gz → 0.6.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (73) hide show
  1. {acadfetch-0.6.1/acadfetch.egg-info → acadfetch-0.6.2}/PKG-INFO +1 -1
  2. {acadfetch-0.6.1 → acadfetch-0.6.2}/acadfetch/client.py +13 -2
  3. {acadfetch-0.6.1 → acadfetch-0.6.2}/acadfetch/consts.py +1 -1
  4. {acadfetch-0.6.1 → acadfetch-0.6.2}/acadfetch/downloader.py +41 -1
  5. {acadfetch-0.6.1 → acadfetch-0.6.2}/acadfetch/errors.py +4 -0
  6. {acadfetch-0.6.1 → acadfetch-0.6.2}/acadfetch/sources/zenodo.py +4 -0
  7. {acadfetch-0.6.1 → acadfetch-0.6.2/acadfetch.egg-info}/PKG-INFO +1 -1
  8. {acadfetch-0.6.1 → acadfetch-0.6.2}/pyproject.toml +1 -1
  9. {acadfetch-0.6.1 → acadfetch-0.6.2}/tests/conftest.py +3 -1
  10. {acadfetch-0.6.1 → acadfetch-0.6.2}/tests/test_downloader.py +57 -0
  11. {acadfetch-0.6.1 → acadfetch-0.6.2}/tests/test_sources.py +11 -0
  12. {acadfetch-0.6.1 → acadfetch-0.6.2}/LICENSE +0 -0
  13. {acadfetch-0.6.1 → acadfetch-0.6.2}/MANIFEST.in +0 -0
  14. {acadfetch-0.6.1 → acadfetch-0.6.2}/README.md +0 -0
  15. {acadfetch-0.6.1 → acadfetch-0.6.2}/acadfetch/__init__.py +0 -0
  16. {acadfetch-0.6.1 → acadfetch-0.6.2}/acadfetch/__main__.py +0 -0
  17. {acadfetch-0.6.1 → acadfetch-0.6.2}/acadfetch/atomic.py +0 -0
  18. {acadfetch-0.6.1 → acadfetch-0.6.2}/acadfetch/auth.py +0 -0
  19. {acadfetch-0.6.1 → acadfetch-0.6.2}/acadfetch/cli.py +0 -0
  20. {acadfetch-0.6.1 → acadfetch-0.6.2}/acadfetch/debug.py +0 -0
  21. {acadfetch-0.6.1 → acadfetch-0.6.2}/acadfetch/manifest.py +0 -0
  22. {acadfetch-0.6.1 → acadfetch-0.6.2}/acadfetch/naming.py +0 -0
  23. {acadfetch-0.6.1 → acadfetch-0.6.2}/acadfetch/progress.py +0 -0
  24. {acadfetch-0.6.1 → acadfetch-0.6.2}/acadfetch/py.typed +0 -0
  25. {acadfetch-0.6.1 → acadfetch-0.6.2}/acadfetch/ratelimit.py +0 -0
  26. {acadfetch-0.6.1 → acadfetch-0.6.2}/acadfetch/remote.py +0 -0
  27. {acadfetch-0.6.1 → acadfetch-0.6.2}/acadfetch/sources/__init__.py +0 -0
  28. {acadfetch-0.6.1 → acadfetch-0.6.2}/acadfetch/sources/arxiv.py +0 -0
  29. {acadfetch-0.6.1 → acadfetch-0.6.2}/acadfetch/sources/base.py +0 -0
  30. {acadfetch-0.6.1 → acadfetch-0.6.2}/acadfetch/sources/direct.py +0 -0
  31. {acadfetch-0.6.1 → acadfetch-0.6.2}/acadfetch/sources/doi.py +0 -0
  32. {acadfetch-0.6.1 → acadfetch-0.6.2}/acadfetch/sources/registry.py +0 -0
  33. {acadfetch-0.6.1 → acadfetch-0.6.2}/acadfetch/sources/removed.py +0 -0
  34. {acadfetch-0.6.1 → acadfetch-0.6.2}/acadfetch/units.py +0 -0
  35. {acadfetch-0.6.1 → acadfetch-0.6.2}/acadfetch/urls.py +0 -0
  36. {acadfetch-0.6.1 → acadfetch-0.6.2}/acadfetch/verify.py +0 -0
  37. {acadfetch-0.6.1 → acadfetch-0.6.2}/acadfetch.egg-info/SOURCES.txt +0 -0
  38. {acadfetch-0.6.1 → acadfetch-0.6.2}/acadfetch.egg-info/dependency_links.txt +0 -0
  39. {acadfetch-0.6.1 → acadfetch-0.6.2}/acadfetch.egg-info/entry_points.txt +0 -0
  40. {acadfetch-0.6.1 → acadfetch-0.6.2}/acadfetch.egg-info/top_level.txt +0 -0
  41. {acadfetch-0.6.1 → acadfetch-0.6.2}/setup.cfg +0 -0
  42. {acadfetch-0.6.1 → acadfetch-0.6.2}/tests/fixtures/README.md +0 -0
  43. {acadfetch-0.6.1 → acadfetch-0.6.2}/tests/fixtures/ca.pem +0 -0
  44. {acadfetch-0.6.1 → acadfetch-0.6.2}/tests/fixtures/figshare_403_block.html +0 -0
  45. {acadfetch-0.6.1 → acadfetch-0.6.2}/tests/fixtures/figshare_403_forbidden.json +0 -0
  46. {acadfetch-0.6.1 → acadfetch-0.6.2}/tests/fixtures/figshare_article.json +0 -0
  47. {acadfetch-0.6.1 → acadfetch-0.6.2}/tests/fixtures/figshare_file.pdf +0 -0
  48. {acadfetch-0.6.1 → acadfetch-0.6.2}/tests/test_atomic.py +0 -0
  49. {acadfetch-0.6.1 → acadfetch-0.6.2}/tests/test_auth.py +0 -0
  50. {acadfetch-0.6.1 → acadfetch-0.6.2}/tests/test_checksum.py +0 -0
  51. {acadfetch-0.6.1 → acadfetch-0.6.2}/tests/test_citation.py +0 -0
  52. {acadfetch-0.6.1 → acadfetch-0.6.2}/tests/test_cli.py +0 -0
  53. {acadfetch-0.6.1 → acadfetch-0.6.2}/tests/test_client.py +0 -0
  54. {acadfetch-0.6.1 → acadfetch-0.6.2}/tests/test_control_names.py +0 -0
  55. {acadfetch-0.6.1 → acadfetch-0.6.2}/tests/test_debug.py +0 -0
  56. {acadfetch-0.6.1 → acadfetch-0.6.2}/tests/test_disk.py +0 -0
  57. {acadfetch-0.6.1 → acadfetch-0.6.2}/tests/test_hash_once.py +0 -0
  58. {acadfetch-0.6.1 → acadfetch-0.6.2}/tests/test_identity.py +0 -0
  59. {acadfetch-0.6.1 → acadfetch-0.6.2}/tests/test_isolation.py +0 -0
  60. {acadfetch-0.6.1 → acadfetch-0.6.2}/tests/test_json.py +0 -0
  61. {acadfetch-0.6.1 → acadfetch-0.6.2}/tests/test_manifest.py +0 -0
  62. {acadfetch-0.6.1 → acadfetch-0.6.2}/tests/test_naming.py +0 -0
  63. {acadfetch-0.6.1 → acadfetch-0.6.2}/tests/test_naming_race.py +0 -0
  64. {acadfetch-0.6.1 → acadfetch-0.6.2}/tests/test_network.py +0 -0
  65. {acadfetch-0.6.1 → acadfetch-0.6.2}/tests/test_packaging.py +0 -0
  66. {acadfetch-0.6.1 → acadfetch-0.6.2}/tests/test_progress.py +0 -0
  67. {acadfetch-0.6.1 → acadfetch-0.6.2}/tests/test_ratelimit.py +0 -0
  68. {acadfetch-0.6.1 → acadfetch-0.6.2}/tests/test_redirect.py +0 -0
  69. {acadfetch-0.6.1 → acadfetch-0.6.2}/tests/test_remote.py +0 -0
  70. {acadfetch-0.6.1 → acadfetch-0.6.2}/tests/test_removed.py +0 -0
  71. {acadfetch-0.6.1 → acadfetch-0.6.2}/tests/test_urls.py +0 -0
  72. {acadfetch-0.6.1 → acadfetch-0.6.2}/tests/test_urls_glob.py +0 -0
  73. {acadfetch-0.6.1 → acadfetch-0.6.2}/tests/test_verify.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: acadfetch
3
- Version: 0.6.1
3
+ Version: 0.6.2
4
4
  Summary: Resumable, rate-limited, checksum-verified bulk downloads from academic data repositories
5
5
  Author: young-ski
6
6
  License-Expression: MIT
@@ -236,7 +236,12 @@ class Client:
236
236
  if exc.code in RETRY_STATUS:
237
237
  pass
238
238
  elif not last_blocked:
239
- raise HttpError(exc.code, url, detail) from exc
239
+ raise HttpError(
240
+ exc.code,
241
+ url,
242
+ detail,
243
+ content_range=exc.headers.get("Content-Range") if exc.headers else None,
244
+ ) from exc
240
245
  elif waf >= block_retries:
241
246
  raise HttpError(
242
247
  403,
@@ -270,7 +275,13 @@ class Client:
270
275
  if last_blocked
271
276
  else last_detail
272
277
  )
273
- raise HttpError(last.code, url, detail, transient=True) from last
278
+ raise HttpError(
279
+ last.code,
280
+ url,
281
+ detail,
282
+ transient=True,
283
+ content_range=last.headers.get("Content-Range") if last.headers else None,
284
+ ) from last
274
285
  raise HttpError(0, url, last_detail or str(last)) from last
275
286
 
276
287
  # -- public ------------------------------------------------------------
@@ -1,6 +1,6 @@
1
1
  """Package-wide constants and version."""
2
2
 
3
- __version__ = "0.6.1"
3
+ __version__ = "0.6.2"
4
4
 
5
5
  USER_AGENT = f"acadfetch/{__version__} (+https://github.com/young-ski/acadfetch)"
6
6
 
@@ -240,6 +240,46 @@ def _chunks(resp, idle_timeout: float, url: str): # noqa: ANN001, ANN202
240
240
  yield chunk
241
241
 
242
242
 
243
+ def _star_total(content_range: str | None) -> int | None:
244
+ """``bytes */N`` → N, for the 416 that carries it."""
245
+ if not content_range:
246
+ return None
247
+ _, _, value = content_range.strip().partition("*/")
248
+ try:
249
+ return int(value.strip())
250
+ except ValueError:
251
+ return None
252
+
253
+
254
+ def _restart_or_finish(
255
+ file: File, part: Path, meta_path: Path, have: int, exc: HttpError
256
+ ) -> Transfer:
257
+ """What a 416 means: the part is the file, or it is stale and must go.
258
+
259
+ "Already complete" used to be assumed whenever the part existed, so a stale
260
+ part (a different, older object) was renamed into place with nothing
261
+ checked. A matching checksum is proof; otherwise the size the server
262
+ reports for the whole object has to equal the part exactly.
263
+ """
264
+ parsed = verify.parse(file.checksum)
265
+ digests: dict[str, str] | None = None
266
+ if parsed is not None:
267
+ digests = verify.hashes(part, {"sha256", parsed[0]})
268
+ if digests[parsed[0]] == parsed[1]:
269
+ return Transfer(0, digests=digests)
270
+ total = _star_total(exc.content_range)
271
+ if total is not None and total == have:
272
+ if digests is None:
273
+ digests = verify.hashes(part, {"sha256"})
274
+ return Transfer(0, digests=digests)
275
+ _drop(part, meta_path)
276
+ if total is None:
277
+ raised, compared = "416 without a size to compare against", "no evidence"
278
+ else:
279
+ raised, compared = f"server says {total} bytes", f"the .part has {have}"
280
+ raise HttpError(416, file.url, f"{raised}, {compared}; restarting", transient=True)
281
+
282
+
243
283
  def _transfer(client: Client, file: File, part: Path, meta_path: Path, opts: Options) -> Transfer:
244
284
  """Move bytes into ``part``, resuming if possible. Returns what happened."""
245
285
  meta = _read_meta(meta_path) if part.exists() else {}
@@ -266,7 +306,7 @@ def _transfer(client: Client, file: File, part: Path, meta_path: Path, opts: Opt
266
306
  resp = client.open(file.url, headers, attempts=1)
267
307
  except HttpError as exc:
268
308
  if exc.status == 416 and have:
269
- return Transfer(0) # already complete
309
+ return _restart_or_finish(file, part, meta_path, have, exc)
270
310
  raise
271
311
 
272
312
  with resp:
@@ -30,6 +30,7 @@ class HttpError(AcFError):
30
30
  detail: str = "",
31
31
  transient: bool = False,
32
32
  reason: str = "",
33
+ content_range: str | None = None,
33
34
  ) -> None:
34
35
  self.status = status
35
36
  self.url = url
@@ -37,6 +38,9 @@ class HttpError(AcFError):
37
38
  #: Machine-readable cause for callers that must branch on it, e.g.
38
39
  #: ``html-page``: the body is a block/login page, not the file asked for.
39
40
  self.reason = reason
41
+ #: ``Content-Range`` of the answer, so a 416 can say how big the object
42
+ #: is supposed to be.
43
+ self.content_range = content_range
40
44
  if status:
41
45
  text = f"HTTP {status} for {url}{': ' + detail if detail else ''}"
42
46
  elif reason == "max-size":
@@ -3,6 +3,10 @@
3
3
  The public REST API exposes ``files[].links.self`` as a redirecting download
4
4
  URL and publishes ``files[].checksum`` as ``md5:<hex>``, which is what makes
5
5
  verification possible here.
6
+
7
+ ``is_latest`` / ``links.latest`` are honoured for an older version DOI. The
8
+ current API does not send ``is_latest`` (checked 2026-11), so the branch is
9
+ dormant; a test pins what it would do if it returned.
6
10
  """
7
11
 
8
12
  from __future__ import annotations
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: acadfetch
3
- Version: 0.6.1
3
+ Version: 0.6.2
4
4
  Summary: Resumable, rate-limited, checksum-verified bulk downloads from academic data repositories
5
5
  Author: young-ski
6
6
  License-Expression: MIT
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "acadfetch"
7
- version = "0.6.1"
7
+ version = "0.6.2"
8
8
  description = "Resumable, rate-limited, checksum-verified bulk downloads from academic data repositories"
9
9
  readme = "README.md"
10
10
  requires-python = ">=3.10"
@@ -55,7 +55,8 @@ class Handler(BaseHTTPRequestHandler):
55
55
  return
56
56
  if cfg["answers_416"] and self.headers.get("Range"):
57
57
  self.send_response(416)
58
- self.send_header("Content-Range", f"bytes */{len(cfg['files'].get(path, b''))}")
58
+ if not cfg.get("416_omits_range"):
59
+ self.send_header("Content-Range", f"bytes */{len(cfg['files'].get(path, b''))}")
59
60
  self.send_header("Content-Length", "0")
60
61
  self.end_headers()
61
62
  return
@@ -147,6 +148,7 @@ def _new_server():
147
148
  "declares": {},
148
149
  "types": {},
149
150
  "answers_416": False,
151
+ "416_omits_range": False,
150
152
  "ranges": [],
151
153
  "stall_after": None,
152
154
  "stall_for": 2.0,
@@ -588,3 +588,60 @@ def test_a_failed_transfer_records_the_status(server, tmp_path):
588
588
  assert res.state == "failed"
589
589
  assert res.http_status == 500
590
590
  assert res.retrieved_at
591
+
592
+
593
+ # -- 416: only evidence makes the kept part the answer ---------------------
594
+
595
+
596
+ def _stale_part(tmp_path, url, body: bytes):
597
+ (tmp_path / "a.bin.part").write_bytes(body)
598
+ (tmp_path / "a.bin.part.meta.json").write_text(f'{{"url": "{url}", "etag": ""v1""}}', "utf-8")
599
+
600
+
601
+ def test_a_stale_part_is_restarted_not_renamed(server, tmp_path):
602
+ """A part from an older object is bigger than the file; 416 must not win."""
603
+ payload = b"new" * 1000
604
+ server.cfg["files"]["/a.bin"] = payload
605
+ url = server.base + "/a.bin"
606
+ _stale_part(tmp_path, url, b"old" * 2000) # 6000 bytes vs 3000
607
+ server.cfg["answers_416"] = True
608
+
609
+ res = downloader.fetch_file(
610
+ Client(rate=1000, retries=1), File(url, "a.bin"), opts(tmp_path, retries=1), set()
611
+ )
612
+
613
+ assert res.state == "downloaded", res.error
614
+ assert (tmp_path / "a.bin").read_bytes() == payload
615
+
616
+
617
+ def test_a_416_without_a_size_and_without_checksum_restarts(server, tmp_path):
618
+ payload = b"fresh bytes"
619
+ server.cfg["files"]["/a.bin"] = payload
620
+ url = server.base + "/a.bin"
621
+ _stale_part(tmp_path, url, b"stale")
622
+ server.cfg["answers_416"] = True
623
+ server.cfg["416_omits_range"] = True
624
+
625
+ res = downloader.fetch_file(
626
+ Client(rate=1000, retries=1), File(url, "a.bin"), opts(tmp_path, retries=1), set()
627
+ )
628
+
629
+ assert res.state == "downloaded", res.error
630
+ assert (tmp_path / "a.bin").read_bytes() == payload
631
+
632
+
633
+ def test_a_416_with_a_matching_checksum_accepts_the_part(server, tmp_path):
634
+ """Proof beats a missing Content-Range."""
635
+ payload = b"proven bytes"
636
+ server.cfg["files"]["/a.bin"] = payload
637
+ url = server.base + "/a.bin"
638
+ _stale_part(tmp_path, url, payload)
639
+ server.cfg["answers_416"] = True
640
+ server.cfg["416_omits_range"] = True
641
+ f = File(url, "a.bin", checksum="sha256:" + hashlib.sha256(payload).hexdigest())
642
+
643
+ res = downloader.fetch_file(Client(rate=1000, retries=0), f, opts(tmp_path, retries=0), set())
644
+
645
+ assert res.state == "downloaded", res.error
646
+ assert res.verified is True
647
+ assert server.cfg["hits"]["/a.bin"] == 1, "no refetch when the checksum proves it"
@@ -320,3 +320,14 @@ def test_two_legacy_papers_no_longer_share_a_name():
320
320
  def test_direct_resolve_rejects_a_non_url():
321
321
  with pytest.raises(TargetError, match="not a URL"):
322
322
  direct.resolve(HeadClient({}), "zenodo:1")
323
+
324
+
325
+ def test_an_older_zenodo_version_follows_latest():
326
+ """The API no longer sends ``is_latest``, but if a source does, follow it."""
327
+ old = dict(ZENODO, is_latest=False, links={"latest": "https://zenodo.org/api/records/99"})
328
+ new = dict(ZENODO, id=99, doi="10.5281/zenodo.99")
329
+ record = zenodo.resolve(FakeClient({"records/1": old, "records/99": new}), "10.5281/zenodo.1")
330
+
331
+ assert record.doi == "10.5281/zenodo.99"
332
+ assert record.files[0].name == "a.csv"
333
+ assert record.target == "10.5281/zenodo.1", "the manifest keeps what was asked for"
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes