acadfetch 0.7.0__tar.gz → 0.7.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (77) hide show
  1. {acadfetch-0.7.0/acadfetch.egg-info → acadfetch-0.7.1}/PKG-INFO +3 -4
  2. {acadfetch-0.7.0 → acadfetch-0.7.1}/README.md +2 -3
  3. {acadfetch-0.7.0 → acadfetch-0.7.1}/acadfetch/client.py +20 -0
  4. {acadfetch-0.7.0 → acadfetch-0.7.1}/acadfetch/consts.py +1 -1
  5. {acadfetch-0.7.0 → acadfetch-0.7.1}/acadfetch/downloader.py +34 -1
  6. {acadfetch-0.7.0 → acadfetch-0.7.1}/acadfetch/pooch.py +9 -39
  7. {acadfetch-0.7.0 → acadfetch-0.7.1/acadfetch.egg-info}/PKG-INFO +3 -4
  8. {acadfetch-0.7.0 → acadfetch-0.7.1}/pyproject.toml +1 -1
  9. {acadfetch-0.7.0 → acadfetch-0.7.1}/LICENSE +0 -0
  10. {acadfetch-0.7.0 → acadfetch-0.7.1}/MANIFEST.in +0 -0
  11. {acadfetch-0.7.0 → acadfetch-0.7.1}/acadfetch/__init__.py +0 -0
  12. {acadfetch-0.7.0 → acadfetch-0.7.1}/acadfetch/__main__.py +0 -0
  13. {acadfetch-0.7.0 → acadfetch-0.7.1}/acadfetch/atomic.py +0 -0
  14. {acadfetch-0.7.0 → acadfetch-0.7.1}/acadfetch/auth.py +0 -0
  15. {acadfetch-0.7.0 → acadfetch-0.7.1}/acadfetch/cli.py +0 -0
  16. {acadfetch-0.7.0 → acadfetch-0.7.1}/acadfetch/debug.py +0 -0
  17. {acadfetch-0.7.0 → acadfetch-0.7.1}/acadfetch/errors.py +0 -0
  18. {acadfetch-0.7.0 → acadfetch-0.7.1}/acadfetch/manifest.py +0 -0
  19. {acadfetch-0.7.0 → acadfetch-0.7.1}/acadfetch/naming.py +0 -0
  20. {acadfetch-0.7.0 → acadfetch-0.7.1}/acadfetch/progress.py +0 -0
  21. {acadfetch-0.7.0 → acadfetch-0.7.1}/acadfetch/py.typed +0 -0
  22. {acadfetch-0.7.0 → acadfetch-0.7.1}/acadfetch/ratelimit.py +0 -0
  23. {acadfetch-0.7.0 → acadfetch-0.7.1}/acadfetch/remote.py +0 -0
  24. {acadfetch-0.7.0 → acadfetch-0.7.1}/acadfetch/sources/__init__.py +0 -0
  25. {acadfetch-0.7.0 → acadfetch-0.7.1}/acadfetch/sources/arxiv.py +0 -0
  26. {acadfetch-0.7.0 → acadfetch-0.7.1}/acadfetch/sources/base.py +0 -0
  27. {acadfetch-0.7.0 → acadfetch-0.7.1}/acadfetch/sources/direct.py +0 -0
  28. {acadfetch-0.7.0 → acadfetch-0.7.1}/acadfetch/sources/doi.py +0 -0
  29. {acadfetch-0.7.0 → acadfetch-0.7.1}/acadfetch/sources/registry.py +0 -0
  30. {acadfetch-0.7.0 → acadfetch-0.7.1}/acadfetch/sources/removed.py +0 -0
  31. {acadfetch-0.7.0 → acadfetch-0.7.1}/acadfetch/sources/zenodo.py +0 -0
  32. {acadfetch-0.7.0 → acadfetch-0.7.1}/acadfetch/units.py +0 -0
  33. {acadfetch-0.7.0 → acadfetch-0.7.1}/acadfetch/urls.py +0 -0
  34. {acadfetch-0.7.0 → acadfetch-0.7.1}/acadfetch/verify.py +0 -0
  35. {acadfetch-0.7.0 → acadfetch-0.7.1}/acadfetch.egg-info/SOURCES.txt +0 -0
  36. {acadfetch-0.7.0 → acadfetch-0.7.1}/acadfetch.egg-info/dependency_links.txt +0 -0
  37. {acadfetch-0.7.0 → acadfetch-0.7.1}/acadfetch.egg-info/entry_points.txt +0 -0
  38. {acadfetch-0.7.0 → acadfetch-0.7.1}/acadfetch.egg-info/top_level.txt +0 -0
  39. {acadfetch-0.7.0 → acadfetch-0.7.1}/setup.cfg +0 -0
  40. {acadfetch-0.7.0 → acadfetch-0.7.1}/tests/conftest.py +0 -0
  41. {acadfetch-0.7.0 → acadfetch-0.7.1}/tests/fixtures/README.md +0 -0
  42. {acadfetch-0.7.0 → acadfetch-0.7.1}/tests/fixtures/ca.pem +0 -0
  43. {acadfetch-0.7.0 → acadfetch-0.7.1}/tests/fixtures/figshare_403_block.html +0 -0
  44. {acadfetch-0.7.0 → acadfetch-0.7.1}/tests/fixtures/figshare_403_forbidden.json +0 -0
  45. {acadfetch-0.7.0 → acadfetch-0.7.1}/tests/fixtures/figshare_article.json +0 -0
  46. {acadfetch-0.7.0 → acadfetch-0.7.1}/tests/fixtures/figshare_file.pdf +0 -0
  47. {acadfetch-0.7.0 → acadfetch-0.7.1}/tests/test_atomic.py +0 -0
  48. {acadfetch-0.7.0 → acadfetch-0.7.1}/tests/test_auth.py +0 -0
  49. {acadfetch-0.7.0 → acadfetch-0.7.1}/tests/test_checksum.py +0 -0
  50. {acadfetch-0.7.0 → acadfetch-0.7.1}/tests/test_citation.py +0 -0
  51. {acadfetch-0.7.0 → acadfetch-0.7.1}/tests/test_claims.py +0 -0
  52. {acadfetch-0.7.0 → acadfetch-0.7.1}/tests/test_cli.py +0 -0
  53. {acadfetch-0.7.0 → acadfetch-0.7.1}/tests/test_client.py +0 -0
  54. {acadfetch-0.7.0 → acadfetch-0.7.1}/tests/test_control_names.py +0 -0
  55. {acadfetch-0.7.0 → acadfetch-0.7.1}/tests/test_debug.py +0 -0
  56. {acadfetch-0.7.0 → acadfetch-0.7.1}/tests/test_disk.py +0 -0
  57. {acadfetch-0.7.0 → acadfetch-0.7.1}/tests/test_downloader.py +0 -0
  58. {acadfetch-0.7.0 → acadfetch-0.7.1}/tests/test_hash_once.py +0 -0
  59. {acadfetch-0.7.0 → acadfetch-0.7.1}/tests/test_identity.py +0 -0
  60. {acadfetch-0.7.0 → acadfetch-0.7.1}/tests/test_isolation.py +0 -0
  61. {acadfetch-0.7.0 → acadfetch-0.7.1}/tests/test_json.py +0 -0
  62. {acadfetch-0.7.0 → acadfetch-0.7.1}/tests/test_lock.py +0 -0
  63. {acadfetch-0.7.0 → acadfetch-0.7.1}/tests/test_manifest.py +0 -0
  64. {acadfetch-0.7.0 → acadfetch-0.7.1}/tests/test_naming.py +0 -0
  65. {acadfetch-0.7.0 → acadfetch-0.7.1}/tests/test_naming_race.py +0 -0
  66. {acadfetch-0.7.0 → acadfetch-0.7.1}/tests/test_network.py +0 -0
  67. {acadfetch-0.7.0 → acadfetch-0.7.1}/tests/test_packaging.py +0 -0
  68. {acadfetch-0.7.0 → acadfetch-0.7.1}/tests/test_pooch.py +0 -0
  69. {acadfetch-0.7.0 → acadfetch-0.7.1}/tests/test_progress.py +0 -0
  70. {acadfetch-0.7.0 → acadfetch-0.7.1}/tests/test_ratelimit.py +0 -0
  71. {acadfetch-0.7.0 → acadfetch-0.7.1}/tests/test_redirect.py +0 -0
  72. {acadfetch-0.7.0 → acadfetch-0.7.1}/tests/test_remote.py +0 -0
  73. {acadfetch-0.7.0 → acadfetch-0.7.1}/tests/test_removed.py +0 -0
  74. {acadfetch-0.7.0 → acadfetch-0.7.1}/tests/test_sources.py +0 -0
  75. {acadfetch-0.7.0 → acadfetch-0.7.1}/tests/test_urls.py +0 -0
  76. {acadfetch-0.7.0 → acadfetch-0.7.1}/tests/test_urls_glob.py +0 -0
  77. {acadfetch-0.7.0 → acadfetch-0.7.1}/tests/test_verify.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: acadfetch
3
- Version: 0.7.0
3
+ Version: 0.7.1
4
4
  Summary: Resumable, rate-limited, checksum-verified bulk downloads from academic data repositories
5
5
  Author: young-ski
6
6
  License-Expression: MIT
@@ -337,9 +337,8 @@ honest way to test this code path.
337
337
  ## Getting involved
338
338
 
339
339
  A [30-second tour](docs/demo.md) shows the whole loop, and
340
- [docs/integrations.md](docs/integrations.md) has Make, CI, aria2, Snakemake and a
341
- pooch downloader
342
- recipes.
340
+ [docs/integrations.md](docs/integrations.md) has Make, CI, aria2, a Snakemake
341
+ storage plugin and a pooch downloader.
343
342
 
344
343
  A nightly job downloads three fixed public records and verifies them, so a
345
344
  rotted adapter opens an issue before a user reports one. It never blocks a
@@ -306,9 +306,8 @@ honest way to test this code path.
306
306
  ## Getting involved
307
307
 
308
308
  A [30-second tour](docs/demo.md) shows the whole loop, and
309
- [docs/integrations.md](docs/integrations.md) has Make, CI, aria2, Snakemake and a
310
- pooch downloader
311
- recipes.
309
+ [docs/integrations.md](docs/integrations.md) has Make, CI, aria2, a Snakemake
310
+ storage plugin and a pooch downloader.
312
311
 
313
312
  A nightly job downloads three fixed public records and verifies them, so a
314
313
  rotted adapter opens an issue before a user reports one. It never blocks a
@@ -325,6 +325,26 @@ class Client:
325
325
  return {}
326
326
  raise
327
327
 
328
+ def available(self, url: str) -> bool:
329
+ """Whether the URL answers at all — the question a cache asks before fetching.
330
+
331
+ A HEAD is cheap, but ``head()`` folds 403/405/501 into "no headers", which
332
+ is indistinguishable from a 200 that sent none. So an empty answer is
333
+ settled with a one-byte range request instead of being read as success:
334
+ guessing wrong here makes a cache believe a file exists when it does not,
335
+ or refetch one that is fine.
336
+ """
337
+ try:
338
+ if self.head(url):
339
+ return True
340
+ except HttpError:
341
+ return False
342
+ try:
343
+ with self.open(url, {"Range": "bytes=0-0"}, attempts=1):
344
+ return True
345
+ except HttpError:
346
+ return False
347
+
328
348
 
329
349
  def _snippet(exc: urllib.error.HTTPError) -> str:
330
350
  try:
@@ -1,6 +1,6 @@
1
1
  """Package-wide constants and version."""
2
2
 
3
- __version__ = "0.7.0"
3
+ __version__ = "0.7.1"
4
4
 
5
5
  USER_AGENT = f"acadfetch/{__version__} (+https://github.com/young-ski/acadfetch)"
6
6
 
@@ -32,7 +32,7 @@ from typing import Any, cast
32
32
  from . import atomic, debug, naming, verify
33
33
  from .client import Client
34
34
  from .consts import CHUNK
35
- from .errors import HttpError, LocalError, PartialError, TargetError
35
+ from .errors import AcFError, HttpError, LocalError, PartialError, TargetError
36
36
  from .progress import Progress
37
37
  from .sources.base import File, Record, Result
38
38
  from .units import human
@@ -1006,3 +1006,36 @@ def check_total(record: Record, limit: int) -> None:
1006
1006
  from .errors import SourceError
1007
1007
 
1008
1008
  raise SourceError(f"record is {human(total)}, over --max-size ({human(limit)})")
1009
+
1010
+
1011
+ def fetch_one(
1012
+ client: Client,
1013
+ url: str,
1014
+ target: Path,
1015
+ *,
1016
+ retries: int = TRANSFER_ATTEMPTS,
1017
+ fsync: bool = True,
1018
+ ) -> None:
1019
+ """Download ``url`` to exactly ``target``, raising instead of returning a row.
1020
+
1021
+ The CLI reports a row per file because one bad file must not hide the rest; a
1022
+ library caller, by contrast, wants an exception and no partial result. The
1023
+ pooch downloader and the Snakemake plugin both ask this question, so they ask
1024
+ it here rather than each rebuilding the row-to-exception translation.
1025
+
1026
+ The name on disk is the caller's, not the repository's: if ``naming`` had to
1027
+ rewrite it, the produced file is moved back to what was asked for.
1028
+ """
1029
+ target = Path(target)
1030
+ target.parent.mkdir(parents=True, exist_ok=True)
1031
+ result = fetch_file(
1032
+ client,
1033
+ File(url=url, name=target.name),
1034
+ Options(out=target.parent, concurrency=1, retries=retries, fsync=fsync),
1035
+ set(),
1036
+ )
1037
+ if result.state == "failed":
1038
+ raise AcFError(result.error or f"could not fetch {url}")
1039
+ produced = Path(result.path or target)
1040
+ if produced != target:
1041
+ atomic.replace(produced, target)
@@ -35,12 +35,10 @@ from collections.abc import Mapping
35
35
  from pathlib import Path
36
36
  from typing import Any
37
37
 
38
- from . import atomic
39
38
  from .client import Client
40
39
  from .consts import DEFAULT_RATE, DEFAULT_RETRIES, DEFAULT_TIMEOUT
41
- from .downloader import CHUNK, Options, fetch_file
42
- from .errors import AcFError, HttpError, TargetError
43
- from .sources.base import File
40
+ from .downloader import CHUNK, fetch_one
41
+ from .errors import TargetError
44
42
 
45
43
  __all__ = ["Downloader", "downloader"]
46
44
 
@@ -100,46 +98,18 @@ class Downloader:
100
98
  return None
101
99
 
102
100
  target = Path(output_file)
103
- target.parent.mkdir(parents=True, exist_ok=True)
104
- result = fetch_file(
101
+ fetch_one(
105
102
  self._client(),
106
- File(url=url, name=target.name),
107
- Options(
108
- out=target.parent,
109
- concurrency=1,
110
- retries=self.retries,
111
- fsync=self.fsync,
112
- ),
113
- set(),
103
+ url,
104
+ target,
105
+ retries=self.retries,
106
+ fsync=self.fsync,
114
107
  )
115
- if result.state == "failed":
116
- # pooch does not catch this, so the sentence the CLI would print is
117
- # the sentence the caller sees.
118
- raise AcFError(result.error or f"could not fetch {url}")
119
- produced = Path(result.path or target)
120
- if produced != target:
121
- # `naming` is allowed to change a name; pooch's own file must win.
122
- atomic.replace(produced, target)
123
108
  return None
124
109
 
125
110
  def available(self, url: str) -> bool:
126
- """Whether the URL answers at all — HEAD first, one ranged byte after.
127
-
128
- ``Client.head`` treats a 403/405/501 as "no headers", which is
129
- indistinguishable from a 200 that sent none, so an empty answer falls
130
- through to a one-byte range request rather than being read as success.
131
- """
132
- client = self._client()
133
- try:
134
- if client.head(url):
135
- return True
136
- except HttpError:
137
- return False
138
- try:
139
- with client.open(url, {"Range": "bytes=0-0"}, attempts=1):
140
- return True
141
- except HttpError:
142
- return False
111
+ """Whether the URL answers at all; see :meth:`acadfetch.client.Client.available`."""
112
+ return self._client().available(url)
143
113
 
144
114
  def _stream(self, url: str, handle: Any) -> None:
145
115
  client = self._client()
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: acadfetch
3
- Version: 0.7.0
3
+ Version: 0.7.1
4
4
  Summary: Resumable, rate-limited, checksum-verified bulk downloads from academic data repositories
5
5
  Author: young-ski
6
6
  License-Expression: MIT
@@ -337,9 +337,8 @@ honest way to test this code path.
337
337
  ## Getting involved
338
338
 
339
339
  A [30-second tour](docs/demo.md) shows the whole loop, and
340
- [docs/integrations.md](docs/integrations.md) has Make, CI, aria2, Snakemake and a
341
- pooch downloader
342
- recipes.
340
+ [docs/integrations.md](docs/integrations.md) has Make, CI, aria2, a Snakemake
341
+ storage plugin and a pooch downloader.
343
342
 
344
343
  A nightly job downloads three fixed public records and verifies them, so a
345
344
  rotted adapter opens an issue before a user reports one. It never blocks a
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "acadfetch"
7
- version = "0.7.0"
7
+ version = "0.7.1"
8
8
  description = "Resumable, rate-limited, checksum-verified bulk downloads from academic data repositories"
9
9
  readme = "README.md"
10
10
  requires-python = ">=3.10"
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes