acadfetch 0.7.0__tar.gz → 0.7.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {acadfetch-0.7.0/acadfetch.egg-info → acadfetch-0.7.1}/PKG-INFO +3 -4
- {acadfetch-0.7.0 → acadfetch-0.7.1}/README.md +2 -3
- {acadfetch-0.7.0 → acadfetch-0.7.1}/acadfetch/client.py +20 -0
- {acadfetch-0.7.0 → acadfetch-0.7.1}/acadfetch/consts.py +1 -1
- {acadfetch-0.7.0 → acadfetch-0.7.1}/acadfetch/downloader.py +34 -1
- {acadfetch-0.7.0 → acadfetch-0.7.1}/acadfetch/pooch.py +9 -39
- {acadfetch-0.7.0 → acadfetch-0.7.1/acadfetch.egg-info}/PKG-INFO +3 -4
- {acadfetch-0.7.0 → acadfetch-0.7.1}/pyproject.toml +1 -1
- {acadfetch-0.7.0 → acadfetch-0.7.1}/LICENSE +0 -0
- {acadfetch-0.7.0 → acadfetch-0.7.1}/MANIFEST.in +0 -0
- {acadfetch-0.7.0 → acadfetch-0.7.1}/acadfetch/__init__.py +0 -0
- {acadfetch-0.7.0 → acadfetch-0.7.1}/acadfetch/__main__.py +0 -0
- {acadfetch-0.7.0 → acadfetch-0.7.1}/acadfetch/atomic.py +0 -0
- {acadfetch-0.7.0 → acadfetch-0.7.1}/acadfetch/auth.py +0 -0
- {acadfetch-0.7.0 → acadfetch-0.7.1}/acadfetch/cli.py +0 -0
- {acadfetch-0.7.0 → acadfetch-0.7.1}/acadfetch/debug.py +0 -0
- {acadfetch-0.7.0 → acadfetch-0.7.1}/acadfetch/errors.py +0 -0
- {acadfetch-0.7.0 → acadfetch-0.7.1}/acadfetch/manifest.py +0 -0
- {acadfetch-0.7.0 → acadfetch-0.7.1}/acadfetch/naming.py +0 -0
- {acadfetch-0.7.0 → acadfetch-0.7.1}/acadfetch/progress.py +0 -0
- {acadfetch-0.7.0 → acadfetch-0.7.1}/acadfetch/py.typed +0 -0
- {acadfetch-0.7.0 → acadfetch-0.7.1}/acadfetch/ratelimit.py +0 -0
- {acadfetch-0.7.0 → acadfetch-0.7.1}/acadfetch/remote.py +0 -0
- {acadfetch-0.7.0 → acadfetch-0.7.1}/acadfetch/sources/__init__.py +0 -0
- {acadfetch-0.7.0 → acadfetch-0.7.1}/acadfetch/sources/arxiv.py +0 -0
- {acadfetch-0.7.0 → acadfetch-0.7.1}/acadfetch/sources/base.py +0 -0
- {acadfetch-0.7.0 → acadfetch-0.7.1}/acadfetch/sources/direct.py +0 -0
- {acadfetch-0.7.0 → acadfetch-0.7.1}/acadfetch/sources/doi.py +0 -0
- {acadfetch-0.7.0 → acadfetch-0.7.1}/acadfetch/sources/registry.py +0 -0
- {acadfetch-0.7.0 → acadfetch-0.7.1}/acadfetch/sources/removed.py +0 -0
- {acadfetch-0.7.0 → acadfetch-0.7.1}/acadfetch/sources/zenodo.py +0 -0
- {acadfetch-0.7.0 → acadfetch-0.7.1}/acadfetch/units.py +0 -0
- {acadfetch-0.7.0 → acadfetch-0.7.1}/acadfetch/urls.py +0 -0
- {acadfetch-0.7.0 → acadfetch-0.7.1}/acadfetch/verify.py +0 -0
- {acadfetch-0.7.0 → acadfetch-0.7.1}/acadfetch.egg-info/SOURCES.txt +0 -0
- {acadfetch-0.7.0 → acadfetch-0.7.1}/acadfetch.egg-info/dependency_links.txt +0 -0
- {acadfetch-0.7.0 → acadfetch-0.7.1}/acadfetch.egg-info/entry_points.txt +0 -0
- {acadfetch-0.7.0 → acadfetch-0.7.1}/acadfetch.egg-info/top_level.txt +0 -0
- {acadfetch-0.7.0 → acadfetch-0.7.1}/setup.cfg +0 -0
- {acadfetch-0.7.0 → acadfetch-0.7.1}/tests/conftest.py +0 -0
- {acadfetch-0.7.0 → acadfetch-0.7.1}/tests/fixtures/README.md +0 -0
- {acadfetch-0.7.0 → acadfetch-0.7.1}/tests/fixtures/ca.pem +0 -0
- {acadfetch-0.7.0 → acadfetch-0.7.1}/tests/fixtures/figshare_403_block.html +0 -0
- {acadfetch-0.7.0 → acadfetch-0.7.1}/tests/fixtures/figshare_403_forbidden.json +0 -0
- {acadfetch-0.7.0 → acadfetch-0.7.1}/tests/fixtures/figshare_article.json +0 -0
- {acadfetch-0.7.0 → acadfetch-0.7.1}/tests/fixtures/figshare_file.pdf +0 -0
- {acadfetch-0.7.0 → acadfetch-0.7.1}/tests/test_atomic.py +0 -0
- {acadfetch-0.7.0 → acadfetch-0.7.1}/tests/test_auth.py +0 -0
- {acadfetch-0.7.0 → acadfetch-0.7.1}/tests/test_checksum.py +0 -0
- {acadfetch-0.7.0 → acadfetch-0.7.1}/tests/test_citation.py +0 -0
- {acadfetch-0.7.0 → acadfetch-0.7.1}/tests/test_claims.py +0 -0
- {acadfetch-0.7.0 → acadfetch-0.7.1}/tests/test_cli.py +0 -0
- {acadfetch-0.7.0 → acadfetch-0.7.1}/tests/test_client.py +0 -0
- {acadfetch-0.7.0 → acadfetch-0.7.1}/tests/test_control_names.py +0 -0
- {acadfetch-0.7.0 → acadfetch-0.7.1}/tests/test_debug.py +0 -0
- {acadfetch-0.7.0 → acadfetch-0.7.1}/tests/test_disk.py +0 -0
- {acadfetch-0.7.0 → acadfetch-0.7.1}/tests/test_downloader.py +0 -0
- {acadfetch-0.7.0 → acadfetch-0.7.1}/tests/test_hash_once.py +0 -0
- {acadfetch-0.7.0 → acadfetch-0.7.1}/tests/test_identity.py +0 -0
- {acadfetch-0.7.0 → acadfetch-0.7.1}/tests/test_isolation.py +0 -0
- {acadfetch-0.7.0 → acadfetch-0.7.1}/tests/test_json.py +0 -0
- {acadfetch-0.7.0 → acadfetch-0.7.1}/tests/test_lock.py +0 -0
- {acadfetch-0.7.0 → acadfetch-0.7.1}/tests/test_manifest.py +0 -0
- {acadfetch-0.7.0 → acadfetch-0.7.1}/tests/test_naming.py +0 -0
- {acadfetch-0.7.0 → acadfetch-0.7.1}/tests/test_naming_race.py +0 -0
- {acadfetch-0.7.0 → acadfetch-0.7.1}/tests/test_network.py +0 -0
- {acadfetch-0.7.0 → acadfetch-0.7.1}/tests/test_packaging.py +0 -0
- {acadfetch-0.7.0 → acadfetch-0.7.1}/tests/test_pooch.py +0 -0
- {acadfetch-0.7.0 → acadfetch-0.7.1}/tests/test_progress.py +0 -0
- {acadfetch-0.7.0 → acadfetch-0.7.1}/tests/test_ratelimit.py +0 -0
- {acadfetch-0.7.0 → acadfetch-0.7.1}/tests/test_redirect.py +0 -0
- {acadfetch-0.7.0 → acadfetch-0.7.1}/tests/test_remote.py +0 -0
- {acadfetch-0.7.0 → acadfetch-0.7.1}/tests/test_removed.py +0 -0
- {acadfetch-0.7.0 → acadfetch-0.7.1}/tests/test_sources.py +0 -0
- {acadfetch-0.7.0 → acadfetch-0.7.1}/tests/test_urls.py +0 -0
- {acadfetch-0.7.0 → acadfetch-0.7.1}/tests/test_urls_glob.py +0 -0
- {acadfetch-0.7.0 → acadfetch-0.7.1}/tests/test_verify.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: acadfetch
|
|
3
|
-
Version: 0.7.
|
|
3
|
+
Version: 0.7.1
|
|
4
4
|
Summary: Resumable, rate-limited, checksum-verified bulk downloads from academic data repositories
|
|
5
5
|
Author: young-ski
|
|
6
6
|
License-Expression: MIT
|
|
@@ -337,9 +337,8 @@ honest way to test this code path.
|
|
|
337
337
|
## Getting involved
|
|
338
338
|
|
|
339
339
|
A [30-second tour](docs/demo.md) shows the whole loop, and
|
|
340
|
-
[docs/integrations.md](docs/integrations.md) has Make, CI, aria2, Snakemake
|
|
341
|
-
pooch downloader
|
|
342
|
-
recipes.
|
|
340
|
+
[docs/integrations.md](docs/integrations.md) has Make, CI, aria2, a Snakemake
|
|
341
|
+
storage plugin and a pooch downloader.
|
|
343
342
|
|
|
344
343
|
A nightly job downloads three fixed public records and verifies them, so a
|
|
345
344
|
rotted adapter opens an issue before a user reports one. It never blocks a
|
|
@@ -306,9 +306,8 @@ honest way to test this code path.
|
|
|
306
306
|
## Getting involved
|
|
307
307
|
|
|
308
308
|
A [30-second tour](docs/demo.md) shows the whole loop, and
|
|
309
|
-
[docs/integrations.md](docs/integrations.md) has Make, CI, aria2, Snakemake
|
|
310
|
-
pooch downloader
|
|
311
|
-
recipes.
|
|
309
|
+
[docs/integrations.md](docs/integrations.md) has Make, CI, aria2, a Snakemake
|
|
310
|
+
storage plugin and a pooch downloader.
|
|
312
311
|
|
|
313
312
|
A nightly job downloads three fixed public records and verifies them, so a
|
|
314
313
|
rotted adapter opens an issue before a user reports one. It never blocks a
|
|
@@ -325,6 +325,26 @@ class Client:
|
|
|
325
325
|
return {}
|
|
326
326
|
raise
|
|
327
327
|
|
|
328
|
+
def available(self, url: str) -> bool:
|
|
329
|
+
"""Whether the URL answers at all — the question a cache asks before fetching.
|
|
330
|
+
|
|
331
|
+
A HEAD is cheap, but ``head()`` folds 403/405/501 into "no headers", which
|
|
332
|
+
is indistinguishable from a 200 that sent none. So an empty answer is
|
|
333
|
+
settled with a one-byte range request instead of being read as success:
|
|
334
|
+
guessing wrong here makes a cache believe a file exists when it does not,
|
|
335
|
+
or refetch one that is fine.
|
|
336
|
+
"""
|
|
337
|
+
try:
|
|
338
|
+
if self.head(url):
|
|
339
|
+
return True
|
|
340
|
+
except HttpError:
|
|
341
|
+
return False
|
|
342
|
+
try:
|
|
343
|
+
with self.open(url, {"Range": "bytes=0-0"}, attempts=1):
|
|
344
|
+
return True
|
|
345
|
+
except HttpError:
|
|
346
|
+
return False
|
|
347
|
+
|
|
328
348
|
|
|
329
349
|
def _snippet(exc: urllib.error.HTTPError) -> str:
|
|
330
350
|
try:
|
|
@@ -32,7 +32,7 @@ from typing import Any, cast
|
|
|
32
32
|
from . import atomic, debug, naming, verify
|
|
33
33
|
from .client import Client
|
|
34
34
|
from .consts import CHUNK
|
|
35
|
-
from .errors import HttpError, LocalError, PartialError, TargetError
|
|
35
|
+
from .errors import AcFError, HttpError, LocalError, PartialError, TargetError
|
|
36
36
|
from .progress import Progress
|
|
37
37
|
from .sources.base import File, Record, Result
|
|
38
38
|
from .units import human
|
|
@@ -1006,3 +1006,36 @@ def check_total(record: Record, limit: int) -> None:
|
|
|
1006
1006
|
from .errors import SourceError
|
|
1007
1007
|
|
|
1008
1008
|
raise SourceError(f"record is {human(total)}, over --max-size ({human(limit)})")
|
|
1009
|
+
|
|
1010
|
+
|
|
1011
|
+
def fetch_one(
|
|
1012
|
+
client: Client,
|
|
1013
|
+
url: str,
|
|
1014
|
+
target: Path,
|
|
1015
|
+
*,
|
|
1016
|
+
retries: int = TRANSFER_ATTEMPTS,
|
|
1017
|
+
fsync: bool = True,
|
|
1018
|
+
) -> None:
|
|
1019
|
+
"""Download ``url`` to exactly ``target``, raising instead of returning a row.
|
|
1020
|
+
|
|
1021
|
+
The CLI reports a row per file because one bad file must not hide the rest; a
|
|
1022
|
+
library caller, by contrast, wants an exception and no partial result. The
|
|
1023
|
+
pooch downloader and the Snakemake plugin both ask this question, so they ask
|
|
1024
|
+
it here rather than each rebuilding the row-to-exception translation.
|
|
1025
|
+
|
|
1026
|
+
The name on disk is the caller's, not the repository's: if ``naming`` had to
|
|
1027
|
+
rewrite it, the produced file is moved back to what was asked for.
|
|
1028
|
+
"""
|
|
1029
|
+
target = Path(target)
|
|
1030
|
+
target.parent.mkdir(parents=True, exist_ok=True)
|
|
1031
|
+
result = fetch_file(
|
|
1032
|
+
client,
|
|
1033
|
+
File(url=url, name=target.name),
|
|
1034
|
+
Options(out=target.parent, concurrency=1, retries=retries, fsync=fsync),
|
|
1035
|
+
set(),
|
|
1036
|
+
)
|
|
1037
|
+
if result.state == "failed":
|
|
1038
|
+
raise AcFError(result.error or f"could not fetch {url}")
|
|
1039
|
+
produced = Path(result.path or target)
|
|
1040
|
+
if produced != target:
|
|
1041
|
+
atomic.replace(produced, target)
|
|
@@ -35,12 +35,10 @@ from collections.abc import Mapping
|
|
|
35
35
|
from pathlib import Path
|
|
36
36
|
from typing import Any
|
|
37
37
|
|
|
38
|
-
from . import atomic
|
|
39
38
|
from .client import Client
|
|
40
39
|
from .consts import DEFAULT_RATE, DEFAULT_RETRIES, DEFAULT_TIMEOUT
|
|
41
|
-
from .downloader import CHUNK,
|
|
42
|
-
from .errors import
|
|
43
|
-
from .sources.base import File
|
|
40
|
+
from .downloader import CHUNK, fetch_one
|
|
41
|
+
from .errors import TargetError
|
|
44
42
|
|
|
45
43
|
__all__ = ["Downloader", "downloader"]
|
|
46
44
|
|
|
@@ -100,46 +98,18 @@ class Downloader:
|
|
|
100
98
|
return None
|
|
101
99
|
|
|
102
100
|
target = Path(output_file)
|
|
103
|
-
|
|
104
|
-
result = fetch_file(
|
|
101
|
+
fetch_one(
|
|
105
102
|
self._client(),
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
retries=self.retries,
|
|
111
|
-
fsync=self.fsync,
|
|
112
|
-
),
|
|
113
|
-
set(),
|
|
103
|
+
url,
|
|
104
|
+
target,
|
|
105
|
+
retries=self.retries,
|
|
106
|
+
fsync=self.fsync,
|
|
114
107
|
)
|
|
115
|
-
if result.state == "failed":
|
|
116
|
-
# pooch does not catch this, so the sentence the CLI would print is
|
|
117
|
-
# the sentence the caller sees.
|
|
118
|
-
raise AcFError(result.error or f"could not fetch {url}")
|
|
119
|
-
produced = Path(result.path or target)
|
|
120
|
-
if produced != target:
|
|
121
|
-
# `naming` is allowed to change a name; pooch's own file must win.
|
|
122
|
-
atomic.replace(produced, target)
|
|
123
108
|
return None
|
|
124
109
|
|
|
125
110
|
def available(self, url: str) -> bool:
|
|
126
|
-
"""Whether the URL answers at all
|
|
127
|
-
|
|
128
|
-
``Client.head`` treats a 403/405/501 as "no headers", which is
|
|
129
|
-
indistinguishable from a 200 that sent none, so an empty answer falls
|
|
130
|
-
through to a one-byte range request rather than being read as success.
|
|
131
|
-
"""
|
|
132
|
-
client = self._client()
|
|
133
|
-
try:
|
|
134
|
-
if client.head(url):
|
|
135
|
-
return True
|
|
136
|
-
except HttpError:
|
|
137
|
-
return False
|
|
138
|
-
try:
|
|
139
|
-
with client.open(url, {"Range": "bytes=0-0"}, attempts=1):
|
|
140
|
-
return True
|
|
141
|
-
except HttpError:
|
|
142
|
-
return False
|
|
111
|
+
"""Whether the URL answers at all; see :meth:`acadfetch.client.Client.available`."""
|
|
112
|
+
return self._client().available(url)
|
|
143
113
|
|
|
144
114
|
def _stream(self, url: str, handle: Any) -> None:
|
|
145
115
|
client = self._client()
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: acadfetch
|
|
3
|
-
Version: 0.7.
|
|
3
|
+
Version: 0.7.1
|
|
4
4
|
Summary: Resumable, rate-limited, checksum-verified bulk downloads from academic data repositories
|
|
5
5
|
Author: young-ski
|
|
6
6
|
License-Expression: MIT
|
|
@@ -337,9 +337,8 @@ honest way to test this code path.
|
|
|
337
337
|
## Getting involved
|
|
338
338
|
|
|
339
339
|
A [30-second tour](docs/demo.md) shows the whole loop, and
|
|
340
|
-
[docs/integrations.md](docs/integrations.md) has Make, CI, aria2, Snakemake
|
|
341
|
-
pooch downloader
|
|
342
|
-
recipes.
|
|
340
|
+
[docs/integrations.md](docs/integrations.md) has Make, CI, aria2, a Snakemake
|
|
341
|
+
storage plugin and a pooch downloader.
|
|
343
342
|
|
|
344
343
|
A nightly job downloads three fixed public records and verifies them, so a
|
|
345
344
|
rotted adapter opens an issue before a user reports one. It never blocks a
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "acadfetch"
|
|
7
|
-
version = "0.7.
|
|
7
|
+
version = "0.7.1"
|
|
8
8
|
description = "Resumable, rate-limited, checksum-verified bulk downloads from academic data repositories"
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.10"
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|