acadfetch 0.6.4__tar.gz → 0.7.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {acadfetch-0.6.4/acadfetch.egg-info → acadfetch-0.7.0}/PKG-INFO +3 -2
- {acadfetch-0.6.4 → acadfetch-0.7.0}/README.md +2 -1
- {acadfetch-0.6.4 → acadfetch-0.7.0}/acadfetch/consts.py +1 -1
- acadfetch-0.7.0/acadfetch/pooch.py +157 -0
- {acadfetch-0.6.4 → acadfetch-0.7.0/acadfetch.egg-info}/PKG-INFO +3 -2
- {acadfetch-0.6.4 → acadfetch-0.7.0}/acadfetch.egg-info/SOURCES.txt +2 -0
- {acadfetch-0.6.4 → acadfetch-0.7.0}/pyproject.toml +1 -1
- acadfetch-0.7.0/tests/test_pooch.py +149 -0
- {acadfetch-0.6.4 → acadfetch-0.7.0}/LICENSE +0 -0
- {acadfetch-0.6.4 → acadfetch-0.7.0}/MANIFEST.in +0 -0
- {acadfetch-0.6.4 → acadfetch-0.7.0}/acadfetch/__init__.py +0 -0
- {acadfetch-0.6.4 → acadfetch-0.7.0}/acadfetch/__main__.py +0 -0
- {acadfetch-0.6.4 → acadfetch-0.7.0}/acadfetch/atomic.py +0 -0
- {acadfetch-0.6.4 → acadfetch-0.7.0}/acadfetch/auth.py +0 -0
- {acadfetch-0.6.4 → acadfetch-0.7.0}/acadfetch/cli.py +0 -0
- {acadfetch-0.6.4 → acadfetch-0.7.0}/acadfetch/client.py +0 -0
- {acadfetch-0.6.4 → acadfetch-0.7.0}/acadfetch/debug.py +0 -0
- {acadfetch-0.6.4 → acadfetch-0.7.0}/acadfetch/downloader.py +0 -0
- {acadfetch-0.6.4 → acadfetch-0.7.0}/acadfetch/errors.py +0 -0
- {acadfetch-0.6.4 → acadfetch-0.7.0}/acadfetch/manifest.py +0 -0
- {acadfetch-0.6.4 → acadfetch-0.7.0}/acadfetch/naming.py +0 -0
- {acadfetch-0.6.4 → acadfetch-0.7.0}/acadfetch/progress.py +0 -0
- {acadfetch-0.6.4 → acadfetch-0.7.0}/acadfetch/py.typed +0 -0
- {acadfetch-0.6.4 → acadfetch-0.7.0}/acadfetch/ratelimit.py +0 -0
- {acadfetch-0.6.4 → acadfetch-0.7.0}/acadfetch/remote.py +0 -0
- {acadfetch-0.6.4 → acadfetch-0.7.0}/acadfetch/sources/__init__.py +0 -0
- {acadfetch-0.6.4 → acadfetch-0.7.0}/acadfetch/sources/arxiv.py +0 -0
- {acadfetch-0.6.4 → acadfetch-0.7.0}/acadfetch/sources/base.py +0 -0
- {acadfetch-0.6.4 → acadfetch-0.7.0}/acadfetch/sources/direct.py +0 -0
- {acadfetch-0.6.4 → acadfetch-0.7.0}/acadfetch/sources/doi.py +0 -0
- {acadfetch-0.6.4 → acadfetch-0.7.0}/acadfetch/sources/registry.py +0 -0
- {acadfetch-0.6.4 → acadfetch-0.7.0}/acadfetch/sources/removed.py +0 -0
- {acadfetch-0.6.4 → acadfetch-0.7.0}/acadfetch/sources/zenodo.py +0 -0
- {acadfetch-0.6.4 → acadfetch-0.7.0}/acadfetch/units.py +0 -0
- {acadfetch-0.6.4 → acadfetch-0.7.0}/acadfetch/urls.py +0 -0
- {acadfetch-0.6.4 → acadfetch-0.7.0}/acadfetch/verify.py +0 -0
- {acadfetch-0.6.4 → acadfetch-0.7.0}/acadfetch.egg-info/dependency_links.txt +0 -0
- {acadfetch-0.6.4 → acadfetch-0.7.0}/acadfetch.egg-info/entry_points.txt +0 -0
- {acadfetch-0.6.4 → acadfetch-0.7.0}/acadfetch.egg-info/top_level.txt +0 -0
- {acadfetch-0.6.4 → acadfetch-0.7.0}/setup.cfg +0 -0
- {acadfetch-0.6.4 → acadfetch-0.7.0}/tests/conftest.py +0 -0
- {acadfetch-0.6.4 → acadfetch-0.7.0}/tests/fixtures/README.md +0 -0
- {acadfetch-0.6.4 → acadfetch-0.7.0}/tests/fixtures/ca.pem +0 -0
- {acadfetch-0.6.4 → acadfetch-0.7.0}/tests/fixtures/figshare_403_block.html +0 -0
- {acadfetch-0.6.4 → acadfetch-0.7.0}/tests/fixtures/figshare_403_forbidden.json +0 -0
- {acadfetch-0.6.4 → acadfetch-0.7.0}/tests/fixtures/figshare_article.json +0 -0
- {acadfetch-0.6.4 → acadfetch-0.7.0}/tests/fixtures/figshare_file.pdf +0 -0
- {acadfetch-0.6.4 → acadfetch-0.7.0}/tests/test_atomic.py +0 -0
- {acadfetch-0.6.4 → acadfetch-0.7.0}/tests/test_auth.py +0 -0
- {acadfetch-0.6.4 → acadfetch-0.7.0}/tests/test_checksum.py +0 -0
- {acadfetch-0.6.4 → acadfetch-0.7.0}/tests/test_citation.py +0 -0
- {acadfetch-0.6.4 → acadfetch-0.7.0}/tests/test_claims.py +0 -0
- {acadfetch-0.6.4 → acadfetch-0.7.0}/tests/test_cli.py +0 -0
- {acadfetch-0.6.4 → acadfetch-0.7.0}/tests/test_client.py +0 -0
- {acadfetch-0.6.4 → acadfetch-0.7.0}/tests/test_control_names.py +0 -0
- {acadfetch-0.6.4 → acadfetch-0.7.0}/tests/test_debug.py +0 -0
- {acadfetch-0.6.4 → acadfetch-0.7.0}/tests/test_disk.py +0 -0
- {acadfetch-0.6.4 → acadfetch-0.7.0}/tests/test_downloader.py +0 -0
- {acadfetch-0.6.4 → acadfetch-0.7.0}/tests/test_hash_once.py +0 -0
- {acadfetch-0.6.4 → acadfetch-0.7.0}/tests/test_identity.py +0 -0
- {acadfetch-0.6.4 → acadfetch-0.7.0}/tests/test_isolation.py +0 -0
- {acadfetch-0.6.4 → acadfetch-0.7.0}/tests/test_json.py +0 -0
- {acadfetch-0.6.4 → acadfetch-0.7.0}/tests/test_lock.py +0 -0
- {acadfetch-0.6.4 → acadfetch-0.7.0}/tests/test_manifest.py +0 -0
- {acadfetch-0.6.4 → acadfetch-0.7.0}/tests/test_naming.py +0 -0
- {acadfetch-0.6.4 → acadfetch-0.7.0}/tests/test_naming_race.py +0 -0
- {acadfetch-0.6.4 → acadfetch-0.7.0}/tests/test_network.py +0 -0
- {acadfetch-0.6.4 → acadfetch-0.7.0}/tests/test_packaging.py +0 -0
- {acadfetch-0.6.4 → acadfetch-0.7.0}/tests/test_progress.py +0 -0
- {acadfetch-0.6.4 → acadfetch-0.7.0}/tests/test_ratelimit.py +0 -0
- {acadfetch-0.6.4 → acadfetch-0.7.0}/tests/test_redirect.py +0 -0
- {acadfetch-0.6.4 → acadfetch-0.7.0}/tests/test_remote.py +0 -0
- {acadfetch-0.6.4 → acadfetch-0.7.0}/tests/test_removed.py +0 -0
- {acadfetch-0.6.4 → acadfetch-0.7.0}/tests/test_sources.py +0 -0
- {acadfetch-0.6.4 → acadfetch-0.7.0}/tests/test_urls.py +0 -0
- {acadfetch-0.6.4 → acadfetch-0.7.0}/tests/test_urls_glob.py +0 -0
- {acadfetch-0.6.4 → acadfetch-0.7.0}/tests/test_verify.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: acadfetch
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.7.0
|
|
4
4
|
Summary: Resumable, rate-limited, checksum-verified bulk downloads from academic data repositories
|
|
5
5
|
Author: young-ski
|
|
6
6
|
License-Expression: MIT
|
|
@@ -337,7 +337,8 @@ honest way to test this code path.
|
|
|
337
337
|
## Getting involved
|
|
338
338
|
|
|
339
339
|
A [30-second tour](docs/demo.md) shows the whole loop, and
|
|
340
|
-
[docs/integrations.md](docs/integrations.md) has Make, CI, aria2 and
|
|
340
|
+
[docs/integrations.md](docs/integrations.md) has Make, CI, aria2, Snakemake and a
|
|
341
|
+
pooch downloader
|
|
341
342
|
recipes.
|
|
342
343
|
|
|
343
344
|
A nightly job downloads three fixed public records and verifies them, so a
|
|
@@ -306,7 +306,8 @@ honest way to test this code path.
|
|
|
306
306
|
## Getting involved
|
|
307
307
|
|
|
308
308
|
A [30-second tour](docs/demo.md) shows the whole loop, and
|
|
309
|
-
[docs/integrations.md](docs/integrations.md) has Make, CI, aria2 and
|
|
309
|
+
[docs/integrations.md](docs/integrations.md) has Make, CI, aria2, Snakemake and a
|
|
310
|
+
pooch downloader
|
|
310
311
|
recipes.
|
|
311
312
|
|
|
312
313
|
A nightly job downloads three fixed public records and verifies them, so a
|
|
@@ -0,0 +1,157 @@
|
|
|
1
|
+
"""A pooch downloader that borrows acadfetch's HTTP client.
|
|
2
|
+
|
|
3
|
+
``pooch`` already caches, verifies and moves a file into place. What it does not
|
|
4
|
+
do is survive a link that stalls at 80%: its own downloader issues a flat GET,
|
|
5
|
+
and a drop means the whole transfer again. This adds the transport only — every
|
|
6
|
+
request goes through :class:`acadfetch.client.Client`, so it honours
|
|
7
|
+
``Retry-After``, retries a 5xx or a WAF block page, keeps a ``.part`` and resumes
|
|
8
|
+
it with ``Range``/``If-Range`` inside the attempt, and drops a credential header
|
|
9
|
+
when a redirect leaves the origin.
|
|
10
|
+
|
|
11
|
+
.. code-block:: python
|
|
12
|
+
|
|
13
|
+
import pooch
|
|
14
|
+
from acadfetch.pooch import downloader
|
|
15
|
+
|
|
16
|
+
path = pooch.retrieve(url, known_hash="md5:…", downloader=downloader)
|
|
17
|
+
|
|
18
|
+
``pooch`` still does the hashing and the move, so this cannot make verification
|
|
19
|
+
weaker — only the arrival more likely. It is a plain callable rather than a
|
|
20
|
+
subclass because a callable is the whole contract ``pooch`` asks for.
|
|
21
|
+
|
|
22
|
+
Two limits worth knowing, both from how ``pooch`` drives a downloader:
|
|
23
|
+
|
|
24
|
+
* it hands each attempt a **fresh temporary file**, so a partial is resumed
|
|
25
|
+
within one call and not across calls — ``pooch`` owns the retry loop between
|
|
26
|
+
them;
|
|
27
|
+
* ``pooch``'s ``progressbar=True`` is implemented inside its own downloader and
|
|
28
|
+
is not passed here, so a transfer through this one is silent. The CLI's
|
|
29
|
+
progress line is a separate thing.
|
|
30
|
+
"""
|
|
31
|
+
|
|
32
|
+
from __future__ import annotations
|
|
33
|
+
|
|
34
|
+
from collections.abc import Mapping
|
|
35
|
+
from pathlib import Path
|
|
36
|
+
from typing import Any
|
|
37
|
+
|
|
38
|
+
from . import atomic
|
|
39
|
+
from .client import Client
|
|
40
|
+
from .consts import DEFAULT_RATE, DEFAULT_RETRIES, DEFAULT_TIMEOUT
|
|
41
|
+
from .downloader import CHUNK, Options, fetch_file
|
|
42
|
+
from .errors import AcFError, HttpError, TargetError
|
|
43
|
+
from .sources.base import File
|
|
44
|
+
|
|
45
|
+
__all__ = ["Downloader", "downloader"]
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
class Downloader:
|
|
49
|
+
"""The callable ``pooch`` wants, configured like the CLI.
|
|
50
|
+
|
|
51
|
+
Every argument maps to a flag of the same name, so a pipeline that already
|
|
52
|
+
tuned the CLI does not have to learn a second vocabulary.
|
|
53
|
+
"""
|
|
54
|
+
|
|
55
|
+
def __init__(
|
|
56
|
+
self,
|
|
57
|
+
*,
|
|
58
|
+
rate: float = DEFAULT_RATE,
|
|
59
|
+
retries: int = DEFAULT_RETRIES,
|
|
60
|
+
timeout: float = DEFAULT_TIMEOUT,
|
|
61
|
+
headers: Mapping[str, str] | None = None,
|
|
62
|
+
block_retries: int = 2,
|
|
63
|
+
allow_http: bool = False,
|
|
64
|
+
fsync: bool = True,
|
|
65
|
+
) -> None:
|
|
66
|
+
self.rate = rate
|
|
67
|
+
self.retries = retries
|
|
68
|
+
self.timeout = timeout
|
|
69
|
+
self.headers = dict(headers or {})
|
|
70
|
+
self.block_retries = block_retries
|
|
71
|
+
self.allow_http = allow_http
|
|
72
|
+
self.fsync = fsync
|
|
73
|
+
|
|
74
|
+
def _client(self) -> Client:
|
|
75
|
+
return Client(
|
|
76
|
+
rate=self.rate,
|
|
77
|
+
retries=self.retries,
|
|
78
|
+
timeout=self.timeout,
|
|
79
|
+
headers=self.headers,
|
|
80
|
+
block_retries=self.block_retries,
|
|
81
|
+
allow_http=self.allow_http,
|
|
82
|
+
)
|
|
83
|
+
|
|
84
|
+
# -- the two things pooch asks for -------------------------------------
|
|
85
|
+
def __call__(
|
|
86
|
+
self,
|
|
87
|
+
url: str,
|
|
88
|
+
output_file: Any = None,
|
|
89
|
+
pooch: Any = None,
|
|
90
|
+
check_only: bool = False,
|
|
91
|
+
) -> bool | None:
|
|
92
|
+
if check_only:
|
|
93
|
+
return self.available(url)
|
|
94
|
+
if output_file is None:
|
|
95
|
+
raise TargetError("pooch asked for a download but passed no output file")
|
|
96
|
+
if hasattr(output_file, "write"):
|
|
97
|
+
# A caller-supplied handle: pooch owns the file, so there is nothing
|
|
98
|
+
# to rename and nothing worth resuming onto.
|
|
99
|
+
self._stream(url, output_file)
|
|
100
|
+
return None
|
|
101
|
+
|
|
102
|
+
target = Path(output_file)
|
|
103
|
+
target.parent.mkdir(parents=True, exist_ok=True)
|
|
104
|
+
result = fetch_file(
|
|
105
|
+
self._client(),
|
|
106
|
+
File(url=url, name=target.name),
|
|
107
|
+
Options(
|
|
108
|
+
out=target.parent,
|
|
109
|
+
concurrency=1,
|
|
110
|
+
retries=self.retries,
|
|
111
|
+
fsync=self.fsync,
|
|
112
|
+
),
|
|
113
|
+
set(),
|
|
114
|
+
)
|
|
115
|
+
if result.state == "failed":
|
|
116
|
+
# pooch does not catch this, so the sentence the CLI would print is
|
|
117
|
+
# the sentence the caller sees.
|
|
118
|
+
raise AcFError(result.error or f"could not fetch {url}")
|
|
119
|
+
produced = Path(result.path or target)
|
|
120
|
+
if produced != target:
|
|
121
|
+
# `naming` is allowed to change a name; pooch's own file must win.
|
|
122
|
+
atomic.replace(produced, target)
|
|
123
|
+
return None
|
|
124
|
+
|
|
125
|
+
def available(self, url: str) -> bool:
|
|
126
|
+
"""Whether the URL answers at all — HEAD first, one ranged byte after.
|
|
127
|
+
|
|
128
|
+
``Client.head`` treats a 403/405/501 as "no headers", which is
|
|
129
|
+
indistinguishable from a 200 that sent none, so an empty answer falls
|
|
130
|
+
through to a one-byte range request rather than being read as success.
|
|
131
|
+
"""
|
|
132
|
+
client = self._client()
|
|
133
|
+
try:
|
|
134
|
+
if client.head(url):
|
|
135
|
+
return True
|
|
136
|
+
except HttpError:
|
|
137
|
+
return False
|
|
138
|
+
try:
|
|
139
|
+
with client.open(url, {"Range": "bytes=0-0"}, attempts=1):
|
|
140
|
+
return True
|
|
141
|
+
except HttpError:
|
|
142
|
+
return False
|
|
143
|
+
|
|
144
|
+
def _stream(self, url: str, handle: Any) -> None:
|
|
145
|
+
client = self._client()
|
|
146
|
+
with client.open(url, attempts=self.retries + 1) as resp:
|
|
147
|
+
reader = getattr(resp, "read1", None) or resp.read
|
|
148
|
+
while True:
|
|
149
|
+
chunk = reader(CHUNK)
|
|
150
|
+
if not chunk:
|
|
151
|
+
break
|
|
152
|
+
handle.write(chunk)
|
|
153
|
+
handle.flush()
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
#: The zero-configuration instance: ``pooch.retrieve(..., downloader=downloader)``.
|
|
157
|
+
downloader = Downloader()
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: acadfetch
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.7.0
|
|
4
4
|
Summary: Resumable, rate-limited, checksum-verified bulk downloads from academic data repositories
|
|
5
5
|
Author: young-ski
|
|
6
6
|
License-Expression: MIT
|
|
@@ -337,7 +337,8 @@ honest way to test this code path.
|
|
|
337
337
|
## Getting involved
|
|
338
338
|
|
|
339
339
|
A [30-second tour](docs/demo.md) shows the whole loop, and
|
|
340
|
-
[docs/integrations.md](docs/integrations.md) has Make, CI, aria2 and
|
|
340
|
+
[docs/integrations.md](docs/integrations.md) has Make, CI, aria2, Snakemake and a
|
|
341
|
+
pooch downloader
|
|
341
342
|
recipes.
|
|
342
343
|
|
|
343
344
|
A nightly job downloads three fixed public records and verifies them, so a
|
|
@@ -14,6 +14,7 @@ acadfetch/downloader.py
|
|
|
14
14
|
acadfetch/errors.py
|
|
15
15
|
acadfetch/manifest.py
|
|
16
16
|
acadfetch/naming.py
|
|
17
|
+
acadfetch/pooch.py
|
|
17
18
|
acadfetch/progress.py
|
|
18
19
|
acadfetch/py.typed
|
|
19
20
|
acadfetch/ratelimit.py
|
|
@@ -56,6 +57,7 @@ tests/test_naming.py
|
|
|
56
57
|
tests/test_naming_race.py
|
|
57
58
|
tests/test_network.py
|
|
58
59
|
tests/test_packaging.py
|
|
60
|
+
tests/test_pooch.py
|
|
59
61
|
tests/test_progress.py
|
|
60
62
|
tests/test_ratelimit.py
|
|
61
63
|
tests/test_redirect.py
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "acadfetch"
|
|
7
|
-
version = "0.
|
|
7
|
+
version = "0.7.0"
|
|
8
8
|
description = "Resumable, rate-limited, checksum-verified bulk downloads from academic data repositories"
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.10"
|
|
@@ -0,0 +1,149 @@
|
|
|
1
|
+
"""The pooch downloader, driven through pooch's own machinery.
|
|
2
|
+
|
|
3
|
+
These run the real ``pooch.retrieve`` and ``Pooch.is_available`` against the local
|
|
4
|
+
test server, so the contract under test is pooch's, not a paraphrase of it. The
|
|
5
|
+
point is that acadfetch only supplies transport: pooch must still do the hashing
|
|
6
|
+
and the move, and must still be the one to fail on a bad hash.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import hashlib
|
|
12
|
+
import io
|
|
13
|
+
from pathlib import Path
|
|
14
|
+
|
|
15
|
+
import pooch
|
|
16
|
+
import pytest
|
|
17
|
+
|
|
18
|
+
from acadfetch import naming
|
|
19
|
+
from acadfetch.client import Client
|
|
20
|
+
from acadfetch.pooch import Downloader, downloader
|
|
21
|
+
|
|
22
|
+
PAYLOAD = b"pooch payload" * 500
|
|
23
|
+
MD5 = hashlib.md5(PAYLOAD).hexdigest()
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def _serve(server, path="/a.bin", payload=PAYLOAD):
|
|
27
|
+
server.cfg["files"][path] = payload
|
|
28
|
+
return server.base + path
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def test_pooch_gets_the_bytes_and_verifies_them(server, tmp_path):
|
|
32
|
+
url = _serve(server)
|
|
33
|
+
|
|
34
|
+
got = pooch.retrieve(url, known_hash=f"md5:{MD5}", downloader=downloader, path=str(tmp_path))
|
|
35
|
+
|
|
36
|
+
assert Path(got).read_bytes() == PAYLOAD
|
|
37
|
+
assert server.cfg["hits"]["/a.bin"] == 1
|
|
38
|
+
# pooch names the cached file "<hash>-a.bin" and moves it into place; we must
|
|
39
|
+
# not have left a .part or anything else behind.
|
|
40
|
+
kept = list(tmp_path.iterdir())
|
|
41
|
+
assert len(kept) == 1 and kept[0].name.endswith("-a.bin"), kept
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def test_a_bad_hash_is_still_pooch_that_refuses_it(server, tmp_path):
|
|
45
|
+
"""The downloader must not be able to talk pooch out of a mismatch."""
|
|
46
|
+
url = _serve(server)
|
|
47
|
+
|
|
48
|
+
with pytest.raises(ValueError) as caught:
|
|
49
|
+
pooch.retrieve(url, known_hash="md5:" + "0" * 32, downloader=downloader, path=str(tmp_path))
|
|
50
|
+
|
|
51
|
+
assert "hash" in str(caught.value).lower()
|
|
52
|
+
assert list(tmp_path.iterdir()) == [], "a rejected download must not be kept"
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def test_a_truncated_response_is_resumed_within_the_attempt(server, tmp_path):
|
|
56
|
+
"""One cut connection must cost a chunk, not the whole file."""
|
|
57
|
+
url = _serve(server)
|
|
58
|
+
server.cfg["truncate_at"] = 100
|
|
59
|
+
server.cfg["truncate_times"] = 1
|
|
60
|
+
|
|
61
|
+
got = pooch.retrieve(
|
|
62
|
+
url, known_hash=f"md5:{MD5}", downloader=Downloader(rate=100), path=str(tmp_path)
|
|
63
|
+
)
|
|
64
|
+
|
|
65
|
+
assert Path(got).read_bytes() == PAYLOAD
|
|
66
|
+
assert server.cfg["ranges"][1] is not None, "the retry should have asked for a range"
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def test_a_missing_file_is_reported_to_the_caller(server, tmp_path):
|
|
70
|
+
url = server.base + "/gone.bin"
|
|
71
|
+
|
|
72
|
+
with pytest.raises(Exception) as caught:
|
|
73
|
+
pooch.retrieve(url, known_hash=None, downloader=downloader, path=str(tmp_path))
|
|
74
|
+
|
|
75
|
+
assert "404" in str(caught.value)
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def test_availability_answers_without_downloading(server, tmp_path):
|
|
79
|
+
_serve(server)
|
|
80
|
+
pool = pooch.Pooch(path=str(tmp_path), base_url=server.base + "/")
|
|
81
|
+
pool.registry["a.bin"] = None
|
|
82
|
+
|
|
83
|
+
assert pool.is_available("a.bin", downloader=downloader) is True
|
|
84
|
+
# A HEAD is expected; a GET of the body is not.
|
|
85
|
+
assert server.cfg["ranges"] == [], "availability must not fetch the body"
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def test_a_missing_file_is_not_available(server, tmp_path):
|
|
89
|
+
pool = pooch.Pooch(path=str(tmp_path), base_url=server.base + "/")
|
|
90
|
+
pool.registry["gone.bin"] = None
|
|
91
|
+
|
|
92
|
+
assert pool.is_available("gone.bin", downloader=downloader) is False
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def test_a_header_survives_to_the_origin(server, tmp_path):
|
|
96
|
+
"""A restricted record is reached by handing the token to the downloader."""
|
|
97
|
+
url = _serve(server)
|
|
98
|
+
configured = Downloader(headers={"Authorization": "Bearer secret"})
|
|
99
|
+
|
|
100
|
+
pooch.retrieve(url, known_hash=f"md5:{MD5}", downloader=configured, path=str(tmp_path))
|
|
101
|
+
|
|
102
|
+
assert server.cfg["auth_seen"][0] == "Bearer secret"
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def test_a_caller_supplied_handle_is_written_through(server, tmp_path):
|
|
106
|
+
"""pooch's docs allow a file-like object; pooch owns it, so we do not rename."""
|
|
107
|
+
url = _serve(server)
|
|
108
|
+
buffer = io.BytesIO()
|
|
109
|
+
|
|
110
|
+
assert downloader(url, buffer) is None
|
|
111
|
+
assert buffer.getvalue() == PAYLOAD
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def test_no_output_file_is_a_clear_error(server):
|
|
115
|
+
with pytest.raises(Exception) as caught:
|
|
116
|
+
downloader(_serve(server))
|
|
117
|
+
|
|
118
|
+
assert "no output file" in str(caught.value)
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def test_a_name_the_sanitiser_changes_still_lands_where_the_caller_asked(server, tmp_path):
|
|
122
|
+
"""`naming` is allowed to rewrite a name; pooch's requested path wins."""
|
|
123
|
+
url = _serve(server)
|
|
124
|
+
target = tmp_path / "data .bin"
|
|
125
|
+
assert target.name != naming.sanitize(target.name), "this test needs a rewritable name"
|
|
126
|
+
|
|
127
|
+
downloader(url, target)
|
|
128
|
+
|
|
129
|
+
assert target.read_bytes() == PAYLOAD
|
|
130
|
+
assert not (tmp_path / naming.sanitize(target.name)).exists()
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
def test_a_head_that_fails_is_not_available(server):
|
|
134
|
+
assert downloader.available(server.base + "/gone.bin") is False
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
def test_an_empty_head_is_not_read_as_available(server, monkeypatch):
|
|
138
|
+
"""A 403/405/501 loses its headers; that must not count as success."""
|
|
139
|
+
monkeypatch.setattr(Client, "head", lambda self, url: {})
|
|
140
|
+
|
|
141
|
+
assert downloader.available(server.base + "/gone.bin") is False
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
def test_an_empty_head_still_finds_a_file_that_is_there(server, monkeypatch):
|
|
145
|
+
"""The fallback exists so a 405 on HEAD does not make a real file look absent."""
|
|
146
|
+
url = _serve(server)
|
|
147
|
+
monkeypatch.setattr(Client, "head", lambda self, url: {})
|
|
148
|
+
|
|
149
|
+
assert downloader.available(url) is True
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|