acadfetch 0.3.0__tar.gz → 0.4.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {acadfetch-0.3.0/acadfetch.egg-info → acadfetch-0.4.0}/PKG-INFO +22 -2
- {acadfetch-0.3.0 → acadfetch-0.4.0}/README.md +21 -1
- {acadfetch-0.3.0 → acadfetch-0.4.0}/acadfetch/cli.py +55 -0
- {acadfetch-0.3.0 → acadfetch-0.4.0}/acadfetch/client.py +13 -1
- {acadfetch-0.3.0 → acadfetch-0.4.0}/acadfetch/consts.py +1 -1
- {acadfetch-0.3.0 → acadfetch-0.4.0/acadfetch.egg-info}/PKG-INFO +22 -2
- {acadfetch-0.3.0 → acadfetch-0.4.0}/acadfetch.egg-info/SOURCES.txt +2 -0
- {acadfetch-0.3.0 → acadfetch-0.4.0}/pyproject.toml +1 -1
- {acadfetch-0.3.0 → acadfetch-0.4.0}/tests/fixtures/README.md +3 -0
- acadfetch-0.4.0/tests/fixtures/ca.pem +17 -0
- {acadfetch-0.3.0 → acadfetch-0.4.0}/tests/test_json.py +20 -0
- acadfetch-0.4.0/tests/test_network.py +100 -0
- {acadfetch-0.3.0 → acadfetch-0.4.0}/LICENSE +0 -0
- {acadfetch-0.3.0 → acadfetch-0.4.0}/MANIFEST.in +0 -0
- {acadfetch-0.3.0 → acadfetch-0.4.0}/acadfetch/__init__.py +0 -0
- {acadfetch-0.3.0 → acadfetch-0.4.0}/acadfetch/__main__.py +0 -0
- {acadfetch-0.3.0 → acadfetch-0.4.0}/acadfetch/atomic.py +0 -0
- {acadfetch-0.3.0 → acadfetch-0.4.0}/acadfetch/auth.py +0 -0
- {acadfetch-0.3.0 → acadfetch-0.4.0}/acadfetch/downloader.py +0 -0
- {acadfetch-0.3.0 → acadfetch-0.4.0}/acadfetch/errors.py +0 -0
- {acadfetch-0.3.0 → acadfetch-0.4.0}/acadfetch/manifest.py +0 -0
- {acadfetch-0.3.0 → acadfetch-0.4.0}/acadfetch/naming.py +0 -0
- {acadfetch-0.3.0 → acadfetch-0.4.0}/acadfetch/progress.py +0 -0
- {acadfetch-0.3.0 → acadfetch-0.4.0}/acadfetch/py.typed +0 -0
- {acadfetch-0.3.0 → acadfetch-0.4.0}/acadfetch/ratelimit.py +0 -0
- {acadfetch-0.3.0 → acadfetch-0.4.0}/acadfetch/remote.py +0 -0
- {acadfetch-0.3.0 → acadfetch-0.4.0}/acadfetch/sources/__init__.py +0 -0
- {acadfetch-0.3.0 → acadfetch-0.4.0}/acadfetch/sources/arxiv.py +0 -0
- {acadfetch-0.3.0 → acadfetch-0.4.0}/acadfetch/sources/base.py +0 -0
- {acadfetch-0.3.0 → acadfetch-0.4.0}/acadfetch/sources/direct.py +0 -0
- {acadfetch-0.3.0 → acadfetch-0.4.0}/acadfetch/sources/doi.py +0 -0
- {acadfetch-0.3.0 → acadfetch-0.4.0}/acadfetch/sources/registry.py +0 -0
- {acadfetch-0.3.0 → acadfetch-0.4.0}/acadfetch/sources/removed.py +0 -0
- {acadfetch-0.3.0 → acadfetch-0.4.0}/acadfetch/sources/zenodo.py +0 -0
- {acadfetch-0.3.0 → acadfetch-0.4.0}/acadfetch/units.py +0 -0
- {acadfetch-0.3.0 → acadfetch-0.4.0}/acadfetch/urls.py +0 -0
- {acadfetch-0.3.0 → acadfetch-0.4.0}/acadfetch/verify.py +0 -0
- {acadfetch-0.3.0 → acadfetch-0.4.0}/acadfetch.egg-info/dependency_links.txt +0 -0
- {acadfetch-0.3.0 → acadfetch-0.4.0}/acadfetch.egg-info/entry_points.txt +0 -0
- {acadfetch-0.3.0 → acadfetch-0.4.0}/acadfetch.egg-info/top_level.txt +0 -0
- {acadfetch-0.3.0 → acadfetch-0.4.0}/setup.cfg +0 -0
- {acadfetch-0.3.0 → acadfetch-0.4.0}/tests/conftest.py +0 -0
- {acadfetch-0.3.0 → acadfetch-0.4.0}/tests/fixtures/figshare_403_block.html +0 -0
- {acadfetch-0.3.0 → acadfetch-0.4.0}/tests/fixtures/figshare_403_forbidden.json +0 -0
- {acadfetch-0.3.0 → acadfetch-0.4.0}/tests/fixtures/figshare_article.json +0 -0
- {acadfetch-0.3.0 → acadfetch-0.4.0}/tests/fixtures/figshare_file.pdf +0 -0
- {acadfetch-0.3.0 → acadfetch-0.4.0}/tests/test_atomic.py +0 -0
- {acadfetch-0.3.0 → acadfetch-0.4.0}/tests/test_auth.py +0 -0
- {acadfetch-0.3.0 → acadfetch-0.4.0}/tests/test_checksum.py +0 -0
- {acadfetch-0.3.0 → acadfetch-0.4.0}/tests/test_cli.py +0 -0
- {acadfetch-0.3.0 → acadfetch-0.4.0}/tests/test_client.py +0 -0
- {acadfetch-0.3.0 → acadfetch-0.4.0}/tests/test_control_names.py +0 -0
- {acadfetch-0.3.0 → acadfetch-0.4.0}/tests/test_disk.py +0 -0
- {acadfetch-0.3.0 → acadfetch-0.4.0}/tests/test_downloader.py +0 -0
- {acadfetch-0.3.0 → acadfetch-0.4.0}/tests/test_identity.py +0 -0
- {acadfetch-0.3.0 → acadfetch-0.4.0}/tests/test_isolation.py +0 -0
- {acadfetch-0.3.0 → acadfetch-0.4.0}/tests/test_manifest.py +0 -0
- {acadfetch-0.3.0 → acadfetch-0.4.0}/tests/test_naming.py +0 -0
- {acadfetch-0.3.0 → acadfetch-0.4.0}/tests/test_naming_race.py +0 -0
- {acadfetch-0.3.0 → acadfetch-0.4.0}/tests/test_packaging.py +0 -0
- {acadfetch-0.3.0 → acadfetch-0.4.0}/tests/test_progress.py +0 -0
- {acadfetch-0.3.0 → acadfetch-0.4.0}/tests/test_ratelimit.py +0 -0
- {acadfetch-0.3.0 → acadfetch-0.4.0}/tests/test_redirect.py +0 -0
- {acadfetch-0.3.0 → acadfetch-0.4.0}/tests/test_remote.py +0 -0
- {acadfetch-0.3.0 → acadfetch-0.4.0}/tests/test_removed.py +0 -0
- {acadfetch-0.3.0 → acadfetch-0.4.0}/tests/test_sources.py +0 -0
- {acadfetch-0.3.0 → acadfetch-0.4.0}/tests/test_urls.py +0 -0
- {acadfetch-0.3.0 → acadfetch-0.4.0}/tests/test_verify.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: acadfetch
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.4.0
|
|
4
4
|
Summary: Resumable, rate-limited, checksum-verified bulk downloads from academic data repositories
|
|
5
5
|
Author: young-ski
|
|
6
6
|
License-Expression: MIT
|
|
@@ -143,7 +143,8 @@ acadfetch fetch zenodo:123456 --json | jq -r '.files[] | select(.state=="failed"
|
|
|
143
143
|
```
|
|
144
144
|
|
|
145
145
|
Exit codes are unchanged, and a failure before any transfer still produces a
|
|
146
|
-
document with `error` set. `report` is the document's schema version (now `2
|
|
146
|
+
document with `error` set. `report` is the document's schema version (now `2`, field reference in
|
|
147
|
+
[docs/json-report.md](docs/json-report.md)):
|
|
147
148
|
fields are added, never removed or given a new meaning, and `state` may gain
|
|
148
149
|
values — treat an unknown one as non-terminal. `verify --json` reports per-file
|
|
149
150
|
`MISMATCH` rows the same way, and its `summary` counts `skipped` rows
|
|
@@ -268,6 +269,21 @@ A CDN in front of a direct URL can answer 403 with a block page. Those are
|
|
|
268
269
|
retried twice with backoff, while a 403 carrying the application's own JSON is
|
|
269
270
|
terminal on the first answer; `--retry-403 N` changes the count.
|
|
270
271
|
|
|
272
|
+
## Networking
|
|
273
|
+
|
|
274
|
+
Proxies, timeouts and a private CA are flags, not luck:
|
|
275
|
+
|
|
276
|
+
```bash
|
|
277
|
+
acadfetch fetch zenodo:123456 --proxy http://proxy.campus:3128 # overrides the environment
|
|
278
|
+
acadfetch fetch zenodo:123456 --no-proxy # ignore HTTP(S)_PROXY
|
|
279
|
+
acadfetch fetch zenodo:123456 --timeout 15 # idle/connect budget, seconds
|
|
280
|
+
acadfetch fetch zenodo:123456 --ca-bundle corp-tls.pem # trust this PEM instead of the store
|
|
281
|
+
```
|
|
282
|
+
|
|
283
|
+
`--proxy` takes `http://` or `https://` and wins over `--no-proxy`; without
|
|
284
|
+
either, `HTTP_PROXY` / `HTTPS_PROXY` / `ALL_PROXY` are honoured by urllib as
|
|
285
|
+
usual. A timeout is per request, connect to first byte.
|
|
286
|
+
|
|
271
287
|
## Politeness
|
|
272
288
|
|
|
273
289
|
Defaults: 3 parallel transfers, 2 requests/second, 4 retries with jittered
|
|
@@ -291,6 +307,10 @@ honest way to test this code path.
|
|
|
291
307
|
|
|
292
308
|
## Getting involved
|
|
293
309
|
|
|
310
|
+
A nightly job downloads three fixed public records and verifies them, so a
|
|
311
|
+
rotted adapter opens an issue before a user reports one. It never blocks a
|
|
312
|
+
merge; the next good night closes the issue.
|
|
313
|
+
|
|
294
314
|
- **Stuck on something** → [docs/troubleshooting.md](docs/troubleshooting.md) first; it is short and indexed by symptom
|
|
295
315
|
- **Usage questions** → [Discussions](https://github.com/young-ski/acadfetch/discussions)
|
|
296
316
|
- **Bugs** → the bug template; say whether `verify` reports `MISMATCH`
|
|
@@ -112,7 +112,8 @@ acadfetch fetch zenodo:123456 --json | jq -r '.files[] | select(.state=="failed"
|
|
|
112
112
|
```
|
|
113
113
|
|
|
114
114
|
Exit codes are unchanged, and a failure before any transfer still produces a
|
|
115
|
-
document with `error` set. `report` is the document's schema version (now `2
|
|
115
|
+
document with `error` set. `report` is the document's schema version (now `2`, field reference in
|
|
116
|
+
[docs/json-report.md](docs/json-report.md)):
|
|
116
117
|
fields are added, never removed or given a new meaning, and `state` may gain
|
|
117
118
|
values — treat an unknown one as non-terminal. `verify --json` reports per-file
|
|
118
119
|
`MISMATCH` rows the same way, and its `summary` counts `skipped` rows
|
|
@@ -237,6 +238,21 @@ A CDN in front of a direct URL can answer 403 with a block page. Those are
|
|
|
237
238
|
retried twice with backoff, while a 403 carrying the application's own JSON is
|
|
238
239
|
terminal on the first answer; `--retry-403 N` changes the count.
|
|
239
240
|
|
|
241
|
+
## Networking
|
|
242
|
+
|
|
243
|
+
Proxies, timeouts and a private CA are flags, not luck:
|
|
244
|
+
|
|
245
|
+
```bash
|
|
246
|
+
acadfetch fetch zenodo:123456 --proxy http://proxy.campus:3128 # overrides the environment
|
|
247
|
+
acadfetch fetch zenodo:123456 --no-proxy # ignore HTTP(S)_PROXY
|
|
248
|
+
acadfetch fetch zenodo:123456 --timeout 15 # idle/connect budget, seconds
|
|
249
|
+
acadfetch fetch zenodo:123456 --ca-bundle corp-tls.pem # trust this PEM instead of the store
|
|
250
|
+
```
|
|
251
|
+
|
|
252
|
+
`--proxy` takes `http://` or `https://` and wins over `--no-proxy`; without
|
|
253
|
+
either, `HTTP_PROXY` / `HTTPS_PROXY` / `ALL_PROXY` are honoured by urllib as
|
|
254
|
+
usual. A timeout is per request, connect to first byte.
|
|
255
|
+
|
|
240
256
|
## Politeness
|
|
241
257
|
|
|
242
258
|
Defaults: 3 parallel transfers, 2 requests/second, 4 retries with jittered
|
|
@@ -260,6 +276,10 @@ honest way to test this code path.
|
|
|
260
276
|
|
|
261
277
|
## Getting involved
|
|
262
278
|
|
|
279
|
+
A nightly job downloads three fixed public records and verifies them, so a
|
|
280
|
+
rotted adapter opens an issue before a user reports one. It never blocks a
|
|
281
|
+
merge; the next good night closes the issue.
|
|
282
|
+
|
|
263
283
|
- **Stuck on something** → [docs/troubleshooting.md](docs/troubleshooting.md) first; it is short and indexed by symptom
|
|
264
284
|
- **Usage questions** → [Discussions](https://github.com/young-ski/acadfetch/discussions)
|
|
265
285
|
- **Bugs** → the bug template; say whether `verify` reports `MISMATCH`
|
|
@@ -100,6 +100,32 @@ def _count(text: str) -> int:
|
|
|
100
100
|
return value
|
|
101
101
|
|
|
102
102
|
|
|
103
|
+
def _timeout(text: str) -> float:
|
|
104
|
+
try:
|
|
105
|
+
value = float(text)
|
|
106
|
+
except ValueError:
|
|
107
|
+
raise argparse.ArgumentTypeError(f"not a number of seconds: {text!r}") from None
|
|
108
|
+
if value <= 0:
|
|
109
|
+
raise argparse.ArgumentTypeError("must be > 0 seconds")
|
|
110
|
+
return value
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def _proxy(text: str) -> str:
|
|
114
|
+
candidate = text if "://" in text else f"http://{text}"
|
|
115
|
+
parts = urlsplit(candidate)
|
|
116
|
+
if parts.scheme not in ("http", "https") or not parts.hostname:
|
|
117
|
+
raise argparse.ArgumentTypeError(
|
|
118
|
+
f"expected http://host:port or https://host:port, got {text!r}"
|
|
119
|
+
)
|
|
120
|
+
return candidate
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def _ca_bundle(text: str) -> str:
|
|
124
|
+
if not Path(text).is_file():
|
|
125
|
+
raise argparse.ArgumentTypeError(f"no such file: {text!r}")
|
|
126
|
+
return text
|
|
127
|
+
|
|
128
|
+
|
|
103
129
|
def _checksum(text: str) -> str:
|
|
104
130
|
"""Validate ``--checksum`` before any request is made."""
|
|
105
131
|
parsed = verify.parse(text)
|
|
@@ -126,6 +152,10 @@ def _client(args: argparse.Namespace) -> Client:
|
|
|
126
152
|
retries=args.retries,
|
|
127
153
|
block_retries=args.retry_403,
|
|
128
154
|
allow_http=args.allow_http,
|
|
155
|
+
timeout=args.timeout,
|
|
156
|
+
proxy={"http": args.proxy, "https": args.proxy} if args.proxy else None,
|
|
157
|
+
no_proxy=args.no_proxy,
|
|
158
|
+
ca_bundle=args.ca_bundle,
|
|
129
159
|
headers=_headers(args.header),
|
|
130
160
|
auth={source: token.headers for source, token in tokens.items()},
|
|
131
161
|
)
|
|
@@ -602,6 +632,31 @@ def build_parser() -> argparse.ArgumentParser:
|
|
|
602
632
|
metavar="NAME:VALUE",
|
|
603
633
|
help="extra request header, repeatable (e.g. 'Authorization: Bearer <token>')",
|
|
604
634
|
)
|
|
635
|
+
p.add_argument(
|
|
636
|
+
"--timeout",
|
|
637
|
+
type=_timeout,
|
|
638
|
+
default=consts.DEFAULT_TIMEOUT,
|
|
639
|
+
metavar="SECONDS",
|
|
640
|
+
help="per-request timeout, connect and idle (default: %(default)s)",
|
|
641
|
+
)
|
|
642
|
+
p.add_argument(
|
|
643
|
+
"--proxy",
|
|
644
|
+
type=_proxy,
|
|
645
|
+
metavar="URL",
|
|
646
|
+
help="http:// or https:// proxy for every request; overrides the "
|
|
647
|
+
"environment and wins over --no-proxy",
|
|
648
|
+
)
|
|
649
|
+
p.add_argument(
|
|
650
|
+
"--no-proxy",
|
|
651
|
+
action="store_true",
|
|
652
|
+
help="ignore HTTP_PROXY / HTTPS_PROXY / ALL_PROXY",
|
|
653
|
+
)
|
|
654
|
+
p.add_argument(
|
|
655
|
+
"--ca-bundle",
|
|
656
|
+
type=_ca_bundle,
|
|
657
|
+
metavar="PATH",
|
|
658
|
+
help="PEM file to trust instead of the system store (corporate TLS inspection)",
|
|
659
|
+
)
|
|
605
660
|
p.add_argument(
|
|
606
661
|
"--allow-http",
|
|
607
662
|
action="store_true",
|
|
@@ -13,6 +13,7 @@ import email.utils
|
|
|
13
13
|
import hashlib
|
|
14
14
|
import json
|
|
15
15
|
import random
|
|
16
|
+
import ssl
|
|
16
17
|
import sys
|
|
17
18
|
import time
|
|
18
19
|
import urllib.error
|
|
@@ -143,6 +144,9 @@ class Client:
|
|
|
143
144
|
auth: Mapping[str, Mapping[str, str]] | None = None,
|
|
144
145
|
block_retries: int = consts.WAF_403_RETRIES,
|
|
145
146
|
allow_http: bool = False,
|
|
147
|
+
proxy: Mapping[str, str] | None = None,
|
|
148
|
+
no_proxy: bool = False,
|
|
149
|
+
ca_bundle: str | None = None,
|
|
146
150
|
) -> None:
|
|
147
151
|
self.bucket = TokenBucket(rate)
|
|
148
152
|
self.retries = max(0, retries)
|
|
@@ -157,7 +161,15 @@ class Client:
|
|
|
157
161
|
self.auth = {k: dict(v) for k, v in (auth or {}).items()}
|
|
158
162
|
self.source: str | None = None
|
|
159
163
|
self.redirects = RedirectPolicy(allow_http=allow_http)
|
|
160
|
-
|
|
164
|
+
handlers: list = [self.redirects]
|
|
165
|
+
# An explicit ProxyHandler replaces urllib's environment handling; the
|
|
166
|
+
# empty dict is how --no-proxy ignores HTTP(S)_PROXY.
|
|
167
|
+
if proxy is not None or no_proxy:
|
|
168
|
+
handlers.append(urllib.request.ProxyHandler(dict(proxy or {})))
|
|
169
|
+
if ca_bundle:
|
|
170
|
+
context = ssl.create_default_context(cafile=ca_bundle)
|
|
171
|
+
handlers.append(urllib.request.HTTPSHandler(context=context))
|
|
172
|
+
self.opener = urllib.request.build_opener(*handlers)
|
|
161
173
|
|
|
162
174
|
# -- internals ---------------------------------------------------------
|
|
163
175
|
def _sleep(self, attempt: int, retry_after: float | None) -> None:
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: acadfetch
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.4.0
|
|
4
4
|
Summary: Resumable, rate-limited, checksum-verified bulk downloads from academic data repositories
|
|
5
5
|
Author: young-ski
|
|
6
6
|
License-Expression: MIT
|
|
@@ -143,7 +143,8 @@ acadfetch fetch zenodo:123456 --json | jq -r '.files[] | select(.state=="failed"
|
|
|
143
143
|
```
|
|
144
144
|
|
|
145
145
|
Exit codes are unchanged, and a failure before any transfer still produces a
|
|
146
|
-
document with `error` set. `report` is the document's schema version (now `2
|
|
146
|
+
document with `error` set. `report` is the document's schema version (now `2`, field reference in
|
|
147
|
+
[docs/json-report.md](docs/json-report.md)):
|
|
147
148
|
fields are added, never removed or given a new meaning, and `state` may gain
|
|
148
149
|
values — treat an unknown one as non-terminal. `verify --json` reports per-file
|
|
149
150
|
`MISMATCH` rows the same way, and its `summary` counts `skipped` rows
|
|
@@ -268,6 +269,21 @@ A CDN in front of a direct URL can answer 403 with a block page. Those are
|
|
|
268
269
|
retried twice with backoff, while a 403 carrying the application's own JSON is
|
|
269
270
|
terminal on the first answer; `--retry-403 N` changes the count.
|
|
270
271
|
|
|
272
|
+
## Networking
|
|
273
|
+
|
|
274
|
+
Proxies, timeouts and a private CA are flags, not luck:
|
|
275
|
+
|
|
276
|
+
```bash
|
|
277
|
+
acadfetch fetch zenodo:123456 --proxy http://proxy.campus:3128 # overrides the environment
|
|
278
|
+
acadfetch fetch zenodo:123456 --no-proxy # ignore HTTP(S)_PROXY
|
|
279
|
+
acadfetch fetch zenodo:123456 --timeout 15 # idle/connect budget, seconds
|
|
280
|
+
acadfetch fetch zenodo:123456 --ca-bundle corp-tls.pem # trust this PEM instead of the store
|
|
281
|
+
```
|
|
282
|
+
|
|
283
|
+
`--proxy` takes `http://` or `https://` and wins over `--no-proxy`; without
|
|
284
|
+
either, `HTTP_PROXY` / `HTTPS_PROXY` / `ALL_PROXY` are honoured by urllib as
|
|
285
|
+
usual. A timeout is per request, connect to first byte.
|
|
286
|
+
|
|
271
287
|
## Politeness
|
|
272
288
|
|
|
273
289
|
Defaults: 3 parallel transfers, 2 requests/second, 4 retries with jittered
|
|
@@ -291,6 +307,10 @@ honest way to test this code path.
|
|
|
291
307
|
|
|
292
308
|
## Getting involved
|
|
293
309
|
|
|
310
|
+
A nightly job downloads three fixed public records and verifies them, so a
|
|
311
|
+
rotted adapter opens an issue before a user reports one. It never blocks a
|
|
312
|
+
merge; the next good night closes the issue.
|
|
313
|
+
|
|
294
314
|
- **Stuck on something** → [docs/troubleshooting.md](docs/troubleshooting.md) first; it is short and indexed by symptom
|
|
295
315
|
- **Usage questions** → [Discussions](https://github.com/young-ski/acadfetch/discussions)
|
|
296
316
|
- **Bugs** → the bug template; say whether `verify` reports `MISMATCH`
|
|
@@ -48,6 +48,7 @@ tests/test_json.py
|
|
|
48
48
|
tests/test_manifest.py
|
|
49
49
|
tests/test_naming.py
|
|
50
50
|
tests/test_naming_race.py
|
|
51
|
+
tests/test_network.py
|
|
51
52
|
tests/test_packaging.py
|
|
52
53
|
tests/test_progress.py
|
|
53
54
|
tests/test_ratelimit.py
|
|
@@ -58,6 +59,7 @@ tests/test_sources.py
|
|
|
58
59
|
tests/test_urls.py
|
|
59
60
|
tests/test_verify.py
|
|
60
61
|
tests/fixtures/README.md
|
|
62
|
+
tests/fixtures/ca.pem
|
|
61
63
|
tests/fixtures/figshare_403_block.html
|
|
62
64
|
tests/fixtures/figshare_403_forbidden.json
|
|
63
65
|
tests/fixtures/figshare_article.json
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "acadfetch"
|
|
7
|
-
version = "0.
|
|
7
|
+
version = "0.4.0"
|
|
8
8
|
description = "Resumable, rate-limited, checksum-verified bulk downloads from academic data repositories"
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.10"
|
|
@@ -18,6 +18,9 @@ restricted file fails fast instead of burning retries.
|
|
|
18
18
|
`tests/test_client.py` and `tests/test_downloader.py` through the conftest
|
|
19
19
|
server; the names keep the provenance.
|
|
20
20
|
|
|
21
|
+
`ca.pem` is one public root certificate (taken from certifi) so a test can pass
|
|
22
|
+
`--ca-bundle` a PEM that really loads; it is not expected to verify any host.
|
|
23
|
+
|
|
21
24
|
`figshare_file.pdf` is a real 193 410-byte PDF whose md5 is
|
|
22
25
|
`bbde85208529e663141a0e8cb6fa9c4a`. It is the only real-bytes fixture, so the
|
|
23
26
|
verify path is proven against a published checksum rather than a synthetic
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
-----BEGIN CERTIFICATE-----
|
|
2
|
+
MIICiTCCAg+gAwIBAgIQH0evqmIAcFBUTAGem2OZKjAKBggqhkjOPQQDAzCBhTEL
|
|
3
|
+
MAkGA1UEBhMCR0IxGzAZBgNVBAgTEkdyZWF0ZXIgTWFuY2hlc3RlcjEQMA4GA1UE
|
|
4
|
+
BxMHU2FsZm9yZDEaMBgGA1UEChMRQ09NT0RPIENBIExpbWl0ZWQxKzApBgNVBAMT
|
|
5
|
+
IkNPTU9ETyBFQ0MgQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkwHhcNMDgwMzA2MDAw
|
|
6
|
+
MDAwWhcNMzgwMTE4MjM1OTU5WjCBhTELMAkGA1UEBhMCR0IxGzAZBgNVBAgTEkdy
|
|
7
|
+
ZWF0ZXIgTWFuY2hlc3RlcjEQMA4GA1UEBxMHU2FsZm9yZDEaMBgGA1UEChMRQ09N
|
|
8
|
+
T0RPIENBIExpbWl0ZWQxKzApBgNVBAMTIkNPTU9ETyBFQ0MgQ2VydGlmaWNhdGlv
|
|
9
|
+
biBBdXRob3JpdHkwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAAQDR3svdcmCFYX7deSR
|
|
10
|
+
FtSrYpn1PlILBs5BAH+X4QokPB0BBO490o0JlwzgdeT6+3eKKvUDYEs2ixYjFq0J
|
|
11
|
+
cfRK9ChQtP6IHG4/bC8vCVlbpVsLM5niwz2J+Wos77LTBumjQjBAMB0GA1UdDgQW
|
|
12
|
+
BBR1cacZSBm8nZ3qQUfflMRId5nTeTAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/
|
|
13
|
+
BAUwAwEB/zAKBggqhkjOPQQDAwNoADBlAjEA7wNbeqy3eApyt4jf/7VGFAkK+qDm
|
|
14
|
+
fQjGGoe9GKhzvSbKYAydzpmfz1wPMOG+FDHqAjAU9JM8SaczepBGR7NjfRObTrdv
|
|
15
|
+
GDeAU/7dIOA1mjbRxwG55tzd8/8dLDoWV9mSOdY=
|
|
16
|
+
-----END CERTIFICATE-----
|
|
17
|
+
|
|
@@ -273,3 +273,23 @@ def test_the_default_directory_drops_the_query(server, monkeypatch, capsys, tmp_
|
|
|
273
273
|
slug = Path(json.loads(out)["out"]).name
|
|
274
274
|
assert slug.startswith("direct-127.0.0.1-") and slug.endswith("-a.bin")
|
|
275
275
|
assert "SECRET" not in slug
|
|
276
|
+
|
|
277
|
+
|
|
278
|
+
def test_every_report_field_is_in_the_contract(served, capsys, tmp_path):
|
|
279
|
+
"""A field added to the document without a line in the contract fails here."""
|
|
280
|
+
docs = Path("docs/json-report.md").read_text("utf-8")
|
|
281
|
+
|
|
282
|
+
_, out, _ = _run(capsys, "fetch", "zenodo:1", "--json", "-o", tmp_path)
|
|
283
|
+
fetch_doc = json.loads(out)
|
|
284
|
+
for key in fetch_doc:
|
|
285
|
+
assert f"`{key}`" in docs, f"fetch field {key!r} is undocumented"
|
|
286
|
+
for key in fetch_doc["files"][0]:
|
|
287
|
+
assert f"`{key}`" in docs, f"file field {key!r} is undocumented"
|
|
288
|
+
|
|
289
|
+
manifest_path = tmp_path / "manifest.json"
|
|
290
|
+
_, vout, _ = _run(capsys, "verify", manifest_path, "--json")
|
|
291
|
+
verify_doc = json.loads(vout)
|
|
292
|
+
for key in verify_doc:
|
|
293
|
+
assert f"`{key}`" in docs, f"verify field {key!r} is undocumented"
|
|
294
|
+
for value in ("escaped", "malformed", "remote_match", "remote_unavailable"):
|
|
295
|
+
assert value in docs, f"{value!r} is undocumented"
|
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import urllib.request
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
|
|
6
|
+
import pytest
|
|
7
|
+
|
|
8
|
+
from acadfetch import cli
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def _args(*extra):
|
|
12
|
+
return cli.build_parser().parse_args(["fetch", "zenodo:1", *extra])
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def _proxies(client):
|
|
16
|
+
"""The explicit proxy mapping, or {} when urllib would keep no handler.
|
|
17
|
+
|
|
18
|
+
A ProxyHandler with an empty mapping registers no method, so build_opener
|
|
19
|
+
drops it; that is exactly what --no-proxy means.
|
|
20
|
+
"""
|
|
21
|
+
found = [h for h in client.opener.handlers if isinstance(h, urllib.request.ProxyHandler)]
|
|
22
|
+
return found[0].proxies if found else {}
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
@pytest.mark.parametrize("bad", ["0", "-1", "abc"])
|
|
26
|
+
def test_a_bad_timeout_is_a_usage_error(bad):
|
|
27
|
+
with pytest.raises(SystemExit) as caught:
|
|
28
|
+
cli.main(["fetch", "zenodo:1", "--timeout", bad])
|
|
29
|
+
assert caught.value.code == cli.EXIT_USAGE
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
@pytest.mark.parametrize("bad", ["ftp://x:1", "http://", "://x"])
|
|
33
|
+
def test_a_bad_proxy_is_a_usage_error(bad):
|
|
34
|
+
with pytest.raises(SystemExit) as caught:
|
|
35
|
+
cli.main(["fetch", "zenodo:1", "--proxy", bad])
|
|
36
|
+
assert caught.value.code == cli.EXIT_USAGE
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def test_a_missing_ca_bundle_is_a_usage_error(tmp_path):
|
|
40
|
+
with pytest.raises(SystemExit) as caught:
|
|
41
|
+
cli.main(["fetch", "zenodo:1", "--ca-bundle", str(tmp_path / "no.pem")])
|
|
42
|
+
assert caught.value.code == cli.EXIT_USAGE
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def test_the_timeout_reaches_the_client():
|
|
46
|
+
assert cli._client(_args("--timeout", "15")).timeout == 15.0
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def test_a_bare_host_port_gets_http():
|
|
50
|
+
assert cli._proxy("127.0.0.1:8899") == "http://127.0.0.1:8899"
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def test_the_proxy_reaches_both_schemes():
|
|
54
|
+
assert _proxies(cli._client(_args("--proxy", "http://127.0.0.1:8899"))) == {
|
|
55
|
+
"http": "http://127.0.0.1:8899",
|
|
56
|
+
"https": "http://127.0.0.1:8899",
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def test_no_proxy_leaves_no_proxy_handler(monkeypatch):
|
|
61
|
+
monkeypatch.setenv("HTTP_PROXY", "http://env.example:1")
|
|
62
|
+
monkeypatch.setenv("HTTPS_PROXY", "http://env.example:1")
|
|
63
|
+
|
|
64
|
+
client = cli._client(_args("--no-proxy"))
|
|
65
|
+
|
|
66
|
+
assert not [h for h in client.opener.handlers if isinstance(h, urllib.request.ProxyHandler)]
|
|
67
|
+
assert _proxies(client) == {}
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def test_the_environment_is_used_by_default(monkeypatch):
|
|
71
|
+
monkeypatch.setenv("HTTPS_PROXY", "http://env.example:1")
|
|
72
|
+
monkeypatch.setenv("HTTP_PROXY", "http://env.example:1")
|
|
73
|
+
|
|
74
|
+
proxies = _proxies(cli._client(_args()))
|
|
75
|
+
assert proxies.get("https") == "http://env.example:1"
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def test_an_explicit_proxy_beats_no_proxy(monkeypatch):
|
|
79
|
+
monkeypatch.setenv("HTTPS_PROXY", "http://env.example:1")
|
|
80
|
+
|
|
81
|
+
proxies = _proxies(cli._client(_args("--proxy", "http://own:1", "--no-proxy")))
|
|
82
|
+
assert proxies == {"http": "http://own:1", "https": "http://own:1"}
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def test_a_ca_bundle_adds_an_https_handler():
|
|
86
|
+
bundle = Path(__file__).parent / "fixtures" / "ca.pem"
|
|
87
|
+
|
|
88
|
+
client = cli._client(_args("--ca-bundle", str(bundle)))
|
|
89
|
+
|
|
90
|
+
assert any(isinstance(h, urllib.request.HTTPSHandler) for h in client.opener.handlers)
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def test_every_network_command_takes_the_flags(capsys):
|
|
94
|
+
for argv in (
|
|
95
|
+
["fetch", "zenodo:1", "--timeout", "5"],
|
|
96
|
+
["manifest", "zenodo:1", "--timeout", "5"],
|
|
97
|
+
["verify", "m.json", "--timeout", "5"],
|
|
98
|
+
):
|
|
99
|
+
args = cli.build_parser().parse_args(argv)
|
|
100
|
+
assert args.timeout == 5.0
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|