acadfetch 0.3.1__tar.gz → 0.4.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {acadfetch-0.3.1/acadfetch.egg-info → acadfetch-0.4.0}/PKG-INFO +16 -1
- {acadfetch-0.3.1 → acadfetch-0.4.0}/README.md +15 -0
- {acadfetch-0.3.1 → acadfetch-0.4.0}/acadfetch/cli.py +55 -0
- {acadfetch-0.3.1 → acadfetch-0.4.0}/acadfetch/client.py +13 -1
- {acadfetch-0.3.1 → acadfetch-0.4.0}/acadfetch/consts.py +1 -1
- {acadfetch-0.3.1 → acadfetch-0.4.0/acadfetch.egg-info}/PKG-INFO +16 -1
- {acadfetch-0.3.1 → acadfetch-0.4.0}/acadfetch.egg-info/SOURCES.txt +2 -0
- {acadfetch-0.3.1 → acadfetch-0.4.0}/pyproject.toml +1 -1
- {acadfetch-0.3.1 → acadfetch-0.4.0}/tests/fixtures/README.md +3 -0
- acadfetch-0.4.0/tests/fixtures/ca.pem +17 -0
- acadfetch-0.4.0/tests/test_network.py +100 -0
- {acadfetch-0.3.1 → acadfetch-0.4.0}/LICENSE +0 -0
- {acadfetch-0.3.1 → acadfetch-0.4.0}/MANIFEST.in +0 -0
- {acadfetch-0.3.1 → acadfetch-0.4.0}/acadfetch/__init__.py +0 -0
- {acadfetch-0.3.1 → acadfetch-0.4.0}/acadfetch/__main__.py +0 -0
- {acadfetch-0.3.1 → acadfetch-0.4.0}/acadfetch/atomic.py +0 -0
- {acadfetch-0.3.1 → acadfetch-0.4.0}/acadfetch/auth.py +0 -0
- {acadfetch-0.3.1 → acadfetch-0.4.0}/acadfetch/downloader.py +0 -0
- {acadfetch-0.3.1 → acadfetch-0.4.0}/acadfetch/errors.py +0 -0
- {acadfetch-0.3.1 → acadfetch-0.4.0}/acadfetch/manifest.py +0 -0
- {acadfetch-0.3.1 → acadfetch-0.4.0}/acadfetch/naming.py +0 -0
- {acadfetch-0.3.1 → acadfetch-0.4.0}/acadfetch/progress.py +0 -0
- {acadfetch-0.3.1 → acadfetch-0.4.0}/acadfetch/py.typed +0 -0
- {acadfetch-0.3.1 → acadfetch-0.4.0}/acadfetch/ratelimit.py +0 -0
- {acadfetch-0.3.1 → acadfetch-0.4.0}/acadfetch/remote.py +0 -0
- {acadfetch-0.3.1 → acadfetch-0.4.0}/acadfetch/sources/__init__.py +0 -0
- {acadfetch-0.3.1 → acadfetch-0.4.0}/acadfetch/sources/arxiv.py +0 -0
- {acadfetch-0.3.1 → acadfetch-0.4.0}/acadfetch/sources/base.py +0 -0
- {acadfetch-0.3.1 → acadfetch-0.4.0}/acadfetch/sources/direct.py +0 -0
- {acadfetch-0.3.1 → acadfetch-0.4.0}/acadfetch/sources/doi.py +0 -0
- {acadfetch-0.3.1 → acadfetch-0.4.0}/acadfetch/sources/registry.py +0 -0
- {acadfetch-0.3.1 → acadfetch-0.4.0}/acadfetch/sources/removed.py +0 -0
- {acadfetch-0.3.1 → acadfetch-0.4.0}/acadfetch/sources/zenodo.py +0 -0
- {acadfetch-0.3.1 → acadfetch-0.4.0}/acadfetch/units.py +0 -0
- {acadfetch-0.3.1 → acadfetch-0.4.0}/acadfetch/urls.py +0 -0
- {acadfetch-0.3.1 → acadfetch-0.4.0}/acadfetch/verify.py +0 -0
- {acadfetch-0.3.1 → acadfetch-0.4.0}/acadfetch.egg-info/dependency_links.txt +0 -0
- {acadfetch-0.3.1 → acadfetch-0.4.0}/acadfetch.egg-info/entry_points.txt +0 -0
- {acadfetch-0.3.1 → acadfetch-0.4.0}/acadfetch.egg-info/top_level.txt +0 -0
- {acadfetch-0.3.1 → acadfetch-0.4.0}/setup.cfg +0 -0
- {acadfetch-0.3.1 → acadfetch-0.4.0}/tests/conftest.py +0 -0
- {acadfetch-0.3.1 → acadfetch-0.4.0}/tests/fixtures/figshare_403_block.html +0 -0
- {acadfetch-0.3.1 → acadfetch-0.4.0}/tests/fixtures/figshare_403_forbidden.json +0 -0
- {acadfetch-0.3.1 → acadfetch-0.4.0}/tests/fixtures/figshare_article.json +0 -0
- {acadfetch-0.3.1 → acadfetch-0.4.0}/tests/fixtures/figshare_file.pdf +0 -0
- {acadfetch-0.3.1 → acadfetch-0.4.0}/tests/test_atomic.py +0 -0
- {acadfetch-0.3.1 → acadfetch-0.4.0}/tests/test_auth.py +0 -0
- {acadfetch-0.3.1 → acadfetch-0.4.0}/tests/test_checksum.py +0 -0
- {acadfetch-0.3.1 → acadfetch-0.4.0}/tests/test_cli.py +0 -0
- {acadfetch-0.3.1 → acadfetch-0.4.0}/tests/test_client.py +0 -0
- {acadfetch-0.3.1 → acadfetch-0.4.0}/tests/test_control_names.py +0 -0
- {acadfetch-0.3.1 → acadfetch-0.4.0}/tests/test_disk.py +0 -0
- {acadfetch-0.3.1 → acadfetch-0.4.0}/tests/test_downloader.py +0 -0
- {acadfetch-0.3.1 → acadfetch-0.4.0}/tests/test_identity.py +0 -0
- {acadfetch-0.3.1 → acadfetch-0.4.0}/tests/test_isolation.py +0 -0
- {acadfetch-0.3.1 → acadfetch-0.4.0}/tests/test_json.py +0 -0
- {acadfetch-0.3.1 → acadfetch-0.4.0}/tests/test_manifest.py +0 -0
- {acadfetch-0.3.1 → acadfetch-0.4.0}/tests/test_naming.py +0 -0
- {acadfetch-0.3.1 → acadfetch-0.4.0}/tests/test_naming_race.py +0 -0
- {acadfetch-0.3.1 → acadfetch-0.4.0}/tests/test_packaging.py +0 -0
- {acadfetch-0.3.1 → acadfetch-0.4.0}/tests/test_progress.py +0 -0
- {acadfetch-0.3.1 → acadfetch-0.4.0}/tests/test_ratelimit.py +0 -0
- {acadfetch-0.3.1 → acadfetch-0.4.0}/tests/test_redirect.py +0 -0
- {acadfetch-0.3.1 → acadfetch-0.4.0}/tests/test_remote.py +0 -0
- {acadfetch-0.3.1 → acadfetch-0.4.0}/tests/test_removed.py +0 -0
- {acadfetch-0.3.1 → acadfetch-0.4.0}/tests/test_sources.py +0 -0
- {acadfetch-0.3.1 → acadfetch-0.4.0}/tests/test_urls.py +0 -0
- {acadfetch-0.3.1 → acadfetch-0.4.0}/tests/test_verify.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: acadfetch
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.4.0
|
|
4
4
|
Summary: Resumable, rate-limited, checksum-verified bulk downloads from academic data repositories
|
|
5
5
|
Author: young-ski
|
|
6
6
|
License-Expression: MIT
|
|
@@ -269,6 +269,21 @@ A CDN in front of a direct URL can answer 403 with a block page. Those are
|
|
|
269
269
|
retried twice with backoff, while a 403 carrying the application's own JSON is
|
|
270
270
|
terminal on the first answer; `--retry-403 N` changes the count.
|
|
271
271
|
|
|
272
|
+
## Networking
|
|
273
|
+
|
|
274
|
+
Proxies, timeouts and a private CA are flags, not luck:
|
|
275
|
+
|
|
276
|
+
```bash
|
|
277
|
+
acadfetch fetch zenodo:123456 --proxy http://proxy.campus:3128 # overrides the environment
|
|
278
|
+
acadfetch fetch zenodo:123456 --no-proxy # ignore HTTP(S)_PROXY
|
|
279
|
+
acadfetch fetch zenodo:123456 --timeout 15 # idle/connect budget, seconds
|
|
280
|
+
acadfetch fetch zenodo:123456 --ca-bundle corp-tls.pem # trust this PEM instead of the store
|
|
281
|
+
```
|
|
282
|
+
|
|
283
|
+
`--proxy` takes `http://` or `https://` and wins over `--no-proxy`; without
|
|
284
|
+
either, `HTTP_PROXY` / `HTTPS_PROXY` / `ALL_PROXY` are honoured by urllib as
|
|
285
|
+
usual. A timeout is per request, connect to first byte.
|
|
286
|
+
|
|
272
287
|
## Politeness
|
|
273
288
|
|
|
274
289
|
Defaults: 3 parallel transfers, 2 requests/second, 4 retries with jittered
|
|
@@ -238,6 +238,21 @@ A CDN in front of a direct URL can answer 403 with a block page. Those are
|
|
|
238
238
|
retried twice with backoff, while a 403 carrying the application's own JSON is
|
|
239
239
|
terminal on the first answer; `--retry-403 N` changes the count.
|
|
240
240
|
|
|
241
|
+
## Networking
|
|
242
|
+
|
|
243
|
+
Proxies, timeouts and a private CA are flags, not luck:
|
|
244
|
+
|
|
245
|
+
```bash
|
|
246
|
+
acadfetch fetch zenodo:123456 --proxy http://proxy.campus:3128 # overrides the environment
|
|
247
|
+
acadfetch fetch zenodo:123456 --no-proxy # ignore HTTP(S)_PROXY
|
|
248
|
+
acadfetch fetch zenodo:123456 --timeout 15 # idle/connect budget, seconds
|
|
249
|
+
acadfetch fetch zenodo:123456 --ca-bundle corp-tls.pem # trust this PEM instead of the store
|
|
250
|
+
```
|
|
251
|
+
|
|
252
|
+
`--proxy` takes `http://` or `https://` and wins over `--no-proxy`; without
|
|
253
|
+
either, `HTTP_PROXY` / `HTTPS_PROXY` / `ALL_PROXY` are honoured by urllib as
|
|
254
|
+
usual. A timeout is per request, connect to first byte.
|
|
255
|
+
|
|
241
256
|
## Politeness
|
|
242
257
|
|
|
243
258
|
Defaults: 3 parallel transfers, 2 requests/second, 4 retries with jittered
|
|
@@ -100,6 +100,32 @@ def _count(text: str) -> int:
|
|
|
100
100
|
return value
|
|
101
101
|
|
|
102
102
|
|
|
103
|
+
def _timeout(text: str) -> float:
|
|
104
|
+
try:
|
|
105
|
+
value = float(text)
|
|
106
|
+
except ValueError:
|
|
107
|
+
raise argparse.ArgumentTypeError(f"not a number of seconds: {text!r}") from None
|
|
108
|
+
if value <= 0:
|
|
109
|
+
raise argparse.ArgumentTypeError("must be > 0 seconds")
|
|
110
|
+
return value
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def _proxy(text: str) -> str:
|
|
114
|
+
candidate = text if "://" in text else f"http://{text}"
|
|
115
|
+
parts = urlsplit(candidate)
|
|
116
|
+
if parts.scheme not in ("http", "https") or not parts.hostname:
|
|
117
|
+
raise argparse.ArgumentTypeError(
|
|
118
|
+
f"expected http://host:port or https://host:port, got {text!r}"
|
|
119
|
+
)
|
|
120
|
+
return candidate
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def _ca_bundle(text: str) -> str:
|
|
124
|
+
if not Path(text).is_file():
|
|
125
|
+
raise argparse.ArgumentTypeError(f"no such file: {text!r}")
|
|
126
|
+
return text
|
|
127
|
+
|
|
128
|
+
|
|
103
129
|
def _checksum(text: str) -> str:
|
|
104
130
|
"""Validate ``--checksum`` before any request is made."""
|
|
105
131
|
parsed = verify.parse(text)
|
|
@@ -126,6 +152,10 @@ def _client(args: argparse.Namespace) -> Client:
|
|
|
126
152
|
retries=args.retries,
|
|
127
153
|
block_retries=args.retry_403,
|
|
128
154
|
allow_http=args.allow_http,
|
|
155
|
+
timeout=args.timeout,
|
|
156
|
+
proxy={"http": args.proxy, "https": args.proxy} if args.proxy else None,
|
|
157
|
+
no_proxy=args.no_proxy,
|
|
158
|
+
ca_bundle=args.ca_bundle,
|
|
129
159
|
headers=_headers(args.header),
|
|
130
160
|
auth={source: token.headers for source, token in tokens.items()},
|
|
131
161
|
)
|
|
@@ -602,6 +632,31 @@ def build_parser() -> argparse.ArgumentParser:
|
|
|
602
632
|
metavar="NAME:VALUE",
|
|
603
633
|
help="extra request header, repeatable (e.g. 'Authorization: Bearer <token>')",
|
|
604
634
|
)
|
|
635
|
+
p.add_argument(
|
|
636
|
+
"--timeout",
|
|
637
|
+
type=_timeout,
|
|
638
|
+
default=consts.DEFAULT_TIMEOUT,
|
|
639
|
+
metavar="SECONDS",
|
|
640
|
+
help="per-request timeout, connect and idle (default: %(default)s)",
|
|
641
|
+
)
|
|
642
|
+
p.add_argument(
|
|
643
|
+
"--proxy",
|
|
644
|
+
type=_proxy,
|
|
645
|
+
metavar="URL",
|
|
646
|
+
help="http:// or https:// proxy for every request; overrides the "
|
|
647
|
+
"environment and wins over --no-proxy",
|
|
648
|
+
)
|
|
649
|
+
p.add_argument(
|
|
650
|
+
"--no-proxy",
|
|
651
|
+
action="store_true",
|
|
652
|
+
help="ignore HTTP_PROXY / HTTPS_PROXY / ALL_PROXY",
|
|
653
|
+
)
|
|
654
|
+
p.add_argument(
|
|
655
|
+
"--ca-bundle",
|
|
656
|
+
type=_ca_bundle,
|
|
657
|
+
metavar="PATH",
|
|
658
|
+
help="PEM file to trust instead of the system store (corporate TLS inspection)",
|
|
659
|
+
)
|
|
605
660
|
p.add_argument(
|
|
606
661
|
"--allow-http",
|
|
607
662
|
action="store_true",
|
|
@@ -13,6 +13,7 @@ import email.utils
|
|
|
13
13
|
import hashlib
|
|
14
14
|
import json
|
|
15
15
|
import random
|
|
16
|
+
import ssl
|
|
16
17
|
import sys
|
|
17
18
|
import time
|
|
18
19
|
import urllib.error
|
|
@@ -143,6 +144,9 @@ class Client:
|
|
|
143
144
|
auth: Mapping[str, Mapping[str, str]] | None = None,
|
|
144
145
|
block_retries: int = consts.WAF_403_RETRIES,
|
|
145
146
|
allow_http: bool = False,
|
|
147
|
+
proxy: Mapping[str, str] | None = None,
|
|
148
|
+
no_proxy: bool = False,
|
|
149
|
+
ca_bundle: str | None = None,
|
|
146
150
|
) -> None:
|
|
147
151
|
self.bucket = TokenBucket(rate)
|
|
148
152
|
self.retries = max(0, retries)
|
|
@@ -157,7 +161,15 @@ class Client:
|
|
|
157
161
|
self.auth = {k: dict(v) for k, v in (auth or {}).items()}
|
|
158
162
|
self.source: str | None = None
|
|
159
163
|
self.redirects = RedirectPolicy(allow_http=allow_http)
|
|
160
|
-
|
|
164
|
+
handlers: list = [self.redirects]
|
|
165
|
+
# An explicit ProxyHandler replaces urllib's environment handling; the
|
|
166
|
+
# empty dict is how --no-proxy ignores HTTP(S)_PROXY.
|
|
167
|
+
if proxy is not None or no_proxy:
|
|
168
|
+
handlers.append(urllib.request.ProxyHandler(dict(proxy or {})))
|
|
169
|
+
if ca_bundle:
|
|
170
|
+
context = ssl.create_default_context(cafile=ca_bundle)
|
|
171
|
+
handlers.append(urllib.request.HTTPSHandler(context=context))
|
|
172
|
+
self.opener = urllib.request.build_opener(*handlers)
|
|
161
173
|
|
|
162
174
|
# -- internals ---------------------------------------------------------
|
|
163
175
|
def _sleep(self, attempt: int, retry_after: float | None) -> None:
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: acadfetch
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.4.0
|
|
4
4
|
Summary: Resumable, rate-limited, checksum-verified bulk downloads from academic data repositories
|
|
5
5
|
Author: young-ski
|
|
6
6
|
License-Expression: MIT
|
|
@@ -269,6 +269,21 @@ A CDN in front of a direct URL can answer 403 with a block page. Those are
|
|
|
269
269
|
retried twice with backoff, while a 403 carrying the application's own JSON is
|
|
270
270
|
terminal on the first answer; `--retry-403 N` changes the count.
|
|
271
271
|
|
|
272
|
+
## Networking
|
|
273
|
+
|
|
274
|
+
Proxies, timeouts and a private CA are flags, not luck:
|
|
275
|
+
|
|
276
|
+
```bash
|
|
277
|
+
acadfetch fetch zenodo:123456 --proxy http://proxy.campus:3128 # overrides the environment
|
|
278
|
+
acadfetch fetch zenodo:123456 --no-proxy # ignore HTTP(S)_PROXY
|
|
279
|
+
acadfetch fetch zenodo:123456 --timeout 15 # idle/connect budget, seconds
|
|
280
|
+
acadfetch fetch zenodo:123456 --ca-bundle corp-tls.pem # trust this PEM instead of the store
|
|
281
|
+
```
|
|
282
|
+
|
|
283
|
+
`--proxy` takes `http://` or `https://` and wins over `--no-proxy`; without
|
|
284
|
+
either, `HTTP_PROXY` / `HTTPS_PROXY` / `ALL_PROXY` are honoured by urllib as
|
|
285
|
+
usual. A timeout is per request, connect to first byte.
|
|
286
|
+
|
|
272
287
|
## Politeness
|
|
273
288
|
|
|
274
289
|
Defaults: 3 parallel transfers, 2 requests/second, 4 retries with jittered
|
|
@@ -48,6 +48,7 @@ tests/test_json.py
|
|
|
48
48
|
tests/test_manifest.py
|
|
49
49
|
tests/test_naming.py
|
|
50
50
|
tests/test_naming_race.py
|
|
51
|
+
tests/test_network.py
|
|
51
52
|
tests/test_packaging.py
|
|
52
53
|
tests/test_progress.py
|
|
53
54
|
tests/test_ratelimit.py
|
|
@@ -58,6 +59,7 @@ tests/test_sources.py
|
|
|
58
59
|
tests/test_urls.py
|
|
59
60
|
tests/test_verify.py
|
|
60
61
|
tests/fixtures/README.md
|
|
62
|
+
tests/fixtures/ca.pem
|
|
61
63
|
tests/fixtures/figshare_403_block.html
|
|
62
64
|
tests/fixtures/figshare_403_forbidden.json
|
|
63
65
|
tests/fixtures/figshare_article.json
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "acadfetch"
|
|
7
|
-
version = "0.
|
|
7
|
+
version = "0.4.0"
|
|
8
8
|
description = "Resumable, rate-limited, checksum-verified bulk downloads from academic data repositories"
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.10"
|
|
@@ -18,6 +18,9 @@ restricted file fails fast instead of burning retries.
|
|
|
18
18
|
`tests/test_client.py` and `tests/test_downloader.py` through the conftest
|
|
19
19
|
server; the names keep the provenance.
|
|
20
20
|
|
|
21
|
+
`ca.pem` is one public root certificate (taken from certifi) so a test can pass
|
|
22
|
+
`--ca-bundle` a PEM that really loads; it is not expected to verify any host.
|
|
23
|
+
|
|
21
24
|
`figshare_file.pdf` is a real 193 410-byte PDF whose md5 is
|
|
22
25
|
`bbde85208529e663141a0e8cb6fa9c4a`. It is the only real-bytes fixture, so the
|
|
23
26
|
verify path is proven against a published checksum rather than a synthetic
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
-----BEGIN CERTIFICATE-----
|
|
2
|
+
MIICiTCCAg+gAwIBAgIQH0evqmIAcFBUTAGem2OZKjAKBggqhkjOPQQDAzCBhTEL
|
|
3
|
+
MAkGA1UEBhMCR0IxGzAZBgNVBAgTEkdyZWF0ZXIgTWFuY2hlc3RlcjEQMA4GA1UE
|
|
4
|
+
BxMHU2FsZm9yZDEaMBgGA1UEChMRQ09NT0RPIENBIExpbWl0ZWQxKzApBgNVBAMT
|
|
5
|
+
IkNPTU9ETyBFQ0MgQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkwHhcNMDgwMzA2MDAw
|
|
6
|
+
MDAwWhcNMzgwMTE4MjM1OTU5WjCBhTELMAkGA1UEBhMCR0IxGzAZBgNVBAgTEkdy
|
|
7
|
+
ZWF0ZXIgTWFuY2hlc3RlcjEQMA4GA1UEBxMHU2FsZm9yZDEaMBgGA1UEChMRQ09N
|
|
8
|
+
T0RPIENBIExpbWl0ZWQxKzApBgNVBAMTIkNPTU9ETyBFQ0MgQ2VydGlmaWNhdGlv
|
|
9
|
+
biBBdXRob3JpdHkwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAAQDR3svdcmCFYX7deSR
|
|
10
|
+
FtSrYpn1PlILBs5BAH+X4QokPB0BBO490o0JlwzgdeT6+3eKKvUDYEs2ixYjFq0J
|
|
11
|
+
cfRK9ChQtP6IHG4/bC8vCVlbpVsLM5niwz2J+Wos77LTBumjQjBAMB0GA1UdDgQW
|
|
12
|
+
BBR1cacZSBm8nZ3qQUfflMRId5nTeTAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/
|
|
13
|
+
BAUwAwEB/zAKBggqhkjOPQQDAwNoADBlAjEA7wNbeqy3eApyt4jf/7VGFAkK+qDm
|
|
14
|
+
fQjGGoe9GKhzvSbKYAydzpmfz1wPMOG+FDHqAjAU9JM8SaczepBGR7NjfRObTrdv
|
|
15
|
+
GDeAU/7dIOA1mjbRxwG55tzd8/8dLDoWV9mSOdY=
|
|
16
|
+
-----END CERTIFICATE-----
|
|
17
|
+
|
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import urllib.request
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
|
|
6
|
+
import pytest
|
|
7
|
+
|
|
8
|
+
from acadfetch import cli
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def _args(*extra):
|
|
12
|
+
return cli.build_parser().parse_args(["fetch", "zenodo:1", *extra])
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def _proxies(client):
|
|
16
|
+
"""The explicit proxy mapping, or {} when urllib would keep no handler.
|
|
17
|
+
|
|
18
|
+
A ProxyHandler with an empty mapping registers no method, so build_opener
|
|
19
|
+
drops it; that is exactly what --no-proxy means.
|
|
20
|
+
"""
|
|
21
|
+
found = [h for h in client.opener.handlers if isinstance(h, urllib.request.ProxyHandler)]
|
|
22
|
+
return found[0].proxies if found else {}
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
@pytest.mark.parametrize("bad", ["0", "-1", "abc"])
|
|
26
|
+
def test_a_bad_timeout_is_a_usage_error(bad):
|
|
27
|
+
with pytest.raises(SystemExit) as caught:
|
|
28
|
+
cli.main(["fetch", "zenodo:1", "--timeout", bad])
|
|
29
|
+
assert caught.value.code == cli.EXIT_USAGE
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
@pytest.mark.parametrize("bad", ["ftp://x:1", "http://", "://x"])
|
|
33
|
+
def test_a_bad_proxy_is_a_usage_error(bad):
|
|
34
|
+
with pytest.raises(SystemExit) as caught:
|
|
35
|
+
cli.main(["fetch", "zenodo:1", "--proxy", bad])
|
|
36
|
+
assert caught.value.code == cli.EXIT_USAGE
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def test_a_missing_ca_bundle_is_a_usage_error(tmp_path):
|
|
40
|
+
with pytest.raises(SystemExit) as caught:
|
|
41
|
+
cli.main(["fetch", "zenodo:1", "--ca-bundle", str(tmp_path / "no.pem")])
|
|
42
|
+
assert caught.value.code == cli.EXIT_USAGE
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def test_the_timeout_reaches_the_client():
|
|
46
|
+
assert cli._client(_args("--timeout", "15")).timeout == 15.0
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def test_a_bare_host_port_gets_http():
|
|
50
|
+
assert cli._proxy("127.0.0.1:8899") == "http://127.0.0.1:8899"
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def test_the_proxy_reaches_both_schemes():
|
|
54
|
+
assert _proxies(cli._client(_args("--proxy", "http://127.0.0.1:8899"))) == {
|
|
55
|
+
"http": "http://127.0.0.1:8899",
|
|
56
|
+
"https": "http://127.0.0.1:8899",
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def test_no_proxy_leaves_no_proxy_handler(monkeypatch):
|
|
61
|
+
monkeypatch.setenv("HTTP_PROXY", "http://env.example:1")
|
|
62
|
+
monkeypatch.setenv("HTTPS_PROXY", "http://env.example:1")
|
|
63
|
+
|
|
64
|
+
client = cli._client(_args("--no-proxy"))
|
|
65
|
+
|
|
66
|
+
assert not [h for h in client.opener.handlers if isinstance(h, urllib.request.ProxyHandler)]
|
|
67
|
+
assert _proxies(client) == {}
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def test_the_environment_is_used_by_default(monkeypatch):
|
|
71
|
+
monkeypatch.setenv("HTTPS_PROXY", "http://env.example:1")
|
|
72
|
+
monkeypatch.setenv("HTTP_PROXY", "http://env.example:1")
|
|
73
|
+
|
|
74
|
+
proxies = _proxies(cli._client(_args()))
|
|
75
|
+
assert proxies.get("https") == "http://env.example:1"
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def test_an_explicit_proxy_beats_no_proxy(monkeypatch):
|
|
79
|
+
monkeypatch.setenv("HTTPS_PROXY", "http://env.example:1")
|
|
80
|
+
|
|
81
|
+
proxies = _proxies(cli._client(_args("--proxy", "http://own:1", "--no-proxy")))
|
|
82
|
+
assert proxies == {"http": "http://own:1", "https": "http://own:1"}
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def test_a_ca_bundle_adds_an_https_handler():
|
|
86
|
+
bundle = Path(__file__).parent / "fixtures" / "ca.pem"
|
|
87
|
+
|
|
88
|
+
client = cli._client(_args("--ca-bundle", str(bundle)))
|
|
89
|
+
|
|
90
|
+
assert any(isinstance(h, urllib.request.HTTPSHandler) for h in client.opener.handlers)
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def test_every_network_command_takes_the_flags(capsys):
|
|
94
|
+
for argv in (
|
|
95
|
+
["fetch", "zenodo:1", "--timeout", "5"],
|
|
96
|
+
["manifest", "zenodo:1", "--timeout", "5"],
|
|
97
|
+
["verify", "m.json", "--timeout", "5"],
|
|
98
|
+
):
|
|
99
|
+
args = cli.build_parser().parse_args(argv)
|
|
100
|
+
assert args.timeout == 5.0
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|