acadfetch 0.4.0__tar.gz → 0.5.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {acadfetch-0.4.0/acadfetch.egg-info → acadfetch-0.5.0}/PKG-INFO +8 -1
- {acadfetch-0.4.0 → acadfetch-0.5.0}/README.md +7 -0
- {acadfetch-0.4.0 → acadfetch-0.5.0}/acadfetch/cli.py +9 -1
- {acadfetch-0.4.0 → acadfetch-0.5.0}/acadfetch/client.py +16 -7
- {acadfetch-0.4.0 → acadfetch-0.5.0}/acadfetch/consts.py +1 -1
- acadfetch-0.5.0/acadfetch/debug.py +59 -0
- {acadfetch-0.4.0 → acadfetch-0.5.0}/acadfetch/downloader.py +2 -1
- {acadfetch-0.4.0 → acadfetch-0.5.0/acadfetch.egg-info}/PKG-INFO +8 -1
- {acadfetch-0.4.0 → acadfetch-0.5.0}/acadfetch.egg-info/SOURCES.txt +2 -0
- {acadfetch-0.4.0 → acadfetch-0.5.0}/pyproject.toml +1 -1
- acadfetch-0.5.0/tests/test_debug.py +120 -0
- {acadfetch-0.4.0 → acadfetch-0.5.0}/LICENSE +0 -0
- {acadfetch-0.4.0 → acadfetch-0.5.0}/MANIFEST.in +0 -0
- {acadfetch-0.4.0 → acadfetch-0.5.0}/acadfetch/__init__.py +0 -0
- {acadfetch-0.4.0 → acadfetch-0.5.0}/acadfetch/__main__.py +0 -0
- {acadfetch-0.4.0 → acadfetch-0.5.0}/acadfetch/atomic.py +0 -0
- {acadfetch-0.4.0 → acadfetch-0.5.0}/acadfetch/auth.py +0 -0
- {acadfetch-0.4.0 → acadfetch-0.5.0}/acadfetch/errors.py +0 -0
- {acadfetch-0.4.0 → acadfetch-0.5.0}/acadfetch/manifest.py +0 -0
- {acadfetch-0.4.0 → acadfetch-0.5.0}/acadfetch/naming.py +0 -0
- {acadfetch-0.4.0 → acadfetch-0.5.0}/acadfetch/progress.py +0 -0
- {acadfetch-0.4.0 → acadfetch-0.5.0}/acadfetch/py.typed +0 -0
- {acadfetch-0.4.0 → acadfetch-0.5.0}/acadfetch/ratelimit.py +0 -0
- {acadfetch-0.4.0 → acadfetch-0.5.0}/acadfetch/remote.py +0 -0
- {acadfetch-0.4.0 → acadfetch-0.5.0}/acadfetch/sources/__init__.py +0 -0
- {acadfetch-0.4.0 → acadfetch-0.5.0}/acadfetch/sources/arxiv.py +0 -0
- {acadfetch-0.4.0 → acadfetch-0.5.0}/acadfetch/sources/base.py +0 -0
- {acadfetch-0.4.0 → acadfetch-0.5.0}/acadfetch/sources/direct.py +0 -0
- {acadfetch-0.4.0 → acadfetch-0.5.0}/acadfetch/sources/doi.py +0 -0
- {acadfetch-0.4.0 → acadfetch-0.5.0}/acadfetch/sources/registry.py +0 -0
- {acadfetch-0.4.0 → acadfetch-0.5.0}/acadfetch/sources/removed.py +0 -0
- {acadfetch-0.4.0 → acadfetch-0.5.0}/acadfetch/sources/zenodo.py +0 -0
- {acadfetch-0.4.0 → acadfetch-0.5.0}/acadfetch/units.py +0 -0
- {acadfetch-0.4.0 → acadfetch-0.5.0}/acadfetch/urls.py +0 -0
- {acadfetch-0.4.0 → acadfetch-0.5.0}/acadfetch/verify.py +0 -0
- {acadfetch-0.4.0 → acadfetch-0.5.0}/acadfetch.egg-info/dependency_links.txt +0 -0
- {acadfetch-0.4.0 → acadfetch-0.5.0}/acadfetch.egg-info/entry_points.txt +0 -0
- {acadfetch-0.4.0 → acadfetch-0.5.0}/acadfetch.egg-info/top_level.txt +0 -0
- {acadfetch-0.4.0 → acadfetch-0.5.0}/setup.cfg +0 -0
- {acadfetch-0.4.0 → acadfetch-0.5.0}/tests/conftest.py +0 -0
- {acadfetch-0.4.0 → acadfetch-0.5.0}/tests/fixtures/README.md +0 -0
- {acadfetch-0.4.0 → acadfetch-0.5.0}/tests/fixtures/ca.pem +0 -0
- {acadfetch-0.4.0 → acadfetch-0.5.0}/tests/fixtures/figshare_403_block.html +0 -0
- {acadfetch-0.4.0 → acadfetch-0.5.0}/tests/fixtures/figshare_403_forbidden.json +0 -0
- {acadfetch-0.4.0 → acadfetch-0.5.0}/tests/fixtures/figshare_article.json +0 -0
- {acadfetch-0.4.0 → acadfetch-0.5.0}/tests/fixtures/figshare_file.pdf +0 -0
- {acadfetch-0.4.0 → acadfetch-0.5.0}/tests/test_atomic.py +0 -0
- {acadfetch-0.4.0 → acadfetch-0.5.0}/tests/test_auth.py +0 -0
- {acadfetch-0.4.0 → acadfetch-0.5.0}/tests/test_checksum.py +0 -0
- {acadfetch-0.4.0 → acadfetch-0.5.0}/tests/test_cli.py +0 -0
- {acadfetch-0.4.0 → acadfetch-0.5.0}/tests/test_client.py +0 -0
- {acadfetch-0.4.0 → acadfetch-0.5.0}/tests/test_control_names.py +0 -0
- {acadfetch-0.4.0 → acadfetch-0.5.0}/tests/test_disk.py +0 -0
- {acadfetch-0.4.0 → acadfetch-0.5.0}/tests/test_downloader.py +0 -0
- {acadfetch-0.4.0 → acadfetch-0.5.0}/tests/test_identity.py +0 -0
- {acadfetch-0.4.0 → acadfetch-0.5.0}/tests/test_isolation.py +0 -0
- {acadfetch-0.4.0 → acadfetch-0.5.0}/tests/test_json.py +0 -0
- {acadfetch-0.4.0 → acadfetch-0.5.0}/tests/test_manifest.py +0 -0
- {acadfetch-0.4.0 → acadfetch-0.5.0}/tests/test_naming.py +0 -0
- {acadfetch-0.4.0 → acadfetch-0.5.0}/tests/test_naming_race.py +0 -0
- {acadfetch-0.4.0 → acadfetch-0.5.0}/tests/test_network.py +0 -0
- {acadfetch-0.4.0 → acadfetch-0.5.0}/tests/test_packaging.py +0 -0
- {acadfetch-0.4.0 → acadfetch-0.5.0}/tests/test_progress.py +0 -0
- {acadfetch-0.4.0 → acadfetch-0.5.0}/tests/test_ratelimit.py +0 -0
- {acadfetch-0.4.0 → acadfetch-0.5.0}/tests/test_redirect.py +0 -0
- {acadfetch-0.4.0 → acadfetch-0.5.0}/tests/test_remote.py +0 -0
- {acadfetch-0.4.0 → acadfetch-0.5.0}/tests/test_removed.py +0 -0
- {acadfetch-0.4.0 → acadfetch-0.5.0}/tests/test_sources.py +0 -0
- {acadfetch-0.4.0 → acadfetch-0.5.0}/tests/test_urls.py +0 -0
- {acadfetch-0.4.0 → acadfetch-0.5.0}/tests/test_verify.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: acadfetch
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.5.0
|
|
4
4
|
Summary: Resumable, rate-limited, checksum-verified bulk downloads from academic data repositories
|
|
5
5
|
Author: young-ski
|
|
6
6
|
License-Expression: MIT
|
|
@@ -284,6 +284,13 @@ acadfetch fetch zenodo:123456 --ca-bundle corp-tls.pem # trust this PEM
|
|
|
284
284
|
either, `HTTP_PROXY` / `HTTPS_PROXY` / `ALL_PROXY` are honoured by urllib as
|
|
285
285
|
usual. A timeout is per request, connect to first byte.
|
|
286
286
|
|
|
287
|
+
## Debugging
|
|
288
|
+
|
|
289
|
+
`--verbose` (or `ACADFETCH_DEBUG=1`) traces each request, each retry and the
|
|
290
|
+
traceback of a failure to stderr; stdout, `--json` documents and exit codes are
|
|
291
|
+
unchanged. Credential header values and presigned query parameters are masked,
|
|
292
|
+
so the trace can be pasted into an issue.
|
|
293
|
+
|
|
287
294
|
## Politeness
|
|
288
295
|
|
|
289
296
|
Defaults: 3 parallel transfers, 2 requests/second, 4 retries with jittered
|
|
@@ -253,6 +253,13 @@ acadfetch fetch zenodo:123456 --ca-bundle corp-tls.pem # trust this PEM
|
|
|
253
253
|
either, `HTTP_PROXY` / `HTTPS_PROXY` / `ALL_PROXY` are honoured by urllib as
|
|
254
254
|
usual. A timeout is per request, connect to first byte.
|
|
255
255
|
|
|
256
|
+
## Debugging
|
|
257
|
+
|
|
258
|
+
`--verbose` (or `ACADFETCH_DEBUG=1`) traces each request, each retry and the
|
|
259
|
+
traceback of a failure to stderr; stdout, `--json` documents and exit codes are
|
|
260
|
+
unchanged. Credential header values and presigned query parameters are masked,
|
|
261
|
+
so the trace can be pasted into an issue.
|
|
262
|
+
|
|
256
263
|
## Politeness
|
|
257
264
|
|
|
258
265
|
Defaults: 3 parallel transfers, 2 requests/second, 4 retries with jittered
|
|
@@ -17,7 +17,7 @@ from datetime import datetime, timezone
|
|
|
17
17
|
from pathlib import Path
|
|
18
18
|
from urllib.parse import urlsplit
|
|
19
19
|
|
|
20
|
-
from . import auth, consts, manifest, naming, urls, verify
|
|
20
|
+
from . import auth, consts, debug, manifest, naming, urls, verify
|
|
21
21
|
from . import remote as remote_check
|
|
22
22
|
from .client import Client
|
|
23
23
|
from .downloader import Options, check_disk, check_total, fetch_record, human
|
|
@@ -632,6 +632,11 @@ def build_parser() -> argparse.ArgumentParser:
|
|
|
632
632
|
metavar="NAME:VALUE",
|
|
633
633
|
help="extra request header, repeatable (e.g. 'Authorization: Bearer <token>')",
|
|
634
634
|
)
|
|
635
|
+
p.add_argument(
|
|
636
|
+
"--verbose",
|
|
637
|
+
action="store_true",
|
|
638
|
+
help="trace requests and retries to stderr, with tracebacks (also ACADFETCH_DEBUG=1)",
|
|
639
|
+
)
|
|
635
640
|
p.add_argument(
|
|
636
641
|
"--timeout",
|
|
637
642
|
type=_timeout,
|
|
@@ -788,6 +793,7 @@ def main(argv: list[str] | None = None) -> int:
|
|
|
788
793
|
if hasattr(stream, "reconfigure"):
|
|
789
794
|
stream.reconfigure(encoding="utf-8", errors="replace")
|
|
790
795
|
args = build_parser().parse_args(argv)
|
|
796
|
+
debug.configure(getattr(args, "verbose", False))
|
|
791
797
|
_recover_target(args)
|
|
792
798
|
if getattr(args, "command", "") == "fetch" and getattr(args, "target", None) is None:
|
|
793
799
|
message = (
|
|
@@ -802,12 +808,14 @@ def main(argv: list[str] | None = None) -> int:
|
|
|
802
808
|
try:
|
|
803
809
|
return args.func(args)
|
|
804
810
|
except AcFError as exc: # TargetError is one
|
|
811
|
+
debug.log_exception(exc)
|
|
805
812
|
# An error sentence carries the URL, and that URL may carry a signature.
|
|
806
813
|
message = _hide_text(str(exc), args)
|
|
807
814
|
print(f"acadfetch: {message}", file=sys.stderr)
|
|
808
815
|
_emit_error(args, message)
|
|
809
816
|
return EXIT_FAIL
|
|
810
817
|
except OSError as exc: # a filesystem surprise, not a bug report
|
|
818
|
+
debug.log_exception(exc)
|
|
811
819
|
message = f"{exc.strerror or exc}" if exc.filename else str(exc)
|
|
812
820
|
print(f"acadfetch: {message}", file=sys.stderr)
|
|
813
821
|
_emit_error(args, message)
|
|
@@ -23,7 +23,7 @@ from collections.abc import Mapping
|
|
|
23
23
|
from datetime import datetime, timezone
|
|
24
24
|
from http.client import HTTPException
|
|
25
25
|
|
|
26
|
-
from . import consts, verify
|
|
26
|
+
from . import consts, debug, urls, verify
|
|
27
27
|
from .errors import HttpError
|
|
28
28
|
from .ratelimit import TokenBucket
|
|
29
29
|
|
|
@@ -176,10 +176,11 @@ class Client:
|
|
|
176
176
|
if retry_after is not None:
|
|
177
177
|
# A literal `Retry-After: 0` would mean no gap at all and a burst;
|
|
178
178
|
# keep a small jittered floor under it.
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
179
|
+
delay = max(retry_after, 0.05 + random.random() * 0.15)
|
|
180
|
+
else:
|
|
181
|
+
delay = min(0.8 * (2**attempt), 30.0) * (0.7 + random.random() * 0.6)
|
|
182
|
+
debug.log(f"retry in {delay:.2f}s")
|
|
183
|
+
time.sleep(delay)
|
|
183
184
|
|
|
184
185
|
def _request(
|
|
185
186
|
self, url: str, headers: Mapping[str, str] | None = None
|
|
@@ -204,6 +205,7 @@ class Client:
|
|
|
204
205
|
headers=None,
|
|
205
206
|
block_retries: int | None = None,
|
|
206
207
|
attempts: int | None = None,
|
|
208
|
+
method: str = "GET",
|
|
207
209
|
): # noqa: ANN001, ANN202
|
|
208
210
|
if block_retries is None:
|
|
209
211
|
block_retries = self.block_retries
|
|
@@ -215,11 +217,16 @@ class Client:
|
|
|
215
217
|
for attempt in range(budget):
|
|
216
218
|
self.bucket.acquire()
|
|
217
219
|
retry_after = None
|
|
220
|
+
req = self._request(url, headers)
|
|
221
|
+
debug.log(f"{method} {urls.redact(url)} headers[{debug.headers_for_log(req.headers)}]")
|
|
218
222
|
try:
|
|
219
|
-
|
|
223
|
+
resp = fn(req)
|
|
224
|
+
debug.log(f"-> {getattr(resp, 'status', 'ok')}")
|
|
225
|
+
return resp
|
|
220
226
|
except urllib.error.HTTPError as exc:
|
|
221
227
|
last = exc
|
|
222
228
|
last_detail = detail = _snippet(exc)
|
|
229
|
+
debug.log(f"-> {exc.code} {detail[:120]}")
|
|
223
230
|
content_type = exc.headers.get("Content-Type") if exc.headers else None
|
|
224
231
|
last_blocked = exc.code == 403 and is_block_page(detail, content_type)
|
|
225
232
|
retry_after = parse_retry_after(
|
|
@@ -239,6 +246,7 @@ class Client:
|
|
|
239
246
|
) from exc
|
|
240
247
|
else:
|
|
241
248
|
waf += 1
|
|
249
|
+
debug.log(f"-> block page, retry {waf}/{block_retries}")
|
|
242
250
|
except HttpError as exc:
|
|
243
251
|
# Raised by a caller-supplied fn, e.g. a body that stopped
|
|
244
252
|
# short. Only the transient ones are worth another request.
|
|
@@ -248,6 +256,7 @@ class Client:
|
|
|
248
256
|
last_detail = str(exc)
|
|
249
257
|
last_blocked = False
|
|
250
258
|
except (urllib.error.URLError, TimeoutError, OSError) as exc:
|
|
259
|
+
debug.log(f"-> {type(exc).__name__}: {exc}")
|
|
251
260
|
last = exc
|
|
252
261
|
last_detail = str(exc)
|
|
253
262
|
last_blocked = False
|
|
@@ -298,7 +307,7 @@ class Client:
|
|
|
298
307
|
|
|
299
308
|
try:
|
|
300
309
|
# HEAD only feeds size/filename; one shot, never worth a retry.
|
|
301
|
-
with self._with_retries(url, _head, block_retries=0, attempts=1) as resp:
|
|
310
|
+
with self._with_retries(url, _head, block_retries=0, attempts=1, method="HEAD") as resp:
|
|
302
311
|
return dict(resp.headers)
|
|
303
312
|
except HttpError as exc:
|
|
304
313
|
if exc.status in (403, 405, 501):
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
"""Opt-in tracing, so a bug report does not have to be a guess.
|
|
2
|
+
|
|
3
|
+
Everything here writes to stderr and swallows its own errors: like progress, a
|
|
4
|
+
log line must never be able to fail a download. Header values that are
|
|
5
|
+
credentials are replaced with an ellipsis, and URLs arrive already redacted by
|
|
6
|
+
the caller.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import contextlib
|
|
12
|
+
import os
|
|
13
|
+
import sys
|
|
14
|
+
import traceback
|
|
15
|
+
from collections.abc import Mapping
|
|
16
|
+
|
|
17
|
+
from . import urls
|
|
18
|
+
|
|
19
|
+
_ON = False
|
|
20
|
+
_CREDENTIAL_HEADERS = frozenset(
|
|
21
|
+
{"authorization", "cookie", "proxy-authorization", "x-api-key", "x-auth-token"}
|
|
22
|
+
)
|
|
23
|
+
_FALSE = frozenset({"", "0", "false", "no", "off"})
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def configure(flag: bool = False, environ: Mapping[str, str] | None = None) -> None:
|
|
27
|
+
"""Turn tracing on from the flag or ``ACADFETCH_DEBUG``; the flag wins."""
|
|
28
|
+
global _ON
|
|
29
|
+
env = os.environ if environ is None else environ
|
|
30
|
+
value = (env.get("ACADFETCH_DEBUG") or "").strip().lower()
|
|
31
|
+
_ON = bool(flag) or value not in _FALSE
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def enabled() -> bool:
|
|
35
|
+
return _ON
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def log(message: str) -> None:
|
|
39
|
+
if not _ON:
|
|
40
|
+
return
|
|
41
|
+
with contextlib.suppress(OSError, ValueError):
|
|
42
|
+
print(f"debug: {message}", file=sys.stderr, flush=True)
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def log_exception(exc: BaseException) -> None:
|
|
46
|
+
if not _ON:
|
|
47
|
+
return
|
|
48
|
+
with contextlib.suppress(OSError, ValueError):
|
|
49
|
+
text = "".join(traceback.format_exception(type(exc), exc, exc.__traceback__))
|
|
50
|
+
sys.stderr.write(urls.redact_text(text))
|
|
51
|
+
sys.stderr.flush()
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def headers_for_log(headers: Mapping[str, str]) -> str:
|
|
55
|
+
"""Header names with safe values; credentials become ``…``."""
|
|
56
|
+
return " ".join(
|
|
57
|
+
f"{name}: {'…' if name.lower() in _CREDENTIAL_HEADERS else value}"
|
|
58
|
+
for name, value in headers.items()
|
|
59
|
+
)
|
|
@@ -25,7 +25,7 @@ from datetime import datetime, timezone
|
|
|
25
25
|
from http.client import HTTPException
|
|
26
26
|
from pathlib import Path
|
|
27
27
|
|
|
28
|
-
from . import atomic, naming, verify
|
|
28
|
+
from . import atomic, debug, naming, verify
|
|
29
29
|
from .client import Client
|
|
30
30
|
from .consts import CHUNK
|
|
31
31
|
from .errors import HttpError, LocalError, PartialError, TargetError
|
|
@@ -700,6 +700,7 @@ def _fetch_record(
|
|
|
700
700
|
|
|
701
701
|
def _crashed(file: File, exc: Exception) -> Result:
|
|
702
702
|
"""An unexpected worker exception, as one failed row instead of a traceback."""
|
|
703
|
+
debug.log_exception(exc)
|
|
703
704
|
return Result(
|
|
704
705
|
file=file,
|
|
705
706
|
state="failed",
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: acadfetch
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.5.0
|
|
4
4
|
Summary: Resumable, rate-limited, checksum-verified bulk downloads from academic data repositories
|
|
5
5
|
Author: young-ski
|
|
6
6
|
License-Expression: MIT
|
|
@@ -284,6 +284,13 @@ acadfetch fetch zenodo:123456 --ca-bundle corp-tls.pem # trust this PEM
|
|
|
284
284
|
either, `HTTP_PROXY` / `HTTPS_PROXY` / `ALL_PROXY` are honoured by urllib as
|
|
285
285
|
usual. A timeout is per request, connect to first byte.
|
|
286
286
|
|
|
287
|
+
## Debugging
|
|
288
|
+
|
|
289
|
+
`--verbose` (or `ACADFETCH_DEBUG=1`) traces each request, each retry and the
|
|
290
|
+
traceback of a failure to stderr; stdout, `--json` documents and exit codes are
|
|
291
|
+
unchanged. Credential header values and presigned query parameters are masked,
|
|
292
|
+
so the trace can be pasted into an issue.
|
|
293
|
+
|
|
287
294
|
## Politeness
|
|
288
295
|
|
|
289
296
|
Defaults: 3 parallel transfers, 2 requests/second, 4 retries with jittered
|
|
@@ -9,6 +9,7 @@ acadfetch/auth.py
|
|
|
9
9
|
acadfetch/cli.py
|
|
10
10
|
acadfetch/client.py
|
|
11
11
|
acadfetch/consts.py
|
|
12
|
+
acadfetch/debug.py
|
|
12
13
|
acadfetch/downloader.py
|
|
13
14
|
acadfetch/errors.py
|
|
14
15
|
acadfetch/manifest.py
|
|
@@ -40,6 +41,7 @@ tests/test_checksum.py
|
|
|
40
41
|
tests/test_cli.py
|
|
41
42
|
tests/test_client.py
|
|
42
43
|
tests/test_control_names.py
|
|
44
|
+
tests/test_debug.py
|
|
43
45
|
tests/test_disk.py
|
|
44
46
|
tests/test_downloader.py
|
|
45
47
|
tests/test_identity.py
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "acadfetch"
|
|
7
|
-
version = "0.
|
|
7
|
+
version = "0.5.0"
|
|
8
8
|
description = "Resumable, rate-limited, checksum-verified bulk downloads from academic data repositories"
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.10"
|
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import pytest
|
|
4
|
+
|
|
5
|
+
from acadfetch import cli, debug
|
|
6
|
+
from acadfetch.client import Client
|
|
7
|
+
from acadfetch.errors import HttpError
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
@pytest.fixture(autouse=True)
|
|
11
|
+
def _reset():
|
|
12
|
+
debug.configure(False)
|
|
13
|
+
yield
|
|
14
|
+
debug.configure(False)
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def test_the_environment_can_turn_it_on():
|
|
18
|
+
debug.configure(False, {"ACADFETCH_DEBUG": "1"})
|
|
19
|
+
assert debug.enabled()
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
@pytest.mark.parametrize("value", ["0", "false", "no", "off", ""])
|
|
23
|
+
def test_the_environment_can_turn_it_off(value):
|
|
24
|
+
debug.configure(False, {"ACADFETCH_DEBUG": value})
|
|
25
|
+
assert not debug.enabled()
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def test_the_flag_wins_over_the_environment():
|
|
29
|
+
debug.configure(True, {"ACADFETCH_DEBUG": "0"})
|
|
30
|
+
assert debug.enabled()
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def test_it_is_silent_when_off(capsys):
|
|
34
|
+
debug.log("nothing")
|
|
35
|
+
debug.log_exception(RuntimeError("boom"))
|
|
36
|
+
assert capsys.readouterr().err == ""
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def _talking_client():
|
|
40
|
+
client = Client(rate=1000, retries=0, auth={"direct": {"Authorization": "Bearer SECRET"}})
|
|
41
|
+
client.source = "direct"
|
|
42
|
+
return client
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def test_headers_are_masked(server, capsys):
|
|
46
|
+
server.cfg["files"]["/a.bin"] = b"x"
|
|
47
|
+
debug.configure(True)
|
|
48
|
+
|
|
49
|
+
_talking_client().head(server.base + "/a.bin")
|
|
50
|
+
|
|
51
|
+
err = capsys.readouterr().err
|
|
52
|
+
assert "HEAD" in err
|
|
53
|
+
assert "Authorization: …" in err
|
|
54
|
+
assert "SECRET" not in err
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def test_a_credential_in_the_url_is_redacted(server, capsys):
|
|
58
|
+
server.cfg["files"]["/a.bin"] = b"x"
|
|
59
|
+
debug.configure(True)
|
|
60
|
+
|
|
61
|
+
_talking_client().head(server.base + "/a.bin?X-Amz-Signature=SIG")
|
|
62
|
+
|
|
63
|
+
err = capsys.readouterr().err
|
|
64
|
+
assert "X-Amz-Signature" not in err
|
|
65
|
+
assert "SIG" not in err
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def test_a_retry_is_traced(server, capsys):
|
|
69
|
+
server.cfg["files"]["/a.bin"] = b"x"
|
|
70
|
+
server.cfg["always_500"].add("/a.bin")
|
|
71
|
+
debug.configure(True)
|
|
72
|
+
|
|
73
|
+
with pytest.raises(HttpError):
|
|
74
|
+
Client(rate=1000, retries=1).open(server.base + "/a.bin")
|
|
75
|
+
|
|
76
|
+
err = capsys.readouterr().err
|
|
77
|
+
assert "GET" in err and "-> 500" in err and "retry in" in err
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def test_a_failure_prints_a_traceback_only_with_the_flag(capsys):
|
|
81
|
+
assert cli.main(["fetch", "nonsense://x"]) == 1
|
|
82
|
+
assert "Traceback" not in capsys.readouterr().err
|
|
83
|
+
|
|
84
|
+
assert cli.main(["fetch", "nonsense://x", "--verbose"]) == 1
|
|
85
|
+
assert "Traceback (most recent call last)" in capsys.readouterr().err
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def test_debug_does_not_change_the_json_document(server, monkeypatch, capsys, tmp_path):
|
|
89
|
+
import json
|
|
90
|
+
|
|
91
|
+
server.cfg["files"]["/a.bin"] = b"payload"
|
|
92
|
+
monkeypatch.setattr(
|
|
93
|
+
cli.registry,
|
|
94
|
+
"resolve",
|
|
95
|
+
lambda _c, _t: __import__("acadfetch.sources.base", fromlist=["Record"]).Record(
|
|
96
|
+
source="direct",
|
|
97
|
+
target=server.base + "/a.bin",
|
|
98
|
+
files=[
|
|
99
|
+
__import__("acadfetch.sources.base", fromlist=["File"]).File(
|
|
100
|
+
server.base + "/a.bin", "a.bin", 7
|
|
101
|
+
)
|
|
102
|
+
],
|
|
103
|
+
),
|
|
104
|
+
)
|
|
105
|
+
|
|
106
|
+
code = cli.main(["fetch", "t", "--json", "--verbose", "-o", str(tmp_path)])
|
|
107
|
+
out, _ = capsys.readouterr()
|
|
108
|
+
|
|
109
|
+
assert code == cli.EXIT_OK
|
|
110
|
+
assert json.loads(out)["summary"]["ok"] is True
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def test_a_traceback_does_not_leak_a_credential(capsys):
|
|
114
|
+
assert (
|
|
115
|
+
cli.main(["fetch", "https://example.invalid/x?X-Amz-Signature=SECRET", "--verbose", "-n"])
|
|
116
|
+
== 1
|
|
117
|
+
)
|
|
118
|
+
err = capsys.readouterr().err
|
|
119
|
+
assert "Traceback (most recent call last)" in err
|
|
120
|
+
assert "SECRET" not in err
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|