acadfetch 0.3.1__tar.gz → 0.5.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (70) hide show
  1. {acadfetch-0.3.1/acadfetch.egg-info → acadfetch-0.5.0}/PKG-INFO +23 -1
  2. {acadfetch-0.3.1 → acadfetch-0.5.0}/README.md +22 -0
  3. {acadfetch-0.3.1 → acadfetch-0.5.0}/acadfetch/cli.py +64 -1
  4. {acadfetch-0.3.1 → acadfetch-0.5.0}/acadfetch/client.py +29 -8
  5. {acadfetch-0.3.1 → acadfetch-0.5.0}/acadfetch/consts.py +1 -1
  6. acadfetch-0.5.0/acadfetch/debug.py +59 -0
  7. {acadfetch-0.3.1 → acadfetch-0.5.0}/acadfetch/downloader.py +2 -1
  8. {acadfetch-0.3.1 → acadfetch-0.5.0/acadfetch.egg-info}/PKG-INFO +23 -1
  9. {acadfetch-0.3.1 → acadfetch-0.5.0}/acadfetch.egg-info/SOURCES.txt +4 -0
  10. {acadfetch-0.3.1 → acadfetch-0.5.0}/pyproject.toml +1 -1
  11. {acadfetch-0.3.1 → acadfetch-0.5.0}/tests/fixtures/README.md +3 -0
  12. acadfetch-0.5.0/tests/fixtures/ca.pem +17 -0
  13. acadfetch-0.5.0/tests/test_debug.py +120 -0
  14. acadfetch-0.5.0/tests/test_network.py +100 -0
  15. {acadfetch-0.3.1 → acadfetch-0.5.0}/LICENSE +0 -0
  16. {acadfetch-0.3.1 → acadfetch-0.5.0}/MANIFEST.in +0 -0
  17. {acadfetch-0.3.1 → acadfetch-0.5.0}/acadfetch/__init__.py +0 -0
  18. {acadfetch-0.3.1 → acadfetch-0.5.0}/acadfetch/__main__.py +0 -0
  19. {acadfetch-0.3.1 → acadfetch-0.5.0}/acadfetch/atomic.py +0 -0
  20. {acadfetch-0.3.1 → acadfetch-0.5.0}/acadfetch/auth.py +0 -0
  21. {acadfetch-0.3.1 → acadfetch-0.5.0}/acadfetch/errors.py +0 -0
  22. {acadfetch-0.3.1 → acadfetch-0.5.0}/acadfetch/manifest.py +0 -0
  23. {acadfetch-0.3.1 → acadfetch-0.5.0}/acadfetch/naming.py +0 -0
  24. {acadfetch-0.3.1 → acadfetch-0.5.0}/acadfetch/progress.py +0 -0
  25. {acadfetch-0.3.1 → acadfetch-0.5.0}/acadfetch/py.typed +0 -0
  26. {acadfetch-0.3.1 → acadfetch-0.5.0}/acadfetch/ratelimit.py +0 -0
  27. {acadfetch-0.3.1 → acadfetch-0.5.0}/acadfetch/remote.py +0 -0
  28. {acadfetch-0.3.1 → acadfetch-0.5.0}/acadfetch/sources/__init__.py +0 -0
  29. {acadfetch-0.3.1 → acadfetch-0.5.0}/acadfetch/sources/arxiv.py +0 -0
  30. {acadfetch-0.3.1 → acadfetch-0.5.0}/acadfetch/sources/base.py +0 -0
  31. {acadfetch-0.3.1 → acadfetch-0.5.0}/acadfetch/sources/direct.py +0 -0
  32. {acadfetch-0.3.1 → acadfetch-0.5.0}/acadfetch/sources/doi.py +0 -0
  33. {acadfetch-0.3.1 → acadfetch-0.5.0}/acadfetch/sources/registry.py +0 -0
  34. {acadfetch-0.3.1 → acadfetch-0.5.0}/acadfetch/sources/removed.py +0 -0
  35. {acadfetch-0.3.1 → acadfetch-0.5.0}/acadfetch/sources/zenodo.py +0 -0
  36. {acadfetch-0.3.1 → acadfetch-0.5.0}/acadfetch/units.py +0 -0
  37. {acadfetch-0.3.1 → acadfetch-0.5.0}/acadfetch/urls.py +0 -0
  38. {acadfetch-0.3.1 → acadfetch-0.5.0}/acadfetch/verify.py +0 -0
  39. {acadfetch-0.3.1 → acadfetch-0.5.0}/acadfetch.egg-info/dependency_links.txt +0 -0
  40. {acadfetch-0.3.1 → acadfetch-0.5.0}/acadfetch.egg-info/entry_points.txt +0 -0
  41. {acadfetch-0.3.1 → acadfetch-0.5.0}/acadfetch.egg-info/top_level.txt +0 -0
  42. {acadfetch-0.3.1 → acadfetch-0.5.0}/setup.cfg +0 -0
  43. {acadfetch-0.3.1 → acadfetch-0.5.0}/tests/conftest.py +0 -0
  44. {acadfetch-0.3.1 → acadfetch-0.5.0}/tests/fixtures/figshare_403_block.html +0 -0
  45. {acadfetch-0.3.1 → acadfetch-0.5.0}/tests/fixtures/figshare_403_forbidden.json +0 -0
  46. {acadfetch-0.3.1 → acadfetch-0.5.0}/tests/fixtures/figshare_article.json +0 -0
  47. {acadfetch-0.3.1 → acadfetch-0.5.0}/tests/fixtures/figshare_file.pdf +0 -0
  48. {acadfetch-0.3.1 → acadfetch-0.5.0}/tests/test_atomic.py +0 -0
  49. {acadfetch-0.3.1 → acadfetch-0.5.0}/tests/test_auth.py +0 -0
  50. {acadfetch-0.3.1 → acadfetch-0.5.0}/tests/test_checksum.py +0 -0
  51. {acadfetch-0.3.1 → acadfetch-0.5.0}/tests/test_cli.py +0 -0
  52. {acadfetch-0.3.1 → acadfetch-0.5.0}/tests/test_client.py +0 -0
  53. {acadfetch-0.3.1 → acadfetch-0.5.0}/tests/test_control_names.py +0 -0
  54. {acadfetch-0.3.1 → acadfetch-0.5.0}/tests/test_disk.py +0 -0
  55. {acadfetch-0.3.1 → acadfetch-0.5.0}/tests/test_downloader.py +0 -0
  56. {acadfetch-0.3.1 → acadfetch-0.5.0}/tests/test_identity.py +0 -0
  57. {acadfetch-0.3.1 → acadfetch-0.5.0}/tests/test_isolation.py +0 -0
  58. {acadfetch-0.3.1 → acadfetch-0.5.0}/tests/test_json.py +0 -0
  59. {acadfetch-0.3.1 → acadfetch-0.5.0}/tests/test_manifest.py +0 -0
  60. {acadfetch-0.3.1 → acadfetch-0.5.0}/tests/test_naming.py +0 -0
  61. {acadfetch-0.3.1 → acadfetch-0.5.0}/tests/test_naming_race.py +0 -0
  62. {acadfetch-0.3.1 → acadfetch-0.5.0}/tests/test_packaging.py +0 -0
  63. {acadfetch-0.3.1 → acadfetch-0.5.0}/tests/test_progress.py +0 -0
  64. {acadfetch-0.3.1 → acadfetch-0.5.0}/tests/test_ratelimit.py +0 -0
  65. {acadfetch-0.3.1 → acadfetch-0.5.0}/tests/test_redirect.py +0 -0
  66. {acadfetch-0.3.1 → acadfetch-0.5.0}/tests/test_remote.py +0 -0
  67. {acadfetch-0.3.1 → acadfetch-0.5.0}/tests/test_removed.py +0 -0
  68. {acadfetch-0.3.1 → acadfetch-0.5.0}/tests/test_sources.py +0 -0
  69. {acadfetch-0.3.1 → acadfetch-0.5.0}/tests/test_urls.py +0 -0
  70. {acadfetch-0.3.1 → acadfetch-0.5.0}/tests/test_verify.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: acadfetch
3
- Version: 0.3.1
3
+ Version: 0.5.0
4
4
  Summary: Resumable, rate-limited, checksum-verified bulk downloads from academic data repositories
5
5
  Author: young-ski
6
6
  License-Expression: MIT
@@ -269,6 +269,28 @@ A CDN in front of a direct URL can answer 403 with a block page. Those are
269
269
  retried twice with backoff, while a 403 carrying the application's own JSON is
270
270
  terminal on the first answer; `--retry-403 N` changes the count.
271
271
 
272
+ ## Networking
273
+
274
+ Proxies, timeouts and a private CA are flags, not luck:
275
+
276
+ ```bash
277
+ acadfetch fetch zenodo:123456 --proxy http://proxy.campus:3128 # overrides the environment
278
+ acadfetch fetch zenodo:123456 --no-proxy # ignore HTTP(S)_PROXY
279
+ acadfetch fetch zenodo:123456 --timeout 15 # idle/connect budget, seconds
280
+ acadfetch fetch zenodo:123456 --ca-bundle corp-tls.pem # trust this PEM instead of the store
281
+ ```
282
+
283
+ `--proxy` takes `http://` or `https://` and wins over `--no-proxy`; without
284
+ either, `HTTP_PROXY` / `HTTPS_PROXY` / `ALL_PROXY` are honoured by urllib as
285
+ usual. A timeout is per request, connect to first byte.
286
+
287
+ ## Debugging
288
+
289
+ `--verbose` (or `ACADFETCH_DEBUG=1`) traces each request, each retry and the
290
+ traceback of a failure to stderr; stdout, `--json` documents and exit codes are
291
+ unchanged. Credential header values and presigned query parameters are masked,
292
+ so the trace can be pasted into an issue.
293
+
272
294
  ## Politeness
273
295
 
274
296
  Defaults: 3 parallel transfers, 2 requests/second, 4 retries with jittered
@@ -238,6 +238,28 @@ A CDN in front of a direct URL can answer 403 with a block page. Those are
238
238
  retried twice with backoff, while a 403 carrying the application's own JSON is
239
239
  terminal on the first answer; `--retry-403 N` changes the count.
240
240
 
241
+ ## Networking
242
+
243
+ Proxies, timeouts and a private CA are flags, not luck:
244
+
245
+ ```bash
246
+ acadfetch fetch zenodo:123456 --proxy http://proxy.campus:3128 # overrides the environment
247
+ acadfetch fetch zenodo:123456 --no-proxy # ignore HTTP(S)_PROXY
248
+ acadfetch fetch zenodo:123456 --timeout 15 # idle/connect budget, seconds
249
+ acadfetch fetch zenodo:123456 --ca-bundle corp-tls.pem # trust this PEM instead of the store
250
+ ```
251
+
252
+ `--proxy` takes `http://` or `https://` and wins over `--no-proxy`; without
253
+ either, `HTTP_PROXY` / `HTTPS_PROXY` / `ALL_PROXY` are honoured by urllib as
254
+ usual. A timeout is per request, connect to first byte.
255
+
256
+ ## Debugging
257
+
258
+ `--verbose` (or `ACADFETCH_DEBUG=1`) traces each request, each retry and the
259
+ traceback of a failure to stderr; stdout, `--json` documents and exit codes are
260
+ unchanged. Credential header values and presigned query parameters are masked,
261
+ so the trace can be pasted into an issue.
262
+
241
263
  ## Politeness
242
264
 
243
265
  Defaults: 3 parallel transfers, 2 requests/second, 4 retries with jittered
@@ -17,7 +17,7 @@ from datetime import datetime, timezone
17
17
  from pathlib import Path
18
18
  from urllib.parse import urlsplit
19
19
 
20
- from . import auth, consts, manifest, naming, urls, verify
20
+ from . import auth, consts, debug, manifest, naming, urls, verify
21
21
  from . import remote as remote_check
22
22
  from .client import Client
23
23
  from .downloader import Options, check_disk, check_total, fetch_record, human
@@ -100,6 +100,32 @@ def _count(text: str) -> int:
100
100
  return value
101
101
 
102
102
 
103
+ def _timeout(text: str) -> float:
104
+ try:
105
+ value = float(text)
106
+ except ValueError:
107
+ raise argparse.ArgumentTypeError(f"not a number of seconds: {text!r}") from None
108
+ if value <= 0:
109
+ raise argparse.ArgumentTypeError("must be > 0 seconds")
110
+ return value
111
+
112
+
113
+ def _proxy(text: str) -> str:
114
+ candidate = text if "://" in text else f"http://{text}"
115
+ parts = urlsplit(candidate)
116
+ if parts.scheme not in ("http", "https") or not parts.hostname:
117
+ raise argparse.ArgumentTypeError(
118
+ f"expected http://host:port or https://host:port, got {text!r}"
119
+ )
120
+ return candidate
121
+
122
+
123
+ def _ca_bundle(text: str) -> str:
124
+ if not Path(text).is_file():
125
+ raise argparse.ArgumentTypeError(f"no such file: {text!r}")
126
+ return text
127
+
128
+
103
129
  def _checksum(text: str) -> str:
104
130
  """Validate ``--checksum`` before any request is made."""
105
131
  parsed = verify.parse(text)
@@ -126,6 +152,10 @@ def _client(args: argparse.Namespace) -> Client:
126
152
  retries=args.retries,
127
153
  block_retries=args.retry_403,
128
154
  allow_http=args.allow_http,
155
+ timeout=args.timeout,
156
+ proxy={"http": args.proxy, "https": args.proxy} if args.proxy else None,
157
+ no_proxy=args.no_proxy,
158
+ ca_bundle=args.ca_bundle,
129
159
  headers=_headers(args.header),
130
160
  auth={source: token.headers for source, token in tokens.items()},
131
161
  )
@@ -602,6 +632,36 @@ def build_parser() -> argparse.ArgumentParser:
602
632
  metavar="NAME:VALUE",
603
633
  help="extra request header, repeatable (e.g. 'Authorization: Bearer <token>')",
604
634
  )
635
+ p.add_argument(
636
+ "--verbose",
637
+ action="store_true",
638
+ help="trace requests and retries to stderr, with tracebacks (also ACADFETCH_DEBUG=1)",
639
+ )
640
+ p.add_argument(
641
+ "--timeout",
642
+ type=_timeout,
643
+ default=consts.DEFAULT_TIMEOUT,
644
+ metavar="SECONDS",
645
+ help="per-request timeout, connect and idle (default: %(default)s)",
646
+ )
647
+ p.add_argument(
648
+ "--proxy",
649
+ type=_proxy,
650
+ metavar="URL",
651
+ help="http:// or https:// proxy for every request; overrides the "
652
+ "environment and wins over --no-proxy",
653
+ )
654
+ p.add_argument(
655
+ "--no-proxy",
656
+ action="store_true",
657
+ help="ignore HTTP_PROXY / HTTPS_PROXY / ALL_PROXY",
658
+ )
659
+ p.add_argument(
660
+ "--ca-bundle",
661
+ type=_ca_bundle,
662
+ metavar="PATH",
663
+ help="PEM file to trust instead of the system store (corporate TLS inspection)",
664
+ )
605
665
  p.add_argument(
606
666
  "--allow-http",
607
667
  action="store_true",
@@ -733,6 +793,7 @@ def main(argv: list[str] | None = None) -> int:
733
793
  if hasattr(stream, "reconfigure"):
734
794
  stream.reconfigure(encoding="utf-8", errors="replace")
735
795
  args = build_parser().parse_args(argv)
796
+ debug.configure(getattr(args, "verbose", False))
736
797
  _recover_target(args)
737
798
  if getattr(args, "command", "") == "fetch" and getattr(args, "target", None) is None:
738
799
  message = (
@@ -747,12 +808,14 @@ def main(argv: list[str] | None = None) -> int:
747
808
  try:
748
809
  return args.func(args)
749
810
  except AcFError as exc: # TargetError is one
811
+ debug.log_exception(exc)
750
812
  # An error sentence carries the URL, and that URL may carry a signature.
751
813
  message = _hide_text(str(exc), args)
752
814
  print(f"acadfetch: {message}", file=sys.stderr)
753
815
  _emit_error(args, message)
754
816
  return EXIT_FAIL
755
817
  except OSError as exc: # a filesystem surprise, not a bug report
818
+ debug.log_exception(exc)
756
819
  message = f"{exc.strerror or exc}" if exc.filename else str(exc)
757
820
  print(f"acadfetch: {message}", file=sys.stderr)
758
821
  _emit_error(args, message)
@@ -13,6 +13,7 @@ import email.utils
13
13
  import hashlib
14
14
  import json
15
15
  import random
16
+ import ssl
16
17
  import sys
17
18
  import time
18
19
  import urllib.error
@@ -22,7 +23,7 @@ from collections.abc import Mapping
22
23
  from datetime import datetime, timezone
23
24
  from http.client import HTTPException
24
25
 
25
- from . import consts, verify
26
+ from . import consts, debug, urls, verify
26
27
  from .errors import HttpError
27
28
  from .ratelimit import TokenBucket
28
29
 
@@ -143,6 +144,9 @@ class Client:
143
144
  auth: Mapping[str, Mapping[str, str]] | None = None,
144
145
  block_retries: int = consts.WAF_403_RETRIES,
145
146
  allow_http: bool = False,
147
+ proxy: Mapping[str, str] | None = None,
148
+ no_proxy: bool = False,
149
+ ca_bundle: str | None = None,
146
150
  ) -> None:
147
151
  self.bucket = TokenBucket(rate)
148
152
  self.retries = max(0, retries)
@@ -157,17 +161,26 @@ class Client:
157
161
  self.auth = {k: dict(v) for k, v in (auth or {}).items()}
158
162
  self.source: str | None = None
159
163
  self.redirects = RedirectPolicy(allow_http=allow_http)
160
- self.opener = urllib.request.build_opener(self.redirects)
164
+ handlers: list = [self.redirects]
165
+ # An explicit ProxyHandler replaces urllib's environment handling; the
166
+ # empty dict is how --no-proxy ignores HTTP(S)_PROXY.
167
+ if proxy is not None or no_proxy:
168
+ handlers.append(urllib.request.ProxyHandler(dict(proxy or {})))
169
+ if ca_bundle:
170
+ context = ssl.create_default_context(cafile=ca_bundle)
171
+ handlers.append(urllib.request.HTTPSHandler(context=context))
172
+ self.opener = urllib.request.build_opener(*handlers)
161
173
 
162
174
  # -- internals ---------------------------------------------------------
163
175
  def _sleep(self, attempt: int, retry_after: float | None) -> None:
164
176
  if retry_after is not None:
165
177
  # A literal `Retry-After: 0` would mean no gap at all and a burst;
166
178
  # keep a small jittered floor under it.
167
- time.sleep(max(retry_after, 0.05 + random.random() * 0.15))
168
- return
169
- backoff = min(0.8 * (2**attempt), 30.0)
170
- time.sleep(backoff * (0.7 + random.random() * 0.6))
179
+ delay = max(retry_after, 0.05 + random.random() * 0.15)
180
+ else:
181
+ delay = min(0.8 * (2**attempt), 30.0) * (0.7 + random.random() * 0.6)
182
+ debug.log(f"retry in {delay:.2f}s")
183
+ time.sleep(delay)
171
184
 
172
185
  def _request(
173
186
  self, url: str, headers: Mapping[str, str] | None = None
@@ -192,6 +205,7 @@ class Client:
192
205
  headers=None,
193
206
  block_retries: int | None = None,
194
207
  attempts: int | None = None,
208
+ method: str = "GET",
195
209
  ): # noqa: ANN001, ANN202
196
210
  if block_retries is None:
197
211
  block_retries = self.block_retries
@@ -203,11 +217,16 @@ class Client:
203
217
  for attempt in range(budget):
204
218
  self.bucket.acquire()
205
219
  retry_after = None
220
+ req = self._request(url, headers)
221
+ debug.log(f"{method} {urls.redact(url)} headers[{debug.headers_for_log(req.headers)}]")
206
222
  try:
207
- return fn(self._request(url, headers))
223
+ resp = fn(req)
224
+ debug.log(f"-> {getattr(resp, 'status', 'ok')}")
225
+ return resp
208
226
  except urllib.error.HTTPError as exc:
209
227
  last = exc
210
228
  last_detail = detail = _snippet(exc)
229
+ debug.log(f"-> {exc.code} {detail[:120]}")
211
230
  content_type = exc.headers.get("Content-Type") if exc.headers else None
212
231
  last_blocked = exc.code == 403 and is_block_page(detail, content_type)
213
232
  retry_after = parse_retry_after(
@@ -227,6 +246,7 @@ class Client:
227
246
  ) from exc
228
247
  else:
229
248
  waf += 1
249
+ debug.log(f"-> block page, retry {waf}/{block_retries}")
230
250
  except HttpError as exc:
231
251
  # Raised by a caller-supplied fn, e.g. a body that stopped
232
252
  # short. Only the transient ones are worth another request.
@@ -236,6 +256,7 @@ class Client:
236
256
  last_detail = str(exc)
237
257
  last_blocked = False
238
258
  except (urllib.error.URLError, TimeoutError, OSError) as exc:
259
+ debug.log(f"-> {type(exc).__name__}: {exc}")
239
260
  last = exc
240
261
  last_detail = str(exc)
241
262
  last_blocked = False
@@ -286,7 +307,7 @@ class Client:
286
307
 
287
308
  try:
288
309
  # HEAD only feeds size/filename; one shot, never worth a retry.
289
- with self._with_retries(url, _head, block_retries=0, attempts=1) as resp:
310
+ with self._with_retries(url, _head, block_retries=0, attempts=1, method="HEAD") as resp:
290
311
  return dict(resp.headers)
291
312
  except HttpError as exc:
292
313
  if exc.status in (403, 405, 501):
@@ -1,6 +1,6 @@
1
1
  """Package-wide constants and version."""
2
2
 
3
- __version__ = "0.3.1"
3
+ __version__ = "0.5.0"
4
4
 
5
5
  USER_AGENT = f"acadfetch/{__version__} (+https://github.com/young-ski/acadfetch)"
6
6
 
@@ -0,0 +1,59 @@
1
+ """Opt-in tracing, so a bug report does not have to be a guess.
2
+
3
+ Everything here writes to stderr and swallows its own errors: like progress, a
4
+ log line must never be able to fail a download. Header values that are
5
+ credentials are replaced with an ellipsis, and URLs arrive already redacted by
6
+ the caller.
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ import contextlib
12
+ import os
13
+ import sys
14
+ import traceback
15
+ from collections.abc import Mapping
16
+
17
+ from . import urls
18
+
19
+ _ON = False
20
+ _CREDENTIAL_HEADERS = frozenset(
21
+ {"authorization", "cookie", "proxy-authorization", "x-api-key", "x-auth-token"}
22
+ )
23
+ _FALSE = frozenset({"", "0", "false", "no", "off"})
24
+
25
+
26
+ def configure(flag: bool = False, environ: Mapping[str, str] | None = None) -> None:
27
+ """Turn tracing on from the flag or ``ACADFETCH_DEBUG``; the flag wins."""
28
+ global _ON
29
+ env = os.environ if environ is None else environ
30
+ value = (env.get("ACADFETCH_DEBUG") or "").strip().lower()
31
+ _ON = bool(flag) or value not in _FALSE
32
+
33
+
34
+ def enabled() -> bool:
35
+ return _ON
36
+
37
+
38
+ def log(message: str) -> None:
39
+ if not _ON:
40
+ return
41
+ with contextlib.suppress(OSError, ValueError):
42
+ print(f"debug: {message}", file=sys.stderr, flush=True)
43
+
44
+
45
+ def log_exception(exc: BaseException) -> None:
46
+ if not _ON:
47
+ return
48
+ with contextlib.suppress(OSError, ValueError):
49
+ text = "".join(traceback.format_exception(type(exc), exc, exc.__traceback__))
50
+ sys.stderr.write(urls.redact_text(text))
51
+ sys.stderr.flush()
52
+
53
+
54
+ def headers_for_log(headers: Mapping[str, str]) -> str:
55
+ """Header names with safe values; credentials become ``…``."""
56
+ return " ".join(
57
+ f"{name}: {'…' if name.lower() in _CREDENTIAL_HEADERS else value}"
58
+ for name, value in headers.items()
59
+ )
@@ -25,7 +25,7 @@ from datetime import datetime, timezone
25
25
  from http.client import HTTPException
26
26
  from pathlib import Path
27
27
 
28
- from . import atomic, naming, verify
28
+ from . import atomic, debug, naming, verify
29
29
  from .client import Client
30
30
  from .consts import CHUNK
31
31
  from .errors import HttpError, LocalError, PartialError, TargetError
@@ -700,6 +700,7 @@ def _fetch_record(
700
700
 
701
701
  def _crashed(file: File, exc: Exception) -> Result:
702
702
  """An unexpected worker exception, as one failed row instead of a traceback."""
703
+ debug.log_exception(exc)
703
704
  return Result(
704
705
  file=file,
705
706
  state="failed",
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: acadfetch
3
- Version: 0.3.1
3
+ Version: 0.5.0
4
4
  Summary: Resumable, rate-limited, checksum-verified bulk downloads from academic data repositories
5
5
  Author: young-ski
6
6
  License-Expression: MIT
@@ -269,6 +269,28 @@ A CDN in front of a direct URL can answer 403 with a block page. Those are
269
269
  retried twice with backoff, while a 403 carrying the application's own JSON is
270
270
  terminal on the first answer; `--retry-403 N` changes the count.
271
271
 
272
+ ## Networking
273
+
274
+ Proxies, timeouts and a private CA are flags, not luck:
275
+
276
+ ```bash
277
+ acadfetch fetch zenodo:123456 --proxy http://proxy.campus:3128 # overrides the environment
278
+ acadfetch fetch zenodo:123456 --no-proxy # ignore HTTP(S)_PROXY
279
+ acadfetch fetch zenodo:123456 --timeout 15 # idle/connect budget, seconds
280
+ acadfetch fetch zenodo:123456 --ca-bundle corp-tls.pem # trust this PEM instead of the store
281
+ ```
282
+
283
+ `--proxy` takes `http://` or `https://` and wins over `--no-proxy`; without
284
+ either, `HTTP_PROXY` / `HTTPS_PROXY` / `ALL_PROXY` are honoured by urllib as
285
+ usual. A timeout is per request, connect to first byte.
286
+
287
+ ## Debugging
288
+
289
+ `--verbose` (or `ACADFETCH_DEBUG=1`) traces each request, each retry and the
290
+ traceback of a failure to stderr; stdout, `--json` documents and exit codes are
291
+ unchanged. Credential header values and presigned query parameters are masked,
292
+ so the trace can be pasted into an issue.
293
+
272
294
  ## Politeness
273
295
 
274
296
  Defaults: 3 parallel transfers, 2 requests/second, 4 retries with jittered
@@ -9,6 +9,7 @@ acadfetch/auth.py
9
9
  acadfetch/cli.py
10
10
  acadfetch/client.py
11
11
  acadfetch/consts.py
12
+ acadfetch/debug.py
12
13
  acadfetch/downloader.py
13
14
  acadfetch/errors.py
14
15
  acadfetch/manifest.py
@@ -40,6 +41,7 @@ tests/test_checksum.py
40
41
  tests/test_cli.py
41
42
  tests/test_client.py
42
43
  tests/test_control_names.py
44
+ tests/test_debug.py
43
45
  tests/test_disk.py
44
46
  tests/test_downloader.py
45
47
  tests/test_identity.py
@@ -48,6 +50,7 @@ tests/test_json.py
48
50
  tests/test_manifest.py
49
51
  tests/test_naming.py
50
52
  tests/test_naming_race.py
53
+ tests/test_network.py
51
54
  tests/test_packaging.py
52
55
  tests/test_progress.py
53
56
  tests/test_ratelimit.py
@@ -58,6 +61,7 @@ tests/test_sources.py
58
61
  tests/test_urls.py
59
62
  tests/test_verify.py
60
63
  tests/fixtures/README.md
64
+ tests/fixtures/ca.pem
61
65
  tests/fixtures/figshare_403_block.html
62
66
  tests/fixtures/figshare_403_forbidden.json
63
67
  tests/fixtures/figshare_article.json
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "acadfetch"
7
- version = "0.3.1"
7
+ version = "0.5.0"
8
8
  description = "Resumable, rate-limited, checksum-verified bulk downloads from academic data repositories"
9
9
  readme = "README.md"
10
10
  requires-python = ">=3.10"
@@ -18,6 +18,9 @@ restricted file fails fast instead of burning retries.
18
18
  `tests/test_client.py` and `tests/test_downloader.py` through the conftest
19
19
  server; the names keep the provenance.
20
20
 
21
+ `ca.pem` is one public root certificate (taken from certifi) so a test can pass
22
+ `--ca-bundle` a PEM that really loads; it is not expected to verify any host.
23
+
21
24
  `figshare_file.pdf` is a real 193 410-byte PDF whose md5 is
22
25
  `bbde85208529e663141a0e8cb6fa9c4a`. It is the only real-bytes fixture, so the
23
26
  verify path is proven against a published checksum rather than a synthetic
@@ -0,0 +1,17 @@
1
+ -----BEGIN CERTIFICATE-----
2
+ MIICiTCCAg+gAwIBAgIQH0evqmIAcFBUTAGem2OZKjAKBggqhkjOPQQDAzCBhTEL
3
+ MAkGA1UEBhMCR0IxGzAZBgNVBAgTEkdyZWF0ZXIgTWFuY2hlc3RlcjEQMA4GA1UE
4
+ BxMHU2FsZm9yZDEaMBgGA1UEChMRQ09NT0RPIENBIExpbWl0ZWQxKzApBgNVBAMT
5
+ IkNPTU9ETyBFQ0MgQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkwHhcNMDgwMzA2MDAw
6
+ MDAwWhcNMzgwMTE4MjM1OTU5WjCBhTELMAkGA1UEBhMCR0IxGzAZBgNVBAgTEkdy
7
+ ZWF0ZXIgTWFuY2hlc3RlcjEQMA4GA1UEBxMHU2FsZm9yZDEaMBgGA1UEChMRQ09N
8
+ T0RPIENBIExpbWl0ZWQxKzApBgNVBAMTIkNPTU9ETyBFQ0MgQ2VydGlmaWNhdGlv
9
+ biBBdXRob3JpdHkwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAAQDR3svdcmCFYX7deSR
10
+ FtSrYpn1PlILBs5BAH+X4QokPB0BBO490o0JlwzgdeT6+3eKKvUDYEs2ixYjFq0J
11
+ cfRK9ChQtP6IHG4/bC8vCVlbpVsLM5niwz2J+Wos77LTBumjQjBAMB0GA1UdDgQW
12
+ BBR1cacZSBm8nZ3qQUfflMRId5nTeTAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/
13
+ BAUwAwEB/zAKBggqhkjOPQQDAwNoADBlAjEA7wNbeqy3eApyt4jf/7VGFAkK+qDm
14
+ fQjGGoe9GKhzvSbKYAydzpmfz1wPMOG+FDHqAjAU9JM8SaczepBGR7NjfRObTrdv
15
+ GDeAU/7dIOA1mjbRxwG55tzd8/8dLDoWV9mSOdY=
16
+ -----END CERTIFICATE-----
17
+
@@ -0,0 +1,120 @@
1
+ from __future__ import annotations
2
+
3
+ import pytest
4
+
5
+ from acadfetch import cli, debug
6
+ from acadfetch.client import Client
7
+ from acadfetch.errors import HttpError
8
+
9
+
10
+ @pytest.fixture(autouse=True)
11
+ def _reset():
12
+ debug.configure(False)
13
+ yield
14
+ debug.configure(False)
15
+
16
+
17
+ def test_the_environment_can_turn_it_on():
18
+ debug.configure(False, {"ACADFETCH_DEBUG": "1"})
19
+ assert debug.enabled()
20
+
21
+
22
+ @pytest.mark.parametrize("value", ["0", "false", "no", "off", ""])
23
+ def test_the_environment_can_turn_it_off(value):
24
+ debug.configure(False, {"ACADFETCH_DEBUG": value})
25
+ assert not debug.enabled()
26
+
27
+
28
+ def test_the_flag_wins_over_the_environment():
29
+ debug.configure(True, {"ACADFETCH_DEBUG": "0"})
30
+ assert debug.enabled()
31
+
32
+
33
+ def test_it_is_silent_when_off(capsys):
34
+ debug.log("nothing")
35
+ debug.log_exception(RuntimeError("boom"))
36
+ assert capsys.readouterr().err == ""
37
+
38
+
39
+ def _talking_client():
40
+ client = Client(rate=1000, retries=0, auth={"direct": {"Authorization": "Bearer SECRET"}})
41
+ client.source = "direct"
42
+ return client
43
+
44
+
45
+ def test_headers_are_masked(server, capsys):
46
+ server.cfg["files"]["/a.bin"] = b"x"
47
+ debug.configure(True)
48
+
49
+ _talking_client().head(server.base + "/a.bin")
50
+
51
+ err = capsys.readouterr().err
52
+ assert "HEAD" in err
53
+ assert "Authorization: …" in err
54
+ assert "SECRET" not in err
55
+
56
+
57
+ def test_a_credential_in_the_url_is_redacted(server, capsys):
58
+ server.cfg["files"]["/a.bin"] = b"x"
59
+ debug.configure(True)
60
+
61
+ _talking_client().head(server.base + "/a.bin?X-Amz-Signature=SIG")
62
+
63
+ err = capsys.readouterr().err
64
+ assert "X-Amz-Signature" not in err
65
+ assert "SIG" not in err
66
+
67
+
68
+ def test_a_retry_is_traced(server, capsys):
69
+ server.cfg["files"]["/a.bin"] = b"x"
70
+ server.cfg["always_500"].add("/a.bin")
71
+ debug.configure(True)
72
+
73
+ with pytest.raises(HttpError):
74
+ Client(rate=1000, retries=1).open(server.base + "/a.bin")
75
+
76
+ err = capsys.readouterr().err
77
+ assert "GET" in err and "-> 500" in err and "retry in" in err
78
+
79
+
80
+ def test_a_failure_prints_a_traceback_only_with_the_flag(capsys):
81
+ assert cli.main(["fetch", "nonsense://x"]) == 1
82
+ assert "Traceback" not in capsys.readouterr().err
83
+
84
+ assert cli.main(["fetch", "nonsense://x", "--verbose"]) == 1
85
+ assert "Traceback (most recent call last)" in capsys.readouterr().err
86
+
87
+
88
+ def test_debug_does_not_change_the_json_document(server, monkeypatch, capsys, tmp_path):
89
+ import json
90
+
91
+ server.cfg["files"]["/a.bin"] = b"payload"
92
+ monkeypatch.setattr(
93
+ cli.registry,
94
+ "resolve",
95
+ lambda _c, _t: __import__("acadfetch.sources.base", fromlist=["Record"]).Record(
96
+ source="direct",
97
+ target=server.base + "/a.bin",
98
+ files=[
99
+ __import__("acadfetch.sources.base", fromlist=["File"]).File(
100
+ server.base + "/a.bin", "a.bin", 7
101
+ )
102
+ ],
103
+ ),
104
+ )
105
+
106
+ code = cli.main(["fetch", "t", "--json", "--verbose", "-o", str(tmp_path)])
107
+ out, _ = capsys.readouterr()
108
+
109
+ assert code == cli.EXIT_OK
110
+ assert json.loads(out)["summary"]["ok"] is True
111
+
112
+
113
+ def test_a_traceback_does_not_leak_a_credential(capsys):
114
+ assert (
115
+ cli.main(["fetch", "https://example.invalid/x?X-Amz-Signature=SECRET", "--verbose", "-n"])
116
+ == 1
117
+ )
118
+ err = capsys.readouterr().err
119
+ assert "Traceback (most recent call last)" in err
120
+ assert "SECRET" not in err
@@ -0,0 +1,100 @@
1
+ from __future__ import annotations
2
+
3
+ import urllib.request
4
+ from pathlib import Path
5
+
6
+ import pytest
7
+
8
+ from acadfetch import cli
9
+
10
+
11
+ def _args(*extra):
12
+ return cli.build_parser().parse_args(["fetch", "zenodo:1", *extra])
13
+
14
+
15
+ def _proxies(client):
16
+ """The explicit proxy mapping, or {} when urllib would keep no handler.
17
+
18
+ A ProxyHandler with an empty mapping registers no method, so build_opener
19
+ drops it; that is exactly what --no-proxy means.
20
+ """
21
+ found = [h for h in client.opener.handlers if isinstance(h, urllib.request.ProxyHandler)]
22
+ return found[0].proxies if found else {}
23
+
24
+
25
+ @pytest.mark.parametrize("bad", ["0", "-1", "abc"])
26
+ def test_a_bad_timeout_is_a_usage_error(bad):
27
+ with pytest.raises(SystemExit) as caught:
28
+ cli.main(["fetch", "zenodo:1", "--timeout", bad])
29
+ assert caught.value.code == cli.EXIT_USAGE
30
+
31
+
32
+ @pytest.mark.parametrize("bad", ["ftp://x:1", "http://", "://x"])
33
+ def test_a_bad_proxy_is_a_usage_error(bad):
34
+ with pytest.raises(SystemExit) as caught:
35
+ cli.main(["fetch", "zenodo:1", "--proxy", bad])
36
+ assert caught.value.code == cli.EXIT_USAGE
37
+
38
+
39
+ def test_a_missing_ca_bundle_is_a_usage_error(tmp_path):
40
+ with pytest.raises(SystemExit) as caught:
41
+ cli.main(["fetch", "zenodo:1", "--ca-bundle", str(tmp_path / "no.pem")])
42
+ assert caught.value.code == cli.EXIT_USAGE
43
+
44
+
45
+ def test_the_timeout_reaches_the_client():
46
+ assert cli._client(_args("--timeout", "15")).timeout == 15.0
47
+
48
+
49
+ def test_a_bare_host_port_gets_http():
50
+ assert cli._proxy("127.0.0.1:8899") == "http://127.0.0.1:8899"
51
+
52
+
53
+ def test_the_proxy_reaches_both_schemes():
54
+ assert _proxies(cli._client(_args("--proxy", "http://127.0.0.1:8899"))) == {
55
+ "http": "http://127.0.0.1:8899",
56
+ "https": "http://127.0.0.1:8899",
57
+ }
58
+
59
+
60
+ def test_no_proxy_leaves_no_proxy_handler(monkeypatch):
61
+ monkeypatch.setenv("HTTP_PROXY", "http://env.example:1")
62
+ monkeypatch.setenv("HTTPS_PROXY", "http://env.example:1")
63
+
64
+ client = cli._client(_args("--no-proxy"))
65
+
66
+ assert not [h for h in client.opener.handlers if isinstance(h, urllib.request.ProxyHandler)]
67
+ assert _proxies(client) == {}
68
+
69
+
70
+ def test_the_environment_is_used_by_default(monkeypatch):
71
+ monkeypatch.setenv("HTTPS_PROXY", "http://env.example:1")
72
+ monkeypatch.setenv("HTTP_PROXY", "http://env.example:1")
73
+
74
+ proxies = _proxies(cli._client(_args()))
75
+ assert proxies.get("https") == "http://env.example:1"
76
+
77
+
78
+ def test_an_explicit_proxy_beats_no_proxy(monkeypatch):
79
+ monkeypatch.setenv("HTTPS_PROXY", "http://env.example:1")
80
+
81
+ proxies = _proxies(cli._client(_args("--proxy", "http://own:1", "--no-proxy")))
82
+ assert proxies == {"http": "http://own:1", "https": "http://own:1"}
83
+
84
+
85
+ def test_a_ca_bundle_adds_an_https_handler():
86
+ bundle = Path(__file__).parent / "fixtures" / "ca.pem"
87
+
88
+ client = cli._client(_args("--ca-bundle", str(bundle)))
89
+
90
+ assert any(isinstance(h, urllib.request.HTTPSHandler) for h in client.opener.handlers)
91
+
92
+
93
+ def test_every_network_command_takes_the_flags(capsys):
94
+ for argv in (
95
+ ["fetch", "zenodo:1", "--timeout", "5"],
96
+ ["manifest", "zenodo:1", "--timeout", "5"],
97
+ ["verify", "m.json", "--timeout", "5"],
98
+ ):
99
+ args = cli.build_parser().parse_args(argv)
100
+ assert args.timeout == 5.0
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes