acadfetch 0.4.0__tar.gz → 0.6.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (72) hide show
  1. {acadfetch-0.4.0/acadfetch.egg-info → acadfetch-0.6.0}/PKG-INFO +28 -2
  2. {acadfetch-0.4.0 → acadfetch-0.6.0}/README.md +27 -1
  3. {acadfetch-0.4.0 → acadfetch-0.6.0}/acadfetch/cli.py +104 -26
  4. {acadfetch-0.4.0 → acadfetch-0.6.0}/acadfetch/client.py +16 -7
  5. {acadfetch-0.4.0 → acadfetch-0.6.0}/acadfetch/consts.py +1 -1
  6. acadfetch-0.6.0/acadfetch/debug.py +59 -0
  7. {acadfetch-0.4.0 → acadfetch-0.6.0}/acadfetch/downloader.py +35 -15
  8. {acadfetch-0.4.0 → acadfetch-0.6.0}/acadfetch/sources/doi.py +2 -1
  9. {acadfetch-0.4.0 → acadfetch-0.6.0/acadfetch.egg-info}/PKG-INFO +28 -2
  10. {acadfetch-0.4.0 → acadfetch-0.6.0}/acadfetch.egg-info/SOURCES.txt +4 -0
  11. {acadfetch-0.4.0 → acadfetch-0.6.0}/pyproject.toml +7 -1
  12. acadfetch-0.6.0/tests/test_citation.py +26 -0
  13. {acadfetch-0.4.0 → acadfetch-0.6.0}/tests/test_cli.py +2 -2
  14. acadfetch-0.6.0/tests/test_debug.py +120 -0
  15. {acadfetch-0.4.0 → acadfetch-0.6.0}/tests/test_sources.py +5 -0
  16. acadfetch-0.6.0/tests/test_urls_glob.py +99 -0
  17. {acadfetch-0.4.0 → acadfetch-0.6.0}/LICENSE +0 -0
  18. {acadfetch-0.4.0 → acadfetch-0.6.0}/MANIFEST.in +0 -0
  19. {acadfetch-0.4.0 → acadfetch-0.6.0}/acadfetch/__init__.py +0 -0
  20. {acadfetch-0.4.0 → acadfetch-0.6.0}/acadfetch/__main__.py +0 -0
  21. {acadfetch-0.4.0 → acadfetch-0.6.0}/acadfetch/atomic.py +0 -0
  22. {acadfetch-0.4.0 → acadfetch-0.6.0}/acadfetch/auth.py +0 -0
  23. {acadfetch-0.4.0 → acadfetch-0.6.0}/acadfetch/errors.py +0 -0
  24. {acadfetch-0.4.0 → acadfetch-0.6.0}/acadfetch/manifest.py +0 -0
  25. {acadfetch-0.4.0 → acadfetch-0.6.0}/acadfetch/naming.py +0 -0
  26. {acadfetch-0.4.0 → acadfetch-0.6.0}/acadfetch/progress.py +0 -0
  27. {acadfetch-0.4.0 → acadfetch-0.6.0}/acadfetch/py.typed +0 -0
  28. {acadfetch-0.4.0 → acadfetch-0.6.0}/acadfetch/ratelimit.py +0 -0
  29. {acadfetch-0.4.0 → acadfetch-0.6.0}/acadfetch/remote.py +0 -0
  30. {acadfetch-0.4.0 → acadfetch-0.6.0}/acadfetch/sources/__init__.py +0 -0
  31. {acadfetch-0.4.0 → acadfetch-0.6.0}/acadfetch/sources/arxiv.py +0 -0
  32. {acadfetch-0.4.0 → acadfetch-0.6.0}/acadfetch/sources/base.py +0 -0
  33. {acadfetch-0.4.0 → acadfetch-0.6.0}/acadfetch/sources/direct.py +0 -0
  34. {acadfetch-0.4.0 → acadfetch-0.6.0}/acadfetch/sources/registry.py +0 -0
  35. {acadfetch-0.4.0 → acadfetch-0.6.0}/acadfetch/sources/removed.py +0 -0
  36. {acadfetch-0.4.0 → acadfetch-0.6.0}/acadfetch/sources/zenodo.py +0 -0
  37. {acadfetch-0.4.0 → acadfetch-0.6.0}/acadfetch/units.py +0 -0
  38. {acadfetch-0.4.0 → acadfetch-0.6.0}/acadfetch/urls.py +0 -0
  39. {acadfetch-0.4.0 → acadfetch-0.6.0}/acadfetch/verify.py +0 -0
  40. {acadfetch-0.4.0 → acadfetch-0.6.0}/acadfetch.egg-info/dependency_links.txt +0 -0
  41. {acadfetch-0.4.0 → acadfetch-0.6.0}/acadfetch.egg-info/entry_points.txt +0 -0
  42. {acadfetch-0.4.0 → acadfetch-0.6.0}/acadfetch.egg-info/top_level.txt +0 -0
  43. {acadfetch-0.4.0 → acadfetch-0.6.0}/setup.cfg +0 -0
  44. {acadfetch-0.4.0 → acadfetch-0.6.0}/tests/conftest.py +0 -0
  45. {acadfetch-0.4.0 → acadfetch-0.6.0}/tests/fixtures/README.md +0 -0
  46. {acadfetch-0.4.0 → acadfetch-0.6.0}/tests/fixtures/ca.pem +0 -0
  47. {acadfetch-0.4.0 → acadfetch-0.6.0}/tests/fixtures/figshare_403_block.html +0 -0
  48. {acadfetch-0.4.0 → acadfetch-0.6.0}/tests/fixtures/figshare_403_forbidden.json +0 -0
  49. {acadfetch-0.4.0 → acadfetch-0.6.0}/tests/fixtures/figshare_article.json +0 -0
  50. {acadfetch-0.4.0 → acadfetch-0.6.0}/tests/fixtures/figshare_file.pdf +0 -0
  51. {acadfetch-0.4.0 → acadfetch-0.6.0}/tests/test_atomic.py +0 -0
  52. {acadfetch-0.4.0 → acadfetch-0.6.0}/tests/test_auth.py +0 -0
  53. {acadfetch-0.4.0 → acadfetch-0.6.0}/tests/test_checksum.py +0 -0
  54. {acadfetch-0.4.0 → acadfetch-0.6.0}/tests/test_client.py +0 -0
  55. {acadfetch-0.4.0 → acadfetch-0.6.0}/tests/test_control_names.py +0 -0
  56. {acadfetch-0.4.0 → acadfetch-0.6.0}/tests/test_disk.py +0 -0
  57. {acadfetch-0.4.0 → acadfetch-0.6.0}/tests/test_downloader.py +0 -0
  58. {acadfetch-0.4.0 → acadfetch-0.6.0}/tests/test_identity.py +0 -0
  59. {acadfetch-0.4.0 → acadfetch-0.6.0}/tests/test_isolation.py +0 -0
  60. {acadfetch-0.4.0 → acadfetch-0.6.0}/tests/test_json.py +0 -0
  61. {acadfetch-0.4.0 → acadfetch-0.6.0}/tests/test_manifest.py +0 -0
  62. {acadfetch-0.4.0 → acadfetch-0.6.0}/tests/test_naming.py +0 -0
  63. {acadfetch-0.4.0 → acadfetch-0.6.0}/tests/test_naming_race.py +0 -0
  64. {acadfetch-0.4.0 → acadfetch-0.6.0}/tests/test_network.py +0 -0
  65. {acadfetch-0.4.0 → acadfetch-0.6.0}/tests/test_packaging.py +0 -0
  66. {acadfetch-0.4.0 → acadfetch-0.6.0}/tests/test_progress.py +0 -0
  67. {acadfetch-0.4.0 → acadfetch-0.6.0}/tests/test_ratelimit.py +0 -0
  68. {acadfetch-0.4.0 → acadfetch-0.6.0}/tests/test_redirect.py +0 -0
  69. {acadfetch-0.4.0 → acadfetch-0.6.0}/tests/test_remote.py +0 -0
  70. {acadfetch-0.4.0 → acadfetch-0.6.0}/tests/test_removed.py +0 -0
  71. {acadfetch-0.4.0 → acadfetch-0.6.0}/tests/test_urls.py +0 -0
  72. {acadfetch-0.4.0 → acadfetch-0.6.0}/tests/test_verify.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: acadfetch
3
- Version: 0.4.0
3
+ Version: 0.6.0
4
4
  Summary: Resumable, rate-limited, checksum-verified bulk downloads from academic data repositories
5
5
  Author: young-ski
6
6
  License-Expression: MIT
@@ -89,7 +89,7 @@ From a checkout, for development:
89
89
 
90
90
  ```bash
91
91
  git clone https://github.com/young-ski/acadfetch && cd acadfetch
92
- python -m pip install -e . pytest && python -m pytest -q
92
+ python -m pip install -e . pytest mypy && python -m pytest -q
93
93
  ```
94
94
 
95
95
  ## Supported platforms
@@ -142,6 +142,15 @@ two never interleave:
142
142
  acadfetch fetch zenodo:123456 --json | jq -r '.files[] | select(.state=="failed") | .name'
143
143
  ```
144
144
 
145
+ To hand the bytes to another downloader, `-w/--urls` writes an aria2-compatible
146
+ list (`-` is stdout) and transfers nothing:
147
+
148
+ ```bash
149
+ acadfetch fetch zenodo:123456 -w - --glob '*.csv' | aria2c -i -
150
+ ```
151
+
152
+ `--glob` takes shell patterns and combines with `--only` (both must pass).
153
+
145
154
  Exit codes are unchanged, and a failure before any transfer still produces a
146
155
  document with `error` set. `report` is the document's schema version (now `2`, field reference in
147
156
  [docs/json-report.md](docs/json-report.md)):
@@ -284,6 +293,13 @@ acadfetch fetch zenodo:123456 --ca-bundle corp-tls.pem # trust this PEM
284
293
  either, `HTTP_PROXY` / `HTTPS_PROXY` / `ALL_PROXY` are honoured by urllib as
285
294
  usual. A timeout is per request, connect to first byte.
286
295
 
296
+ ## Debugging
297
+
298
+ `--verbose` (or `ACADFETCH_DEBUG=1`) traces each request, each retry and the
299
+ traceback of a failure to stderr; stdout, `--json` documents and exit codes are
300
+ unchanged. Credential header values and presigned query parameters are masked,
301
+ so the trace can be pasted into an issue.
302
+
287
303
  ## Politeness
288
304
 
289
305
  Defaults: 3 parallel transfers, 2 requests/second, 4 retries with jittered
@@ -307,6 +323,10 @@ honest way to test this code path.
307
323
 
308
324
  ## Getting involved
309
325
 
326
+ A [30-second tour](docs/demo.md) shows the whole loop, and
327
+ [docs/integrations.md](docs/integrations.md) has Make, CI, aria2 and Snakemake
328
+ recipes.
329
+
310
330
  A nightly job downloads three fixed public records and verifies them, so a
311
331
  rotted adapter opens an issue before a user reports one. It never blocks a
312
332
  merge; the next good night closes the issue.
@@ -346,6 +366,12 @@ acadfetch verify ./data/manifest.json
346
366
 
347
367
  MIT。
348
368
 
369
+ ## Citing
370
+
371
+ `acadfetch --cite` prints how to cite the tool; the machine-readable form is
372
+ [`CITATION.cff`](CITATION.cff). For the data, cite the repository record — the
373
+ manifest is the part that lets a reader re-check it.
374
+
349
375
  ## Audit trail
350
376
 
351
377
  How the 3.15 support claim was reached, including what it did **not** prove —
@@ -58,7 +58,7 @@ From a checkout, for development:
58
58
 
59
59
  ```bash
60
60
  git clone https://github.com/young-ski/acadfetch && cd acadfetch
61
- python -m pip install -e . pytest && python -m pytest -q
61
+ python -m pip install -e . pytest mypy && python -m pytest -q
62
62
  ```
63
63
 
64
64
  ## Supported platforms
@@ -111,6 +111,15 @@ two never interleave:
111
111
  acadfetch fetch zenodo:123456 --json | jq -r '.files[] | select(.state=="failed") | .name'
112
112
  ```
113
113
 
114
+ To hand the bytes to another downloader, `-w/--urls` writes an aria2-compatible
115
+ list (`-` is stdout) and transfers nothing:
116
+
117
+ ```bash
118
+ acadfetch fetch zenodo:123456 -w - --glob '*.csv' | aria2c -i -
119
+ ```
120
+
121
+ `--glob` takes shell patterns and combines with `--only` (both must pass).
122
+
114
123
  Exit codes are unchanged, and a failure before any transfer still produces a
115
124
  document with `error` set. `report` is the document's schema version (now `2`, field reference in
116
125
  [docs/json-report.md](docs/json-report.md)):
@@ -253,6 +262,13 @@ acadfetch fetch zenodo:123456 --ca-bundle corp-tls.pem # trust this PEM
253
262
  either, `HTTP_PROXY` / `HTTPS_PROXY` / `ALL_PROXY` are honoured by urllib as
254
263
  usual. A timeout is per request, connect to first byte.
255
264
 
265
+ ## Debugging
266
+
267
+ `--verbose` (or `ACADFETCH_DEBUG=1`) traces each request, each retry and the
268
+ traceback of a failure to stderr; stdout, `--json` documents and exit codes are
269
+ unchanged. Credential header values and presigned query parameters are masked,
270
+ so the trace can be pasted into an issue.
271
+
256
272
  ## Politeness
257
273
 
258
274
  Defaults: 3 parallel transfers, 2 requests/second, 4 retries with jittered
@@ -276,6 +292,10 @@ honest way to test this code path.
276
292
 
277
293
  ## Getting involved
278
294
 
295
+ A [30-second tour](docs/demo.md) shows the whole loop, and
296
+ [docs/integrations.md](docs/integrations.md) has Make, CI, aria2 and Snakemake
297
+ recipes.
298
+
279
299
  A nightly job downloads three fixed public records and verifies them, so a
280
300
  rotted adapter opens an issue before a user reports one. It never blocks a
281
301
  merge; the next good night closes the issue.
@@ -315,6 +335,12 @@ acadfetch verify ./data/manifest.json
315
335
 
316
336
  MIT。
317
337
 
338
+ ## Citing
339
+
340
+ `acadfetch --cite` prints how to cite the tool; the machine-readable form is
341
+ [`CITATION.cff`](CITATION.cff). For the data, cite the repository record — the
342
+ manifest is the part that lets a reader re-check it.
343
+
318
344
  ## Audit trail
319
345
 
320
346
  How the 3.15 support claim was reached, including what it did **not** prove —
@@ -15,12 +15,14 @@ import time
15
15
  from dataclasses import replace
16
16
  from datetime import datetime, timezone
17
17
  from pathlib import Path
18
+ from typing import NoReturn
18
19
  from urllib.parse import urlsplit
19
20
 
20
- from . import auth, consts, manifest, naming, urls, verify
21
+ from . import auth, consts, debug, manifest, naming, urls, verify
21
22
  from . import remote as remote_check
22
23
  from .client import Client
23
24
  from .downloader import Options, check_disk, check_total, fetch_record, human
25
+ from .downloader import matches as downloader_matches
24
26
  from .errors import AcFError, PartialError, TargetError
25
27
  from .progress import Progress
26
28
  from .sources import doi, registry, removed
@@ -224,6 +226,15 @@ def _report(args: argparse.Namespace) -> dict:
224
226
  }
225
227
 
226
228
 
229
+ CITATION = """To cite acadfetch itself:
230
+
231
+ young-ski. acadfetch (version {version}). MIT. {url}
232
+
233
+ To cite the data, cite the repository record and keep its manifest.json beside
234
+ the analysis: the manifest is what lets anyone re-check the bytes later.
235
+ `docs/manifest.md` explains what it does and does not prove."""
236
+
237
+
227
238
  def _hide(text: str, args: argparse.Namespace) -> str:
228
239
  """Redact credential query parameters unless --record-full-url asked for
229
240
  the original string."""
@@ -266,35 +277,56 @@ def _emit(doc: dict) -> None:
266
277
  print(json.dumps(doc, ensure_ascii=False))
267
278
 
268
279
 
269
- def _print_header(record, hide=urls.redact) -> None: # noqa: ANN001
270
- print(f"title : {record.title or '(untitled)'}")
280
+ def _print_header(record, hide=urls.redact, stream=None) -> None: # noqa: ANN001
281
+ out = sys.stdout if stream is None else stream
282
+ print(f"title : {record.title or '(untitled)'}", file=out)
271
283
  if record.doi:
272
- print(f"doi : {record.doi}")
273
- print(f"license : {record.license or 'UNKNOWN - check before redistributing'}")
284
+ print(f"doi : {record.doi}", file=out)
285
+ print(f"license : {record.license or 'UNKNOWN - check before redistributing'}", file=out)
274
286
  total = record.total_bytes
275
- print(f"files : {len(record.files)}" + (f", {human(total)}" if total is not None else ""))
287
+ print(
288
+ f"files : {len(record.files)}" + (f", {human(total)}" if total is not None else ""),
289
+ file=out,
290
+ )
276
291
  if record.landing:
277
- print(f"landing : {hide(record.landing)}")
292
+ print(f"landing : {hide(record.landing)}", file=out)
293
+
294
+
295
+ def _write_urls(files, out_dir, target) -> None: # noqa: ANN001
296
+ lines = []
297
+ for file in files:
298
+ lines.append(file.url)
299
+ lines.append(f" out={naming.sanitize(file.name)}")
300
+ lines.append(f" dir={out_dir}")
301
+ text = "\n".join(lines) + "\n"
302
+ if target == "-":
303
+ sys.stdout.write(text)
304
+ return
305
+ Path(target).write_text(text, "utf-8")
306
+ print(f"{len(files)} url(s) -> {target}", file=sys.stderr)
278
307
 
279
308
 
280
- def _no_match(record, only, args) -> int: # noqa: ANN001
281
- """An unknown --only name is a failure that can name the candidates."""
309
+ def _no_match(record, args) -> int: # noqa: ANN001
310
+ """A filter that matched nothing is a failure that can name the candidates."""
282
311
  available = ", ".join(naming.sanitize(f.name) for f in record.files[:8]) or "(none)"
283
- message = f"no file matched --only {', '.join(only)}; available: {available}"
312
+ wanted = " ".join(
313
+ part
314
+ for part in (
315
+ f"--only {', '.join(args.only)}" if args.only else "",
316
+ f"--glob {', '.join(args.globs)}" if args.globs else "",
317
+ )
318
+ if part
319
+ )
320
+ message = f"no file matched {wanted}; available: {available}"
284
321
  print(f"acadfetch: {message}", file=sys.stderr)
285
322
  _emit_error(args, message)
286
323
  return EXIT_FAIL
287
324
 
288
325
 
289
- def _selected(record, only: list[str] | None): # noqa: ANN001, ANN202
290
- if not only:
326
+ def _selected(record, args): # noqa: ANN001, ANN202
327
+ if not args.only and not args.globs:
291
328
  return record.files
292
- wanted = {n.lower() for n in only}
293
- return [
294
- f
295
- for f in record.files
296
- if naming.sanitize(f.name).lower() in wanted or f.name.lower() in wanted
297
- ]
329
+ return [f for f in record.files if downloader_matches(f, args.only, args.globs)]
298
330
 
299
331
 
300
332
  def cmd_fetch(args: argparse.Namespace) -> int:
@@ -315,15 +347,27 @@ def cmd_fetch(args: argparse.Namespace) -> int:
315
347
  license=record.license,
316
348
  landing=_hide(record.landing, args),
317
349
  )
350
+ # A URL list on stdout must stay machine-readable, so the header and the
351
+ # token line move to stderr for --urls.
352
+ header = sys.stderr if args.urls else sys.stdout
318
353
  if not args.json:
319
- _print_header(record, lambda value: _hide(value, args))
354
+ _print_header(record, lambda value: _hide(value, args), stream=header)
320
355
  token = _token_for(args, record)
321
356
  if token and not args.json:
322
- print(f"auth : token from {token}")
357
+ print(f"auth : token from {token}", file=header)
323
358
 
324
359
  out_dir = Path(args.out or f"acadfetch-out/{record.source}-{_slug(record)}")
325
360
  doc["out"] = str(out_dir)
326
361
  doc["dry_run"] = bool(args.dry_run)
362
+
363
+ if args.urls:
364
+ if args.json:
365
+ raise TargetError("--urls writes a text list; drop --json")
366
+ chosen = _selected(record, args)
367
+ if not chosen:
368
+ return _no_match(record, args)
369
+ _write_urls(chosen, out_dir, args.urls)
370
+ return EXIT_OK
327
371
  # Say whether a credential was stripped, so a script can tell the manifest
328
372
  # is not replayable as-is.
329
373
  doc["url_redacted"] = record.target != _hide(record.target, args) or record.landing != _hide(
@@ -344,9 +388,9 @@ def cmd_fetch(args: argparse.Namespace) -> int:
344
388
  )
345
389
 
346
390
  if args.dry_run:
347
- chosen = _selected(record, args.only)
391
+ chosen = _selected(record, args)
348
392
  if not chosen:
349
- return _no_match(record, args.only, args)
393
+ return _no_match(record, args)
350
394
  if args.max_size:
351
395
  check_total(replace(record, files=chosen), args.max_size)
352
396
  try:
@@ -387,9 +431,9 @@ def cmd_fetch(args: argparse.Namespace) -> int:
387
431
  return EXIT_OK
388
432
 
389
433
  try:
390
- results = fetch_record(record, opts, client, only=args.only)
434
+ results = fetch_record(record, opts, client, only=args.only, globs=args.globs)
391
435
  except PartialError:
392
- return _no_match(record, args.only, args)
436
+ return _no_match(record, args)
393
437
  finally:
394
438
  progress.finish()
395
439
 
@@ -423,7 +467,7 @@ def cmd_fetch(args: argparse.Namespace) -> int:
423
467
  token,
424
468
  # Which files this run asked for: a --only run must not look like a
425
469
  # run that fetched everything and then lost most of it.
426
- selected_ids={id(f) for f in _selected(record, args.only)},
470
+ selected_ids={id(f) for f in _selected(record, args)},
427
471
  redact=not args.record_full_url,
428
472
  )
429
473
  code = EXIT_OK
@@ -590,7 +634,7 @@ def _slug(record) -> str: # noqa: ANN001
590
634
  class Parser(argparse.ArgumentParser):
591
635
  """Usage errors are not "partial": exit 64 keeps 2 meaning a partial run."""
592
636
 
593
- def error(self, message: str) -> None: # noqa: D102
637
+ def error(self, message: str) -> NoReturn: # noqa: D102
594
638
  self.print_usage(sys.stderr)
595
639
  self.exit(EXIT_USAGE, f"{self.prog}: error: {message}\n")
596
640
 
@@ -603,6 +647,11 @@ def build_parser() -> argparse.ArgumentParser:
603
647
  formatter_class=argparse.RawDescriptionHelpFormatter,
604
648
  )
605
649
  parser.add_argument("--version", action="version", version=f"acadfetch {consts.__version__}")
650
+ parser.add_argument(
651
+ "--cite",
652
+ action="store_true",
653
+ help="print how to cite the tool, then exit",
654
+ )
606
655
  sub = parser.add_subparsers(dest="command", required=True)
607
656
 
608
657
  def network_flags(p: argparse.ArgumentParser) -> None:
@@ -632,6 +681,11 @@ def build_parser() -> argparse.ArgumentParser:
632
681
  metavar="NAME:VALUE",
633
682
  help="extra request header, repeatable (e.g. 'Authorization: Bearer <token>')",
634
683
  )
684
+ p.add_argument(
685
+ "--verbose",
686
+ action="store_true",
687
+ help="trace requests and retries to stderr, with tracebacks (also ACADFETCH_DEBUG=1)",
688
+ )
635
689
  p.add_argument(
636
690
  "--timeout",
637
691
  type=_timeout,
@@ -703,6 +757,19 @@ def build_parser() -> argparse.ArgumentParser:
703
757
  metavar="NAME",
704
758
  help="fetch only these file names",
705
759
  )
760
+ fetch.add_argument(
761
+ "--glob",
762
+ nargs="+",
763
+ dest="globs",
764
+ metavar="PATTERN",
765
+ help="fetch only files whose name matches a shell pattern, e.g. '*.pdf'",
766
+ )
767
+ fetch.add_argument(
768
+ "-w",
769
+ "--urls",
770
+ metavar="PATH",
771
+ help="write an aria2-compatible URL list instead of downloading; '-' is stdout",
772
+ )
706
773
  fetch.add_argument("--manifest-out", help="also write the manifest here")
707
774
  fetch.add_argument("-n", "--dry-run", action="store_true", help="list what would be fetched")
708
775
  fetch.add_argument("-q", "--quiet", action="store_true", help="no progress line")
@@ -787,7 +854,16 @@ def main(argv: list[str] | None = None) -> int:
787
854
  for stream in (sys.stdout, sys.stderr):
788
855
  if hasattr(stream, "reconfigure"):
789
856
  stream.reconfigure(encoding="utf-8", errors="replace")
857
+ argv_list = list(sys.argv[1:] if argv is None else argv)
858
+ if "--cite" in argv_list:
859
+ print(
860
+ CITATION.format(
861
+ version=consts.__version__, url="https://github.com/young-ski/acadfetch"
862
+ )
863
+ )
864
+ return EXIT_OK
790
865
  args = build_parser().parse_args(argv)
866
+ debug.configure(getattr(args, "verbose", False))
791
867
  _recover_target(args)
792
868
  if getattr(args, "command", "") == "fetch" and getattr(args, "target", None) is None:
793
869
  message = (
@@ -802,12 +878,14 @@ def main(argv: list[str] | None = None) -> int:
802
878
  try:
803
879
  return args.func(args)
804
880
  except AcFError as exc: # TargetError is one
881
+ debug.log_exception(exc)
805
882
  # An error sentence carries the URL, and that URL may carry a signature.
806
883
  message = _hide_text(str(exc), args)
807
884
  print(f"acadfetch: {message}", file=sys.stderr)
808
885
  _emit_error(args, message)
809
886
  return EXIT_FAIL
810
887
  except OSError as exc: # a filesystem surprise, not a bug report
888
+ debug.log_exception(exc)
811
889
  message = f"{exc.strerror or exc}" if exc.filename else str(exc)
812
890
  print(f"acadfetch: {message}", file=sys.stderr)
813
891
  _emit_error(args, message)
@@ -23,7 +23,7 @@ from collections.abc import Mapping
23
23
  from datetime import datetime, timezone
24
24
  from http.client import HTTPException
25
25
 
26
- from . import consts, verify
26
+ from . import consts, debug, urls, verify
27
27
  from .errors import HttpError
28
28
  from .ratelimit import TokenBucket
29
29
 
@@ -176,10 +176,11 @@ class Client:
176
176
  if retry_after is not None:
177
177
  # A literal `Retry-After: 0` would mean no gap at all and a burst;
178
178
  # keep a small jittered floor under it.
179
- time.sleep(max(retry_after, 0.05 + random.random() * 0.15))
180
- return
181
- backoff = min(0.8 * (2**attempt), 30.0)
182
- time.sleep(backoff * (0.7 + random.random() * 0.6))
179
+ delay = max(retry_after, 0.05 + random.random() * 0.15)
180
+ else:
181
+ delay = min(0.8 * (2**attempt), 30.0) * (0.7 + random.random() * 0.6)
182
+ debug.log(f"retry in {delay:.2f}s")
183
+ time.sleep(delay)
183
184
 
184
185
  def _request(
185
186
  self, url: str, headers: Mapping[str, str] | None = None
@@ -204,6 +205,7 @@ class Client:
204
205
  headers=None,
205
206
  block_retries: int | None = None,
206
207
  attempts: int | None = None,
208
+ method: str = "GET",
207
209
  ): # noqa: ANN001, ANN202
208
210
  if block_retries is None:
209
211
  block_retries = self.block_retries
@@ -215,11 +217,16 @@ class Client:
215
217
  for attempt in range(budget):
216
218
  self.bucket.acquire()
217
219
  retry_after = None
220
+ req = self._request(url, headers)
221
+ debug.log(f"{method} {urls.redact(url)} headers[{debug.headers_for_log(req.headers)}]")
218
222
  try:
219
- return fn(self._request(url, headers))
223
+ resp = fn(req)
224
+ debug.log(f"-> {getattr(resp, 'status', 'ok')}")
225
+ return resp
220
226
  except urllib.error.HTTPError as exc:
221
227
  last = exc
222
228
  last_detail = detail = _snippet(exc)
229
+ debug.log(f"-> {exc.code} {detail[:120]}")
223
230
  content_type = exc.headers.get("Content-Type") if exc.headers else None
224
231
  last_blocked = exc.code == 403 and is_block_page(detail, content_type)
225
232
  retry_after = parse_retry_after(
@@ -239,6 +246,7 @@ class Client:
239
246
  ) from exc
240
247
  else:
241
248
  waf += 1
249
+ debug.log(f"-> block page, retry {waf}/{block_retries}")
242
250
  except HttpError as exc:
243
251
  # Raised by a caller-supplied fn, e.g. a body that stopped
244
252
  # short. Only the transient ones are worth another request.
@@ -248,6 +256,7 @@ class Client:
248
256
  last_detail = str(exc)
249
257
  last_blocked = False
250
258
  except (urllib.error.URLError, TimeoutError, OSError) as exc:
259
+ debug.log(f"-> {type(exc).__name__}: {exc}")
251
260
  last = exc
252
261
  last_detail = str(exc)
253
262
  last_blocked = False
@@ -298,7 +307,7 @@ class Client:
298
307
 
299
308
  try:
300
309
  # HEAD only feeds size/filename; one shot, never worth a retry.
301
- with self._with_retries(url, _head, block_retries=0, attempts=1) as resp:
310
+ with self._with_retries(url, _head, block_retries=0, attempts=1, method="HEAD") as resp:
302
311
  return dict(resp.headers)
303
312
  except HttpError as exc:
304
313
  if exc.status in (403, 405, 501):
@@ -1,6 +1,6 @@
1
1
  """Package-wide constants and version."""
2
2
 
3
- __version__ = "0.4.0"
3
+ __version__ = "0.6.0"
4
4
 
5
5
  USER_AGENT = f"acadfetch/{__version__} (+https://github.com/young-ski/acadfetch)"
6
6
 
@@ -0,0 +1,59 @@
1
+ """Opt-in tracing, so a bug report does not have to be a guess.
2
+
3
+ Everything here writes to stderr and swallows its own errors: like progress, a
4
+ log line must never be able to fail a download. Header values that are
5
+ credentials are replaced with an ellipsis, and URLs arrive already redacted by
6
+ the caller.
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ import contextlib
12
+ import os
13
+ import sys
14
+ import traceback
15
+ from collections.abc import Mapping
16
+
17
+ from . import urls
18
+
19
+ _ON = False
20
+ _CREDENTIAL_HEADERS = frozenset(
21
+ {"authorization", "cookie", "proxy-authorization", "x-api-key", "x-auth-token"}
22
+ )
23
+ _FALSE = frozenset({"", "0", "false", "no", "off"})
24
+
25
+
26
+ def configure(flag: bool = False, environ: Mapping[str, str] | None = None) -> None:
27
+ """Turn tracing on from the flag or ``ACADFETCH_DEBUG``; the flag wins."""
28
+ global _ON
29
+ env = os.environ if environ is None else environ
30
+ value = (env.get("ACADFETCH_DEBUG") or "").strip().lower()
31
+ _ON = bool(flag) or value not in _FALSE
32
+
33
+
34
+ def enabled() -> bool:
35
+ return _ON
36
+
37
+
38
+ def log(message: str) -> None:
39
+ if not _ON:
40
+ return
41
+ with contextlib.suppress(OSError, ValueError):
42
+ print(f"debug: {message}", file=sys.stderr, flush=True)
43
+
44
+
45
+ def log_exception(exc: BaseException) -> None:
46
+ if not _ON:
47
+ return
48
+ with contextlib.suppress(OSError, ValueError):
49
+ text = "".join(traceback.format_exception(type(exc), exc, exc.__traceback__))
50
+ sys.stderr.write(urls.redact_text(text))
51
+ sys.stderr.flush()
52
+
53
+
54
+ def headers_for_log(headers: Mapping[str, str]) -> str:
55
+ """Header names with safe values; credentials become ``…``."""
56
+ return " ".join(
57
+ f"{name}: {'…' if name.lower() in _CREDENTIAL_HEADERS else value}"
58
+ for name, value in headers.items()
59
+ )
@@ -13,10 +13,12 @@ Resume is built on three ideas:
13
13
  from __future__ import annotations
14
14
 
15
15
  import contextlib
16
+ import fnmatch
16
17
  import json
17
18
  import os
18
19
  import random
19
20
  import shutil
21
+ import socket
20
22
  import threading
21
23
  import time
22
24
  from concurrent.futures import ThreadPoolExecutor, as_completed
@@ -24,8 +26,9 @@ from dataclasses import dataclass, field
24
26
  from datetime import datetime, timezone
25
27
  from http.client import HTTPException
26
28
  from pathlib import Path
29
+ from typing import cast
27
30
 
28
- from . import atomic, naming, verify
31
+ from . import atomic, debug, naming, verify
29
32
  from .client import Client
30
33
  from .consts import CHUNK
31
34
  from .errors import HttpError, LocalError, PartialError, TargetError
@@ -172,10 +175,10 @@ def _range_start(value: str | None) -> int | None:
172
175
  return None
173
176
 
174
177
 
175
- def _socket_of(resp) -> object | None: # noqa: ANN001
178
+ def _socket_of(resp) -> socket.socket | None: # noqa: ANN001
176
179
  """The socket under a urllib response, if we can reach it."""
177
180
  raw = getattr(getattr(resp, "fp", None), "raw", None)
178
- return getattr(raw, "_sock", None)
181
+ return cast("socket.socket | None", getattr(raw, "_sock", None))
179
182
 
180
183
 
181
184
  def _is_error_page(content_type: str, name: str) -> bool:
@@ -200,7 +203,7 @@ def _clear_timeout_latch(resp) -> None: # noqa: ANN001
200
203
  behaviour instead of breaking the run.
201
204
  """
202
205
  stream = getattr(getattr(resp, "fp", None), "raw", None)
203
- if getattr(stream, "_timeout_occurred", False):
206
+ if stream is not None and getattr(stream, "_timeout_occurred", False):
204
207
  stream._timeout_occurred = False
205
208
 
206
209
 
@@ -371,7 +374,7 @@ def _trusted_urls(manifest_path: Path) -> dict[str, str]:
371
374
  if not isinstance(rows, list):
372
375
  return {}
373
376
  return {
374
- row["name"]: row.get("url")
377
+ row["name"]: str(row.get("url") or "")
375
378
  for row in rows
376
379
  if isinstance(row, dict) and isinstance(row.get("name"), str)
377
380
  }
@@ -541,6 +544,7 @@ def _attempt(
541
544
  time.sleep(min(2**attempt, 10) * (0.7 + random.random() * 0.6))
542
545
  resumed_from = part.stat().st_size if part.exists() else 0
543
546
  else:
547
+ assert last is not None # the loop only ends after a failure
544
548
  return Result(
545
549
  file=file,
546
550
  state="failed",
@@ -606,8 +610,24 @@ def _attempt(
606
610
  )
607
611
 
608
612
 
613
+ def matches(file: File, only: list[str] | None, globs: list[str] | None) -> bool:
614
+ """Whether a file passes the ``--only`` / ``--glob`` filters."""
615
+ name = naming.sanitize(file.name).lower()
616
+ if (
617
+ only
618
+ and name not in {n.lower() for n in only}
619
+ and file.name.lower() not in {n.lower() for n in only}
620
+ ):
621
+ return False
622
+ return not globs or any(fnmatch.fnmatch(name, glob.lower()) for glob in globs)
623
+
624
+
609
625
  def fetch_record(
610
- record: Record, opts: Options, client: Client, only: list[str] | None = None
626
+ record: Record,
627
+ opts: Options,
628
+ client: Client,
629
+ only: list[str] | None = None,
630
+ globs: list[str] | None = None,
611
631
  ) -> list[Result]:
612
632
  """Fetch every file of ``record``, surviving a Ctrl-C.
613
633
 
@@ -616,25 +636,24 @@ def fetch_record(
616
636
  """
617
637
  _STOP.clear()
618
638
  try:
619
- return _fetch_record(record, opts, client, only)
639
+ return _fetch_record(record, opts, client, only, globs)
620
640
  finally:
621
641
  _STOP.clear()
622
642
 
623
643
 
624
644
  def _fetch_record(
625
- record: Record, opts: Options, client: Client, only: list[str] | None = None
645
+ record: Record,
646
+ opts: Options,
647
+ client: Client,
648
+ only: list[str] | None = None,
649
+ globs: list[str] | None = None,
626
650
  ) -> list[Result]:
627
651
  if opts.out.exists() and not opts.out.is_dir():
628
652
  raise TargetError(f"-o must be a directory, and {opts.out} is a file")
629
653
  opts.out.mkdir(parents=True, exist_ok=True)
630
654
  files = record.files
631
- if only:
632
- wanted = {n.lower() for n in only}
633
- files = [
634
- f
635
- for f in files
636
- if naming.sanitize(f.name).lower() in wanted or f.name.lower() in wanted
637
- ]
655
+ if only or globs:
656
+ files = [f for f in files if matches(f, only, globs)]
638
657
  if not files:
639
658
  raise PartialError([])
640
659
 
@@ -700,6 +719,7 @@ def _fetch_record(
700
719
 
701
720
  def _crashed(file: File, exc: Exception) -> Result:
702
721
  """An unexpected worker exception, as one failed row instead of a traceback."""
722
+ debug.log_exception(exc)
703
723
  return Result(
704
724
  file=file,
705
725
  state="failed",
@@ -13,6 +13,7 @@ from urllib.parse import quote, urlsplit
13
13
 
14
14
  from ..client import Client
15
15
  from ..errors import HttpError, SourceError, TargetError
16
+ from .base import Record
16
17
 
17
18
  HANDLE = "https://doi.org/api/handles/{doi}"
18
19
  _DOI = re.compile(r"^10\.\d{4,9}/\S+$")
@@ -52,7 +53,7 @@ def _page_like(client: Client, url: str, content_type: str) -> bool:
52
53
  return head.lstrip()[:1] == b"<"
53
54
 
54
55
 
55
- def resolve(client: Client, target: str) -> object:
56
+ def resolve(client: Client, target: str) -> Record:
56
57
  url = resolve_url(client, target)
57
58
  from . import registry # late import: registry imports this module
58
59
 
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: acadfetch
3
- Version: 0.4.0
3
+ Version: 0.6.0
4
4
  Summary: Resumable, rate-limited, checksum-verified bulk downloads from academic data repositories
5
5
  Author: young-ski
6
6
  License-Expression: MIT
@@ -89,7 +89,7 @@ From a checkout, for development:
89
89
 
90
90
  ```bash
91
91
  git clone https://github.com/young-ski/acadfetch && cd acadfetch
92
- python -m pip install -e . pytest && python -m pytest -q
92
+ python -m pip install -e . pytest mypy && python -m pytest -q
93
93
  ```
94
94
 
95
95
  ## Supported platforms
@@ -142,6 +142,15 @@ two never interleave:
142
142
  acadfetch fetch zenodo:123456 --json | jq -r '.files[] | select(.state=="failed") | .name'
143
143
  ```
144
144
 
145
+ To hand the bytes to another downloader, `-w/--urls` writes an aria2-compatible
146
+ list (`-` is stdout) and transfers nothing:
147
+
148
+ ```bash
149
+ acadfetch fetch zenodo:123456 -w - --glob '*.csv' | aria2c -i -
150
+ ```
151
+
152
+ `--glob` takes shell patterns and combines with `--only` (both must pass).
153
+
145
154
  Exit codes are unchanged, and a failure before any transfer still produces a
146
155
  document with `error` set. `report` is the document's schema version (now `2`, field reference in
147
156
  [docs/json-report.md](docs/json-report.md)):
@@ -284,6 +293,13 @@ acadfetch fetch zenodo:123456 --ca-bundle corp-tls.pem # trust this PEM
284
293
  either, `HTTP_PROXY` / `HTTPS_PROXY` / `ALL_PROXY` are honoured by urllib as
285
294
  usual. A timeout is per request, connect to first byte.
286
295
 
296
+ ## Debugging
297
+
298
+ `--verbose` (or `ACADFETCH_DEBUG=1`) traces each request, each retry and the
299
+ traceback of a failure to stderr; stdout, `--json` documents and exit codes are
300
+ unchanged. Credential header values and presigned query parameters are masked,
301
+ so the trace can be pasted into an issue.
302
+
287
303
  ## Politeness
288
304
 
289
305
  Defaults: 3 parallel transfers, 2 requests/second, 4 retries with jittered
@@ -307,6 +323,10 @@ honest way to test this code path.
307
323
 
308
324
  ## Getting involved
309
325
 
326
+ A [30-second tour](docs/demo.md) shows the whole loop, and
327
+ [docs/integrations.md](docs/integrations.md) has Make, CI, aria2 and Snakemake
328
+ recipes.
329
+
310
330
  A nightly job downloads three fixed public records and verifies them, so a
311
331
  rotted adapter opens an issue before a user reports one. It never blocks a
312
332
  merge; the next good night closes the issue.
@@ -346,6 +366,12 @@ acadfetch verify ./data/manifest.json
346
366
 
347
367
  MIT。
348
368
 
369
+ ## Citing
370
+
371
+ `acadfetch --cite` prints how to cite the tool; the machine-readable form is
372
+ [`CITATION.cff`](CITATION.cff). For the data, cite the repository record — the
373
+ manifest is the part that lets a reader re-check it.
374
+
349
375
  ## Audit trail
350
376
 
351
377
  How the 3.15 support claim was reached, including what it did **not** prove —
@@ -9,6 +9,7 @@ acadfetch/auth.py
9
9
  acadfetch/cli.py
10
10
  acadfetch/client.py
11
11
  acadfetch/consts.py
12
+ acadfetch/debug.py
12
13
  acadfetch/downloader.py
13
14
  acadfetch/errors.py
14
15
  acadfetch/manifest.py
@@ -37,9 +38,11 @@ tests/conftest.py
37
38
  tests/test_atomic.py
38
39
  tests/test_auth.py
39
40
  tests/test_checksum.py
41
+ tests/test_citation.py
40
42
  tests/test_cli.py
41
43
  tests/test_client.py
42
44
  tests/test_control_names.py
45
+ tests/test_debug.py
43
46
  tests/test_disk.py
44
47
  tests/test_downloader.py
45
48
  tests/test_identity.py
@@ -57,6 +60,7 @@ tests/test_remote.py
57
60
  tests/test_removed.py
58
61
  tests/test_sources.py
59
62
  tests/test_urls.py
63
+ tests/test_urls_glob.py
60
64
  tests/test_verify.py
61
65
  tests/fixtures/README.md
62
66
  tests/fixtures/ca.pem
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "acadfetch"
7
- version = "0.4.0"
7
+ version = "0.6.0"
8
8
  description = "Resumable, rate-limited, checksum-verified bulk downloads from academic data repositories"
9
9
  readme = "README.md"
10
10
  requires-python = ">=3.10"
@@ -46,6 +46,12 @@ include = ["acadfetch*"]
46
46
  testpaths = ["tests"]
47
47
  addopts = "-q"
48
48
 
49
+ [tool.mypy]
50
+ python_version = "3.10"
51
+ files = ["acadfetch"]
52
+ warn_unused_ignores = true
53
+ warn_redundant_casts = true
54
+
49
55
  [tool.ruff]
50
56
  line-length = 100
51
57
  target-version = "py310"
@@ -0,0 +1,26 @@
1
+ from __future__ import annotations
2
+
3
+ from pathlib import Path
4
+
5
+ from acadfetch import cli
6
+
7
+
8
+ def test_the_citation_file_is_cff():
9
+ text = Path("CITATION.cff").read_text("utf-8")
10
+ assert text.startswith("cff-version: 1.2.0")
11
+ assert "type: software" in text
12
+ assert "license: MIT" in text
13
+ assert "https://github.com/young-ski/acadfetch" in text
14
+
15
+
16
+ def test_cite_prints_the_tool_citation(capsys):
17
+ assert cli.main(["--cite"]) == cli.EXIT_OK
18
+ out = capsys.readouterr().out
19
+ assert "acadfetch (version" in out
20
+ assert "manifest.json" in out
21
+
22
+
23
+ def test_cite_needs_no_subcommand(capsys):
24
+ """argparse would otherwise demand one."""
25
+ assert cli.main(["--cite"]) == cli.EXIT_OK
26
+ assert "To cite" in capsys.readouterr().out
@@ -226,7 +226,7 @@ def test_retries_flag_reaches_the_transfer_layer(monkeypatch, capsys):
226
226
  _stub_resolve(monkeypatch)
227
227
  seen = {}
228
228
 
229
- def fake_fetch(record, opts, client, only=None):
229
+ def fake_fetch(record, opts, client, only=None, globs=None):
230
230
  seen["retries"] = opts.retries
231
231
  return []
232
232
 
@@ -331,7 +331,7 @@ def test_interrupt_exits_130_and_still_writes_the_manifest(monkeypatch, capsys,
331
331
 
332
332
  _stub_resolve(monkeypatch)
333
333
 
334
- def fake_fetch(record, opts, client, only=None):
334
+ def fake_fetch(record, opts, client, only=None, globs=None):
335
335
  opts.interrupted = True
336
336
  return [Result(record.files[0], "failed", error="interrupted")]
337
337
 
@@ -0,0 +1,120 @@
1
+ from __future__ import annotations
2
+
3
+ import pytest
4
+
5
+ from acadfetch import cli, debug
6
+ from acadfetch.client import Client
7
+ from acadfetch.errors import HttpError
8
+
9
+
10
+ @pytest.fixture(autouse=True)
11
+ def _reset():
12
+ debug.configure(False)
13
+ yield
14
+ debug.configure(False)
15
+
16
+
17
+ def test_the_environment_can_turn_it_on():
18
+ debug.configure(False, {"ACADFETCH_DEBUG": "1"})
19
+ assert debug.enabled()
20
+
21
+
22
+ @pytest.mark.parametrize("value", ["0", "false", "no", "off", ""])
23
+ def test_the_environment_can_turn_it_off(value):
24
+ debug.configure(False, {"ACADFETCH_DEBUG": value})
25
+ assert not debug.enabled()
26
+
27
+
28
+ def test_the_flag_wins_over_the_environment():
29
+ debug.configure(True, {"ACADFETCH_DEBUG": "0"})
30
+ assert debug.enabled()
31
+
32
+
33
+ def test_it_is_silent_when_off(capsys):
34
+ debug.log("nothing")
35
+ debug.log_exception(RuntimeError("boom"))
36
+ assert capsys.readouterr().err == ""
37
+
38
+
39
+ def _talking_client():
40
+ client = Client(rate=1000, retries=0, auth={"direct": {"Authorization": "Bearer SECRET"}})
41
+ client.source = "direct"
42
+ return client
43
+
44
+
45
+ def test_headers_are_masked(server, capsys):
46
+ server.cfg["files"]["/a.bin"] = b"x"
47
+ debug.configure(True)
48
+
49
+ _talking_client().head(server.base + "/a.bin")
50
+
51
+ err = capsys.readouterr().err
52
+ assert "HEAD" in err
53
+ assert "Authorization: …" in err
54
+ assert "SECRET" not in err
55
+
56
+
57
+ def test_a_credential_in_the_url_is_redacted(server, capsys):
58
+ server.cfg["files"]["/a.bin"] = b"x"
59
+ debug.configure(True)
60
+
61
+ _talking_client().head(server.base + "/a.bin?X-Amz-Signature=SIG")
62
+
63
+ err = capsys.readouterr().err
64
+ assert "X-Amz-Signature" not in err
65
+ assert "SIG" not in err
66
+
67
+
68
+ def test_a_retry_is_traced(server, capsys):
69
+ server.cfg["files"]["/a.bin"] = b"x"
70
+ server.cfg["always_500"].add("/a.bin")
71
+ debug.configure(True)
72
+
73
+ with pytest.raises(HttpError):
74
+ Client(rate=1000, retries=1).open(server.base + "/a.bin")
75
+
76
+ err = capsys.readouterr().err
77
+ assert "GET" in err and "-> 500" in err and "retry in" in err
78
+
79
+
80
+ def test_a_failure_prints_a_traceback_only_with_the_flag(capsys):
81
+ assert cli.main(["fetch", "nonsense://x"]) == 1
82
+ assert "Traceback" not in capsys.readouterr().err
83
+
84
+ assert cli.main(["fetch", "nonsense://x", "--verbose"]) == 1
85
+ assert "Traceback (most recent call last)" in capsys.readouterr().err
86
+
87
+
88
+ def test_debug_does_not_change_the_json_document(server, monkeypatch, capsys, tmp_path):
89
+ import json
90
+
91
+ server.cfg["files"]["/a.bin"] = b"payload"
92
+ monkeypatch.setattr(
93
+ cli.registry,
94
+ "resolve",
95
+ lambda _c, _t: __import__("acadfetch.sources.base", fromlist=["Record"]).Record(
96
+ source="direct",
97
+ target=server.base + "/a.bin",
98
+ files=[
99
+ __import__("acadfetch.sources.base", fromlist=["File"]).File(
100
+ server.base + "/a.bin", "a.bin", 7
101
+ )
102
+ ],
103
+ ),
104
+ )
105
+
106
+ code = cli.main(["fetch", "t", "--json", "--verbose", "-o", str(tmp_path)])
107
+ out, _ = capsys.readouterr()
108
+
109
+ assert code == cli.EXIT_OK
110
+ assert json.loads(out)["summary"]["ok"] is True
111
+
112
+
113
+ def test_a_traceback_does_not_leak_a_credential(capsys):
114
+ assert (
115
+ cli.main(["fetch", "https://example.invalid/x?X-Amz-Signature=SECRET", "--verbose", "-n"])
116
+ == 1
117
+ )
118
+ err = capsys.readouterr().err
119
+ assert "Traceback (most recent call last)" in err
120
+ assert "SECRET" not in err
@@ -315,3 +315,8 @@ def test_two_legacy_papers_no_longer_share_a_name():
315
315
  "arxiv:solv-int/9701002v1",
316
316
  )
317
317
  assert first.files[0].name != second.files[0].name
318
+
319
+
320
+ def test_direct_resolve_rejects_a_non_url():
321
+ with pytest.raises(TargetError, match="not a URL"):
322
+ direct.resolve(HeadClient({}), "zenodo:1")
@@ -0,0 +1,99 @@
1
+ from __future__ import annotations
2
+
3
+ import json
4
+
5
+ import pytest
6
+
7
+ from acadfetch import cli
8
+ from acadfetch.sources.base import File, Record
9
+
10
+
11
+ def _run(capsys, *argv):
12
+ code = cli.main([str(a) for a in argv])
13
+ out, err = capsys.readouterr()
14
+ return code, out, err
15
+
16
+
17
+ @pytest.fixture
18
+ def two(server, monkeypatch):
19
+ server.cfg["files"]["/a.pdf"] = b"PDF"
20
+ server.cfg["files"]["/b.csv"] = b"CSV"
21
+ record = Record(
22
+ source="direct",
23
+ target="t",
24
+ files=[File(server.base + "/a.pdf", "a.pdf", 3), File(server.base + "/b.csv", "b.csv", 3)],
25
+ )
26
+ monkeypatch.setattr(cli.registry, "resolve", lambda _c, _t: record)
27
+ return record
28
+
29
+
30
+ def test_glob_narrows_the_plan(two, capsys):
31
+ code, out, _ = _run(capsys, "fetch", "t", "-n", "--json", "--glob", "*.pdf")
32
+
33
+ assert code == cli.EXIT_OK
34
+ assert [row["name"] for row in json.loads(out)["files"]] == ["a.pdf"]
35
+
36
+
37
+ def test_glob_and_only_are_both_applied(two, capsys):
38
+ code, _, err = _run(capsys, "fetch", "t", "-n", "--glob", "*.pdf", "--only", "b.csv")
39
+
40
+ assert code == cli.EXIT_FAIL
41
+ assert "no file matched" in err
42
+
43
+
44
+ def test_a_glob_that_matches_nothing_names_the_candidates(two, capsys):
45
+ code, _, err = _run(capsys, "fetch", "t", "-n", "--glob", "*.zip")
46
+
47
+ assert code == cli.EXIT_FAIL
48
+ assert "--glob *.zip" in err
49
+ assert "a.pdf" in err
50
+
51
+
52
+ def test_glob_only_downloads_the_match(two, server, capsys, tmp_path):
53
+ code, _, _ = _run(capsys, "fetch", "t", "-q", "--glob", "*.csv", "-o", tmp_path)
54
+
55
+ assert code == cli.EXIT_OK
56
+ assert (tmp_path / "b.csv").exists()
57
+ assert not (tmp_path / "a.pdf").exists()
58
+
59
+
60
+ def test_urls_to_stdout_are_aria2_shaped(two, capsys, tmp_path):
61
+ code, out, _ = _run(capsys, "fetch", "t", "-w", "-", "-o", tmp_path)
62
+
63
+ assert code == cli.EXIT_OK
64
+ lines = out.splitlines()
65
+ assert lines[0].endswith("/a.pdf")
66
+ assert lines[1] == " out=a.pdf"
67
+ assert lines[2] == f" dir={tmp_path}"
68
+ assert "out=b.csv" in out
69
+
70
+
71
+ def test_urls_to_a_file(two, capsys, tmp_path):
72
+ target = tmp_path / "list.txt"
73
+
74
+ code, out, err = _run(capsys, "fetch", "t", "-w", target, "-o", tmp_path)
75
+
76
+ assert code == cli.EXIT_OK
77
+ assert out == ""
78
+ assert "2 url(s)" in err
79
+ assert "out=a.pdf" in target.read_text("utf-8")
80
+
81
+
82
+ def test_urls_honour_glob(two, capsys, tmp_path):
83
+ code, out, _ = _run(capsys, "fetch", "t", "-w", "-", "--glob", "*.pdf", "-o", tmp_path)
84
+
85
+ assert code == cli.EXIT_OK
86
+ assert "a.pdf" in out and "b.csv" not in out
87
+
88
+
89
+ def test_urls_and_json_are_mutually_exclusive(two, capsys):
90
+ code, _, err = _run(capsys, "fetch", "t", "-w", "-", "--json")
91
+
92
+ assert code == cli.EXIT_FAIL
93
+ assert "drop --json" in err
94
+
95
+
96
+ def test_no_bytes_are_fetched_for_a_list(two, server, capsys, tmp_path):
97
+ _run(capsys, "fetch", "t", "-w", "-", "-o", tmp_path)
98
+
99
+ assert server.cfg["hits"] == {}, "a list must not transfer"
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes