acadfetch 0.7.2__tar.gz → 0.8.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (78) hide show
  1. {acadfetch-0.7.2/acadfetch.egg-info → acadfetch-0.8.0}/PKG-INFO +33 -2
  2. {acadfetch-0.7.2 → acadfetch-0.8.0}/README.md +32 -1
  3. {acadfetch-0.7.2 → acadfetch-0.8.0}/acadfetch/cli.py +302 -74
  4. {acadfetch-0.7.2 → acadfetch-0.8.0}/acadfetch/consts.py +1 -1
  5. {acadfetch-0.7.2 → acadfetch-0.8.0/acadfetch.egg-info}/PKG-INFO +33 -2
  6. {acadfetch-0.7.2 → acadfetch-0.8.0}/acadfetch.egg-info/SOURCES.txt +1 -0
  7. {acadfetch-0.7.2 → acadfetch-0.8.0}/pyproject.toml +1 -1
  8. acadfetch-0.8.0/tests/test_batch.py +262 -0
  9. {acadfetch-0.7.2 → acadfetch-0.8.0}/LICENSE +0 -0
  10. {acadfetch-0.7.2 → acadfetch-0.8.0}/MANIFEST.in +0 -0
  11. {acadfetch-0.7.2 → acadfetch-0.8.0}/acadfetch/__init__.py +0 -0
  12. {acadfetch-0.7.2 → acadfetch-0.8.0}/acadfetch/__main__.py +0 -0
  13. {acadfetch-0.7.2 → acadfetch-0.8.0}/acadfetch/atomic.py +0 -0
  14. {acadfetch-0.7.2 → acadfetch-0.8.0}/acadfetch/auth.py +0 -0
  15. {acadfetch-0.7.2 → acadfetch-0.8.0}/acadfetch/client.py +0 -0
  16. {acadfetch-0.7.2 → acadfetch-0.8.0}/acadfetch/debug.py +0 -0
  17. {acadfetch-0.7.2 → acadfetch-0.8.0}/acadfetch/downloader.py +0 -0
  18. {acadfetch-0.7.2 → acadfetch-0.8.0}/acadfetch/errors.py +0 -0
  19. {acadfetch-0.7.2 → acadfetch-0.8.0}/acadfetch/manifest.py +0 -0
  20. {acadfetch-0.7.2 → acadfetch-0.8.0}/acadfetch/naming.py +0 -0
  21. {acadfetch-0.7.2 → acadfetch-0.8.0}/acadfetch/pooch.py +0 -0
  22. {acadfetch-0.7.2 → acadfetch-0.8.0}/acadfetch/progress.py +0 -0
  23. {acadfetch-0.7.2 → acadfetch-0.8.0}/acadfetch/py.typed +0 -0
  24. {acadfetch-0.7.2 → acadfetch-0.8.0}/acadfetch/ratelimit.py +0 -0
  25. {acadfetch-0.7.2 → acadfetch-0.8.0}/acadfetch/remote.py +0 -0
  26. {acadfetch-0.7.2 → acadfetch-0.8.0}/acadfetch/sources/__init__.py +0 -0
  27. {acadfetch-0.7.2 → acadfetch-0.8.0}/acadfetch/sources/arxiv.py +0 -0
  28. {acadfetch-0.7.2 → acadfetch-0.8.0}/acadfetch/sources/base.py +0 -0
  29. {acadfetch-0.7.2 → acadfetch-0.8.0}/acadfetch/sources/direct.py +0 -0
  30. {acadfetch-0.7.2 → acadfetch-0.8.0}/acadfetch/sources/doi.py +0 -0
  31. {acadfetch-0.7.2 → acadfetch-0.8.0}/acadfetch/sources/registry.py +0 -0
  32. {acadfetch-0.7.2 → acadfetch-0.8.0}/acadfetch/sources/removed.py +0 -0
  33. {acadfetch-0.7.2 → acadfetch-0.8.0}/acadfetch/sources/zenodo.py +0 -0
  34. {acadfetch-0.7.2 → acadfetch-0.8.0}/acadfetch/units.py +0 -0
  35. {acadfetch-0.7.2 → acadfetch-0.8.0}/acadfetch/urls.py +0 -0
  36. {acadfetch-0.7.2 → acadfetch-0.8.0}/acadfetch/verify.py +0 -0
  37. {acadfetch-0.7.2 → acadfetch-0.8.0}/acadfetch.egg-info/dependency_links.txt +0 -0
  38. {acadfetch-0.7.2 → acadfetch-0.8.0}/acadfetch.egg-info/entry_points.txt +0 -0
  39. {acadfetch-0.7.2 → acadfetch-0.8.0}/acadfetch.egg-info/top_level.txt +0 -0
  40. {acadfetch-0.7.2 → acadfetch-0.8.0}/setup.cfg +0 -0
  41. {acadfetch-0.7.2 → acadfetch-0.8.0}/tests/conftest.py +0 -0
  42. {acadfetch-0.7.2 → acadfetch-0.8.0}/tests/fixtures/README.md +0 -0
  43. {acadfetch-0.7.2 → acadfetch-0.8.0}/tests/fixtures/ca.pem +0 -0
  44. {acadfetch-0.7.2 → acadfetch-0.8.0}/tests/fixtures/figshare_403_block.html +0 -0
  45. {acadfetch-0.7.2 → acadfetch-0.8.0}/tests/fixtures/figshare_403_forbidden.json +0 -0
  46. {acadfetch-0.7.2 → acadfetch-0.8.0}/tests/fixtures/figshare_article.json +0 -0
  47. {acadfetch-0.7.2 → acadfetch-0.8.0}/tests/fixtures/figshare_file.pdf +0 -0
  48. {acadfetch-0.7.2 → acadfetch-0.8.0}/tests/test_atomic.py +0 -0
  49. {acadfetch-0.7.2 → acadfetch-0.8.0}/tests/test_auth.py +0 -0
  50. {acadfetch-0.7.2 → acadfetch-0.8.0}/tests/test_checksum.py +0 -0
  51. {acadfetch-0.7.2 → acadfetch-0.8.0}/tests/test_citation.py +0 -0
  52. {acadfetch-0.7.2 → acadfetch-0.8.0}/tests/test_claims.py +0 -0
  53. {acadfetch-0.7.2 → acadfetch-0.8.0}/tests/test_cli.py +0 -0
  54. {acadfetch-0.7.2 → acadfetch-0.8.0}/tests/test_client.py +0 -0
  55. {acadfetch-0.7.2 → acadfetch-0.8.0}/tests/test_control_names.py +0 -0
  56. {acadfetch-0.7.2 → acadfetch-0.8.0}/tests/test_debug.py +0 -0
  57. {acadfetch-0.7.2 → acadfetch-0.8.0}/tests/test_disk.py +0 -0
  58. {acadfetch-0.7.2 → acadfetch-0.8.0}/tests/test_downloader.py +0 -0
  59. {acadfetch-0.7.2 → acadfetch-0.8.0}/tests/test_hash_once.py +0 -0
  60. {acadfetch-0.7.2 → acadfetch-0.8.0}/tests/test_identity.py +0 -0
  61. {acadfetch-0.7.2 → acadfetch-0.8.0}/tests/test_isolation.py +0 -0
  62. {acadfetch-0.7.2 → acadfetch-0.8.0}/tests/test_json.py +0 -0
  63. {acadfetch-0.7.2 → acadfetch-0.8.0}/tests/test_lock.py +0 -0
  64. {acadfetch-0.7.2 → acadfetch-0.8.0}/tests/test_manifest.py +0 -0
  65. {acadfetch-0.7.2 → acadfetch-0.8.0}/tests/test_naming.py +0 -0
  66. {acadfetch-0.7.2 → acadfetch-0.8.0}/tests/test_naming_race.py +0 -0
  67. {acadfetch-0.7.2 → acadfetch-0.8.0}/tests/test_network.py +0 -0
  68. {acadfetch-0.7.2 → acadfetch-0.8.0}/tests/test_packaging.py +0 -0
  69. {acadfetch-0.7.2 → acadfetch-0.8.0}/tests/test_pooch.py +0 -0
  70. {acadfetch-0.7.2 → acadfetch-0.8.0}/tests/test_progress.py +0 -0
  71. {acadfetch-0.7.2 → acadfetch-0.8.0}/tests/test_ratelimit.py +0 -0
  72. {acadfetch-0.7.2 → acadfetch-0.8.0}/tests/test_redirect.py +0 -0
  73. {acadfetch-0.7.2 → acadfetch-0.8.0}/tests/test_remote.py +0 -0
  74. {acadfetch-0.7.2 → acadfetch-0.8.0}/tests/test_removed.py +0 -0
  75. {acadfetch-0.7.2 → acadfetch-0.8.0}/tests/test_sources.py +0 -0
  76. {acadfetch-0.7.2 → acadfetch-0.8.0}/tests/test_urls.py +0 -0
  77. {acadfetch-0.7.2 → acadfetch-0.8.0}/tests/test_urls_glob.py +0 -0
  78. {acadfetch-0.7.2 → acadfetch-0.8.0}/tests/test_verify.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: acadfetch
3
- Version: 0.7.2
3
+ Version: 0.8.0
4
4
  Summary: Resumable, rate-limited, checksum-verified bulk downloads from academic data repositories
5
5
  Author: young-ski
6
6
  License-Expression: MIT
@@ -114,6 +114,23 @@ acadfetch fetch arxiv:1706.03762 # an arXiv id
114
114
  acadfetch fetch https://ndownloader.figshare.com/files/70600233/report.pdf # any URL
115
115
  ```
116
116
 
117
+ Or a whole list, with the failures kept for the next run:
118
+
119
+ ```bash
120
+ acadfetch batch identifiers.txt # one identifier per line
121
+ acadfetch batch identifiers.txt -o ./data --failed-out todo.txt
122
+ ```
123
+
124
+ Blank lines and `#` comments are skipped, so a list can carry notes. Every
125
+ identifier still gets a directory of its own under `-o` (default
126
+ `acadfetch-out`); one bad line does not stop the rest, and the identifiers that
127
+ failed land in `failed.txt` (or `--failed-out`), verbatim and one per line — so
128
+ the retry is `acadfetch batch failed.txt`. The file is replaced even when
129
+ nothing failed, so a stale list never lies. Exit codes keep their meaning: `0`
130
+ all ok, `1` all failed, `2` partial, `130` interrupted, and an interrupt puts the
131
+ lines that were never attempted on the list too. Records are attempted one after
132
+ another; `-j` still applies inside a record.
133
+
117
134
  The first two are answered by Zenodo and arXiv and work from any network those
118
135
  are reachable on. The third is a direct URL, so the host decides: figshare's CDN
119
136
  answers 403 with a WAF block page from some networks — the same behaviour that
@@ -188,7 +205,8 @@ checksum matches, so an interrupted or failed run never leaves something that
188
205
  it resumes. The one case where a run must start over is a server that ignores
189
206
  `Range`; the bytes already on disk are then moved aside and kept until a whole
190
207
  new copy has arrived, so a restart that dies early costs nothing. `acadfetch-out/`
191
- is worth adding to a project's `.gitignore`; this repository already has it.
208
+ is worth adding to a project's `.gitignore`; this repository already has it, and
209
+ names the default `failed.txt` there as well.
192
210
 
193
211
  The default directory names the target, not only its filename — `data.csv` from
194
212
  two hosts is two datasets, and they do not share a directory. A file already
@@ -405,6 +423,19 @@ acadfetch verify ./data/manifest.json # 事后复验;对不上就非
405
423
  > figshare 适配器正是因为这个)。acadfetch 会明说这是拦截页、而不是把 HTML 存成
406
424
  > PDF,并把下一条命令给你。
407
425
 
426
+ 批量清单则一条一条来,失败的留下:
427
+
428
+ ```bash
429
+ acadfetch batch identifiers.txt # 每行一个目标
430
+ acadfetch batch identifiers.txt -o ./data --failed-out todo.txt
431
+ ```
432
+
433
+ 空行与 `#` 注释会被跳过;每个目标仍在 `-o`(默认 `acadfetch-out`)下拿到自己
434
+ 的目录,一行坏掉不影响其余。失败的目标原样写进 `failed.txt`(或
435
+ `--failed-out`),重跑就是 `acadfetch batch failed.txt`;没失败也会清空该文件,
436
+ 不会留下上次的假清单。退出码含义不变:`0` 全成、`1` 全败、`2` 部分、`130`
437
+ 被中断 —— 中断时还没轮到的行也会进清单。
438
+
408
439
  ### 它替你保住的东西
409
440
 
410
441
  | 情况 | 它怎么做 |
@@ -83,6 +83,23 @@ acadfetch fetch arxiv:1706.03762 # an arXiv id
83
83
  acadfetch fetch https://ndownloader.figshare.com/files/70600233/report.pdf # any URL
84
84
  ```
85
85
 
86
+ Or a whole list, with the failures kept for the next run:
87
+
88
+ ```bash
89
+ acadfetch batch identifiers.txt # one identifier per line
90
+ acadfetch batch identifiers.txt -o ./data --failed-out todo.txt
91
+ ```
92
+
93
+ Blank lines and `#` comments are skipped, so a list can carry notes. Every
94
+ identifier still gets a directory of its own under `-o` (default
95
+ `acadfetch-out`); one bad line does not stop the rest, and the identifiers that
96
+ failed land in `failed.txt` (or `--failed-out`), verbatim and one per line — so
97
+ the retry is `acadfetch batch failed.txt`. The file is replaced even when
98
+ nothing failed, so a stale list never lies. Exit codes keep their meaning: `0`
99
+ all ok, `1` all failed, `2` partial, `130` interrupted, and an interrupt puts the
100
+ lines that were never attempted on the list too. Records are attempted one after
101
+ another; `-j` still applies inside a record.
102
+
86
103
  The first two are answered by Zenodo and arXiv and work from any network those
87
104
  are reachable on. The third is a direct URL, so the host decides: figshare's CDN
88
105
  answers 403 with a WAF block page from some networks — the same behaviour that
@@ -157,7 +174,8 @@ checksum matches, so an interrupted or failed run never leaves something that
157
174
  it resumes. The one case where a run must start over is a server that ignores
158
175
  `Range`; the bytes already on disk are then moved aside and kept until a whole
159
176
  new copy has arrived, so a restart that dies early costs nothing. `acadfetch-out/`
160
- is worth adding to a project's `.gitignore`; this repository already has it.
177
+ is worth adding to a project's `.gitignore`; this repository already has it, and
178
+ names the default `failed.txt` there as well.
161
179
 
162
180
  The default directory names the target, not only its filename — `data.csv` from
163
181
  two hosts is two datasets, and they do not share a directory. A file already
@@ -374,6 +392,19 @@ acadfetch verify ./data/manifest.json # 事后复验;对不上就非
374
392
  > figshare 适配器正是因为这个)。acadfetch 会明说这是拦截页、而不是把 HTML 存成
375
393
  > PDF,并把下一条命令给你。
376
394
 
395
+ 批量清单则一条一条来,失败的留下:
396
+
397
+ ```bash
398
+ acadfetch batch identifiers.txt # 每行一个目标
399
+ acadfetch batch identifiers.txt -o ./data --failed-out todo.txt
400
+ ```
401
+
402
+ 空行与 `#` 注释会被跳过;每个目标仍在 `-o`(默认 `acadfetch-out`)下拿到自己
403
+ 的目录,一行坏掉不影响其余。失败的目标原样写进 `failed.txt`(或
404
+ `--failed-out`),重跑就是 `acadfetch batch failed.txt`;没失败也会清空该文件,
405
+ 不会留下上次的假清单。退出码含义不变:`0` 全成、`1` 全败、`2` 部分、`130`
406
+ 被中断 —— 中断时还没轮到的行也会进清单。
407
+
377
408
  ### 它替你保住的东西
378
409
 
379
410
  | 情况 | 它怎么做 |
@@ -18,7 +18,7 @@ from pathlib import Path
18
18
  from typing import NoReturn
19
19
  from urllib.parse import urlsplit
20
20
 
21
- from . import auth, consts, debug, manifest, naming, urls, verify
21
+ from . import atomic, auth, consts, debug, manifest, naming, urls, verify
22
22
  from . import remote as remote_check
23
23
  from .client import Client
24
24
  from .downloader import Options, check_disk, check_total, fetch_record, human
@@ -183,8 +183,11 @@ def _recover_target(args: argparse.Namespace) -> None:
183
183
  """``--only a.pdf <target>`` makes argparse eat the target; put it back.
184
184
 
185
185
  Checked only when the target is missing, so a file that happens to look
186
- like an id keeps working as long as the target is spelled out.
186
+ like an id keeps working as long as the target is spelled out. ``batch``
187
+ has no positional target, so its ``--only`` must not be mistaken for one.
187
188
  """
189
+ if getattr(args, "command", "") != "fetch":
190
+ return
188
191
  if getattr(args, "target", None) is not None or not getattr(args, "only", None):
189
192
  return
190
193
  if len(args.only) > 1 and _looks_like_target(args.only[-1]):
@@ -306,7 +309,7 @@ def _write_urls(files, out_dir, target) -> None: # noqa: ANN001
306
309
  print(f"{len(files)} url(s) -> {target}", file=sys.stderr)
307
310
 
308
311
 
309
- def _no_match(record, args) -> int: # noqa: ANN001
312
+ def _no_match(record, args, emit: bool = True) -> int: # noqa: ANN001
310
313
  """A filter that matched nothing is a failure that can name the candidates."""
311
314
  available = ", ".join(naming.sanitize(f.name) for f in record.files[:8]) or "(none)"
312
315
  wanted = " ".join(
@@ -319,7 +322,8 @@ def _no_match(record, args) -> int: # noqa: ANN001
319
322
  )
320
323
  message = f"no file matched {wanted}; available: {available}"
321
324
  print(f"acadfetch: {message}", file=sys.stderr)
322
- _emit_error(args, message)
325
+ if emit:
326
+ _emit_error(args, message)
323
327
  return EXIT_FAIL
324
328
 
325
329
 
@@ -330,8 +334,17 @@ def _selected(record, args): # noqa: ANN001, ANN202
330
334
 
331
335
 
332
336
  def cmd_fetch(args: argparse.Namespace) -> int:
337
+ code, _ = _fetch_one(args, _client(args))
338
+ return code
339
+
340
+
341
+ def _fetch_one(args: argparse.Namespace, client: Client, emit: bool = True) -> tuple[int, dict]:
342
+ """Resolve and fetch one target. Returns the exit code and the report.
343
+
344
+ ``emit`` is false when the caller owns stdout (``batch``): it collects these
345
+ documents and prints one of its own at the end.
346
+ """
333
347
  started = time.monotonic()
334
- client = _client(args)
335
348
  record = _resolve(args, client)
336
349
  if args.checksum:
337
350
  # Explicit wins over whatever the source published, and over nothing.
@@ -340,6 +353,8 @@ def cmd_fetch(args: argparse.Namespace) -> int:
340
353
  files=[replace(f, checksum=args.checksum, checksum_source="cli") for f in record.files],
341
354
  )
342
355
  doc = _report(args)
356
+ # The document keeps its shape even when the caller is a batch.
357
+ doc["command"] = "fetch"
343
358
  doc.update(
344
359
  target=_hide(record.target, args),
345
360
  source=record.source,
@@ -356,7 +371,14 @@ def cmd_fetch(args: argparse.Namespace) -> int:
356
371
  if token and not args.json:
357
372
  print(f"auth : token from {token}", file=header)
358
373
 
359
- out_dir = Path(args.out or f"acadfetch-out/{record.source}-{_slug(record)}")
374
+ # A batch passes its base directory in -o: every identifier still gets a
375
+ # directory of its own, because two records must not share one.
376
+ if args.out and getattr(args, "command", "") == "batch":
377
+ out_dir = Path(args.out) / f"{record.source}-{_slug(record)}"
378
+ elif args.out:
379
+ out_dir = Path(args.out)
380
+ else:
381
+ out_dir = Path(f"acadfetch-out/{record.source}-{_slug(record)}")
360
382
  doc["out"] = str(out_dir)
361
383
  doc["dry_run"] = bool(args.dry_run)
362
384
 
@@ -365,9 +387,9 @@ def cmd_fetch(args: argparse.Namespace) -> int:
365
387
  raise TargetError("--urls writes a text list; drop --json")
366
388
  chosen = _selected(record, args)
367
389
  if not chosen:
368
- return _no_match(record, args)
390
+ return _no_match(record, args, emit), doc
369
391
  _write_urls(chosen, out_dir, args.urls)
370
- return EXIT_OK
392
+ return EXIT_OK, doc
371
393
  # Say whether a credential was stripped, so a script can tell the manifest
372
394
  # is not replayable as-is.
373
395
  doc["url_redacted"] = record.target != _hide(record.target, args) or record.landing != _hide(
@@ -390,7 +412,7 @@ def cmd_fetch(args: argparse.Namespace) -> int:
390
412
  if args.dry_run:
391
413
  chosen = _selected(record, args)
392
414
  if not chosen:
393
- return _no_match(record, args)
415
+ return _no_match(record, args, emit), doc
394
416
  if args.max_size:
395
417
  check_total(replace(record, files=chosen), args.max_size)
396
418
  try:
@@ -418,8 +440,9 @@ def cmd_fetch(args: argparse.Namespace) -> int:
418
440
  doc["disk"] = disk
419
441
  doc["summary"] = {"total": len(chosen), "downloaded": 0, "skipped": 0, "failed": 0}
420
442
  doc["elapsed_s"] = round(time.monotonic() - started, 3)
421
- _emit(doc)
422
- return EXIT_OK
443
+ if emit:
444
+ _emit(doc)
445
+ return EXIT_OK, doc
423
446
  print(f"would write to {out_dir}")
424
447
  if needed:
425
448
  print(
@@ -431,12 +454,12 @@ def cmd_fetch(args: argparse.Namespace) -> int:
431
454
  # size, so say that instead of printing a question mark.
432
455
  size = human(file.size) if file.size is not None else "size not published"
433
456
  print(f" would fetch {naming.sanitize(file.name)} ({size})")
434
- return EXIT_OK
457
+ return EXIT_OK, doc
435
458
 
436
459
  try:
437
460
  results = fetch_record(record, opts, client, only=args.only, globs=args.globs)
438
461
  except PartialError:
439
- return _no_match(record, args)
462
+ return _no_match(record, args, emit), doc
440
463
  finally:
441
464
  progress.finish()
442
465
 
@@ -503,9 +526,9 @@ def cmd_fetch(args: argparse.Namespace) -> int:
503
526
  elif failures:
504
527
  code = EXIT_PARTIAL
505
528
  doc["summary"]["ok"] = code == EXIT_OK
506
- if args.json:
529
+ if emit and args.json:
507
530
  _emit(doc)
508
- return code
531
+ return code, doc
509
532
 
510
533
 
511
534
  def cmd_verify(args: argparse.Namespace) -> int:
@@ -644,6 +667,176 @@ def _slug(record) -> str: # noqa: ANN001
644
667
  return f"{host}-{digest}-{name}"
645
668
 
646
669
 
670
+ def _read_identifiers(path: Path) -> list[tuple[int, str]]:
671
+ """Non-blank, non-comment lines, with the line number they came from.
672
+
673
+ ``utf-8-sig`` because a Windows editor happily adds a BOM, and a BOM on the
674
+ first identifier otherwise makes exactly that one unresolvable. ``#`` starts
675
+ a comment so a list can carry notes and still be fed back as a retry list.
676
+ """
677
+ try:
678
+ text = path.read_text("utf-8-sig")
679
+ except FileNotFoundError:
680
+ raise TargetError(f"no such identifiers file: {path}") from None
681
+ except IsADirectoryError:
682
+ raise TargetError(f"{path} is a directory") from None
683
+ except UnicodeDecodeError as exc:
684
+ raise TargetError(f"{path} is not UTF-8: {exc}") from None
685
+ return [
686
+ (number, item)
687
+ for number, line in enumerate(text.splitlines(), 1)
688
+ if (item := line.strip()) and not item.startswith("#")
689
+ ]
690
+
691
+
692
+ def _write_failed(path: Path, targets: list[str]) -> None:
693
+ """Replace the failure list, whole and one identifier per line.
694
+
695
+ Written even when nothing failed: an empty file says "nothing to retry",
696
+ and last run's list would say the opposite. Same-name ``.part`` then a
697
+ replace, so a crash never leaves half a list that reads as complete.
698
+ """
699
+ text = "".join(f"{target}\n" for target in targets)
700
+ if str(path) == "-":
701
+ sys.stdout.write(text)
702
+ return
703
+ path.parent.mkdir(parents=True, exist_ok=True)
704
+ tmp = path.with_name(path.name + ".part")
705
+ tmp.write_text(text, "utf-8")
706
+ atomic.replace(tmp, path)
707
+
708
+
709
+ def cmd_batch(args: argparse.Namespace) -> int:
710
+ """Fetch every identifier of a file; keep the failures for the next run.
711
+
712
+ One bad identifier is one line of output, not the end of the run. The exit
713
+ code keeps its meaning: 0 all ok, 1 all failed, 2 partial, 130 interrupted -
714
+ and on an interrupt the identifiers that were never attempted join the list,
715
+ so a single rerun covers them too.
716
+ """
717
+ started = time.monotonic()
718
+ source_path = Path(args.identifiers)
719
+ failed_path = Path(args.failed_out)
720
+ # _fetch_one reads both; neither is a batch-level decision any more.
721
+ args.urls = None
722
+ args.manifest_out = None
723
+ if args.json and args.failed_out == "-":
724
+ raise TargetError("--json claims stdout; write --failed-out to a file")
725
+ base = args.out or "acadfetch-out"
726
+ doc = _report(args)
727
+ doc.update(identifiers=str(source_path), out=base, failed_out=str(failed_path))
728
+ items: list[dict] = []
729
+ doc["items"] = items
730
+ entries = _read_identifiers(source_path)
731
+ summary = doc["summary"] = {
732
+ "total": len(entries),
733
+ "ok": 0,
734
+ "failed": 0,
735
+ "interrupted": 0,
736
+ "ok_run": False,
737
+ }
738
+ doc["elapsed_s"] = 0.0
739
+
740
+ if not entries:
741
+ message = f"{source_path} lists no identifiers"
742
+ print(f"acadfetch: {message}", file=sys.stderr)
743
+ doc["error"] = message
744
+ _write_failed(failed_path, [])
745
+ if args.json:
746
+ _emit(doc)
747
+ return EXIT_FAIL
748
+
749
+ client = _client(args) # one client: the rate limit holds across the batch
750
+ if not args.json:
751
+ print(f"{len(entries)} identifier(s) -> {base}")
752
+ failures: list[str] = []
753
+ for index, (line_no, target) in enumerate(entries, 1):
754
+ args.target = target
755
+ label = _hide_text(target, args)
756
+ if not args.json:
757
+ print(f"==> [{index}/{len(entries)}] {label}")
758
+ item = {
759
+ "line": line_no,
760
+ "target": label,
761
+ "code": EXIT_OK,
762
+ "out": None,
763
+ "error": None,
764
+ "report": None,
765
+ }
766
+ items.append(item)
767
+ code = EXIT_OK
768
+ try:
769
+ code, report_doc = _fetch_one(args, client, emit=False)
770
+ except KeyboardInterrupt:
771
+ code, report_doc = EXIT_INTERRUPT, None
772
+ item["error"] = "interrupted"
773
+ except (AcFError, OSError) as exc:
774
+ debug.log_exception(exc)
775
+ code, report_doc = EXIT_FAIL, None
776
+ item["error"] = _hide_text(str(exc), args)
777
+ print(f"acadfetch: {item['error']}", file=sys.stderr)
778
+ else:
779
+ assert report_doc is not None
780
+ item["out"] = report_doc["out"]
781
+ item["error"] = report_doc["error"] or None
782
+ item["report"] = report_doc
783
+ item["code"] = code
784
+ item["ok"] = code == EXIT_OK
785
+ if code != EXIT_OK:
786
+ failures.append(target)
787
+ if code == EXIT_INTERRUPT:
788
+ print("\ninterrupted - partial files kept, rerun to resume", file=sys.stderr)
789
+ # The lines never reached belong on the retry list, not in an
790
+ # unaccounted gap between "done" and "not asked for".
791
+ for rest_line, rest_target in entries[index:]:
792
+ failures.append(rest_target)
793
+ items.append(
794
+ {
795
+ "line": rest_line,
796
+ "target": _hide_text(rest_target, args),
797
+ "code": EXIT_INTERRUPT,
798
+ "ok": False,
799
+ "out": None,
800
+ "error": "not attempted",
801
+ "report": None,
802
+ }
803
+ )
804
+ summary["interrupted"] = len(items) - index + 1
805
+ break
806
+
807
+ summary["failed"] = len(failures)
808
+ summary["ok"] = len(items) - len(failures)
809
+ code = EXIT_OK
810
+ if summary["failed"] == summary["total"]:
811
+ code = EXIT_FAIL
812
+ elif summary["failed"]:
813
+ code = EXIT_PARTIAL
814
+ if summary["interrupted"]:
815
+ code = EXIT_INTERRUPT
816
+ try:
817
+ _write_failed(failed_path, failures)
818
+ except OSError as exc:
819
+ # The downloads stand; only the retry list is missing.
820
+ print(f"acadfetch: could not write {failed_path}: {exc}", file=sys.stderr)
821
+ doc["error"] = f"could not write {failed_path}: {exc}"
822
+ if code == EXIT_OK:
823
+ code = EXIT_PARTIAL
824
+ summary["ok_run"] = code == EXIT_OK
825
+ doc["elapsed_s"] = round(time.monotonic() - started, 3)
826
+ if not args.json:
827
+ print()
828
+ print(
829
+ f"{summary['ok']}/{summary['total']} identifier(s) ok, "
830
+ f"{summary['failed']} failed -> {failed_path}"
831
+ )
832
+ for row in items:
833
+ if row["code"] != EXIT_OK:
834
+ print(f" line {row['line']}: {row['target']} ({row['error'] or 'failed'})")
835
+ if args.json:
836
+ _emit(doc)
837
+ return code
838
+
839
+
647
840
  class Parser(argparse.ArgumentParser):
648
841
  """Usage errors are not "partial": exit 64 keeps 2 meaning a partial run."""
649
842
 
@@ -735,6 +928,79 @@ def build_parser() -> argparse.ArgumentParser:
735
928
  help="ignore $ZENODO_TOKEN",
736
929
  )
737
930
 
931
+ def transfer_flags(p: argparse.ArgumentParser, *, batch: bool = False) -> None:
932
+ """Flags that mean the same thing to ``fetch`` and to ``batch``."""
933
+ if batch:
934
+ p.add_argument(
935
+ "-o",
936
+ "--out",
937
+ help="base directory; every identifier still gets a directory of its "
938
+ "own (default: acadfetch-out)",
939
+ )
940
+ else:
941
+ p.add_argument(
942
+ "-o", "--out", help="output directory (default: acadfetch-out/<source>-<id>)"
943
+ )
944
+ p.add_argument(
945
+ "-j",
946
+ "--concurrency",
947
+ type=_positive,
948
+ default=consts.DEFAULT_CONCURRENCY,
949
+ help="parallel transfers, per record (default: %(default)s)",
950
+ )
951
+ p.add_argument("--max-size", type=parse_size, help="abort above this total, e.g. 500MB")
952
+ p.add_argument(
953
+ "--checksum",
954
+ type=_checksum,
955
+ metavar="ALGO:HEX",
956
+ help="verify against this checksum instead (md5:<hex> or sha256:<hex>), "
957
+ "for sources that publish none; required when migrating a figshare target",
958
+ )
959
+ p.add_argument(
960
+ "--only",
961
+ nargs="+",
962
+ metavar="NAME",
963
+ help="fetch only these file names",
964
+ )
965
+ p.add_argument(
966
+ "--glob",
967
+ nargs="+",
968
+ dest="globs",
969
+ metavar="PATTERN",
970
+ help="fetch only files whose name matches a shell pattern, e.g. '*.pdf'",
971
+ )
972
+ if not batch:
973
+ p.add_argument(
974
+ "-w",
975
+ "--urls",
976
+ metavar="PATH",
977
+ help="write an aria2-compatible URL list instead of downloading; '-' is stdout",
978
+ )
979
+ p.add_argument("--manifest-out", help="also write the manifest here")
980
+ p.add_argument("-n", "--dry-run", action="store_true", help="list what would be fetched")
981
+ p.add_argument("-q", "--quiet", action="store_true", help="no progress line")
982
+ p.add_argument(
983
+ "--force",
984
+ action="store_true",
985
+ help="fetch again even when a matching file is already there",
986
+ )
987
+ p.add_argument(
988
+ "--no-fsync",
989
+ action="store_true",
990
+ help="skip the flush before each rename (faster, weaker against a power cut)",
991
+ )
992
+ p.add_argument(
993
+ "--json",
994
+ action="store_true",
995
+ help="print one JSON report to stdout, progress to stderr",
996
+ )
997
+ p.add_argument(
998
+ "--record-full-url",
999
+ action="store_true",
1000
+ help="keep credential query parameters (X-Amz-Signature, token, …) in "
1001
+ "the manifest and on stdout instead of stripping them",
1002
+ )
1003
+
738
1004
  fetch = sub.add_parser(
739
1005
  "fetch",
740
1006
  help="download every file of a record",
@@ -746,69 +1012,31 @@ def build_parser() -> argparse.ArgumentParser:
746
1012
  nargs="?",
747
1013
  help="DOI, zenodo:<id>, arxiv:<id>, or URL",
748
1014
  )
749
- fetch.add_argument(
750
- "-o", "--out", help="output directory (default: acadfetch-out/<source>-<id>)"
751
- )
752
- fetch.add_argument(
753
- "-j",
754
- "--concurrency",
755
- type=_positive,
756
- default=consts.DEFAULT_CONCURRENCY,
757
- help="parallel transfers (default: %(default)s)",
758
- )
759
- fetch.add_argument("--max-size", type=parse_size, help="abort above this total, e.g. 500MB")
760
- fetch.add_argument(
761
- "--checksum",
762
- type=_checksum,
763
- metavar="ALGO:HEX",
764
- help="verify against this checksum instead (md5:<hex> or sha256:<hex>), "
765
- "for sources that publish none; required when migrating a figshare target",
766
- )
767
- fetch.add_argument(
768
- "--only",
769
- nargs="+",
770
- metavar="NAME",
771
- help="fetch only these file names",
1015
+ transfer_flags(fetch)
1016
+ network_flags(fetch)
1017
+ fetch.set_defaults(func=cmd_fetch)
1018
+
1019
+ bat = sub.add_parser(
1020
+ "batch",
1021
+ help="download every identifier in a file, listing the failures",
1022
+ epilog=EPILOG,
1023
+ formatter_class=argparse.RawDescriptionHelpFormatter,
772
1024
  )
773
- fetch.add_argument(
774
- "--glob",
775
- nargs="+",
776
- dest="globs",
777
- metavar="PATTERN",
778
- help="fetch only files whose name matches a shell pattern, e.g. '*.pdf'",
1025
+ bat.add_argument(
1026
+ "identifiers",
1027
+ metavar="FILE",
1028
+ help="one identifier per line; blank lines and '#' comments are skipped",
779
1029
  )
780
- fetch.add_argument(
781
- "-w",
782
- "--urls",
1030
+ transfer_flags(bat, batch=True)
1031
+ bat.add_argument(
1032
+ "--failed-out",
1033
+ default="failed.txt",
783
1034
  metavar="PATH",
784
- help="write an aria2-compatible URL list instead of downloading; '-' is stdout",
785
- )
786
- fetch.add_argument("--manifest-out", help="also write the manifest here")
787
- fetch.add_argument("-n", "--dry-run", action="store_true", help="list what would be fetched")
788
- fetch.add_argument("-q", "--quiet", action="store_true", help="no progress line")
789
- fetch.add_argument(
790
- "--force",
791
- action="store_true",
792
- help="fetch again even when a matching file is already there",
1035
+ help="write the identifiers that failed here, one per line, for a rerun; "
1036
+ "'-' is stdout (default: failed.txt)",
793
1037
  )
794
- fetch.add_argument(
795
- "--no-fsync",
796
- action="store_true",
797
- help="skip the flush before each rename (faster, weaker against a power cut)",
798
- )
799
- fetch.add_argument(
800
- "--json",
801
- action="store_true",
802
- help="print one JSON report to stdout, progress to stderr",
803
- )
804
- fetch.add_argument(
805
- "--record-full-url",
806
- action="store_true",
807
- help="keep credential query parameters (X-Amz-Signature, token, …) in "
808
- "the manifest and on stdout instead of stripping them",
809
- )
810
- network_flags(fetch)
811
- fetch.set_defaults(func=cmd_fetch)
1038
+ network_flags(bat)
1039
+ bat.set_defaults(func=cmd_batch)
812
1040
 
813
1041
  man = sub.add_parser("manifest", help="write a manifest without downloading")
814
1042
  man.add_argument("target")
@@ -1,6 +1,6 @@
1
1
  """Package-wide constants and version."""
2
2
 
3
- __version__ = "0.7.2"
3
+ __version__ = "0.8.0"
4
4
 
5
5
  USER_AGENT = f"acadfetch/{__version__} (+https://github.com/young-ski/acadfetch)"
6
6
 
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: acadfetch
3
- Version: 0.7.2
3
+ Version: 0.8.0
4
4
  Summary: Resumable, rate-limited, checksum-verified bulk downloads from academic data repositories
5
5
  Author: young-ski
6
6
  License-Expression: MIT
@@ -114,6 +114,23 @@ acadfetch fetch arxiv:1706.03762 # an arXiv id
114
114
  acadfetch fetch https://ndownloader.figshare.com/files/70600233/report.pdf # any URL
115
115
  ```
116
116
 
117
+ Or a whole list, with the failures kept for the next run:
118
+
119
+ ```bash
120
+ acadfetch batch identifiers.txt # one identifier per line
121
+ acadfetch batch identifiers.txt -o ./data --failed-out todo.txt
122
+ ```
123
+
124
+ Blank lines and `#` comments are skipped, so a list can carry notes. Every
125
+ identifier still gets a directory of its own under `-o` (default
126
+ `acadfetch-out`); one bad line does not stop the rest, and the identifiers that
127
+ failed land in `failed.txt` (or `--failed-out`), verbatim and one per line — so
128
+ the retry is `acadfetch batch failed.txt`. The file is replaced even when
129
+ nothing failed, so a stale list never lies. Exit codes keep their meaning: `0`
130
+ all ok, `1` all failed, `2` partial, `130` interrupted, and an interrupt puts the
131
+ lines that were never attempted on the list too. Records are attempted one after
132
+ another; `-j` still applies inside a record.
133
+
117
134
  The first two are answered by Zenodo and arXiv and work from any network those
118
135
  are reachable on. The third is a direct URL, so the host decides: figshare's CDN
119
136
  answers 403 with a WAF block page from some networks — the same behaviour that
@@ -188,7 +205,8 @@ checksum matches, so an interrupted or failed run never leaves something that
188
205
  it resumes. The one case where a run must start over is a server that ignores
189
206
  `Range`; the bytes already on disk are then moved aside and kept until a whole
190
207
  new copy has arrived, so a restart that dies early costs nothing. `acadfetch-out/`
191
- is worth adding to a project's `.gitignore`; this repository already has it.
208
+ is worth adding to a project's `.gitignore`; this repository already has it, and
209
+ names the default `failed.txt` there as well.
192
210
 
193
211
  The default directory names the target, not only its filename — `data.csv` from
194
212
  two hosts is two datasets, and they do not share a directory. A file already
@@ -405,6 +423,19 @@ acadfetch verify ./data/manifest.json # 事后复验;对不上就非
405
423
  > figshare 适配器正是因为这个)。acadfetch 会明说这是拦截页、而不是把 HTML 存成
406
424
  > PDF,并把下一条命令给你。
407
425
 
426
+ 批量清单则一条一条来,失败的留下:
427
+
428
+ ```bash
429
+ acadfetch batch identifiers.txt # 每行一个目标
430
+ acadfetch batch identifiers.txt -o ./data --failed-out todo.txt
431
+ ```
432
+
433
+ 空行与 `#` 注释会被跳过;每个目标仍在 `-o`(默认 `acadfetch-out`)下拿到自己
434
+ 的目录,一行坏掉不影响其余。失败的目标原样写进 `failed.txt`(或
435
+ `--failed-out`),重跑就是 `acadfetch batch failed.txt`;没失败也会清空该文件,
436
+ 不会留下上次的假清单。退出码含义不变:`0` 全成、`1` 全败、`2` 部分、`130`
437
+ 被中断 —— 中断时还没轮到的行也会进清单。
438
+
408
439
  ### 它替你保住的东西
409
440
 
410
441
  | 情况 | 它怎么做 |
@@ -38,6 +38,7 @@ acadfetch/sources/zenodo.py
38
38
  tests/conftest.py
39
39
  tests/test_atomic.py
40
40
  tests/test_auth.py
41
+ tests/test_batch.py
41
42
  tests/test_checksum.py
42
43
  tests/test_citation.py
43
44
  tests/test_claims.py
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "acadfetch"
7
- version = "0.7.2"
7
+ version = "0.8.0"
8
8
  description = "Resumable, rate-limited, checksum-verified bulk downloads from academic data repositories"
9
9
  readme = "README.md"
10
10
  requires-python = ">=3.10"
@@ -0,0 +1,262 @@
1
+ """``batch``: one list in, one failure list out.
2
+
3
+ The command exists so a hundred identifiers is one command, not a shell loop
4
+ that loses the failures. These tests hold it to the two things that make that
5
+ true: a bad line never stops the run, and the failed lines land in a file that
6
+ can be handed straight back.
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ import json
12
+ from pathlib import Path
13
+
14
+ import pytest
15
+
16
+ from acadfetch import cli
17
+ from acadfetch.sources import registry
18
+ from acadfetch.sources.base import File, Record
19
+
20
+
21
+ @pytest.fixture
22
+ def served(server):
23
+ server.cfg["files"]["/a.bin"] = b"alpha payload"
24
+ server.cfg["files"]["/b.bin"] = b"beta"
25
+ return server
26
+
27
+
28
+ def _list(tmp_path: Path, *lines: str) -> Path:
29
+ path = tmp_path / "identifiers.txt"
30
+ path.write_text("\n".join(lines) + "\n", "utf-8")
31
+ return path
32
+
33
+
34
+ def _run(capsys, *argv):
35
+ code = cli.main([str(a) for a in argv])
36
+ out, err = capsys.readouterr()
37
+ return code, out, err
38
+
39
+
40
+ def _downloaded(root: Path) -> set[str]:
41
+ return {path.name for path in root.rglob("*") if path.is_file()}
42
+
43
+
44
+ def test_downloads_every_identifier_and_replaces_the_failed_list(
45
+ served, capsys, tmp_path, monkeypatch
46
+ ):
47
+ monkeypatch.chdir(tmp_path)
48
+ (tmp_path / "failed.txt").write_text("stale-from-last-time\n", "utf-8")
49
+ listing = _list(tmp_path, served.base + "/a.bin", served.base + "/b.bin")
50
+
51
+ code, out, _ = _run(capsys, "batch", listing, "-q")
52
+
53
+ assert code == cli.EXIT_OK
54
+ assert "2/2 identifier(s) ok" in out
55
+ assert (tmp_path / "failed.txt").read_text("utf-8") == ""
56
+ assert {"a.bin", "b.bin"} <= _downloaded(tmp_path / "acadfetch-out")
57
+
58
+
59
+ def test_a_failed_identifier_is_listed_and_the_run_goes_on(served, capsys, tmp_path, monkeypatch):
60
+ monkeypatch.chdir(tmp_path)
61
+ good = served.base + "/a.bin"
62
+ bad = served.base + "/missing.bin"
63
+ listing = _list(tmp_path, good, bad)
64
+
65
+ code, out, err = _run(capsys, "batch", listing, "-q")
66
+
67
+ assert code == cli.EXIT_PARTIAL
68
+ assert (tmp_path / "failed.txt").read_text("utf-8") == bad + "\n"
69
+ assert "a.bin" in _downloaded(tmp_path / "acadfetch-out")
70
+ assert "line 2" in out and "missing.bin" in out
71
+ assert "missing.bin" in err
72
+ assert "1/2 identifier(s) ok, 1 failed" in out
73
+
74
+
75
+ def test_all_failures_are_exit_1(served, capsys, tmp_path, monkeypatch):
76
+ monkeypatch.chdir(tmp_path)
77
+ listing = _list(tmp_path, served.base + "/nope1", served.base + "/nope2")
78
+
79
+ code, _, _ = _run(capsys, "batch", listing, "-q")
80
+
81
+ assert code == cli.EXIT_FAIL
82
+ assert (tmp_path / "failed.txt").read_text("utf-8").count("\n") == 2
83
+
84
+
85
+ def test_comments_blank_lines_bom_and_crlf_are_skipped(served, capsys, tmp_path, monkeypatch):
86
+ """A hand-edited list is the normal case, including one saved by Notepad."""
87
+ monkeypatch.chdir(tmp_path)
88
+ url = served.base + "/a.bin"
89
+ raw = "\ufeff# records for the thesis\n\n " + url + " \r\n# and a note\r\n"
90
+ listing = tmp_path / "identifiers.txt"
91
+ listing.write_bytes(raw.encode("utf-8"))
92
+
93
+ code, _, _ = _run(capsys, "batch", listing, "-q", "--json")
94
+
95
+ assert code == cli.EXIT_OK
96
+ assert (tmp_path / "failed.txt").read_text("utf-8") == ""
97
+
98
+
99
+ def test_an_empty_list_is_exit_1_and_says_so(capsys, tmp_path, monkeypatch):
100
+ monkeypatch.chdir(tmp_path)
101
+ listing = _list(tmp_path, "# nothing yet")
102
+
103
+ code, _, err = _run(capsys, "batch", listing)
104
+
105
+ assert code == cli.EXIT_FAIL
106
+ assert "lists no identifiers" in err
107
+ assert (tmp_path / "failed.txt").exists()
108
+ assert (tmp_path / "failed.txt").read_text("utf-8") == ""
109
+
110
+
111
+ def test_a_missing_list_is_one_line(capsys, tmp_path, monkeypatch):
112
+ monkeypatch.chdir(tmp_path)
113
+ code, _, err = _run(capsys, "batch", tmp_path / "nope.txt")
114
+ assert code == cli.EXIT_FAIL
115
+ assert "nope.txt" in err
116
+ assert "Traceback" not in err
117
+
118
+
119
+ def test_failed_list_is_a_working_retry_input(served, capsys, tmp_path, monkeypatch):
120
+ monkeypatch.chdir(tmp_path)
121
+ listing = _list(tmp_path, served.base + "/a.bin", served.base + "/missing.bin")
122
+ assert _run(capsys, "batch", listing, "-q")[0] == cli.EXIT_PARTIAL
123
+ retry = tmp_path / "failed.txt"
124
+
125
+ # The server now has the file; the list written last time is the input.
126
+ served.cfg["files"]["/missing.bin"] = b"now here"
127
+ code, _, _ = _run(capsys, "batch", retry, "-q")
128
+
129
+ assert code == cli.EXIT_OK
130
+ assert retry.read_text("utf-8") == ""
131
+
132
+
133
+ def test_each_identifier_keeps_its_own_directory(served, capsys, tmp_path, monkeypatch):
134
+ """Two files can share a name; the leaf directory is what keeps them apart."""
135
+ monkeypatch.chdir(tmp_path)
136
+ served.cfg["files"]["/one/same.bin"] = b"one"
137
+ served.cfg["files"]["/two/same.bin"] = b"two"
138
+ listing = _list(tmp_path, served.base + "/one/same.bin", served.base + "/two/same.bin")
139
+
140
+ assert _run(capsys, "batch", listing, "-q", "-o", tmp_path / "data")[0] == cli.EXIT_OK
141
+
142
+ leaves = sorted(p.parent for p in (tmp_path / "data").rglob("same.bin"))
143
+ assert len(leaves) == 2
144
+ assert sorted(p.read_bytes() for p in (tmp_path / "data").rglob("same.bin")) == [b"one", b"two"]
145
+
146
+
147
+ def test_the_failed_list_keeps_the_identifier_verbatim(served, capsys, tmp_path, monkeypatch):
148
+ """A redacted list cannot be replayed; the failure file must not be redacted."""
149
+ monkeypatch.chdir(tmp_path)
150
+ url = served.base + "/missing.bin?token=shh"
151
+ listing = _list(tmp_path, url)
152
+
153
+ assert _run(capsys, "batch", listing, "-q")[0] == cli.EXIT_FAIL
154
+ assert (tmp_path / "failed.txt").read_text("utf-8") == url + "\n"
155
+
156
+
157
+ def test_only_does_not_eat_the_list(served, capsys, tmp_path, monkeypatch):
158
+ """``--only`` is nargs=+; the fetch parser's recovery must not fire here."""
159
+ monkeypatch.chdir(tmp_path)
160
+ listing = _list(tmp_path, served.base + "/a.bin")
161
+ code, _, _ = _run(capsys, "batch", listing, "-q", "--only", "a.bin")
162
+ assert code == cli.EXIT_OK
163
+
164
+
165
+ def test_json_is_one_document_with_a_row_per_identifier(served, capsys, tmp_path, monkeypatch):
166
+ monkeypatch.chdir(tmp_path)
167
+ good = served.base + "/a.bin"
168
+ bad = served.base + "/missing.bin"
169
+ listing = _list(tmp_path, good, bad)
170
+
171
+ code, out, err = _run(capsys, "batch", listing, "-q", "--json")
172
+
173
+ doc = json.loads(out) # all of stdout parses
174
+ assert code == cli.EXIT_PARTIAL
175
+ assert doc["command"] == "batch"
176
+ assert doc["failed_out"] == "failed.txt"
177
+ assert [row["line"] for row in doc["items"]] == [1, 2]
178
+ assert [row["ok"] for row in doc["items"]] == [True, False]
179
+ assert doc["items"][1]["error"]
180
+ assert doc["items"][0]["report"]["command"] == "fetch"
181
+ assert doc["summary"]["failed"] == 1
182
+ assert doc["summary"]["ok_run"] is False
183
+ assert "missing.bin" in err # the per-identifier error still reaches stderr
184
+
185
+
186
+ def test_json_and_a_stdout_failed_list_are_refused(served, capsys, tmp_path, monkeypatch):
187
+ monkeypatch.chdir(tmp_path)
188
+ listing = _list(tmp_path, served.base + "/a.bin")
189
+ code, out, err = _run(capsys, "batch", listing, "-q", "--json", "--failed-out", "-")
190
+ assert code == cli.EXIT_FAIL
191
+ assert "claims stdout" in json.loads(out)["error"]
192
+ assert "claims stdout" in err
193
+
194
+
195
+ def test_interrupt_lists_the_current_and_the_unattempted_lines(monkeypatch, capsys, tmp_path):
196
+ """Rerunning the failure file must cover what the interrupt skipped."""
197
+ monkeypatch.chdir(tmp_path)
198
+
199
+ def fake_resolve(client, target):
200
+ client.source = "zenodo"
201
+ return Record(source="zenodo", target=target, files=[File("https://x/" + target, target)])
202
+
203
+ calls: list[str] = []
204
+
205
+ def fake_fetch(record, opts, client, only=None, globs=None):
206
+ calls.append(record.target)
207
+ if record.target == "two":
208
+ raise KeyboardInterrupt
209
+ return []
210
+
211
+ monkeypatch.setattr(registry, "resolve", fake_resolve)
212
+ monkeypatch.setattr(cli, "fetch_record", fake_fetch)
213
+ listing = _list(tmp_path, "one", "two", "three")
214
+
215
+ code = cli.main(["batch", str(listing), "-q", "-o", str(tmp_path / "out")])
216
+
217
+ assert code == cli.EXIT_INTERRUPT
218
+ assert calls == ["one", "two"]
219
+ assert (tmp_path / "failed.txt").read_text("utf-8") == "two\nthree\n"
220
+
221
+
222
+ def test_a_filter_matching_nothing_is_that_identifier_failing(
223
+ served, capsys, tmp_path, monkeypatch
224
+ ):
225
+ monkeypatch.chdir(tmp_path)
226
+ listing = _list(tmp_path, served.base + "/a.bin", served.base + "/b.bin")
227
+
228
+ code, _, err = _run(capsys, "batch", listing, "-q", "--only", "nope")
229
+
230
+ assert code == cli.EXIT_FAIL
231
+ assert err.count("no file matched") == 2
232
+ assert (tmp_path / "failed.txt").read_text("utf-8").count("\n") == 2
233
+
234
+
235
+ def test_failed_out_creates_the_directory(served, capsys, tmp_path, monkeypatch):
236
+ monkeypatch.chdir(tmp_path)
237
+ listing = _list(tmp_path, served.base + "/missing.bin")
238
+
239
+ assert _run(capsys, "batch", listing, "-q", "--failed-out", "logs/failed.txt")[0] == (
240
+ cli.EXIT_FAIL
241
+ )
242
+ assert "missing.bin" in (tmp_path / "logs" / "failed.txt").read_text("utf-8")
243
+
244
+
245
+ def test_failed_out_dash_puts_the_list_on_stdout(served, capsys, tmp_path, monkeypatch):
246
+ monkeypatch.chdir(tmp_path)
247
+ bad = served.base + "/missing.bin"
248
+ listing = _list(tmp_path, bad)
249
+
250
+ code, out, _ = _run(capsys, "batch", listing, "-q", "--failed-out", "-")
251
+
252
+ assert code == cli.EXIT_FAIL
253
+ assert bad in out.splitlines(), "the list is the last thing printed"
254
+ assert not (tmp_path / "failed.txt").exists()
255
+
256
+
257
+ def test_out_base_is_used_for_every_identifier(served, capsys, tmp_path, monkeypatch):
258
+ monkeypatch.chdir(tmp_path)
259
+ listing = _list(tmp_path, served.base + "/a.bin")
260
+ assert _run(capsys, "batch", listing, "-q", "-o", tmp_path / "data")[0] == cli.EXIT_OK
261
+ assert list((tmp_path / "data").glob("direct-*/a.bin"))
262
+ assert not (tmp_path / "acadfetch-out").exists()
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes