acadfetch 0.7.2__tar.gz → 0.8.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {acadfetch-0.7.2/acadfetch.egg-info → acadfetch-0.8.0}/PKG-INFO +33 -2
- {acadfetch-0.7.2 → acadfetch-0.8.0}/README.md +32 -1
- {acadfetch-0.7.2 → acadfetch-0.8.0}/acadfetch/cli.py +302 -74
- {acadfetch-0.7.2 → acadfetch-0.8.0}/acadfetch/consts.py +1 -1
- {acadfetch-0.7.2 → acadfetch-0.8.0/acadfetch.egg-info}/PKG-INFO +33 -2
- {acadfetch-0.7.2 → acadfetch-0.8.0}/acadfetch.egg-info/SOURCES.txt +1 -0
- {acadfetch-0.7.2 → acadfetch-0.8.0}/pyproject.toml +1 -1
- acadfetch-0.8.0/tests/test_batch.py +262 -0
- {acadfetch-0.7.2 → acadfetch-0.8.0}/LICENSE +0 -0
- {acadfetch-0.7.2 → acadfetch-0.8.0}/MANIFEST.in +0 -0
- {acadfetch-0.7.2 → acadfetch-0.8.0}/acadfetch/__init__.py +0 -0
- {acadfetch-0.7.2 → acadfetch-0.8.0}/acadfetch/__main__.py +0 -0
- {acadfetch-0.7.2 → acadfetch-0.8.0}/acadfetch/atomic.py +0 -0
- {acadfetch-0.7.2 → acadfetch-0.8.0}/acadfetch/auth.py +0 -0
- {acadfetch-0.7.2 → acadfetch-0.8.0}/acadfetch/client.py +0 -0
- {acadfetch-0.7.2 → acadfetch-0.8.0}/acadfetch/debug.py +0 -0
- {acadfetch-0.7.2 → acadfetch-0.8.0}/acadfetch/downloader.py +0 -0
- {acadfetch-0.7.2 → acadfetch-0.8.0}/acadfetch/errors.py +0 -0
- {acadfetch-0.7.2 → acadfetch-0.8.0}/acadfetch/manifest.py +0 -0
- {acadfetch-0.7.2 → acadfetch-0.8.0}/acadfetch/naming.py +0 -0
- {acadfetch-0.7.2 → acadfetch-0.8.0}/acadfetch/pooch.py +0 -0
- {acadfetch-0.7.2 → acadfetch-0.8.0}/acadfetch/progress.py +0 -0
- {acadfetch-0.7.2 → acadfetch-0.8.0}/acadfetch/py.typed +0 -0
- {acadfetch-0.7.2 → acadfetch-0.8.0}/acadfetch/ratelimit.py +0 -0
- {acadfetch-0.7.2 → acadfetch-0.8.0}/acadfetch/remote.py +0 -0
- {acadfetch-0.7.2 → acadfetch-0.8.0}/acadfetch/sources/__init__.py +0 -0
- {acadfetch-0.7.2 → acadfetch-0.8.0}/acadfetch/sources/arxiv.py +0 -0
- {acadfetch-0.7.2 → acadfetch-0.8.0}/acadfetch/sources/base.py +0 -0
- {acadfetch-0.7.2 → acadfetch-0.8.0}/acadfetch/sources/direct.py +0 -0
- {acadfetch-0.7.2 → acadfetch-0.8.0}/acadfetch/sources/doi.py +0 -0
- {acadfetch-0.7.2 → acadfetch-0.8.0}/acadfetch/sources/registry.py +0 -0
- {acadfetch-0.7.2 → acadfetch-0.8.0}/acadfetch/sources/removed.py +0 -0
- {acadfetch-0.7.2 → acadfetch-0.8.0}/acadfetch/sources/zenodo.py +0 -0
- {acadfetch-0.7.2 → acadfetch-0.8.0}/acadfetch/units.py +0 -0
- {acadfetch-0.7.2 → acadfetch-0.8.0}/acadfetch/urls.py +0 -0
- {acadfetch-0.7.2 → acadfetch-0.8.0}/acadfetch/verify.py +0 -0
- {acadfetch-0.7.2 → acadfetch-0.8.0}/acadfetch.egg-info/dependency_links.txt +0 -0
- {acadfetch-0.7.2 → acadfetch-0.8.0}/acadfetch.egg-info/entry_points.txt +0 -0
- {acadfetch-0.7.2 → acadfetch-0.8.0}/acadfetch.egg-info/top_level.txt +0 -0
- {acadfetch-0.7.2 → acadfetch-0.8.0}/setup.cfg +0 -0
- {acadfetch-0.7.2 → acadfetch-0.8.0}/tests/conftest.py +0 -0
- {acadfetch-0.7.2 → acadfetch-0.8.0}/tests/fixtures/README.md +0 -0
- {acadfetch-0.7.2 → acadfetch-0.8.0}/tests/fixtures/ca.pem +0 -0
- {acadfetch-0.7.2 → acadfetch-0.8.0}/tests/fixtures/figshare_403_block.html +0 -0
- {acadfetch-0.7.2 → acadfetch-0.8.0}/tests/fixtures/figshare_403_forbidden.json +0 -0
- {acadfetch-0.7.2 → acadfetch-0.8.0}/tests/fixtures/figshare_article.json +0 -0
- {acadfetch-0.7.2 → acadfetch-0.8.0}/tests/fixtures/figshare_file.pdf +0 -0
- {acadfetch-0.7.2 → acadfetch-0.8.0}/tests/test_atomic.py +0 -0
- {acadfetch-0.7.2 → acadfetch-0.8.0}/tests/test_auth.py +0 -0
- {acadfetch-0.7.2 → acadfetch-0.8.0}/tests/test_checksum.py +0 -0
- {acadfetch-0.7.2 → acadfetch-0.8.0}/tests/test_citation.py +0 -0
- {acadfetch-0.7.2 → acadfetch-0.8.0}/tests/test_claims.py +0 -0
- {acadfetch-0.7.2 → acadfetch-0.8.0}/tests/test_cli.py +0 -0
- {acadfetch-0.7.2 → acadfetch-0.8.0}/tests/test_client.py +0 -0
- {acadfetch-0.7.2 → acadfetch-0.8.0}/tests/test_control_names.py +0 -0
- {acadfetch-0.7.2 → acadfetch-0.8.0}/tests/test_debug.py +0 -0
- {acadfetch-0.7.2 → acadfetch-0.8.0}/tests/test_disk.py +0 -0
- {acadfetch-0.7.2 → acadfetch-0.8.0}/tests/test_downloader.py +0 -0
- {acadfetch-0.7.2 → acadfetch-0.8.0}/tests/test_hash_once.py +0 -0
- {acadfetch-0.7.2 → acadfetch-0.8.0}/tests/test_identity.py +0 -0
- {acadfetch-0.7.2 → acadfetch-0.8.0}/tests/test_isolation.py +0 -0
- {acadfetch-0.7.2 → acadfetch-0.8.0}/tests/test_json.py +0 -0
- {acadfetch-0.7.2 → acadfetch-0.8.0}/tests/test_lock.py +0 -0
- {acadfetch-0.7.2 → acadfetch-0.8.0}/tests/test_manifest.py +0 -0
- {acadfetch-0.7.2 → acadfetch-0.8.0}/tests/test_naming.py +0 -0
- {acadfetch-0.7.2 → acadfetch-0.8.0}/tests/test_naming_race.py +0 -0
- {acadfetch-0.7.2 → acadfetch-0.8.0}/tests/test_network.py +0 -0
- {acadfetch-0.7.2 → acadfetch-0.8.0}/tests/test_packaging.py +0 -0
- {acadfetch-0.7.2 → acadfetch-0.8.0}/tests/test_pooch.py +0 -0
- {acadfetch-0.7.2 → acadfetch-0.8.0}/tests/test_progress.py +0 -0
- {acadfetch-0.7.2 → acadfetch-0.8.0}/tests/test_ratelimit.py +0 -0
- {acadfetch-0.7.2 → acadfetch-0.8.0}/tests/test_redirect.py +0 -0
- {acadfetch-0.7.2 → acadfetch-0.8.0}/tests/test_remote.py +0 -0
- {acadfetch-0.7.2 → acadfetch-0.8.0}/tests/test_removed.py +0 -0
- {acadfetch-0.7.2 → acadfetch-0.8.0}/tests/test_sources.py +0 -0
- {acadfetch-0.7.2 → acadfetch-0.8.0}/tests/test_urls.py +0 -0
- {acadfetch-0.7.2 → acadfetch-0.8.0}/tests/test_urls_glob.py +0 -0
- {acadfetch-0.7.2 → acadfetch-0.8.0}/tests/test_verify.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: acadfetch
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.8.0
|
|
4
4
|
Summary: Resumable, rate-limited, checksum-verified bulk downloads from academic data repositories
|
|
5
5
|
Author: young-ski
|
|
6
6
|
License-Expression: MIT
|
|
@@ -114,6 +114,23 @@ acadfetch fetch arxiv:1706.03762 # an arXiv id
|
|
|
114
114
|
acadfetch fetch https://ndownloader.figshare.com/files/70600233/report.pdf # any URL
|
|
115
115
|
```
|
|
116
116
|
|
|
117
|
+
Or a whole list, with the failures kept for the next run:
|
|
118
|
+
|
|
119
|
+
```bash
|
|
120
|
+
acadfetch batch identifiers.txt # one identifier per line
|
|
121
|
+
acadfetch batch identifiers.txt -o ./data --failed-out todo.txt
|
|
122
|
+
```
|
|
123
|
+
|
|
124
|
+
Blank lines and `#` comments are skipped, so a list can carry notes. Every
|
|
125
|
+
identifier still gets a directory of its own under `-o` (default
|
|
126
|
+
`acadfetch-out`); one bad line does not stop the rest, and the identifiers that
|
|
127
|
+
failed land in `failed.txt` (or `--failed-out`), verbatim and one per line — so
|
|
128
|
+
the retry is `acadfetch batch failed.txt`. The file is replaced even when
|
|
129
|
+
nothing failed, so a stale list never lies. Exit codes keep their meaning: `0`
|
|
130
|
+
all ok, `1` all failed, `2` partial, `130` interrupted, and an interrupt puts the
|
|
131
|
+
lines that were never attempted on the list too. Records are attempted one after
|
|
132
|
+
another; `-j` still applies inside a record.
|
|
133
|
+
|
|
117
134
|
The first two are answered by Zenodo and arXiv and work from any network those
|
|
118
135
|
are reachable on. The third is a direct URL, so the host decides: figshare's CDN
|
|
119
136
|
answers 403 with a WAF block page from some networks — the same behaviour that
|
|
@@ -188,7 +205,8 @@ checksum matches, so an interrupted or failed run never leaves something that
|
|
|
188
205
|
it resumes. The one case where a run must start over is a server that ignores
|
|
189
206
|
`Range`; the bytes already on disk are then moved aside and kept until a whole
|
|
190
207
|
new copy has arrived, so a restart that dies early costs nothing. `acadfetch-out/`
|
|
191
|
-
is worth adding to a project's `.gitignore`; this repository already has it
|
|
208
|
+
is worth adding to a project's `.gitignore`; this repository already has it, and
|
|
209
|
+
names the default `failed.txt` there as well.
|
|
192
210
|
|
|
193
211
|
The default directory names the target, not only its filename — `data.csv` from
|
|
194
212
|
two hosts is two datasets, and they do not share a directory. A file already
|
|
@@ -405,6 +423,19 @@ acadfetch verify ./data/manifest.json # 事后复验;对不上就非
|
|
|
405
423
|
> figshare 适配器正是因为这个)。acadfetch 会明说这是拦截页、而不是把 HTML 存成
|
|
406
424
|
> PDF,并把下一条命令给你。
|
|
407
425
|
|
|
426
|
+
批量清单则一条一条来,失败的留下:
|
|
427
|
+
|
|
428
|
+
```bash
|
|
429
|
+
acadfetch batch identifiers.txt # 每行一个目标
|
|
430
|
+
acadfetch batch identifiers.txt -o ./data --failed-out todo.txt
|
|
431
|
+
```
|
|
432
|
+
|
|
433
|
+
空行与 `#` 注释会被跳过;每个目标仍在 `-o`(默认 `acadfetch-out`)下拿到自己
|
|
434
|
+
的目录,一行坏掉不影响其余。失败的目标原样写进 `failed.txt`(或
|
|
435
|
+
`--failed-out`),重跑就是 `acadfetch batch failed.txt`;没失败也会清空该文件,
|
|
436
|
+
不会留下上次的假清单。退出码含义不变:`0` 全成、`1` 全败、`2` 部分、`130`
|
|
437
|
+
被中断 —— 中断时还没轮到的行也会进清单。
|
|
438
|
+
|
|
408
439
|
### 它替你保住的东西
|
|
409
440
|
|
|
410
441
|
| 情况 | 它怎么做 |
|
|
@@ -83,6 +83,23 @@ acadfetch fetch arxiv:1706.03762 # an arXiv id
|
|
|
83
83
|
acadfetch fetch https://ndownloader.figshare.com/files/70600233/report.pdf # any URL
|
|
84
84
|
```
|
|
85
85
|
|
|
86
|
+
Or a whole list, with the failures kept for the next run:
|
|
87
|
+
|
|
88
|
+
```bash
|
|
89
|
+
acadfetch batch identifiers.txt # one identifier per line
|
|
90
|
+
acadfetch batch identifiers.txt -o ./data --failed-out todo.txt
|
|
91
|
+
```
|
|
92
|
+
|
|
93
|
+
Blank lines and `#` comments are skipped, so a list can carry notes. Every
|
|
94
|
+
identifier still gets a directory of its own under `-o` (default
|
|
95
|
+
`acadfetch-out`); one bad line does not stop the rest, and the identifiers that
|
|
96
|
+
failed land in `failed.txt` (or `--failed-out`), verbatim and one per line — so
|
|
97
|
+
the retry is `acadfetch batch failed.txt`. The file is replaced even when
|
|
98
|
+
nothing failed, so a stale list never lies. Exit codes keep their meaning: `0`
|
|
99
|
+
all ok, `1` all failed, `2` partial, `130` interrupted, and an interrupt puts the
|
|
100
|
+
lines that were never attempted on the list too. Records are attempted one after
|
|
101
|
+
another; `-j` still applies inside a record.
|
|
102
|
+
|
|
86
103
|
The first two are answered by Zenodo and arXiv and work from any network those
|
|
87
104
|
are reachable on. The third is a direct URL, so the host decides: figshare's CDN
|
|
88
105
|
answers 403 with a WAF block page from some networks — the same behaviour that
|
|
@@ -157,7 +174,8 @@ checksum matches, so an interrupted or failed run never leaves something that
|
|
|
157
174
|
it resumes. The one case where a run must start over is a server that ignores
|
|
158
175
|
`Range`; the bytes already on disk are then moved aside and kept until a whole
|
|
159
176
|
new copy has arrived, so a restart that dies early costs nothing. `acadfetch-out/`
|
|
160
|
-
is worth adding to a project's `.gitignore`; this repository already has it
|
|
177
|
+
is worth adding to a project's `.gitignore`; this repository already has it, and
|
|
178
|
+
names the default `failed.txt` there as well.
|
|
161
179
|
|
|
162
180
|
The default directory names the target, not only its filename — `data.csv` from
|
|
163
181
|
two hosts is two datasets, and they do not share a directory. A file already
|
|
@@ -374,6 +392,19 @@ acadfetch verify ./data/manifest.json # 事后复验;对不上就非
|
|
|
374
392
|
> figshare 适配器正是因为这个)。acadfetch 会明说这是拦截页、而不是把 HTML 存成
|
|
375
393
|
> PDF,并把下一条命令给你。
|
|
376
394
|
|
|
395
|
+
批量清单则一条一条来,失败的留下:
|
|
396
|
+
|
|
397
|
+
```bash
|
|
398
|
+
acadfetch batch identifiers.txt # 每行一个目标
|
|
399
|
+
acadfetch batch identifiers.txt -o ./data --failed-out todo.txt
|
|
400
|
+
```
|
|
401
|
+
|
|
402
|
+
空行与 `#` 注释会被跳过;每个目标仍在 `-o`(默认 `acadfetch-out`)下拿到自己
|
|
403
|
+
的目录,一行坏掉不影响其余。失败的目标原样写进 `failed.txt`(或
|
|
404
|
+
`--failed-out`),重跑就是 `acadfetch batch failed.txt`;没失败也会清空该文件,
|
|
405
|
+
不会留下上次的假清单。退出码含义不变:`0` 全成、`1` 全败、`2` 部分、`130`
|
|
406
|
+
被中断 —— 中断时还没轮到的行也会进清单。
|
|
407
|
+
|
|
377
408
|
### 它替你保住的东西
|
|
378
409
|
|
|
379
410
|
| 情况 | 它怎么做 |
|
|
@@ -18,7 +18,7 @@ from pathlib import Path
|
|
|
18
18
|
from typing import NoReturn
|
|
19
19
|
from urllib.parse import urlsplit
|
|
20
20
|
|
|
21
|
-
from . import auth, consts, debug, manifest, naming, urls, verify
|
|
21
|
+
from . import atomic, auth, consts, debug, manifest, naming, urls, verify
|
|
22
22
|
from . import remote as remote_check
|
|
23
23
|
from .client import Client
|
|
24
24
|
from .downloader import Options, check_disk, check_total, fetch_record, human
|
|
@@ -183,8 +183,11 @@ def _recover_target(args: argparse.Namespace) -> None:
|
|
|
183
183
|
"""``--only a.pdf <target>`` makes argparse eat the target; put it back.
|
|
184
184
|
|
|
185
185
|
Checked only when the target is missing, so a file that happens to look
|
|
186
|
-
like an id keeps working as long as the target is spelled out.
|
|
186
|
+
like an id keeps working as long as the target is spelled out. ``batch``
|
|
187
|
+
has no positional target, so its ``--only`` must not be mistaken for one.
|
|
187
188
|
"""
|
|
189
|
+
if getattr(args, "command", "") != "fetch":
|
|
190
|
+
return
|
|
188
191
|
if getattr(args, "target", None) is not None or not getattr(args, "only", None):
|
|
189
192
|
return
|
|
190
193
|
if len(args.only) > 1 and _looks_like_target(args.only[-1]):
|
|
@@ -306,7 +309,7 @@ def _write_urls(files, out_dir, target) -> None: # noqa: ANN001
|
|
|
306
309
|
print(f"{len(files)} url(s) -> {target}", file=sys.stderr)
|
|
307
310
|
|
|
308
311
|
|
|
309
|
-
def _no_match(record, args) -> int: # noqa: ANN001
|
|
312
|
+
def _no_match(record, args, emit: bool = True) -> int: # noqa: ANN001
|
|
310
313
|
"""A filter that matched nothing is a failure that can name the candidates."""
|
|
311
314
|
available = ", ".join(naming.sanitize(f.name) for f in record.files[:8]) or "(none)"
|
|
312
315
|
wanted = " ".join(
|
|
@@ -319,7 +322,8 @@ def _no_match(record, args) -> int: # noqa: ANN001
|
|
|
319
322
|
)
|
|
320
323
|
message = f"no file matched {wanted}; available: {available}"
|
|
321
324
|
print(f"acadfetch: {message}", file=sys.stderr)
|
|
322
|
-
|
|
325
|
+
if emit:
|
|
326
|
+
_emit_error(args, message)
|
|
323
327
|
return EXIT_FAIL
|
|
324
328
|
|
|
325
329
|
|
|
@@ -330,8 +334,17 @@ def _selected(record, args): # noqa: ANN001, ANN202
|
|
|
330
334
|
|
|
331
335
|
|
|
332
336
|
def cmd_fetch(args: argparse.Namespace) -> int:
|
|
337
|
+
code, _ = _fetch_one(args, _client(args))
|
|
338
|
+
return code
|
|
339
|
+
|
|
340
|
+
|
|
341
|
+
def _fetch_one(args: argparse.Namespace, client: Client, emit: bool = True) -> tuple[int, dict]:
|
|
342
|
+
"""Resolve and fetch one target. Returns the exit code and the report.
|
|
343
|
+
|
|
344
|
+
``emit`` is false when the caller owns stdout (``batch``): it collects these
|
|
345
|
+
documents and prints one of its own at the end.
|
|
346
|
+
"""
|
|
333
347
|
started = time.monotonic()
|
|
334
|
-
client = _client(args)
|
|
335
348
|
record = _resolve(args, client)
|
|
336
349
|
if args.checksum:
|
|
337
350
|
# Explicit wins over whatever the source published, and over nothing.
|
|
@@ -340,6 +353,8 @@ def cmd_fetch(args: argparse.Namespace) -> int:
|
|
|
340
353
|
files=[replace(f, checksum=args.checksum, checksum_source="cli") for f in record.files],
|
|
341
354
|
)
|
|
342
355
|
doc = _report(args)
|
|
356
|
+
# The document keeps its shape even when the caller is a batch.
|
|
357
|
+
doc["command"] = "fetch"
|
|
343
358
|
doc.update(
|
|
344
359
|
target=_hide(record.target, args),
|
|
345
360
|
source=record.source,
|
|
@@ -356,7 +371,14 @@ def cmd_fetch(args: argparse.Namespace) -> int:
|
|
|
356
371
|
if token and not args.json:
|
|
357
372
|
print(f"auth : token from {token}", file=header)
|
|
358
373
|
|
|
359
|
-
|
|
374
|
+
# A batch passes its base directory in -o: every identifier still gets a
|
|
375
|
+
# directory of its own, because two records must not share one.
|
|
376
|
+
if args.out and getattr(args, "command", "") == "batch":
|
|
377
|
+
out_dir = Path(args.out) / f"{record.source}-{_slug(record)}"
|
|
378
|
+
elif args.out:
|
|
379
|
+
out_dir = Path(args.out)
|
|
380
|
+
else:
|
|
381
|
+
out_dir = Path(f"acadfetch-out/{record.source}-{_slug(record)}")
|
|
360
382
|
doc["out"] = str(out_dir)
|
|
361
383
|
doc["dry_run"] = bool(args.dry_run)
|
|
362
384
|
|
|
@@ -365,9 +387,9 @@ def cmd_fetch(args: argparse.Namespace) -> int:
|
|
|
365
387
|
raise TargetError("--urls writes a text list; drop --json")
|
|
366
388
|
chosen = _selected(record, args)
|
|
367
389
|
if not chosen:
|
|
368
|
-
return _no_match(record, args)
|
|
390
|
+
return _no_match(record, args, emit), doc
|
|
369
391
|
_write_urls(chosen, out_dir, args.urls)
|
|
370
|
-
return EXIT_OK
|
|
392
|
+
return EXIT_OK, doc
|
|
371
393
|
# Say whether a credential was stripped, so a script can tell the manifest
|
|
372
394
|
# is not replayable as-is.
|
|
373
395
|
doc["url_redacted"] = record.target != _hide(record.target, args) or record.landing != _hide(
|
|
@@ -390,7 +412,7 @@ def cmd_fetch(args: argparse.Namespace) -> int:
|
|
|
390
412
|
if args.dry_run:
|
|
391
413
|
chosen = _selected(record, args)
|
|
392
414
|
if not chosen:
|
|
393
|
-
return _no_match(record, args)
|
|
415
|
+
return _no_match(record, args, emit), doc
|
|
394
416
|
if args.max_size:
|
|
395
417
|
check_total(replace(record, files=chosen), args.max_size)
|
|
396
418
|
try:
|
|
@@ -418,8 +440,9 @@ def cmd_fetch(args: argparse.Namespace) -> int:
|
|
|
418
440
|
doc["disk"] = disk
|
|
419
441
|
doc["summary"] = {"total": len(chosen), "downloaded": 0, "skipped": 0, "failed": 0}
|
|
420
442
|
doc["elapsed_s"] = round(time.monotonic() - started, 3)
|
|
421
|
-
|
|
422
|
-
|
|
443
|
+
if emit:
|
|
444
|
+
_emit(doc)
|
|
445
|
+
return EXIT_OK, doc
|
|
423
446
|
print(f"would write to {out_dir}")
|
|
424
447
|
if needed:
|
|
425
448
|
print(
|
|
@@ -431,12 +454,12 @@ def cmd_fetch(args: argparse.Namespace) -> int:
|
|
|
431
454
|
# size, so say that instead of printing a question mark.
|
|
432
455
|
size = human(file.size) if file.size is not None else "size not published"
|
|
433
456
|
print(f" would fetch {naming.sanitize(file.name)} ({size})")
|
|
434
|
-
return EXIT_OK
|
|
457
|
+
return EXIT_OK, doc
|
|
435
458
|
|
|
436
459
|
try:
|
|
437
460
|
results = fetch_record(record, opts, client, only=args.only, globs=args.globs)
|
|
438
461
|
except PartialError:
|
|
439
|
-
return _no_match(record, args)
|
|
462
|
+
return _no_match(record, args, emit), doc
|
|
440
463
|
finally:
|
|
441
464
|
progress.finish()
|
|
442
465
|
|
|
@@ -503,9 +526,9 @@ def cmd_fetch(args: argparse.Namespace) -> int:
|
|
|
503
526
|
elif failures:
|
|
504
527
|
code = EXIT_PARTIAL
|
|
505
528
|
doc["summary"]["ok"] = code == EXIT_OK
|
|
506
|
-
if args.json:
|
|
529
|
+
if emit and args.json:
|
|
507
530
|
_emit(doc)
|
|
508
|
-
return code
|
|
531
|
+
return code, doc
|
|
509
532
|
|
|
510
533
|
|
|
511
534
|
def cmd_verify(args: argparse.Namespace) -> int:
|
|
@@ -644,6 +667,176 @@ def _slug(record) -> str: # noqa: ANN001
|
|
|
644
667
|
return f"{host}-{digest}-{name}"
|
|
645
668
|
|
|
646
669
|
|
|
670
|
+
def _read_identifiers(path: Path) -> list[tuple[int, str]]:
|
|
671
|
+
"""Non-blank, non-comment lines, with the line number they came from.
|
|
672
|
+
|
|
673
|
+
``utf-8-sig`` because a Windows editor happily adds a BOM, and a BOM on the
|
|
674
|
+
first identifier otherwise makes exactly that one unresolvable. ``#`` starts
|
|
675
|
+
a comment so a list can carry notes and still be fed back as a retry list.
|
|
676
|
+
"""
|
|
677
|
+
try:
|
|
678
|
+
text = path.read_text("utf-8-sig")
|
|
679
|
+
except FileNotFoundError:
|
|
680
|
+
raise TargetError(f"no such identifiers file: {path}") from None
|
|
681
|
+
except IsADirectoryError:
|
|
682
|
+
raise TargetError(f"{path} is a directory") from None
|
|
683
|
+
except UnicodeDecodeError as exc:
|
|
684
|
+
raise TargetError(f"{path} is not UTF-8: {exc}") from None
|
|
685
|
+
return [
|
|
686
|
+
(number, item)
|
|
687
|
+
for number, line in enumerate(text.splitlines(), 1)
|
|
688
|
+
if (item := line.strip()) and not item.startswith("#")
|
|
689
|
+
]
|
|
690
|
+
|
|
691
|
+
|
|
692
|
+
def _write_failed(path: Path, targets: list[str]) -> None:
|
|
693
|
+
"""Replace the failure list, whole and one identifier per line.
|
|
694
|
+
|
|
695
|
+
Written even when nothing failed: an empty file says "nothing to retry",
|
|
696
|
+
and last run's list would say the opposite. Same-name ``.part`` then a
|
|
697
|
+
replace, so a crash never leaves half a list that reads as complete.
|
|
698
|
+
"""
|
|
699
|
+
text = "".join(f"{target}\n" for target in targets)
|
|
700
|
+
if str(path) == "-":
|
|
701
|
+
sys.stdout.write(text)
|
|
702
|
+
return
|
|
703
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
704
|
+
tmp = path.with_name(path.name + ".part")
|
|
705
|
+
tmp.write_text(text, "utf-8")
|
|
706
|
+
atomic.replace(tmp, path)
|
|
707
|
+
|
|
708
|
+
|
|
709
|
+
def cmd_batch(args: argparse.Namespace) -> int:
|
|
710
|
+
"""Fetch every identifier of a file; keep the failures for the next run.
|
|
711
|
+
|
|
712
|
+
One bad identifier is one line of output, not the end of the run. The exit
|
|
713
|
+
code keeps its meaning: 0 all ok, 1 all failed, 2 partial, 130 interrupted -
|
|
714
|
+
and on an interrupt the identifiers that were never attempted join the list,
|
|
715
|
+
so a single rerun covers them too.
|
|
716
|
+
"""
|
|
717
|
+
started = time.monotonic()
|
|
718
|
+
source_path = Path(args.identifiers)
|
|
719
|
+
failed_path = Path(args.failed_out)
|
|
720
|
+
# _fetch_one reads both; neither is a batch-level decision any more.
|
|
721
|
+
args.urls = None
|
|
722
|
+
args.manifest_out = None
|
|
723
|
+
if args.json and args.failed_out == "-":
|
|
724
|
+
raise TargetError("--json claims stdout; write --failed-out to a file")
|
|
725
|
+
base = args.out or "acadfetch-out"
|
|
726
|
+
doc = _report(args)
|
|
727
|
+
doc.update(identifiers=str(source_path), out=base, failed_out=str(failed_path))
|
|
728
|
+
items: list[dict] = []
|
|
729
|
+
doc["items"] = items
|
|
730
|
+
entries = _read_identifiers(source_path)
|
|
731
|
+
summary = doc["summary"] = {
|
|
732
|
+
"total": len(entries),
|
|
733
|
+
"ok": 0,
|
|
734
|
+
"failed": 0,
|
|
735
|
+
"interrupted": 0,
|
|
736
|
+
"ok_run": False,
|
|
737
|
+
}
|
|
738
|
+
doc["elapsed_s"] = 0.0
|
|
739
|
+
|
|
740
|
+
if not entries:
|
|
741
|
+
message = f"{source_path} lists no identifiers"
|
|
742
|
+
print(f"acadfetch: {message}", file=sys.stderr)
|
|
743
|
+
doc["error"] = message
|
|
744
|
+
_write_failed(failed_path, [])
|
|
745
|
+
if args.json:
|
|
746
|
+
_emit(doc)
|
|
747
|
+
return EXIT_FAIL
|
|
748
|
+
|
|
749
|
+
client = _client(args) # one client: the rate limit holds across the batch
|
|
750
|
+
if not args.json:
|
|
751
|
+
print(f"{len(entries)} identifier(s) -> {base}")
|
|
752
|
+
failures: list[str] = []
|
|
753
|
+
for index, (line_no, target) in enumerate(entries, 1):
|
|
754
|
+
args.target = target
|
|
755
|
+
label = _hide_text(target, args)
|
|
756
|
+
if not args.json:
|
|
757
|
+
print(f"==> [{index}/{len(entries)}] {label}")
|
|
758
|
+
item = {
|
|
759
|
+
"line": line_no,
|
|
760
|
+
"target": label,
|
|
761
|
+
"code": EXIT_OK,
|
|
762
|
+
"out": None,
|
|
763
|
+
"error": None,
|
|
764
|
+
"report": None,
|
|
765
|
+
}
|
|
766
|
+
items.append(item)
|
|
767
|
+
code = EXIT_OK
|
|
768
|
+
try:
|
|
769
|
+
code, report_doc = _fetch_one(args, client, emit=False)
|
|
770
|
+
except KeyboardInterrupt:
|
|
771
|
+
code, report_doc = EXIT_INTERRUPT, None
|
|
772
|
+
item["error"] = "interrupted"
|
|
773
|
+
except (AcFError, OSError) as exc:
|
|
774
|
+
debug.log_exception(exc)
|
|
775
|
+
code, report_doc = EXIT_FAIL, None
|
|
776
|
+
item["error"] = _hide_text(str(exc), args)
|
|
777
|
+
print(f"acadfetch: {item['error']}", file=sys.stderr)
|
|
778
|
+
else:
|
|
779
|
+
assert report_doc is not None
|
|
780
|
+
item["out"] = report_doc["out"]
|
|
781
|
+
item["error"] = report_doc["error"] or None
|
|
782
|
+
item["report"] = report_doc
|
|
783
|
+
item["code"] = code
|
|
784
|
+
item["ok"] = code == EXIT_OK
|
|
785
|
+
if code != EXIT_OK:
|
|
786
|
+
failures.append(target)
|
|
787
|
+
if code == EXIT_INTERRUPT:
|
|
788
|
+
print("\ninterrupted - partial files kept, rerun to resume", file=sys.stderr)
|
|
789
|
+
# The lines never reached belong on the retry list, not in an
|
|
790
|
+
# unaccounted gap between "done" and "not asked for".
|
|
791
|
+
for rest_line, rest_target in entries[index:]:
|
|
792
|
+
failures.append(rest_target)
|
|
793
|
+
items.append(
|
|
794
|
+
{
|
|
795
|
+
"line": rest_line,
|
|
796
|
+
"target": _hide_text(rest_target, args),
|
|
797
|
+
"code": EXIT_INTERRUPT,
|
|
798
|
+
"ok": False,
|
|
799
|
+
"out": None,
|
|
800
|
+
"error": "not attempted",
|
|
801
|
+
"report": None,
|
|
802
|
+
}
|
|
803
|
+
)
|
|
804
|
+
summary["interrupted"] = len(items) - index + 1
|
|
805
|
+
break
|
|
806
|
+
|
|
807
|
+
summary["failed"] = len(failures)
|
|
808
|
+
summary["ok"] = len(items) - len(failures)
|
|
809
|
+
code = EXIT_OK
|
|
810
|
+
if summary["failed"] == summary["total"]:
|
|
811
|
+
code = EXIT_FAIL
|
|
812
|
+
elif summary["failed"]:
|
|
813
|
+
code = EXIT_PARTIAL
|
|
814
|
+
if summary["interrupted"]:
|
|
815
|
+
code = EXIT_INTERRUPT
|
|
816
|
+
try:
|
|
817
|
+
_write_failed(failed_path, failures)
|
|
818
|
+
except OSError as exc:
|
|
819
|
+
# The downloads stand; only the retry list is missing.
|
|
820
|
+
print(f"acadfetch: could not write {failed_path}: {exc}", file=sys.stderr)
|
|
821
|
+
doc["error"] = f"could not write {failed_path}: {exc}"
|
|
822
|
+
if code == EXIT_OK:
|
|
823
|
+
code = EXIT_PARTIAL
|
|
824
|
+
summary["ok_run"] = code == EXIT_OK
|
|
825
|
+
doc["elapsed_s"] = round(time.monotonic() - started, 3)
|
|
826
|
+
if not args.json:
|
|
827
|
+
print()
|
|
828
|
+
print(
|
|
829
|
+
f"{summary['ok']}/{summary['total']} identifier(s) ok, "
|
|
830
|
+
f"{summary['failed']} failed -> {failed_path}"
|
|
831
|
+
)
|
|
832
|
+
for row in items:
|
|
833
|
+
if row["code"] != EXIT_OK:
|
|
834
|
+
print(f" line {row['line']}: {row['target']} ({row['error'] or 'failed'})")
|
|
835
|
+
if args.json:
|
|
836
|
+
_emit(doc)
|
|
837
|
+
return code
|
|
838
|
+
|
|
839
|
+
|
|
647
840
|
class Parser(argparse.ArgumentParser):
|
|
648
841
|
"""Usage errors are not "partial": exit 64 keeps 2 meaning a partial run."""
|
|
649
842
|
|
|
@@ -735,6 +928,79 @@ def build_parser() -> argparse.ArgumentParser:
|
|
|
735
928
|
help="ignore $ZENODO_TOKEN",
|
|
736
929
|
)
|
|
737
930
|
|
|
931
|
+
def transfer_flags(p: argparse.ArgumentParser, *, batch: bool = False) -> None:
|
|
932
|
+
"""Flags that mean the same thing to ``fetch`` and to ``batch``."""
|
|
933
|
+
if batch:
|
|
934
|
+
p.add_argument(
|
|
935
|
+
"-o",
|
|
936
|
+
"--out",
|
|
937
|
+
help="base directory; every identifier still gets a directory of its "
|
|
938
|
+
"own (default: acadfetch-out)",
|
|
939
|
+
)
|
|
940
|
+
else:
|
|
941
|
+
p.add_argument(
|
|
942
|
+
"-o", "--out", help="output directory (default: acadfetch-out/<source>-<id>)"
|
|
943
|
+
)
|
|
944
|
+
p.add_argument(
|
|
945
|
+
"-j",
|
|
946
|
+
"--concurrency",
|
|
947
|
+
type=_positive,
|
|
948
|
+
default=consts.DEFAULT_CONCURRENCY,
|
|
949
|
+
help="parallel transfers, per record (default: %(default)s)",
|
|
950
|
+
)
|
|
951
|
+
p.add_argument("--max-size", type=parse_size, help="abort above this total, e.g. 500MB")
|
|
952
|
+
p.add_argument(
|
|
953
|
+
"--checksum",
|
|
954
|
+
type=_checksum,
|
|
955
|
+
metavar="ALGO:HEX",
|
|
956
|
+
help="verify against this checksum instead (md5:<hex> or sha256:<hex>), "
|
|
957
|
+
"for sources that publish none; required when migrating a figshare target",
|
|
958
|
+
)
|
|
959
|
+
p.add_argument(
|
|
960
|
+
"--only",
|
|
961
|
+
nargs="+",
|
|
962
|
+
metavar="NAME",
|
|
963
|
+
help="fetch only these file names",
|
|
964
|
+
)
|
|
965
|
+
p.add_argument(
|
|
966
|
+
"--glob",
|
|
967
|
+
nargs="+",
|
|
968
|
+
dest="globs",
|
|
969
|
+
metavar="PATTERN",
|
|
970
|
+
help="fetch only files whose name matches a shell pattern, e.g. '*.pdf'",
|
|
971
|
+
)
|
|
972
|
+
if not batch:
|
|
973
|
+
p.add_argument(
|
|
974
|
+
"-w",
|
|
975
|
+
"--urls",
|
|
976
|
+
metavar="PATH",
|
|
977
|
+
help="write an aria2-compatible URL list instead of downloading; '-' is stdout",
|
|
978
|
+
)
|
|
979
|
+
p.add_argument("--manifest-out", help="also write the manifest here")
|
|
980
|
+
p.add_argument("-n", "--dry-run", action="store_true", help="list what would be fetched")
|
|
981
|
+
p.add_argument("-q", "--quiet", action="store_true", help="no progress line")
|
|
982
|
+
p.add_argument(
|
|
983
|
+
"--force",
|
|
984
|
+
action="store_true",
|
|
985
|
+
help="fetch again even when a matching file is already there",
|
|
986
|
+
)
|
|
987
|
+
p.add_argument(
|
|
988
|
+
"--no-fsync",
|
|
989
|
+
action="store_true",
|
|
990
|
+
help="skip the flush before each rename (faster, weaker against a power cut)",
|
|
991
|
+
)
|
|
992
|
+
p.add_argument(
|
|
993
|
+
"--json",
|
|
994
|
+
action="store_true",
|
|
995
|
+
help="print one JSON report to stdout, progress to stderr",
|
|
996
|
+
)
|
|
997
|
+
p.add_argument(
|
|
998
|
+
"--record-full-url",
|
|
999
|
+
action="store_true",
|
|
1000
|
+
help="keep credential query parameters (X-Amz-Signature, token, …) in "
|
|
1001
|
+
"the manifest and on stdout instead of stripping them",
|
|
1002
|
+
)
|
|
1003
|
+
|
|
738
1004
|
fetch = sub.add_parser(
|
|
739
1005
|
"fetch",
|
|
740
1006
|
help="download every file of a record",
|
|
@@ -746,69 +1012,31 @@ def build_parser() -> argparse.ArgumentParser:
|
|
|
746
1012
|
nargs="?",
|
|
747
1013
|
help="DOI, zenodo:<id>, arxiv:<id>, or URL",
|
|
748
1014
|
)
|
|
749
|
-
fetch
|
|
750
|
-
|
|
751
|
-
)
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
"
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
)
|
|
759
|
-
fetch.add_argument("--max-size", type=parse_size, help="abort above this total, e.g. 500MB")
|
|
760
|
-
fetch.add_argument(
|
|
761
|
-
"--checksum",
|
|
762
|
-
type=_checksum,
|
|
763
|
-
metavar="ALGO:HEX",
|
|
764
|
-
help="verify against this checksum instead (md5:<hex> or sha256:<hex>), "
|
|
765
|
-
"for sources that publish none; required when migrating a figshare target",
|
|
766
|
-
)
|
|
767
|
-
fetch.add_argument(
|
|
768
|
-
"--only",
|
|
769
|
-
nargs="+",
|
|
770
|
-
metavar="NAME",
|
|
771
|
-
help="fetch only these file names",
|
|
1015
|
+
transfer_flags(fetch)
|
|
1016
|
+
network_flags(fetch)
|
|
1017
|
+
fetch.set_defaults(func=cmd_fetch)
|
|
1018
|
+
|
|
1019
|
+
bat = sub.add_parser(
|
|
1020
|
+
"batch",
|
|
1021
|
+
help="download every identifier in a file, listing the failures",
|
|
1022
|
+
epilog=EPILOG,
|
|
1023
|
+
formatter_class=argparse.RawDescriptionHelpFormatter,
|
|
772
1024
|
)
|
|
773
|
-
|
|
774
|
-
"
|
|
775
|
-
|
|
776
|
-
|
|
777
|
-
metavar="PATTERN",
|
|
778
|
-
help="fetch only files whose name matches a shell pattern, e.g. '*.pdf'",
|
|
1025
|
+
bat.add_argument(
|
|
1026
|
+
"identifiers",
|
|
1027
|
+
metavar="FILE",
|
|
1028
|
+
help="one identifier per line; blank lines and '#' comments are skipped",
|
|
779
1029
|
)
|
|
780
|
-
|
|
781
|
-
|
|
782
|
-
"--
|
|
1030
|
+
transfer_flags(bat, batch=True)
|
|
1031
|
+
bat.add_argument(
|
|
1032
|
+
"--failed-out",
|
|
1033
|
+
default="failed.txt",
|
|
783
1034
|
metavar="PATH",
|
|
784
|
-
help="write
|
|
785
|
-
|
|
786
|
-
fetch.add_argument("--manifest-out", help="also write the manifest here")
|
|
787
|
-
fetch.add_argument("-n", "--dry-run", action="store_true", help="list what would be fetched")
|
|
788
|
-
fetch.add_argument("-q", "--quiet", action="store_true", help="no progress line")
|
|
789
|
-
fetch.add_argument(
|
|
790
|
-
"--force",
|
|
791
|
-
action="store_true",
|
|
792
|
-
help="fetch again even when a matching file is already there",
|
|
1035
|
+
help="write the identifiers that failed here, one per line, for a rerun; "
|
|
1036
|
+
"'-' is stdout (default: failed.txt)",
|
|
793
1037
|
)
|
|
794
|
-
|
|
795
|
-
|
|
796
|
-
action="store_true",
|
|
797
|
-
help="skip the flush before each rename (faster, weaker against a power cut)",
|
|
798
|
-
)
|
|
799
|
-
fetch.add_argument(
|
|
800
|
-
"--json",
|
|
801
|
-
action="store_true",
|
|
802
|
-
help="print one JSON report to stdout, progress to stderr",
|
|
803
|
-
)
|
|
804
|
-
fetch.add_argument(
|
|
805
|
-
"--record-full-url",
|
|
806
|
-
action="store_true",
|
|
807
|
-
help="keep credential query parameters (X-Amz-Signature, token, …) in "
|
|
808
|
-
"the manifest and on stdout instead of stripping them",
|
|
809
|
-
)
|
|
810
|
-
network_flags(fetch)
|
|
811
|
-
fetch.set_defaults(func=cmd_fetch)
|
|
1038
|
+
network_flags(bat)
|
|
1039
|
+
bat.set_defaults(func=cmd_batch)
|
|
812
1040
|
|
|
813
1041
|
man = sub.add_parser("manifest", help="write a manifest without downloading")
|
|
814
1042
|
man.add_argument("target")
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: acadfetch
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.8.0
|
|
4
4
|
Summary: Resumable, rate-limited, checksum-verified bulk downloads from academic data repositories
|
|
5
5
|
Author: young-ski
|
|
6
6
|
License-Expression: MIT
|
|
@@ -114,6 +114,23 @@ acadfetch fetch arxiv:1706.03762 # an arXiv id
|
|
|
114
114
|
acadfetch fetch https://ndownloader.figshare.com/files/70600233/report.pdf # any URL
|
|
115
115
|
```
|
|
116
116
|
|
|
117
|
+
Or a whole list, with the failures kept for the next run:
|
|
118
|
+
|
|
119
|
+
```bash
|
|
120
|
+
acadfetch batch identifiers.txt # one identifier per line
|
|
121
|
+
acadfetch batch identifiers.txt -o ./data --failed-out todo.txt
|
|
122
|
+
```
|
|
123
|
+
|
|
124
|
+
Blank lines and `#` comments are skipped, so a list can carry notes. Every
|
|
125
|
+
identifier still gets a directory of its own under `-o` (default
|
|
126
|
+
`acadfetch-out`); one bad line does not stop the rest, and the identifiers that
|
|
127
|
+
failed land in `failed.txt` (or `--failed-out`), verbatim and one per line — so
|
|
128
|
+
the retry is `acadfetch batch failed.txt`. The file is replaced even when
|
|
129
|
+
nothing failed, so a stale list never lies. Exit codes keep their meaning: `0`
|
|
130
|
+
all ok, `1` all failed, `2` partial, `130` interrupted, and an interrupt puts the
|
|
131
|
+
lines that were never attempted on the list too. Records are attempted one after
|
|
132
|
+
another; `-j` still applies inside a record.
|
|
133
|
+
|
|
117
134
|
The first two are answered by Zenodo and arXiv and work from any network those
|
|
118
135
|
are reachable on. The third is a direct URL, so the host decides: figshare's CDN
|
|
119
136
|
answers 403 with a WAF block page from some networks — the same behaviour that
|
|
@@ -188,7 +205,8 @@ checksum matches, so an interrupted or failed run never leaves something that
|
|
|
188
205
|
it resumes. The one case where a run must start over is a server that ignores
|
|
189
206
|
`Range`; the bytes already on disk are then moved aside and kept until a whole
|
|
190
207
|
new copy has arrived, so a restart that dies early costs nothing. `acadfetch-out/`
|
|
191
|
-
is worth adding to a project's `.gitignore`; this repository already has it
|
|
208
|
+
is worth adding to a project's `.gitignore`; this repository already has it, and
|
|
209
|
+
names the default `failed.txt` there as well.
|
|
192
210
|
|
|
193
211
|
The default directory names the target, not only its filename — `data.csv` from
|
|
194
212
|
two hosts is two datasets, and they do not share a directory. A file already
|
|
@@ -405,6 +423,19 @@ acadfetch verify ./data/manifest.json # 事后复验;对不上就非
|
|
|
405
423
|
> figshare 适配器正是因为这个)。acadfetch 会明说这是拦截页、而不是把 HTML 存成
|
|
406
424
|
> PDF,并把下一条命令给你。
|
|
407
425
|
|
|
426
|
+
批量清单则一条一条来,失败的留下:
|
|
427
|
+
|
|
428
|
+
```bash
|
|
429
|
+
acadfetch batch identifiers.txt # 每行一个目标
|
|
430
|
+
acadfetch batch identifiers.txt -o ./data --failed-out todo.txt
|
|
431
|
+
```
|
|
432
|
+
|
|
433
|
+
空行与 `#` 注释会被跳过;每个目标仍在 `-o`(默认 `acadfetch-out`)下拿到自己
|
|
434
|
+
的目录,一行坏掉不影响其余。失败的目标原样写进 `failed.txt`(或
|
|
435
|
+
`--failed-out`),重跑就是 `acadfetch batch failed.txt`;没失败也会清空该文件,
|
|
436
|
+
不会留下上次的假清单。退出码含义不变:`0` 全成、`1` 全败、`2` 部分、`130`
|
|
437
|
+
被中断 —— 中断时还没轮到的行也会进清单。
|
|
438
|
+
|
|
408
439
|
### 它替你保住的东西
|
|
409
440
|
|
|
410
441
|
| 情况 | 它怎么做 |
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "acadfetch"
|
|
7
|
-
version = "0.
|
|
7
|
+
version = "0.8.0"
|
|
8
8
|
description = "Resumable, rate-limited, checksum-verified bulk downloads from academic data repositories"
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.10"
|
|
@@ -0,0 +1,262 @@
|
|
|
1
|
+
"""``batch``: one list in, one failure list out.
|
|
2
|
+
|
|
3
|
+
The command exists so a hundred identifiers is one command, not a shell loop
|
|
4
|
+
that loses the failures. These tests hold it to the two things that make that
|
|
5
|
+
true: a bad line never stops the run, and the failed lines land in a file that
|
|
6
|
+
can be handed straight back.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import json
|
|
12
|
+
from pathlib import Path
|
|
13
|
+
|
|
14
|
+
import pytest
|
|
15
|
+
|
|
16
|
+
from acadfetch import cli
|
|
17
|
+
from acadfetch.sources import registry
|
|
18
|
+
from acadfetch.sources.base import File, Record
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
@pytest.fixture
|
|
22
|
+
def served(server):
|
|
23
|
+
server.cfg["files"]["/a.bin"] = b"alpha payload"
|
|
24
|
+
server.cfg["files"]["/b.bin"] = b"beta"
|
|
25
|
+
return server
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def _list(tmp_path: Path, *lines: str) -> Path:
|
|
29
|
+
path = tmp_path / "identifiers.txt"
|
|
30
|
+
path.write_text("\n".join(lines) + "\n", "utf-8")
|
|
31
|
+
return path
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def _run(capsys, *argv):
|
|
35
|
+
code = cli.main([str(a) for a in argv])
|
|
36
|
+
out, err = capsys.readouterr()
|
|
37
|
+
return code, out, err
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def _downloaded(root: Path) -> set[str]:
|
|
41
|
+
return {path.name for path in root.rglob("*") if path.is_file()}
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def test_downloads_every_identifier_and_replaces_the_failed_list(
|
|
45
|
+
served, capsys, tmp_path, monkeypatch
|
|
46
|
+
):
|
|
47
|
+
monkeypatch.chdir(tmp_path)
|
|
48
|
+
(tmp_path / "failed.txt").write_text("stale-from-last-time\n", "utf-8")
|
|
49
|
+
listing = _list(tmp_path, served.base + "/a.bin", served.base + "/b.bin")
|
|
50
|
+
|
|
51
|
+
code, out, _ = _run(capsys, "batch", listing, "-q")
|
|
52
|
+
|
|
53
|
+
assert code == cli.EXIT_OK
|
|
54
|
+
assert "2/2 identifier(s) ok" in out
|
|
55
|
+
assert (tmp_path / "failed.txt").read_text("utf-8") == ""
|
|
56
|
+
assert {"a.bin", "b.bin"} <= _downloaded(tmp_path / "acadfetch-out")
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def test_a_failed_identifier_is_listed_and_the_run_goes_on(served, capsys, tmp_path, monkeypatch):
|
|
60
|
+
monkeypatch.chdir(tmp_path)
|
|
61
|
+
good = served.base + "/a.bin"
|
|
62
|
+
bad = served.base + "/missing.bin"
|
|
63
|
+
listing = _list(tmp_path, good, bad)
|
|
64
|
+
|
|
65
|
+
code, out, err = _run(capsys, "batch", listing, "-q")
|
|
66
|
+
|
|
67
|
+
assert code == cli.EXIT_PARTIAL
|
|
68
|
+
assert (tmp_path / "failed.txt").read_text("utf-8") == bad + "\n"
|
|
69
|
+
assert "a.bin" in _downloaded(tmp_path / "acadfetch-out")
|
|
70
|
+
assert "line 2" in out and "missing.bin" in out
|
|
71
|
+
assert "missing.bin" in err
|
|
72
|
+
assert "1/2 identifier(s) ok, 1 failed" in out
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def test_all_failures_are_exit_1(served, capsys, tmp_path, monkeypatch):
|
|
76
|
+
monkeypatch.chdir(tmp_path)
|
|
77
|
+
listing = _list(tmp_path, served.base + "/nope1", served.base + "/nope2")
|
|
78
|
+
|
|
79
|
+
code, _, _ = _run(capsys, "batch", listing, "-q")
|
|
80
|
+
|
|
81
|
+
assert code == cli.EXIT_FAIL
|
|
82
|
+
assert (tmp_path / "failed.txt").read_text("utf-8").count("\n") == 2
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def test_comments_blank_lines_bom_and_crlf_are_skipped(served, capsys, tmp_path, monkeypatch):
|
|
86
|
+
"""A hand-edited list is the normal case, including one saved by Notepad."""
|
|
87
|
+
monkeypatch.chdir(tmp_path)
|
|
88
|
+
url = served.base + "/a.bin"
|
|
89
|
+
raw = "\ufeff# records for the thesis\n\n " + url + " \r\n# and a note\r\n"
|
|
90
|
+
listing = tmp_path / "identifiers.txt"
|
|
91
|
+
listing.write_bytes(raw.encode("utf-8"))
|
|
92
|
+
|
|
93
|
+
code, _, _ = _run(capsys, "batch", listing, "-q", "--json")
|
|
94
|
+
|
|
95
|
+
assert code == cli.EXIT_OK
|
|
96
|
+
assert (tmp_path / "failed.txt").read_text("utf-8") == ""
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def test_an_empty_list_is_exit_1_and_says_so(capsys, tmp_path, monkeypatch):
|
|
100
|
+
monkeypatch.chdir(tmp_path)
|
|
101
|
+
listing = _list(tmp_path, "# nothing yet")
|
|
102
|
+
|
|
103
|
+
code, _, err = _run(capsys, "batch", listing)
|
|
104
|
+
|
|
105
|
+
assert code == cli.EXIT_FAIL
|
|
106
|
+
assert "lists no identifiers" in err
|
|
107
|
+
assert (tmp_path / "failed.txt").exists()
|
|
108
|
+
assert (tmp_path / "failed.txt").read_text("utf-8") == ""
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def test_a_missing_list_is_one_line(capsys, tmp_path, monkeypatch):
|
|
112
|
+
monkeypatch.chdir(tmp_path)
|
|
113
|
+
code, _, err = _run(capsys, "batch", tmp_path / "nope.txt")
|
|
114
|
+
assert code == cli.EXIT_FAIL
|
|
115
|
+
assert "nope.txt" in err
|
|
116
|
+
assert "Traceback" not in err
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
def test_failed_list_is_a_working_retry_input(served, capsys, tmp_path, monkeypatch):
|
|
120
|
+
monkeypatch.chdir(tmp_path)
|
|
121
|
+
listing = _list(tmp_path, served.base + "/a.bin", served.base + "/missing.bin")
|
|
122
|
+
assert _run(capsys, "batch", listing, "-q")[0] == cli.EXIT_PARTIAL
|
|
123
|
+
retry = tmp_path / "failed.txt"
|
|
124
|
+
|
|
125
|
+
# The server now has the file; the list written last time is the input.
|
|
126
|
+
served.cfg["files"]["/missing.bin"] = b"now here"
|
|
127
|
+
code, _, _ = _run(capsys, "batch", retry, "-q")
|
|
128
|
+
|
|
129
|
+
assert code == cli.EXIT_OK
|
|
130
|
+
assert retry.read_text("utf-8") == ""
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
def test_each_identifier_keeps_its_own_directory(served, capsys, tmp_path, monkeypatch):
|
|
134
|
+
"""Two files can share a name; the leaf directory is what keeps them apart."""
|
|
135
|
+
monkeypatch.chdir(tmp_path)
|
|
136
|
+
served.cfg["files"]["/one/same.bin"] = b"one"
|
|
137
|
+
served.cfg["files"]["/two/same.bin"] = b"two"
|
|
138
|
+
listing = _list(tmp_path, served.base + "/one/same.bin", served.base + "/two/same.bin")
|
|
139
|
+
|
|
140
|
+
assert _run(capsys, "batch", listing, "-q", "-o", tmp_path / "data")[0] == cli.EXIT_OK
|
|
141
|
+
|
|
142
|
+
leaves = sorted(p.parent for p in (tmp_path / "data").rglob("same.bin"))
|
|
143
|
+
assert len(leaves) == 2
|
|
144
|
+
assert sorted(p.read_bytes() for p in (tmp_path / "data").rglob("same.bin")) == [b"one", b"two"]
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
def test_the_failed_list_keeps_the_identifier_verbatim(served, capsys, tmp_path, monkeypatch):
|
|
148
|
+
"""A redacted list cannot be replayed; the failure file must not be redacted."""
|
|
149
|
+
monkeypatch.chdir(tmp_path)
|
|
150
|
+
url = served.base + "/missing.bin?token=shh"
|
|
151
|
+
listing = _list(tmp_path, url)
|
|
152
|
+
|
|
153
|
+
assert _run(capsys, "batch", listing, "-q")[0] == cli.EXIT_FAIL
|
|
154
|
+
assert (tmp_path / "failed.txt").read_text("utf-8") == url + "\n"
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
def test_only_does_not_eat_the_list(served, capsys, tmp_path, monkeypatch):
|
|
158
|
+
"""``--only`` is nargs=+; the fetch parser's recovery must not fire here."""
|
|
159
|
+
monkeypatch.chdir(tmp_path)
|
|
160
|
+
listing = _list(tmp_path, served.base + "/a.bin")
|
|
161
|
+
code, _, _ = _run(capsys, "batch", listing, "-q", "--only", "a.bin")
|
|
162
|
+
assert code == cli.EXIT_OK
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
def test_json_is_one_document_with_a_row_per_identifier(served, capsys, tmp_path, monkeypatch):
|
|
166
|
+
monkeypatch.chdir(tmp_path)
|
|
167
|
+
good = served.base + "/a.bin"
|
|
168
|
+
bad = served.base + "/missing.bin"
|
|
169
|
+
listing = _list(tmp_path, good, bad)
|
|
170
|
+
|
|
171
|
+
code, out, err = _run(capsys, "batch", listing, "-q", "--json")
|
|
172
|
+
|
|
173
|
+
doc = json.loads(out) # all of stdout parses
|
|
174
|
+
assert code == cli.EXIT_PARTIAL
|
|
175
|
+
assert doc["command"] == "batch"
|
|
176
|
+
assert doc["failed_out"] == "failed.txt"
|
|
177
|
+
assert [row["line"] for row in doc["items"]] == [1, 2]
|
|
178
|
+
assert [row["ok"] for row in doc["items"]] == [True, False]
|
|
179
|
+
assert doc["items"][1]["error"]
|
|
180
|
+
assert doc["items"][0]["report"]["command"] == "fetch"
|
|
181
|
+
assert doc["summary"]["failed"] == 1
|
|
182
|
+
assert doc["summary"]["ok_run"] is False
|
|
183
|
+
assert "missing.bin" in err # the per-identifier error still reaches stderr
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
def test_json_and_a_stdout_failed_list_are_refused(served, capsys, tmp_path, monkeypatch):
|
|
187
|
+
monkeypatch.chdir(tmp_path)
|
|
188
|
+
listing = _list(tmp_path, served.base + "/a.bin")
|
|
189
|
+
code, out, err = _run(capsys, "batch", listing, "-q", "--json", "--failed-out", "-")
|
|
190
|
+
assert code == cli.EXIT_FAIL
|
|
191
|
+
assert "claims stdout" in json.loads(out)["error"]
|
|
192
|
+
assert "claims stdout" in err
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
def test_interrupt_lists_the_current_and_the_unattempted_lines(monkeypatch, capsys, tmp_path):
|
|
196
|
+
"""Rerunning the failure file must cover what the interrupt skipped."""
|
|
197
|
+
monkeypatch.chdir(tmp_path)
|
|
198
|
+
|
|
199
|
+
def fake_resolve(client, target):
|
|
200
|
+
client.source = "zenodo"
|
|
201
|
+
return Record(source="zenodo", target=target, files=[File("https://x/" + target, target)])
|
|
202
|
+
|
|
203
|
+
calls: list[str] = []
|
|
204
|
+
|
|
205
|
+
def fake_fetch(record, opts, client, only=None, globs=None):
|
|
206
|
+
calls.append(record.target)
|
|
207
|
+
if record.target == "two":
|
|
208
|
+
raise KeyboardInterrupt
|
|
209
|
+
return []
|
|
210
|
+
|
|
211
|
+
monkeypatch.setattr(registry, "resolve", fake_resolve)
|
|
212
|
+
monkeypatch.setattr(cli, "fetch_record", fake_fetch)
|
|
213
|
+
listing = _list(tmp_path, "one", "two", "three")
|
|
214
|
+
|
|
215
|
+
code = cli.main(["batch", str(listing), "-q", "-o", str(tmp_path / "out")])
|
|
216
|
+
|
|
217
|
+
assert code == cli.EXIT_INTERRUPT
|
|
218
|
+
assert calls == ["one", "two"]
|
|
219
|
+
assert (tmp_path / "failed.txt").read_text("utf-8") == "two\nthree\n"
|
|
220
|
+
|
|
221
|
+
|
|
222
|
+
def test_a_filter_matching_nothing_is_that_identifier_failing(
|
|
223
|
+
served, capsys, tmp_path, monkeypatch
|
|
224
|
+
):
|
|
225
|
+
monkeypatch.chdir(tmp_path)
|
|
226
|
+
listing = _list(tmp_path, served.base + "/a.bin", served.base + "/b.bin")
|
|
227
|
+
|
|
228
|
+
code, _, err = _run(capsys, "batch", listing, "-q", "--only", "nope")
|
|
229
|
+
|
|
230
|
+
assert code == cli.EXIT_FAIL
|
|
231
|
+
assert err.count("no file matched") == 2
|
|
232
|
+
assert (tmp_path / "failed.txt").read_text("utf-8").count("\n") == 2
|
|
233
|
+
|
|
234
|
+
|
|
235
|
+
def test_failed_out_creates_the_directory(served, capsys, tmp_path, monkeypatch):
|
|
236
|
+
monkeypatch.chdir(tmp_path)
|
|
237
|
+
listing = _list(tmp_path, served.base + "/missing.bin")
|
|
238
|
+
|
|
239
|
+
assert _run(capsys, "batch", listing, "-q", "--failed-out", "logs/failed.txt")[0] == (
|
|
240
|
+
cli.EXIT_FAIL
|
|
241
|
+
)
|
|
242
|
+
assert "missing.bin" in (tmp_path / "logs" / "failed.txt").read_text("utf-8")
|
|
243
|
+
|
|
244
|
+
|
|
245
|
+
def test_failed_out_dash_puts_the_list_on_stdout(served, capsys, tmp_path, monkeypatch):
|
|
246
|
+
monkeypatch.chdir(tmp_path)
|
|
247
|
+
bad = served.base + "/missing.bin"
|
|
248
|
+
listing = _list(tmp_path, bad)
|
|
249
|
+
|
|
250
|
+
code, out, _ = _run(capsys, "batch", listing, "-q", "--failed-out", "-")
|
|
251
|
+
|
|
252
|
+
assert code == cli.EXIT_FAIL
|
|
253
|
+
assert bad in out.splitlines(), "the list is the last thing printed"
|
|
254
|
+
assert not (tmp_path / "failed.txt").exists()
|
|
255
|
+
|
|
256
|
+
|
|
257
|
+
def test_out_base_is_used_for_every_identifier(served, capsys, tmp_path, monkeypatch):
|
|
258
|
+
monkeypatch.chdir(tmp_path)
|
|
259
|
+
listing = _list(tmp_path, served.base + "/a.bin")
|
|
260
|
+
assert _run(capsys, "batch", listing, "-q", "-o", tmp_path / "data")[0] == cli.EXIT_OK
|
|
261
|
+
assert list((tmp_path / "data").glob("direct-*/a.bin"))
|
|
262
|
+
assert not (tmp_path / "acadfetch-out").exists()
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|