acadfetch 0.4.0__tar.gz → 0.6.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {acadfetch-0.4.0/acadfetch.egg-info → acadfetch-0.6.0}/PKG-INFO +28 -2
- {acadfetch-0.4.0 → acadfetch-0.6.0}/README.md +27 -1
- {acadfetch-0.4.0 → acadfetch-0.6.0}/acadfetch/cli.py +104 -26
- {acadfetch-0.4.0 → acadfetch-0.6.0}/acadfetch/client.py +16 -7
- {acadfetch-0.4.0 → acadfetch-0.6.0}/acadfetch/consts.py +1 -1
- acadfetch-0.6.0/acadfetch/debug.py +59 -0
- {acadfetch-0.4.0 → acadfetch-0.6.0}/acadfetch/downloader.py +35 -15
- {acadfetch-0.4.0 → acadfetch-0.6.0}/acadfetch/sources/doi.py +2 -1
- {acadfetch-0.4.0 → acadfetch-0.6.0/acadfetch.egg-info}/PKG-INFO +28 -2
- {acadfetch-0.4.0 → acadfetch-0.6.0}/acadfetch.egg-info/SOURCES.txt +4 -0
- {acadfetch-0.4.0 → acadfetch-0.6.0}/pyproject.toml +7 -1
- acadfetch-0.6.0/tests/test_citation.py +26 -0
- {acadfetch-0.4.0 → acadfetch-0.6.0}/tests/test_cli.py +2 -2
- acadfetch-0.6.0/tests/test_debug.py +120 -0
- {acadfetch-0.4.0 → acadfetch-0.6.0}/tests/test_sources.py +5 -0
- acadfetch-0.6.0/tests/test_urls_glob.py +99 -0
- {acadfetch-0.4.0 → acadfetch-0.6.0}/LICENSE +0 -0
- {acadfetch-0.4.0 → acadfetch-0.6.0}/MANIFEST.in +0 -0
- {acadfetch-0.4.0 → acadfetch-0.6.0}/acadfetch/__init__.py +0 -0
- {acadfetch-0.4.0 → acadfetch-0.6.0}/acadfetch/__main__.py +0 -0
- {acadfetch-0.4.0 → acadfetch-0.6.0}/acadfetch/atomic.py +0 -0
- {acadfetch-0.4.0 → acadfetch-0.6.0}/acadfetch/auth.py +0 -0
- {acadfetch-0.4.0 → acadfetch-0.6.0}/acadfetch/errors.py +0 -0
- {acadfetch-0.4.0 → acadfetch-0.6.0}/acadfetch/manifest.py +0 -0
- {acadfetch-0.4.0 → acadfetch-0.6.0}/acadfetch/naming.py +0 -0
- {acadfetch-0.4.0 → acadfetch-0.6.0}/acadfetch/progress.py +0 -0
- {acadfetch-0.4.0 → acadfetch-0.6.0}/acadfetch/py.typed +0 -0
- {acadfetch-0.4.0 → acadfetch-0.6.0}/acadfetch/ratelimit.py +0 -0
- {acadfetch-0.4.0 → acadfetch-0.6.0}/acadfetch/remote.py +0 -0
- {acadfetch-0.4.0 → acadfetch-0.6.0}/acadfetch/sources/__init__.py +0 -0
- {acadfetch-0.4.0 → acadfetch-0.6.0}/acadfetch/sources/arxiv.py +0 -0
- {acadfetch-0.4.0 → acadfetch-0.6.0}/acadfetch/sources/base.py +0 -0
- {acadfetch-0.4.0 → acadfetch-0.6.0}/acadfetch/sources/direct.py +0 -0
- {acadfetch-0.4.0 → acadfetch-0.6.0}/acadfetch/sources/registry.py +0 -0
- {acadfetch-0.4.0 → acadfetch-0.6.0}/acadfetch/sources/removed.py +0 -0
- {acadfetch-0.4.0 → acadfetch-0.6.0}/acadfetch/sources/zenodo.py +0 -0
- {acadfetch-0.4.0 → acadfetch-0.6.0}/acadfetch/units.py +0 -0
- {acadfetch-0.4.0 → acadfetch-0.6.0}/acadfetch/urls.py +0 -0
- {acadfetch-0.4.0 → acadfetch-0.6.0}/acadfetch/verify.py +0 -0
- {acadfetch-0.4.0 → acadfetch-0.6.0}/acadfetch.egg-info/dependency_links.txt +0 -0
- {acadfetch-0.4.0 → acadfetch-0.6.0}/acadfetch.egg-info/entry_points.txt +0 -0
- {acadfetch-0.4.0 → acadfetch-0.6.0}/acadfetch.egg-info/top_level.txt +0 -0
- {acadfetch-0.4.0 → acadfetch-0.6.0}/setup.cfg +0 -0
- {acadfetch-0.4.0 → acadfetch-0.6.0}/tests/conftest.py +0 -0
- {acadfetch-0.4.0 → acadfetch-0.6.0}/tests/fixtures/README.md +0 -0
- {acadfetch-0.4.0 → acadfetch-0.6.0}/tests/fixtures/ca.pem +0 -0
- {acadfetch-0.4.0 → acadfetch-0.6.0}/tests/fixtures/figshare_403_block.html +0 -0
- {acadfetch-0.4.0 → acadfetch-0.6.0}/tests/fixtures/figshare_403_forbidden.json +0 -0
- {acadfetch-0.4.0 → acadfetch-0.6.0}/tests/fixtures/figshare_article.json +0 -0
- {acadfetch-0.4.0 → acadfetch-0.6.0}/tests/fixtures/figshare_file.pdf +0 -0
- {acadfetch-0.4.0 → acadfetch-0.6.0}/tests/test_atomic.py +0 -0
- {acadfetch-0.4.0 → acadfetch-0.6.0}/tests/test_auth.py +0 -0
- {acadfetch-0.4.0 → acadfetch-0.6.0}/tests/test_checksum.py +0 -0
- {acadfetch-0.4.0 → acadfetch-0.6.0}/tests/test_client.py +0 -0
- {acadfetch-0.4.0 → acadfetch-0.6.0}/tests/test_control_names.py +0 -0
- {acadfetch-0.4.0 → acadfetch-0.6.0}/tests/test_disk.py +0 -0
- {acadfetch-0.4.0 → acadfetch-0.6.0}/tests/test_downloader.py +0 -0
- {acadfetch-0.4.0 → acadfetch-0.6.0}/tests/test_identity.py +0 -0
- {acadfetch-0.4.0 → acadfetch-0.6.0}/tests/test_isolation.py +0 -0
- {acadfetch-0.4.0 → acadfetch-0.6.0}/tests/test_json.py +0 -0
- {acadfetch-0.4.0 → acadfetch-0.6.0}/tests/test_manifest.py +0 -0
- {acadfetch-0.4.0 → acadfetch-0.6.0}/tests/test_naming.py +0 -0
- {acadfetch-0.4.0 → acadfetch-0.6.0}/tests/test_naming_race.py +0 -0
- {acadfetch-0.4.0 → acadfetch-0.6.0}/tests/test_network.py +0 -0
- {acadfetch-0.4.0 → acadfetch-0.6.0}/tests/test_packaging.py +0 -0
- {acadfetch-0.4.0 → acadfetch-0.6.0}/tests/test_progress.py +0 -0
- {acadfetch-0.4.0 → acadfetch-0.6.0}/tests/test_ratelimit.py +0 -0
- {acadfetch-0.4.0 → acadfetch-0.6.0}/tests/test_redirect.py +0 -0
- {acadfetch-0.4.0 → acadfetch-0.6.0}/tests/test_remote.py +0 -0
- {acadfetch-0.4.0 → acadfetch-0.6.0}/tests/test_removed.py +0 -0
- {acadfetch-0.4.0 → acadfetch-0.6.0}/tests/test_urls.py +0 -0
- {acadfetch-0.4.0 → acadfetch-0.6.0}/tests/test_verify.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: acadfetch
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.6.0
|
|
4
4
|
Summary: Resumable, rate-limited, checksum-verified bulk downloads from academic data repositories
|
|
5
5
|
Author: young-ski
|
|
6
6
|
License-Expression: MIT
|
|
@@ -89,7 +89,7 @@ From a checkout, for development:
|
|
|
89
89
|
|
|
90
90
|
```bash
|
|
91
91
|
git clone https://github.com/young-ski/acadfetch && cd acadfetch
|
|
92
|
-
python -m pip install -e . pytest && python -m pytest -q
|
|
92
|
+
python -m pip install -e . pytest mypy && python -m pytest -q
|
|
93
93
|
```
|
|
94
94
|
|
|
95
95
|
## Supported platforms
|
|
@@ -142,6 +142,15 @@ two never interleave:
|
|
|
142
142
|
acadfetch fetch zenodo:123456 --json | jq -r '.files[] | select(.state=="failed") | .name'
|
|
143
143
|
```
|
|
144
144
|
|
|
145
|
+
To hand the bytes to another downloader, `-w/--urls` writes an aria2-compatible
|
|
146
|
+
list (`-` is stdout) and transfers nothing:
|
|
147
|
+
|
|
148
|
+
```bash
|
|
149
|
+
acadfetch fetch zenodo:123456 -w - --glob '*.csv' | aria2c -i -
|
|
150
|
+
```
|
|
151
|
+
|
|
152
|
+
`--glob` takes shell patterns and combines with `--only` (both must pass).
|
|
153
|
+
|
|
145
154
|
Exit codes are unchanged, and a failure before any transfer still produces a
|
|
146
155
|
document with `error` set. `report` is the document's schema version (now `2`, field reference in
|
|
147
156
|
[docs/json-report.md](docs/json-report.md)):
|
|
@@ -284,6 +293,13 @@ acadfetch fetch zenodo:123456 --ca-bundle corp-tls.pem # trust this PEM
|
|
|
284
293
|
either, `HTTP_PROXY` / `HTTPS_PROXY` / `ALL_PROXY` are honoured by urllib as
|
|
285
294
|
usual. A timeout is per request, connect to first byte.
|
|
286
295
|
|
|
296
|
+
## Debugging
|
|
297
|
+
|
|
298
|
+
`--verbose` (or `ACADFETCH_DEBUG=1`) traces each request, each retry and the
|
|
299
|
+
traceback of a failure to stderr; stdout, `--json` documents and exit codes are
|
|
300
|
+
unchanged. Credential header values and presigned query parameters are masked,
|
|
301
|
+
so the trace can be pasted into an issue.
|
|
302
|
+
|
|
287
303
|
## Politeness
|
|
288
304
|
|
|
289
305
|
Defaults: 3 parallel transfers, 2 requests/second, 4 retries with jittered
|
|
@@ -307,6 +323,10 @@ honest way to test this code path.
|
|
|
307
323
|
|
|
308
324
|
## Getting involved
|
|
309
325
|
|
|
326
|
+
A [30-second tour](docs/demo.md) shows the whole loop, and
|
|
327
|
+
[docs/integrations.md](docs/integrations.md) has Make, CI, aria2 and Snakemake
|
|
328
|
+
recipes.
|
|
329
|
+
|
|
310
330
|
A nightly job downloads three fixed public records and verifies them, so a
|
|
311
331
|
rotted adapter opens an issue before a user reports one. It never blocks a
|
|
312
332
|
merge; the next good night closes the issue.
|
|
@@ -346,6 +366,12 @@ acadfetch verify ./data/manifest.json
|
|
|
346
366
|
|
|
347
367
|
MIT。
|
|
348
368
|
|
|
369
|
+
## Citing
|
|
370
|
+
|
|
371
|
+
`acadfetch --cite` prints how to cite the tool; the machine-readable form is
|
|
372
|
+
[`CITATION.cff`](CITATION.cff). For the data, cite the repository record — the
|
|
373
|
+
manifest is the part that lets a reader re-check it.
|
|
374
|
+
|
|
349
375
|
## Audit trail
|
|
350
376
|
|
|
351
377
|
How the 3.15 support claim was reached, including what it did **not** prove —
|
|
@@ -58,7 +58,7 @@ From a checkout, for development:
|
|
|
58
58
|
|
|
59
59
|
```bash
|
|
60
60
|
git clone https://github.com/young-ski/acadfetch && cd acadfetch
|
|
61
|
-
python -m pip install -e . pytest && python -m pytest -q
|
|
61
|
+
python -m pip install -e . pytest mypy && python -m pytest -q
|
|
62
62
|
```
|
|
63
63
|
|
|
64
64
|
## Supported platforms
|
|
@@ -111,6 +111,15 @@ two never interleave:
|
|
|
111
111
|
acadfetch fetch zenodo:123456 --json | jq -r '.files[] | select(.state=="failed") | .name'
|
|
112
112
|
```
|
|
113
113
|
|
|
114
|
+
To hand the bytes to another downloader, `-w/--urls` writes an aria2-compatible
|
|
115
|
+
list (`-` is stdout) and transfers nothing:
|
|
116
|
+
|
|
117
|
+
```bash
|
|
118
|
+
acadfetch fetch zenodo:123456 -w - --glob '*.csv' | aria2c -i -
|
|
119
|
+
```
|
|
120
|
+
|
|
121
|
+
`--glob` takes shell patterns and combines with `--only` (both must pass).
|
|
122
|
+
|
|
114
123
|
Exit codes are unchanged, and a failure before any transfer still produces a
|
|
115
124
|
document with `error` set. `report` is the document's schema version (now `2`, field reference in
|
|
116
125
|
[docs/json-report.md](docs/json-report.md)):
|
|
@@ -253,6 +262,13 @@ acadfetch fetch zenodo:123456 --ca-bundle corp-tls.pem # trust this PEM
|
|
|
253
262
|
either, `HTTP_PROXY` / `HTTPS_PROXY` / `ALL_PROXY` are honoured by urllib as
|
|
254
263
|
usual. A timeout is per request, connect to first byte.
|
|
255
264
|
|
|
265
|
+
## Debugging
|
|
266
|
+
|
|
267
|
+
`--verbose` (or `ACADFETCH_DEBUG=1`) traces each request, each retry and the
|
|
268
|
+
traceback of a failure to stderr; stdout, `--json` documents and exit codes are
|
|
269
|
+
unchanged. Credential header values and presigned query parameters are masked,
|
|
270
|
+
so the trace can be pasted into an issue.
|
|
271
|
+
|
|
256
272
|
## Politeness
|
|
257
273
|
|
|
258
274
|
Defaults: 3 parallel transfers, 2 requests/second, 4 retries with jittered
|
|
@@ -276,6 +292,10 @@ honest way to test this code path.
|
|
|
276
292
|
|
|
277
293
|
## Getting involved
|
|
278
294
|
|
|
295
|
+
A [30-second tour](docs/demo.md) shows the whole loop, and
|
|
296
|
+
[docs/integrations.md](docs/integrations.md) has Make, CI, aria2 and Snakemake
|
|
297
|
+
recipes.
|
|
298
|
+
|
|
279
299
|
A nightly job downloads three fixed public records and verifies them, so a
|
|
280
300
|
rotted adapter opens an issue before a user reports one. It never blocks a
|
|
281
301
|
merge; the next good night closes the issue.
|
|
@@ -315,6 +335,12 @@ acadfetch verify ./data/manifest.json
|
|
|
315
335
|
|
|
316
336
|
MIT。
|
|
317
337
|
|
|
338
|
+
## Citing
|
|
339
|
+
|
|
340
|
+
`acadfetch --cite` prints how to cite the tool; the machine-readable form is
|
|
341
|
+
[`CITATION.cff`](CITATION.cff). For the data, cite the repository record — the
|
|
342
|
+
manifest is the part that lets a reader re-check it.
|
|
343
|
+
|
|
318
344
|
## Audit trail
|
|
319
345
|
|
|
320
346
|
How the 3.15 support claim was reached, including what it did **not** prove —
|
|
@@ -15,12 +15,14 @@ import time
|
|
|
15
15
|
from dataclasses import replace
|
|
16
16
|
from datetime import datetime, timezone
|
|
17
17
|
from pathlib import Path
|
|
18
|
+
from typing import NoReturn
|
|
18
19
|
from urllib.parse import urlsplit
|
|
19
20
|
|
|
20
|
-
from . import auth, consts, manifest, naming, urls, verify
|
|
21
|
+
from . import auth, consts, debug, manifest, naming, urls, verify
|
|
21
22
|
from . import remote as remote_check
|
|
22
23
|
from .client import Client
|
|
23
24
|
from .downloader import Options, check_disk, check_total, fetch_record, human
|
|
25
|
+
from .downloader import matches as downloader_matches
|
|
24
26
|
from .errors import AcFError, PartialError, TargetError
|
|
25
27
|
from .progress import Progress
|
|
26
28
|
from .sources import doi, registry, removed
|
|
@@ -224,6 +226,15 @@ def _report(args: argparse.Namespace) -> dict:
|
|
|
224
226
|
}
|
|
225
227
|
|
|
226
228
|
|
|
229
|
+
CITATION = """To cite acadfetch itself:
|
|
230
|
+
|
|
231
|
+
young-ski. acadfetch (version {version}). MIT. {url}
|
|
232
|
+
|
|
233
|
+
To cite the data, cite the repository record and keep its manifest.json beside
|
|
234
|
+
the analysis: the manifest is what lets anyone re-check the bytes later.
|
|
235
|
+
`docs/manifest.md` explains what it does and does not prove."""
|
|
236
|
+
|
|
237
|
+
|
|
227
238
|
def _hide(text: str, args: argparse.Namespace) -> str:
|
|
228
239
|
"""Redact credential query parameters unless --record-full-url asked for
|
|
229
240
|
the original string."""
|
|
@@ -266,35 +277,56 @@ def _emit(doc: dict) -> None:
|
|
|
266
277
|
print(json.dumps(doc, ensure_ascii=False))
|
|
267
278
|
|
|
268
279
|
|
|
269
|
-
def _print_header(record, hide=urls.redact) -> None: # noqa: ANN001
|
|
270
|
-
|
|
280
|
+
def _print_header(record, hide=urls.redact, stream=None) -> None: # noqa: ANN001
|
|
281
|
+
out = sys.stdout if stream is None else stream
|
|
282
|
+
print(f"title : {record.title or '(untitled)'}", file=out)
|
|
271
283
|
if record.doi:
|
|
272
|
-
print(f"doi : {record.doi}")
|
|
273
|
-
print(f"license : {record.license or 'UNKNOWN - check before redistributing'}")
|
|
284
|
+
print(f"doi : {record.doi}", file=out)
|
|
285
|
+
print(f"license : {record.license or 'UNKNOWN - check before redistributing'}", file=out)
|
|
274
286
|
total = record.total_bytes
|
|
275
|
-
print(
|
|
287
|
+
print(
|
|
288
|
+
f"files : {len(record.files)}" + (f", {human(total)}" if total is not None else ""),
|
|
289
|
+
file=out,
|
|
290
|
+
)
|
|
276
291
|
if record.landing:
|
|
277
|
-
print(f"landing : {hide(record.landing)}")
|
|
292
|
+
print(f"landing : {hide(record.landing)}", file=out)
|
|
293
|
+
|
|
294
|
+
|
|
295
|
+
def _write_urls(files, out_dir, target) -> None: # noqa: ANN001
|
|
296
|
+
lines = []
|
|
297
|
+
for file in files:
|
|
298
|
+
lines.append(file.url)
|
|
299
|
+
lines.append(f" out={naming.sanitize(file.name)}")
|
|
300
|
+
lines.append(f" dir={out_dir}")
|
|
301
|
+
text = "\n".join(lines) + "\n"
|
|
302
|
+
if target == "-":
|
|
303
|
+
sys.stdout.write(text)
|
|
304
|
+
return
|
|
305
|
+
Path(target).write_text(text, "utf-8")
|
|
306
|
+
print(f"{len(files)} url(s) -> {target}", file=sys.stderr)
|
|
278
307
|
|
|
279
308
|
|
|
280
|
-
def _no_match(record,
|
|
281
|
-
"""
|
|
309
|
+
def _no_match(record, args) -> int: # noqa: ANN001
|
|
310
|
+
"""A filter that matched nothing is a failure that can name the candidates."""
|
|
282
311
|
available = ", ".join(naming.sanitize(f.name) for f in record.files[:8]) or "(none)"
|
|
283
|
-
|
|
312
|
+
wanted = " ".join(
|
|
313
|
+
part
|
|
314
|
+
for part in (
|
|
315
|
+
f"--only {', '.join(args.only)}" if args.only else "",
|
|
316
|
+
f"--glob {', '.join(args.globs)}" if args.globs else "",
|
|
317
|
+
)
|
|
318
|
+
if part
|
|
319
|
+
)
|
|
320
|
+
message = f"no file matched {wanted}; available: {available}"
|
|
284
321
|
print(f"acadfetch: {message}", file=sys.stderr)
|
|
285
322
|
_emit_error(args, message)
|
|
286
323
|
return EXIT_FAIL
|
|
287
324
|
|
|
288
325
|
|
|
289
|
-
def _selected(record,
|
|
290
|
-
if not only:
|
|
326
|
+
def _selected(record, args): # noqa: ANN001, ANN202
|
|
327
|
+
if not args.only and not args.globs:
|
|
291
328
|
return record.files
|
|
292
|
-
|
|
293
|
-
return [
|
|
294
|
-
f
|
|
295
|
-
for f in record.files
|
|
296
|
-
if naming.sanitize(f.name).lower() in wanted or f.name.lower() in wanted
|
|
297
|
-
]
|
|
329
|
+
return [f for f in record.files if downloader_matches(f, args.only, args.globs)]
|
|
298
330
|
|
|
299
331
|
|
|
300
332
|
def cmd_fetch(args: argparse.Namespace) -> int:
|
|
@@ -315,15 +347,27 @@ def cmd_fetch(args: argparse.Namespace) -> int:
|
|
|
315
347
|
license=record.license,
|
|
316
348
|
landing=_hide(record.landing, args),
|
|
317
349
|
)
|
|
350
|
+
# A URL list on stdout must stay machine-readable, so the header and the
|
|
351
|
+
# token line move to stderr for --urls.
|
|
352
|
+
header = sys.stderr if args.urls else sys.stdout
|
|
318
353
|
if not args.json:
|
|
319
|
-
_print_header(record, lambda value: _hide(value, args))
|
|
354
|
+
_print_header(record, lambda value: _hide(value, args), stream=header)
|
|
320
355
|
token = _token_for(args, record)
|
|
321
356
|
if token and not args.json:
|
|
322
|
-
print(f"auth : token from {token}")
|
|
357
|
+
print(f"auth : token from {token}", file=header)
|
|
323
358
|
|
|
324
359
|
out_dir = Path(args.out or f"acadfetch-out/{record.source}-{_slug(record)}")
|
|
325
360
|
doc["out"] = str(out_dir)
|
|
326
361
|
doc["dry_run"] = bool(args.dry_run)
|
|
362
|
+
|
|
363
|
+
if args.urls:
|
|
364
|
+
if args.json:
|
|
365
|
+
raise TargetError("--urls writes a text list; drop --json")
|
|
366
|
+
chosen = _selected(record, args)
|
|
367
|
+
if not chosen:
|
|
368
|
+
return _no_match(record, args)
|
|
369
|
+
_write_urls(chosen, out_dir, args.urls)
|
|
370
|
+
return EXIT_OK
|
|
327
371
|
# Say whether a credential was stripped, so a script can tell the manifest
|
|
328
372
|
# is not replayable as-is.
|
|
329
373
|
doc["url_redacted"] = record.target != _hide(record.target, args) or record.landing != _hide(
|
|
@@ -344,9 +388,9 @@ def cmd_fetch(args: argparse.Namespace) -> int:
|
|
|
344
388
|
)
|
|
345
389
|
|
|
346
390
|
if args.dry_run:
|
|
347
|
-
chosen = _selected(record, args
|
|
391
|
+
chosen = _selected(record, args)
|
|
348
392
|
if not chosen:
|
|
349
|
-
return _no_match(record, args
|
|
393
|
+
return _no_match(record, args)
|
|
350
394
|
if args.max_size:
|
|
351
395
|
check_total(replace(record, files=chosen), args.max_size)
|
|
352
396
|
try:
|
|
@@ -387,9 +431,9 @@ def cmd_fetch(args: argparse.Namespace) -> int:
|
|
|
387
431
|
return EXIT_OK
|
|
388
432
|
|
|
389
433
|
try:
|
|
390
|
-
results = fetch_record(record, opts, client, only=args.only)
|
|
434
|
+
results = fetch_record(record, opts, client, only=args.only, globs=args.globs)
|
|
391
435
|
except PartialError:
|
|
392
|
-
return _no_match(record, args
|
|
436
|
+
return _no_match(record, args)
|
|
393
437
|
finally:
|
|
394
438
|
progress.finish()
|
|
395
439
|
|
|
@@ -423,7 +467,7 @@ def cmd_fetch(args: argparse.Namespace) -> int:
|
|
|
423
467
|
token,
|
|
424
468
|
# Which files this run asked for: a --only run must not look like a
|
|
425
469
|
# run that fetched everything and then lost most of it.
|
|
426
|
-
selected_ids={id(f) for f in _selected(record, args
|
|
470
|
+
selected_ids={id(f) for f in _selected(record, args)},
|
|
427
471
|
redact=not args.record_full_url,
|
|
428
472
|
)
|
|
429
473
|
code = EXIT_OK
|
|
@@ -590,7 +634,7 @@ def _slug(record) -> str: # noqa: ANN001
|
|
|
590
634
|
class Parser(argparse.ArgumentParser):
|
|
591
635
|
"""Usage errors are not "partial": exit 64 keeps 2 meaning a partial run."""
|
|
592
636
|
|
|
593
|
-
def error(self, message: str) ->
|
|
637
|
+
def error(self, message: str) -> NoReturn: # noqa: D102
|
|
594
638
|
self.print_usage(sys.stderr)
|
|
595
639
|
self.exit(EXIT_USAGE, f"{self.prog}: error: {message}\n")
|
|
596
640
|
|
|
@@ -603,6 +647,11 @@ def build_parser() -> argparse.ArgumentParser:
|
|
|
603
647
|
formatter_class=argparse.RawDescriptionHelpFormatter,
|
|
604
648
|
)
|
|
605
649
|
parser.add_argument("--version", action="version", version=f"acadfetch {consts.__version__}")
|
|
650
|
+
parser.add_argument(
|
|
651
|
+
"--cite",
|
|
652
|
+
action="store_true",
|
|
653
|
+
help="print how to cite the tool, then exit",
|
|
654
|
+
)
|
|
606
655
|
sub = parser.add_subparsers(dest="command", required=True)
|
|
607
656
|
|
|
608
657
|
def network_flags(p: argparse.ArgumentParser) -> None:
|
|
@@ -632,6 +681,11 @@ def build_parser() -> argparse.ArgumentParser:
|
|
|
632
681
|
metavar="NAME:VALUE",
|
|
633
682
|
help="extra request header, repeatable (e.g. 'Authorization: Bearer <token>')",
|
|
634
683
|
)
|
|
684
|
+
p.add_argument(
|
|
685
|
+
"--verbose",
|
|
686
|
+
action="store_true",
|
|
687
|
+
help="trace requests and retries to stderr, with tracebacks (also ACADFETCH_DEBUG=1)",
|
|
688
|
+
)
|
|
635
689
|
p.add_argument(
|
|
636
690
|
"--timeout",
|
|
637
691
|
type=_timeout,
|
|
@@ -703,6 +757,19 @@ def build_parser() -> argparse.ArgumentParser:
|
|
|
703
757
|
metavar="NAME",
|
|
704
758
|
help="fetch only these file names",
|
|
705
759
|
)
|
|
760
|
+
fetch.add_argument(
|
|
761
|
+
"--glob",
|
|
762
|
+
nargs="+",
|
|
763
|
+
dest="globs",
|
|
764
|
+
metavar="PATTERN",
|
|
765
|
+
help="fetch only files whose name matches a shell pattern, e.g. '*.pdf'",
|
|
766
|
+
)
|
|
767
|
+
fetch.add_argument(
|
|
768
|
+
"-w",
|
|
769
|
+
"--urls",
|
|
770
|
+
metavar="PATH",
|
|
771
|
+
help="write an aria2-compatible URL list instead of downloading; '-' is stdout",
|
|
772
|
+
)
|
|
706
773
|
fetch.add_argument("--manifest-out", help="also write the manifest here")
|
|
707
774
|
fetch.add_argument("-n", "--dry-run", action="store_true", help="list what would be fetched")
|
|
708
775
|
fetch.add_argument("-q", "--quiet", action="store_true", help="no progress line")
|
|
@@ -787,7 +854,16 @@ def main(argv: list[str] | None = None) -> int:
|
|
|
787
854
|
for stream in (sys.stdout, sys.stderr):
|
|
788
855
|
if hasattr(stream, "reconfigure"):
|
|
789
856
|
stream.reconfigure(encoding="utf-8", errors="replace")
|
|
857
|
+
argv_list = list(sys.argv[1:] if argv is None else argv)
|
|
858
|
+
if "--cite" in argv_list:
|
|
859
|
+
print(
|
|
860
|
+
CITATION.format(
|
|
861
|
+
version=consts.__version__, url="https://github.com/young-ski/acadfetch"
|
|
862
|
+
)
|
|
863
|
+
)
|
|
864
|
+
return EXIT_OK
|
|
790
865
|
args = build_parser().parse_args(argv)
|
|
866
|
+
debug.configure(getattr(args, "verbose", False))
|
|
791
867
|
_recover_target(args)
|
|
792
868
|
if getattr(args, "command", "") == "fetch" and getattr(args, "target", None) is None:
|
|
793
869
|
message = (
|
|
@@ -802,12 +878,14 @@ def main(argv: list[str] | None = None) -> int:
|
|
|
802
878
|
try:
|
|
803
879
|
return args.func(args)
|
|
804
880
|
except AcFError as exc: # TargetError is one
|
|
881
|
+
debug.log_exception(exc)
|
|
805
882
|
# An error sentence carries the URL, and that URL may carry a signature.
|
|
806
883
|
message = _hide_text(str(exc), args)
|
|
807
884
|
print(f"acadfetch: {message}", file=sys.stderr)
|
|
808
885
|
_emit_error(args, message)
|
|
809
886
|
return EXIT_FAIL
|
|
810
887
|
except OSError as exc: # a filesystem surprise, not a bug report
|
|
888
|
+
debug.log_exception(exc)
|
|
811
889
|
message = f"{exc.strerror or exc}" if exc.filename else str(exc)
|
|
812
890
|
print(f"acadfetch: {message}", file=sys.stderr)
|
|
813
891
|
_emit_error(args, message)
|
|
@@ -23,7 +23,7 @@ from collections.abc import Mapping
|
|
|
23
23
|
from datetime import datetime, timezone
|
|
24
24
|
from http.client import HTTPException
|
|
25
25
|
|
|
26
|
-
from . import consts, verify
|
|
26
|
+
from . import consts, debug, urls, verify
|
|
27
27
|
from .errors import HttpError
|
|
28
28
|
from .ratelimit import TokenBucket
|
|
29
29
|
|
|
@@ -176,10 +176,11 @@ class Client:
|
|
|
176
176
|
if retry_after is not None:
|
|
177
177
|
# A literal `Retry-After: 0` would mean no gap at all and a burst;
|
|
178
178
|
# keep a small jittered floor under it.
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
179
|
+
delay = max(retry_after, 0.05 + random.random() * 0.15)
|
|
180
|
+
else:
|
|
181
|
+
delay = min(0.8 * (2**attempt), 30.0) * (0.7 + random.random() * 0.6)
|
|
182
|
+
debug.log(f"retry in {delay:.2f}s")
|
|
183
|
+
time.sleep(delay)
|
|
183
184
|
|
|
184
185
|
def _request(
|
|
185
186
|
self, url: str, headers: Mapping[str, str] | None = None
|
|
@@ -204,6 +205,7 @@ class Client:
|
|
|
204
205
|
headers=None,
|
|
205
206
|
block_retries: int | None = None,
|
|
206
207
|
attempts: int | None = None,
|
|
208
|
+
method: str = "GET",
|
|
207
209
|
): # noqa: ANN001, ANN202
|
|
208
210
|
if block_retries is None:
|
|
209
211
|
block_retries = self.block_retries
|
|
@@ -215,11 +217,16 @@ class Client:
|
|
|
215
217
|
for attempt in range(budget):
|
|
216
218
|
self.bucket.acquire()
|
|
217
219
|
retry_after = None
|
|
220
|
+
req = self._request(url, headers)
|
|
221
|
+
debug.log(f"{method} {urls.redact(url)} headers[{debug.headers_for_log(req.headers)}]")
|
|
218
222
|
try:
|
|
219
|
-
|
|
223
|
+
resp = fn(req)
|
|
224
|
+
debug.log(f"-> {getattr(resp, 'status', 'ok')}")
|
|
225
|
+
return resp
|
|
220
226
|
except urllib.error.HTTPError as exc:
|
|
221
227
|
last = exc
|
|
222
228
|
last_detail = detail = _snippet(exc)
|
|
229
|
+
debug.log(f"-> {exc.code} {detail[:120]}")
|
|
223
230
|
content_type = exc.headers.get("Content-Type") if exc.headers else None
|
|
224
231
|
last_blocked = exc.code == 403 and is_block_page(detail, content_type)
|
|
225
232
|
retry_after = parse_retry_after(
|
|
@@ -239,6 +246,7 @@ class Client:
|
|
|
239
246
|
) from exc
|
|
240
247
|
else:
|
|
241
248
|
waf += 1
|
|
249
|
+
debug.log(f"-> block page, retry {waf}/{block_retries}")
|
|
242
250
|
except HttpError as exc:
|
|
243
251
|
# Raised by a caller-supplied fn, e.g. a body that stopped
|
|
244
252
|
# short. Only the transient ones are worth another request.
|
|
@@ -248,6 +256,7 @@ class Client:
|
|
|
248
256
|
last_detail = str(exc)
|
|
249
257
|
last_blocked = False
|
|
250
258
|
except (urllib.error.URLError, TimeoutError, OSError) as exc:
|
|
259
|
+
debug.log(f"-> {type(exc).__name__}: {exc}")
|
|
251
260
|
last = exc
|
|
252
261
|
last_detail = str(exc)
|
|
253
262
|
last_blocked = False
|
|
@@ -298,7 +307,7 @@ class Client:
|
|
|
298
307
|
|
|
299
308
|
try:
|
|
300
309
|
# HEAD only feeds size/filename; one shot, never worth a retry.
|
|
301
|
-
with self._with_retries(url, _head, block_retries=0, attempts=1) as resp:
|
|
310
|
+
with self._with_retries(url, _head, block_retries=0, attempts=1, method="HEAD") as resp:
|
|
302
311
|
return dict(resp.headers)
|
|
303
312
|
except HttpError as exc:
|
|
304
313
|
if exc.status in (403, 405, 501):
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
"""Opt-in tracing, so a bug report does not have to be a guess.
|
|
2
|
+
|
|
3
|
+
Everything here writes to stderr and swallows its own errors: like progress, a
|
|
4
|
+
log line must never be able to fail a download. Header values that are
|
|
5
|
+
credentials are replaced with an ellipsis, and URLs arrive already redacted by
|
|
6
|
+
the caller.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import contextlib
|
|
12
|
+
import os
|
|
13
|
+
import sys
|
|
14
|
+
import traceback
|
|
15
|
+
from collections.abc import Mapping
|
|
16
|
+
|
|
17
|
+
from . import urls
|
|
18
|
+
|
|
19
|
+
_ON = False
|
|
20
|
+
_CREDENTIAL_HEADERS = frozenset(
|
|
21
|
+
{"authorization", "cookie", "proxy-authorization", "x-api-key", "x-auth-token"}
|
|
22
|
+
)
|
|
23
|
+
_FALSE = frozenset({"", "0", "false", "no", "off"})
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def configure(flag: bool = False, environ: Mapping[str, str] | None = None) -> None:
|
|
27
|
+
"""Turn tracing on from the flag or ``ACADFETCH_DEBUG``; the flag wins."""
|
|
28
|
+
global _ON
|
|
29
|
+
env = os.environ if environ is None else environ
|
|
30
|
+
value = (env.get("ACADFETCH_DEBUG") or "").strip().lower()
|
|
31
|
+
_ON = bool(flag) or value not in _FALSE
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def enabled() -> bool:
|
|
35
|
+
return _ON
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def log(message: str) -> None:
|
|
39
|
+
if not _ON:
|
|
40
|
+
return
|
|
41
|
+
with contextlib.suppress(OSError, ValueError):
|
|
42
|
+
print(f"debug: {message}", file=sys.stderr, flush=True)
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def log_exception(exc: BaseException) -> None:
|
|
46
|
+
if not _ON:
|
|
47
|
+
return
|
|
48
|
+
with contextlib.suppress(OSError, ValueError):
|
|
49
|
+
text = "".join(traceback.format_exception(type(exc), exc, exc.__traceback__))
|
|
50
|
+
sys.stderr.write(urls.redact_text(text))
|
|
51
|
+
sys.stderr.flush()
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def headers_for_log(headers: Mapping[str, str]) -> str:
|
|
55
|
+
"""Header names with safe values; credentials become ``…``."""
|
|
56
|
+
return " ".join(
|
|
57
|
+
f"{name}: {'…' if name.lower() in _CREDENTIAL_HEADERS else value}"
|
|
58
|
+
for name, value in headers.items()
|
|
59
|
+
)
|
|
@@ -13,10 +13,12 @@ Resume is built on three ideas:
|
|
|
13
13
|
from __future__ import annotations
|
|
14
14
|
|
|
15
15
|
import contextlib
|
|
16
|
+
import fnmatch
|
|
16
17
|
import json
|
|
17
18
|
import os
|
|
18
19
|
import random
|
|
19
20
|
import shutil
|
|
21
|
+
import socket
|
|
20
22
|
import threading
|
|
21
23
|
import time
|
|
22
24
|
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
@@ -24,8 +26,9 @@ from dataclasses import dataclass, field
|
|
|
24
26
|
from datetime import datetime, timezone
|
|
25
27
|
from http.client import HTTPException
|
|
26
28
|
from pathlib import Path
|
|
29
|
+
from typing import cast
|
|
27
30
|
|
|
28
|
-
from . import atomic, naming, verify
|
|
31
|
+
from . import atomic, debug, naming, verify
|
|
29
32
|
from .client import Client
|
|
30
33
|
from .consts import CHUNK
|
|
31
34
|
from .errors import HttpError, LocalError, PartialError, TargetError
|
|
@@ -172,10 +175,10 @@ def _range_start(value: str | None) -> int | None:
|
|
|
172
175
|
return None
|
|
173
176
|
|
|
174
177
|
|
|
175
|
-
def _socket_of(resp) ->
|
|
178
|
+
def _socket_of(resp) -> socket.socket | None: # noqa: ANN001
|
|
176
179
|
"""The socket under a urllib response, if we can reach it."""
|
|
177
180
|
raw = getattr(getattr(resp, "fp", None), "raw", None)
|
|
178
|
-
return getattr(raw, "_sock", None)
|
|
181
|
+
return cast("socket.socket | None", getattr(raw, "_sock", None))
|
|
179
182
|
|
|
180
183
|
|
|
181
184
|
def _is_error_page(content_type: str, name: str) -> bool:
|
|
@@ -200,7 +203,7 @@ def _clear_timeout_latch(resp) -> None: # noqa: ANN001
|
|
|
200
203
|
behaviour instead of breaking the run.
|
|
201
204
|
"""
|
|
202
205
|
stream = getattr(getattr(resp, "fp", None), "raw", None)
|
|
203
|
-
if getattr(stream, "_timeout_occurred", False):
|
|
206
|
+
if stream is not None and getattr(stream, "_timeout_occurred", False):
|
|
204
207
|
stream._timeout_occurred = False
|
|
205
208
|
|
|
206
209
|
|
|
@@ -371,7 +374,7 @@ def _trusted_urls(manifest_path: Path) -> dict[str, str]:
|
|
|
371
374
|
if not isinstance(rows, list):
|
|
372
375
|
return {}
|
|
373
376
|
return {
|
|
374
|
-
row["name"]: row.get("url")
|
|
377
|
+
row["name"]: str(row.get("url") or "")
|
|
375
378
|
for row in rows
|
|
376
379
|
if isinstance(row, dict) and isinstance(row.get("name"), str)
|
|
377
380
|
}
|
|
@@ -541,6 +544,7 @@ def _attempt(
|
|
|
541
544
|
time.sleep(min(2**attempt, 10) * (0.7 + random.random() * 0.6))
|
|
542
545
|
resumed_from = part.stat().st_size if part.exists() else 0
|
|
543
546
|
else:
|
|
547
|
+
assert last is not None # the loop only ends after a failure
|
|
544
548
|
return Result(
|
|
545
549
|
file=file,
|
|
546
550
|
state="failed",
|
|
@@ -606,8 +610,24 @@ def _attempt(
|
|
|
606
610
|
)
|
|
607
611
|
|
|
608
612
|
|
|
613
|
+
def matches(file: File, only: list[str] | None, globs: list[str] | None) -> bool:
|
|
614
|
+
"""Whether a file passes the ``--only`` / ``--glob`` filters."""
|
|
615
|
+
name = naming.sanitize(file.name).lower()
|
|
616
|
+
if (
|
|
617
|
+
only
|
|
618
|
+
and name not in {n.lower() for n in only}
|
|
619
|
+
and file.name.lower() not in {n.lower() for n in only}
|
|
620
|
+
):
|
|
621
|
+
return False
|
|
622
|
+
return not globs or any(fnmatch.fnmatch(name, glob.lower()) for glob in globs)
|
|
623
|
+
|
|
624
|
+
|
|
609
625
|
def fetch_record(
|
|
610
|
-
record: Record,
|
|
626
|
+
record: Record,
|
|
627
|
+
opts: Options,
|
|
628
|
+
client: Client,
|
|
629
|
+
only: list[str] | None = None,
|
|
630
|
+
globs: list[str] | None = None,
|
|
611
631
|
) -> list[Result]:
|
|
612
632
|
"""Fetch every file of ``record``, surviving a Ctrl-C.
|
|
613
633
|
|
|
@@ -616,25 +636,24 @@ def fetch_record(
|
|
|
616
636
|
"""
|
|
617
637
|
_STOP.clear()
|
|
618
638
|
try:
|
|
619
|
-
return _fetch_record(record, opts, client, only)
|
|
639
|
+
return _fetch_record(record, opts, client, only, globs)
|
|
620
640
|
finally:
|
|
621
641
|
_STOP.clear()
|
|
622
642
|
|
|
623
643
|
|
|
624
644
|
def _fetch_record(
|
|
625
|
-
record: Record,
|
|
645
|
+
record: Record,
|
|
646
|
+
opts: Options,
|
|
647
|
+
client: Client,
|
|
648
|
+
only: list[str] | None = None,
|
|
649
|
+
globs: list[str] | None = None,
|
|
626
650
|
) -> list[Result]:
|
|
627
651
|
if opts.out.exists() and not opts.out.is_dir():
|
|
628
652
|
raise TargetError(f"-o must be a directory, and {opts.out} is a file")
|
|
629
653
|
opts.out.mkdir(parents=True, exist_ok=True)
|
|
630
654
|
files = record.files
|
|
631
|
-
if only:
|
|
632
|
-
|
|
633
|
-
files = [
|
|
634
|
-
f
|
|
635
|
-
for f in files
|
|
636
|
-
if naming.sanitize(f.name).lower() in wanted or f.name.lower() in wanted
|
|
637
|
-
]
|
|
655
|
+
if only or globs:
|
|
656
|
+
files = [f for f in files if matches(f, only, globs)]
|
|
638
657
|
if not files:
|
|
639
658
|
raise PartialError([])
|
|
640
659
|
|
|
@@ -700,6 +719,7 @@ def _fetch_record(
|
|
|
700
719
|
|
|
701
720
|
def _crashed(file: File, exc: Exception) -> Result:
|
|
702
721
|
"""An unexpected worker exception, as one failed row instead of a traceback."""
|
|
722
|
+
debug.log_exception(exc)
|
|
703
723
|
return Result(
|
|
704
724
|
file=file,
|
|
705
725
|
state="failed",
|
|
@@ -13,6 +13,7 @@ from urllib.parse import quote, urlsplit
|
|
|
13
13
|
|
|
14
14
|
from ..client import Client
|
|
15
15
|
from ..errors import HttpError, SourceError, TargetError
|
|
16
|
+
from .base import Record
|
|
16
17
|
|
|
17
18
|
HANDLE = "https://doi.org/api/handles/{doi}"
|
|
18
19
|
_DOI = re.compile(r"^10\.\d{4,9}/\S+$")
|
|
@@ -52,7 +53,7 @@ def _page_like(client: Client, url: str, content_type: str) -> bool:
|
|
|
52
53
|
return head.lstrip()[:1] == b"<"
|
|
53
54
|
|
|
54
55
|
|
|
55
|
-
def resolve(client: Client, target: str) ->
|
|
56
|
+
def resolve(client: Client, target: str) -> Record:
|
|
56
57
|
url = resolve_url(client, target)
|
|
57
58
|
from . import registry # late import: registry imports this module
|
|
58
59
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: acadfetch
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.6.0
|
|
4
4
|
Summary: Resumable, rate-limited, checksum-verified bulk downloads from academic data repositories
|
|
5
5
|
Author: young-ski
|
|
6
6
|
License-Expression: MIT
|
|
@@ -89,7 +89,7 @@ From a checkout, for development:
|
|
|
89
89
|
|
|
90
90
|
```bash
|
|
91
91
|
git clone https://github.com/young-ski/acadfetch && cd acadfetch
|
|
92
|
-
python -m pip install -e . pytest && python -m pytest -q
|
|
92
|
+
python -m pip install -e . pytest mypy && python -m pytest -q
|
|
93
93
|
```
|
|
94
94
|
|
|
95
95
|
## Supported platforms
|
|
@@ -142,6 +142,15 @@ two never interleave:
|
|
|
142
142
|
acadfetch fetch zenodo:123456 --json | jq -r '.files[] | select(.state=="failed") | .name'
|
|
143
143
|
```
|
|
144
144
|
|
|
145
|
+
To hand the bytes to another downloader, `-w/--urls` writes an aria2-compatible
|
|
146
|
+
list (`-` is stdout) and transfers nothing:
|
|
147
|
+
|
|
148
|
+
```bash
|
|
149
|
+
acadfetch fetch zenodo:123456 -w - --glob '*.csv' | aria2c -i -
|
|
150
|
+
```
|
|
151
|
+
|
|
152
|
+
`--glob` takes shell patterns and combines with `--only` (both must pass).
|
|
153
|
+
|
|
145
154
|
Exit codes are unchanged, and a failure before any transfer still produces a
|
|
146
155
|
document with `error` set. `report` is the document's schema version (now `2`, field reference in
|
|
147
156
|
[docs/json-report.md](docs/json-report.md)):
|
|
@@ -284,6 +293,13 @@ acadfetch fetch zenodo:123456 --ca-bundle corp-tls.pem # trust this PEM
|
|
|
284
293
|
either, `HTTP_PROXY` / `HTTPS_PROXY` / `ALL_PROXY` are honoured by urllib as
|
|
285
294
|
usual. A timeout is per request, connect to first byte.
|
|
286
295
|
|
|
296
|
+
## Debugging
|
|
297
|
+
|
|
298
|
+
`--verbose` (or `ACADFETCH_DEBUG=1`) traces each request, each retry and the
|
|
299
|
+
traceback of a failure to stderr; stdout, `--json` documents and exit codes are
|
|
300
|
+
unchanged. Credential header values and presigned query parameters are masked,
|
|
301
|
+
so the trace can be pasted into an issue.
|
|
302
|
+
|
|
287
303
|
## Politeness
|
|
288
304
|
|
|
289
305
|
Defaults: 3 parallel transfers, 2 requests/second, 4 retries with jittered
|
|
@@ -307,6 +323,10 @@ honest way to test this code path.
|
|
|
307
323
|
|
|
308
324
|
## Getting involved
|
|
309
325
|
|
|
326
|
+
A [30-second tour](docs/demo.md) shows the whole loop, and
|
|
327
|
+
[docs/integrations.md](docs/integrations.md) has Make, CI, aria2 and Snakemake
|
|
328
|
+
recipes.
|
|
329
|
+
|
|
310
330
|
A nightly job downloads three fixed public records and verifies them, so a
|
|
311
331
|
rotted adapter opens an issue before a user reports one. It never blocks a
|
|
312
332
|
merge; the next good night closes the issue.
|
|
@@ -346,6 +366,12 @@ acadfetch verify ./data/manifest.json
|
|
|
346
366
|
|
|
347
367
|
MIT。
|
|
348
368
|
|
|
369
|
+
## Citing
|
|
370
|
+
|
|
371
|
+
`acadfetch --cite` prints how to cite the tool; the machine-readable form is
|
|
372
|
+
[`CITATION.cff`](CITATION.cff). For the data, cite the repository record — the
|
|
373
|
+
manifest is the part that lets a reader re-check it.
|
|
374
|
+
|
|
349
375
|
## Audit trail
|
|
350
376
|
|
|
351
377
|
How the 3.15 support claim was reached, including what it did **not** prove —
|
|
@@ -9,6 +9,7 @@ acadfetch/auth.py
|
|
|
9
9
|
acadfetch/cli.py
|
|
10
10
|
acadfetch/client.py
|
|
11
11
|
acadfetch/consts.py
|
|
12
|
+
acadfetch/debug.py
|
|
12
13
|
acadfetch/downloader.py
|
|
13
14
|
acadfetch/errors.py
|
|
14
15
|
acadfetch/manifest.py
|
|
@@ -37,9 +38,11 @@ tests/conftest.py
|
|
|
37
38
|
tests/test_atomic.py
|
|
38
39
|
tests/test_auth.py
|
|
39
40
|
tests/test_checksum.py
|
|
41
|
+
tests/test_citation.py
|
|
40
42
|
tests/test_cli.py
|
|
41
43
|
tests/test_client.py
|
|
42
44
|
tests/test_control_names.py
|
|
45
|
+
tests/test_debug.py
|
|
43
46
|
tests/test_disk.py
|
|
44
47
|
tests/test_downloader.py
|
|
45
48
|
tests/test_identity.py
|
|
@@ -57,6 +60,7 @@ tests/test_remote.py
|
|
|
57
60
|
tests/test_removed.py
|
|
58
61
|
tests/test_sources.py
|
|
59
62
|
tests/test_urls.py
|
|
63
|
+
tests/test_urls_glob.py
|
|
60
64
|
tests/test_verify.py
|
|
61
65
|
tests/fixtures/README.md
|
|
62
66
|
tests/fixtures/ca.pem
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "acadfetch"
|
|
7
|
-
version = "0.
|
|
7
|
+
version = "0.6.0"
|
|
8
8
|
description = "Resumable, rate-limited, checksum-verified bulk downloads from academic data repositories"
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.10"
|
|
@@ -46,6 +46,12 @@ include = ["acadfetch*"]
|
|
|
46
46
|
testpaths = ["tests"]
|
|
47
47
|
addopts = "-q"
|
|
48
48
|
|
|
49
|
+
[tool.mypy]
|
|
50
|
+
python_version = "3.10"
|
|
51
|
+
files = ["acadfetch"]
|
|
52
|
+
warn_unused_ignores = true
|
|
53
|
+
warn_redundant_casts = true
|
|
54
|
+
|
|
49
55
|
[tool.ruff]
|
|
50
56
|
line-length = 100
|
|
51
57
|
target-version = "py310"
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
|
|
5
|
+
from acadfetch import cli
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def test_the_citation_file_is_cff():
|
|
9
|
+
text = Path("CITATION.cff").read_text("utf-8")
|
|
10
|
+
assert text.startswith("cff-version: 1.2.0")
|
|
11
|
+
assert "type: software" in text
|
|
12
|
+
assert "license: MIT" in text
|
|
13
|
+
assert "https://github.com/young-ski/acadfetch" in text
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def test_cite_prints_the_tool_citation(capsys):
|
|
17
|
+
assert cli.main(["--cite"]) == cli.EXIT_OK
|
|
18
|
+
out = capsys.readouterr().out
|
|
19
|
+
assert "acadfetch (version" in out
|
|
20
|
+
assert "manifest.json" in out
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def test_cite_needs_no_subcommand(capsys):
|
|
24
|
+
"""argparse would otherwise demand one."""
|
|
25
|
+
assert cli.main(["--cite"]) == cli.EXIT_OK
|
|
26
|
+
assert "To cite" in capsys.readouterr().out
|
|
@@ -226,7 +226,7 @@ def test_retries_flag_reaches_the_transfer_layer(monkeypatch, capsys):
|
|
|
226
226
|
_stub_resolve(monkeypatch)
|
|
227
227
|
seen = {}
|
|
228
228
|
|
|
229
|
-
def fake_fetch(record, opts, client, only=None):
|
|
229
|
+
def fake_fetch(record, opts, client, only=None, globs=None):
|
|
230
230
|
seen["retries"] = opts.retries
|
|
231
231
|
return []
|
|
232
232
|
|
|
@@ -331,7 +331,7 @@ def test_interrupt_exits_130_and_still_writes_the_manifest(monkeypatch, capsys,
|
|
|
331
331
|
|
|
332
332
|
_stub_resolve(monkeypatch)
|
|
333
333
|
|
|
334
|
-
def fake_fetch(record, opts, client, only=None):
|
|
334
|
+
def fake_fetch(record, opts, client, only=None, globs=None):
|
|
335
335
|
opts.interrupted = True
|
|
336
336
|
return [Result(record.files[0], "failed", error="interrupted")]
|
|
337
337
|
|
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import pytest
|
|
4
|
+
|
|
5
|
+
from acadfetch import cli, debug
|
|
6
|
+
from acadfetch.client import Client
|
|
7
|
+
from acadfetch.errors import HttpError
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
@pytest.fixture(autouse=True)
|
|
11
|
+
def _reset():
|
|
12
|
+
debug.configure(False)
|
|
13
|
+
yield
|
|
14
|
+
debug.configure(False)
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def test_the_environment_can_turn_it_on():
|
|
18
|
+
debug.configure(False, {"ACADFETCH_DEBUG": "1"})
|
|
19
|
+
assert debug.enabled()
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
@pytest.mark.parametrize("value", ["0", "false", "no", "off", ""])
|
|
23
|
+
def test_the_environment_can_turn_it_off(value):
|
|
24
|
+
debug.configure(False, {"ACADFETCH_DEBUG": value})
|
|
25
|
+
assert not debug.enabled()
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def test_the_flag_wins_over_the_environment():
|
|
29
|
+
debug.configure(True, {"ACADFETCH_DEBUG": "0"})
|
|
30
|
+
assert debug.enabled()
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def test_it_is_silent_when_off(capsys):
|
|
34
|
+
debug.log("nothing")
|
|
35
|
+
debug.log_exception(RuntimeError("boom"))
|
|
36
|
+
assert capsys.readouterr().err == ""
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def _talking_client():
|
|
40
|
+
client = Client(rate=1000, retries=0, auth={"direct": {"Authorization": "Bearer SECRET"}})
|
|
41
|
+
client.source = "direct"
|
|
42
|
+
return client
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def test_headers_are_masked(server, capsys):
|
|
46
|
+
server.cfg["files"]["/a.bin"] = b"x"
|
|
47
|
+
debug.configure(True)
|
|
48
|
+
|
|
49
|
+
_talking_client().head(server.base + "/a.bin")
|
|
50
|
+
|
|
51
|
+
err = capsys.readouterr().err
|
|
52
|
+
assert "HEAD" in err
|
|
53
|
+
assert "Authorization: …" in err
|
|
54
|
+
assert "SECRET" not in err
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def test_a_credential_in_the_url_is_redacted(server, capsys):
|
|
58
|
+
server.cfg["files"]["/a.bin"] = b"x"
|
|
59
|
+
debug.configure(True)
|
|
60
|
+
|
|
61
|
+
_talking_client().head(server.base + "/a.bin?X-Amz-Signature=SIG")
|
|
62
|
+
|
|
63
|
+
err = capsys.readouterr().err
|
|
64
|
+
assert "X-Amz-Signature" not in err
|
|
65
|
+
assert "SIG" not in err
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def test_a_retry_is_traced(server, capsys):
|
|
69
|
+
server.cfg["files"]["/a.bin"] = b"x"
|
|
70
|
+
server.cfg["always_500"].add("/a.bin")
|
|
71
|
+
debug.configure(True)
|
|
72
|
+
|
|
73
|
+
with pytest.raises(HttpError):
|
|
74
|
+
Client(rate=1000, retries=1).open(server.base + "/a.bin")
|
|
75
|
+
|
|
76
|
+
err = capsys.readouterr().err
|
|
77
|
+
assert "GET" in err and "-> 500" in err and "retry in" in err
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def test_a_failure_prints_a_traceback_only_with_the_flag(capsys):
|
|
81
|
+
assert cli.main(["fetch", "nonsense://x"]) == 1
|
|
82
|
+
assert "Traceback" not in capsys.readouterr().err
|
|
83
|
+
|
|
84
|
+
assert cli.main(["fetch", "nonsense://x", "--verbose"]) == 1
|
|
85
|
+
assert "Traceback (most recent call last)" in capsys.readouterr().err
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def test_debug_does_not_change_the_json_document(server, monkeypatch, capsys, tmp_path):
|
|
89
|
+
import json
|
|
90
|
+
|
|
91
|
+
server.cfg["files"]["/a.bin"] = b"payload"
|
|
92
|
+
monkeypatch.setattr(
|
|
93
|
+
cli.registry,
|
|
94
|
+
"resolve",
|
|
95
|
+
lambda _c, _t: __import__("acadfetch.sources.base", fromlist=["Record"]).Record(
|
|
96
|
+
source="direct",
|
|
97
|
+
target=server.base + "/a.bin",
|
|
98
|
+
files=[
|
|
99
|
+
__import__("acadfetch.sources.base", fromlist=["File"]).File(
|
|
100
|
+
server.base + "/a.bin", "a.bin", 7
|
|
101
|
+
)
|
|
102
|
+
],
|
|
103
|
+
),
|
|
104
|
+
)
|
|
105
|
+
|
|
106
|
+
code = cli.main(["fetch", "t", "--json", "--verbose", "-o", str(tmp_path)])
|
|
107
|
+
out, _ = capsys.readouterr()
|
|
108
|
+
|
|
109
|
+
assert code == cli.EXIT_OK
|
|
110
|
+
assert json.loads(out)["summary"]["ok"] is True
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def test_a_traceback_does_not_leak_a_credential(capsys):
|
|
114
|
+
assert (
|
|
115
|
+
cli.main(["fetch", "https://example.invalid/x?X-Amz-Signature=SECRET", "--verbose", "-n"])
|
|
116
|
+
== 1
|
|
117
|
+
)
|
|
118
|
+
err = capsys.readouterr().err
|
|
119
|
+
assert "Traceback (most recent call last)" in err
|
|
120
|
+
assert "SECRET" not in err
|
|
@@ -315,3 +315,8 @@ def test_two_legacy_papers_no_longer_share_a_name():
|
|
|
315
315
|
"arxiv:solv-int/9701002v1",
|
|
316
316
|
)
|
|
317
317
|
assert first.files[0].name != second.files[0].name
|
|
318
|
+
|
|
319
|
+
|
|
320
|
+
def test_direct_resolve_rejects_a_non_url():
|
|
321
|
+
with pytest.raises(TargetError, match="not a URL"):
|
|
322
|
+
direct.resolve(HeadClient({}), "zenodo:1")
|
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
|
|
5
|
+
import pytest
|
|
6
|
+
|
|
7
|
+
from acadfetch import cli
|
|
8
|
+
from acadfetch.sources.base import File, Record
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def _run(capsys, *argv):
|
|
12
|
+
code = cli.main([str(a) for a in argv])
|
|
13
|
+
out, err = capsys.readouterr()
|
|
14
|
+
return code, out, err
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
@pytest.fixture
|
|
18
|
+
def two(server, monkeypatch):
|
|
19
|
+
server.cfg["files"]["/a.pdf"] = b"PDF"
|
|
20
|
+
server.cfg["files"]["/b.csv"] = b"CSV"
|
|
21
|
+
record = Record(
|
|
22
|
+
source="direct",
|
|
23
|
+
target="t",
|
|
24
|
+
files=[File(server.base + "/a.pdf", "a.pdf", 3), File(server.base + "/b.csv", "b.csv", 3)],
|
|
25
|
+
)
|
|
26
|
+
monkeypatch.setattr(cli.registry, "resolve", lambda _c, _t: record)
|
|
27
|
+
return record
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def test_glob_narrows_the_plan(two, capsys):
|
|
31
|
+
code, out, _ = _run(capsys, "fetch", "t", "-n", "--json", "--glob", "*.pdf")
|
|
32
|
+
|
|
33
|
+
assert code == cli.EXIT_OK
|
|
34
|
+
assert [row["name"] for row in json.loads(out)["files"]] == ["a.pdf"]
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def test_glob_and_only_are_both_applied(two, capsys):
|
|
38
|
+
code, _, err = _run(capsys, "fetch", "t", "-n", "--glob", "*.pdf", "--only", "b.csv")
|
|
39
|
+
|
|
40
|
+
assert code == cli.EXIT_FAIL
|
|
41
|
+
assert "no file matched" in err
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def test_a_glob_that_matches_nothing_names_the_candidates(two, capsys):
|
|
45
|
+
code, _, err = _run(capsys, "fetch", "t", "-n", "--glob", "*.zip")
|
|
46
|
+
|
|
47
|
+
assert code == cli.EXIT_FAIL
|
|
48
|
+
assert "--glob *.zip" in err
|
|
49
|
+
assert "a.pdf" in err
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def test_glob_only_downloads_the_match(two, server, capsys, tmp_path):
|
|
53
|
+
code, _, _ = _run(capsys, "fetch", "t", "-q", "--glob", "*.csv", "-o", tmp_path)
|
|
54
|
+
|
|
55
|
+
assert code == cli.EXIT_OK
|
|
56
|
+
assert (tmp_path / "b.csv").exists()
|
|
57
|
+
assert not (tmp_path / "a.pdf").exists()
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def test_urls_to_stdout_are_aria2_shaped(two, capsys, tmp_path):
|
|
61
|
+
code, out, _ = _run(capsys, "fetch", "t", "-w", "-", "-o", tmp_path)
|
|
62
|
+
|
|
63
|
+
assert code == cli.EXIT_OK
|
|
64
|
+
lines = out.splitlines()
|
|
65
|
+
assert lines[0].endswith("/a.pdf")
|
|
66
|
+
assert lines[1] == " out=a.pdf"
|
|
67
|
+
assert lines[2] == f" dir={tmp_path}"
|
|
68
|
+
assert "out=b.csv" in out
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def test_urls_to_a_file(two, capsys, tmp_path):
|
|
72
|
+
target = tmp_path / "list.txt"
|
|
73
|
+
|
|
74
|
+
code, out, err = _run(capsys, "fetch", "t", "-w", target, "-o", tmp_path)
|
|
75
|
+
|
|
76
|
+
assert code == cli.EXIT_OK
|
|
77
|
+
assert out == ""
|
|
78
|
+
assert "2 url(s)" in err
|
|
79
|
+
assert "out=a.pdf" in target.read_text("utf-8")
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def test_urls_honour_glob(two, capsys, tmp_path):
|
|
83
|
+
code, out, _ = _run(capsys, "fetch", "t", "-w", "-", "--glob", "*.pdf", "-o", tmp_path)
|
|
84
|
+
|
|
85
|
+
assert code == cli.EXIT_OK
|
|
86
|
+
assert "a.pdf" in out and "b.csv" not in out
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def test_urls_and_json_are_mutually_exclusive(two, capsys):
|
|
90
|
+
code, _, err = _run(capsys, "fetch", "t", "-w", "-", "--json")
|
|
91
|
+
|
|
92
|
+
assert code == cli.EXIT_FAIL
|
|
93
|
+
assert "drop --json" in err
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def test_no_bytes_are_fetched_for_a_list(two, server, capsys, tmp_path):
|
|
97
|
+
_run(capsys, "fetch", "t", "-w", "-", "-o", tmp_path)
|
|
98
|
+
|
|
99
|
+
assert server.cfg["hits"] == {}, "a list must not transfer"
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|