acadfetch 0.7.0__tar.gz → 0.7.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {acadfetch-0.7.0/acadfetch.egg-info → acadfetch-0.7.2}/PKG-INFO +100 -7
- {acadfetch-0.7.0 → acadfetch-0.7.2}/README.md +99 -6
- {acadfetch-0.7.0 → acadfetch-0.7.2}/acadfetch/client.py +20 -0
- {acadfetch-0.7.0 → acadfetch-0.7.2}/acadfetch/consts.py +1 -1
- {acadfetch-0.7.0 → acadfetch-0.7.2}/acadfetch/downloader.py +34 -1
- {acadfetch-0.7.0 → acadfetch-0.7.2}/acadfetch/pooch.py +9 -39
- {acadfetch-0.7.0 → acadfetch-0.7.2/acadfetch.egg-info}/PKG-INFO +100 -7
- {acadfetch-0.7.0 → acadfetch-0.7.2}/pyproject.toml +1 -1
- {acadfetch-0.7.0 → acadfetch-0.7.2}/LICENSE +0 -0
- {acadfetch-0.7.0 → acadfetch-0.7.2}/MANIFEST.in +0 -0
- {acadfetch-0.7.0 → acadfetch-0.7.2}/acadfetch/__init__.py +0 -0
- {acadfetch-0.7.0 → acadfetch-0.7.2}/acadfetch/__main__.py +0 -0
- {acadfetch-0.7.0 → acadfetch-0.7.2}/acadfetch/atomic.py +0 -0
- {acadfetch-0.7.0 → acadfetch-0.7.2}/acadfetch/auth.py +0 -0
- {acadfetch-0.7.0 → acadfetch-0.7.2}/acadfetch/cli.py +0 -0
- {acadfetch-0.7.0 → acadfetch-0.7.2}/acadfetch/debug.py +0 -0
- {acadfetch-0.7.0 → acadfetch-0.7.2}/acadfetch/errors.py +0 -0
- {acadfetch-0.7.0 → acadfetch-0.7.2}/acadfetch/manifest.py +0 -0
- {acadfetch-0.7.0 → acadfetch-0.7.2}/acadfetch/naming.py +0 -0
- {acadfetch-0.7.0 → acadfetch-0.7.2}/acadfetch/progress.py +0 -0
- {acadfetch-0.7.0 → acadfetch-0.7.2}/acadfetch/py.typed +0 -0
- {acadfetch-0.7.0 → acadfetch-0.7.2}/acadfetch/ratelimit.py +0 -0
- {acadfetch-0.7.0 → acadfetch-0.7.2}/acadfetch/remote.py +0 -0
- {acadfetch-0.7.0 → acadfetch-0.7.2}/acadfetch/sources/__init__.py +0 -0
- {acadfetch-0.7.0 → acadfetch-0.7.2}/acadfetch/sources/arxiv.py +0 -0
- {acadfetch-0.7.0 → acadfetch-0.7.2}/acadfetch/sources/base.py +0 -0
- {acadfetch-0.7.0 → acadfetch-0.7.2}/acadfetch/sources/direct.py +0 -0
- {acadfetch-0.7.0 → acadfetch-0.7.2}/acadfetch/sources/doi.py +0 -0
- {acadfetch-0.7.0 → acadfetch-0.7.2}/acadfetch/sources/registry.py +0 -0
- {acadfetch-0.7.0 → acadfetch-0.7.2}/acadfetch/sources/removed.py +0 -0
- {acadfetch-0.7.0 → acadfetch-0.7.2}/acadfetch/sources/zenodo.py +0 -0
- {acadfetch-0.7.0 → acadfetch-0.7.2}/acadfetch/units.py +0 -0
- {acadfetch-0.7.0 → acadfetch-0.7.2}/acadfetch/urls.py +0 -0
- {acadfetch-0.7.0 → acadfetch-0.7.2}/acadfetch/verify.py +0 -0
- {acadfetch-0.7.0 → acadfetch-0.7.2}/acadfetch.egg-info/SOURCES.txt +0 -0
- {acadfetch-0.7.0 → acadfetch-0.7.2}/acadfetch.egg-info/dependency_links.txt +0 -0
- {acadfetch-0.7.0 → acadfetch-0.7.2}/acadfetch.egg-info/entry_points.txt +0 -0
- {acadfetch-0.7.0 → acadfetch-0.7.2}/acadfetch.egg-info/top_level.txt +0 -0
- {acadfetch-0.7.0 → acadfetch-0.7.2}/setup.cfg +0 -0
- {acadfetch-0.7.0 → acadfetch-0.7.2}/tests/conftest.py +0 -0
- {acadfetch-0.7.0 → acadfetch-0.7.2}/tests/fixtures/README.md +0 -0
- {acadfetch-0.7.0 → acadfetch-0.7.2}/tests/fixtures/ca.pem +0 -0
- {acadfetch-0.7.0 → acadfetch-0.7.2}/tests/fixtures/figshare_403_block.html +0 -0
- {acadfetch-0.7.0 → acadfetch-0.7.2}/tests/fixtures/figshare_403_forbidden.json +0 -0
- {acadfetch-0.7.0 → acadfetch-0.7.2}/tests/fixtures/figshare_article.json +0 -0
- {acadfetch-0.7.0 → acadfetch-0.7.2}/tests/fixtures/figshare_file.pdf +0 -0
- {acadfetch-0.7.0 → acadfetch-0.7.2}/tests/test_atomic.py +0 -0
- {acadfetch-0.7.0 → acadfetch-0.7.2}/tests/test_auth.py +0 -0
- {acadfetch-0.7.0 → acadfetch-0.7.2}/tests/test_checksum.py +0 -0
- {acadfetch-0.7.0 → acadfetch-0.7.2}/tests/test_citation.py +0 -0
- {acadfetch-0.7.0 → acadfetch-0.7.2}/tests/test_claims.py +0 -0
- {acadfetch-0.7.0 → acadfetch-0.7.2}/tests/test_cli.py +0 -0
- {acadfetch-0.7.0 → acadfetch-0.7.2}/tests/test_client.py +0 -0
- {acadfetch-0.7.0 → acadfetch-0.7.2}/tests/test_control_names.py +0 -0
- {acadfetch-0.7.0 → acadfetch-0.7.2}/tests/test_debug.py +0 -0
- {acadfetch-0.7.0 → acadfetch-0.7.2}/tests/test_disk.py +0 -0
- {acadfetch-0.7.0 → acadfetch-0.7.2}/tests/test_downloader.py +0 -0
- {acadfetch-0.7.0 → acadfetch-0.7.2}/tests/test_hash_once.py +0 -0
- {acadfetch-0.7.0 → acadfetch-0.7.2}/tests/test_identity.py +0 -0
- {acadfetch-0.7.0 → acadfetch-0.7.2}/tests/test_isolation.py +0 -0
- {acadfetch-0.7.0 → acadfetch-0.7.2}/tests/test_json.py +0 -0
- {acadfetch-0.7.0 → acadfetch-0.7.2}/tests/test_lock.py +0 -0
- {acadfetch-0.7.0 → acadfetch-0.7.2}/tests/test_manifest.py +0 -0
- {acadfetch-0.7.0 → acadfetch-0.7.2}/tests/test_naming.py +0 -0
- {acadfetch-0.7.0 → acadfetch-0.7.2}/tests/test_naming_race.py +0 -0
- {acadfetch-0.7.0 → acadfetch-0.7.2}/tests/test_network.py +0 -0
- {acadfetch-0.7.0 → acadfetch-0.7.2}/tests/test_packaging.py +0 -0
- {acadfetch-0.7.0 → acadfetch-0.7.2}/tests/test_pooch.py +0 -0
- {acadfetch-0.7.0 → acadfetch-0.7.2}/tests/test_progress.py +0 -0
- {acadfetch-0.7.0 → acadfetch-0.7.2}/tests/test_ratelimit.py +0 -0
- {acadfetch-0.7.0 → acadfetch-0.7.2}/tests/test_redirect.py +0 -0
- {acadfetch-0.7.0 → acadfetch-0.7.2}/tests/test_remote.py +0 -0
- {acadfetch-0.7.0 → acadfetch-0.7.2}/tests/test_removed.py +0 -0
- {acadfetch-0.7.0 → acadfetch-0.7.2}/tests/test_sources.py +0 -0
- {acadfetch-0.7.0 → acadfetch-0.7.2}/tests/test_urls.py +0 -0
- {acadfetch-0.7.0 → acadfetch-0.7.2}/tests/test_urls_glob.py +0 -0
- {acadfetch-0.7.0 → acadfetch-0.7.2}/tests/test_verify.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: acadfetch
|
|
3
|
-
Version: 0.7.
|
|
3
|
+
Version: 0.7.2
|
|
4
4
|
Summary: Resumable, rate-limited, checksum-verified bulk downloads from academic data repositories
|
|
5
5
|
Author: young-ski
|
|
6
6
|
License-Expression: MIT
|
|
@@ -35,6 +35,8 @@ Dynamic: license-file
|
|
|
35
35
|
[](https://pypi.org/project/acadfetch/)
|
|
36
36
|
[](LICENSE)
|
|
37
37
|
|
|
38
|
+
**English** · [简体中文](#中文)
|
|
39
|
+
|
|
38
40
|
Batch-download files from academic data repositories — resumably, rate-limited, and
|
|
39
41
|
checked against the checksums the repository published.
|
|
40
42
|
|
|
@@ -337,9 +339,8 @@ honest way to test this code path.
|
|
|
337
339
|
## Getting involved
|
|
338
340
|
|
|
339
341
|
A [30-second tour](docs/demo.md) shows the whole loop, and
|
|
340
|
-
[docs/integrations.md](docs/integrations.md) has Make, CI, aria2, Snakemake
|
|
341
|
-
pooch downloader
|
|
342
|
-
recipes.
|
|
342
|
+
[docs/integrations.md](docs/integrations.md) has Make, CI, aria2, a Snakemake
|
|
343
|
+
storage plugin and a pooch downloader.
|
|
343
344
|
|
|
344
345
|
A nightly job downloads three fixed public records and verifies them, so a
|
|
345
346
|
rotted adapter opens an issue before a user reports one. It never blocks a
|
|
@@ -373,12 +374,104 @@ merge; the next good night closes the issue.
|
|
|
373
374
|
学术数据仓库的批量下载器:断点续传、全局限速、按仓库公布的校验和验证,并落
|
|
374
375
|
`manifest.json`(DOI、许可协议、每文件 SHA-256)供日后复现。
|
|
375
376
|
|
|
377
|
+
它不是搜索器、不抓付费墙、也不是引文管理器 —— 它只管"取数据"这一步,并让这一步
|
|
378
|
+
事后可证明。
|
|
379
|
+
|
|
380
|
+
### 安装
|
|
381
|
+
|
|
382
|
+
Python 3.10–3.15,零运行时依赖。
|
|
383
|
+
|
|
384
|
+
```bash
|
|
385
|
+
uvx acadfetch fetch 10.5281/zenodo.23612969 # 不安装,直接跑
|
|
386
|
+
```
|
|
387
|
+
|
|
388
|
+
```bash
|
|
389
|
+
pipx install acadfetch # 或者装到 PATH 上
|
|
390
|
+
```
|
|
391
|
+
|
|
392
|
+
### 快速开始
|
|
393
|
+
|
|
394
|
+
`10.5281/zenodo.23612969` 是真实记录,可以直接复制;其余 id 都是占位符。
|
|
395
|
+
|
|
376
396
|
```bash
|
|
377
|
-
acadfetch fetch 10.5281/zenodo.
|
|
378
|
-
acadfetch
|
|
397
|
+
acadfetch fetch 10.5281/zenodo.23612969 -n # 先看:落到哪、传多少、许可是什么
|
|
398
|
+
acadfetch fetch 10.5281/zenodo.23612969 -o ./data
|
|
399
|
+
acadfetch verify ./data/manifest.json # 事后复验;对不上就非零退出
|
|
379
400
|
```
|
|
380
401
|
|
|
381
|
-
|
|
402
|
+
目标也可以是 `arxiv:1706.03762`、任意 DOI,或任意 `https://` 直链。
|
|
403
|
+
|
|
404
|
+
> 直链成败由对方主机决定:figshare 的 CDN 在某些网络会回 403 拦截页(当年删掉
|
|
405
|
+
> figshare 适配器正是因为这个)。acadfetch 会明说这是拦截页、而不是把 HTML 存成
|
|
406
|
+
> PDF,并把下一条命令给你。
|
|
407
|
+
|
|
408
|
+
### 它替你保住的东西
|
|
409
|
+
|
|
410
|
+
| 情况 | 它怎么做 |
|
|
411
|
+
| --- | --- |
|
|
412
|
+
| 传到 80% 断了 | `.part` 保留,重跑用 `Range`/`If-Range` 接着传;服务器若无视 `Range`,旧字节先挪到一边,整份新文件到齐才替换 |
|
|
413
|
+
| 悄悄损坏 | 按仓库公布的 md5/sha256 校验,不符就重下;仍不符则 `verified: false` 并删掉坏字节 |
|
|
414
|
+
| "我到底下的是哪个版本" | `manifest.json` 记 DOI、许可、每文件 SHA-256、最终 URL、HTTP 状态与时间 |
|
|
415
|
+
| 被限流吃 403 | 全局令牌桶限速,遵守 `Retry-After`,拦截页按次数重试 |
|
|
416
|
+
|
|
417
|
+
默认 3 并发、2 请求/秒、4 次重试。Zenodo 和 arXiv 靠募捐运营,`--rate 1 -j 1`
|
|
418
|
+
多花你几分钟,对面少受点罪。
|
|
419
|
+
|
|
420
|
+
### 进流水线
|
|
421
|
+
|
|
422
|
+
```bash
|
|
423
|
+
acadfetch fetch 10.5281/zenodo.23612969 --json # 一个 JSON 文档给 stdout,进度走 stderr
|
|
424
|
+
acadfetch fetch 10.5281/zenodo.23612969 -w - # 输出 aria2 可用 URL 列表,不传字节
|
|
425
|
+
```
|
|
426
|
+
|
|
427
|
+
`--json` 的字段契约见 [docs/json-report.md](docs/json-report.md);`report` 是文档
|
|
428
|
+
版本号,只加字段、不改语义。
|
|
429
|
+
|
|
430
|
+
### 退出码
|
|
431
|
+
|
|
432
|
+
`0` 成功、`1` 失败、`2` 部分成功、`64` 用法错误、`130` 被中断(`.part` 已保留,
|
|
433
|
+
重跑即可续)。
|
|
434
|
+
|
|
435
|
+
### 令牌
|
|
436
|
+
|
|
437
|
+
`ZENODO_TOKEN` 从环境读取,**只发给 Zenodo**;跨主机跳转时剥掉,https 降级到
|
|
438
|
+
http 时也会剥掉并告警。manifest 只记变量名和后四位。
|
|
439
|
+
|
|
440
|
+
```bash
|
|
441
|
+
export ZENODO_TOKEN=...
|
|
442
|
+
acadfetch fetch zenodo:123456
|
|
443
|
+
```
|
|
444
|
+
|
|
445
|
+
### 支持的源
|
|
446
|
+
|
|
447
|
+
| 源 | 目标形式 | 校验和 |
|
|
448
|
+
| --- | --- | --- |
|
|
449
|
+
| Zenodo | DOI `10.5281/zenodo.N`、`zenodo:N`、记录 URL | md5 |
|
|
450
|
+
| arXiv | `arxiv:ID`、裸 ID、abs/pdf URL | 无(只比字节数) |
|
|
451
|
+
| doi.org | 任意 DOI,解析后再分派 | 视源而定 |
|
|
452
|
+
| 直链 | 任意 `http(s)` URL | 响应头里有就用 |
|
|
453
|
+
|
|
454
|
+
figshare 适配器在 0.2.0 已移除;源不公布校验和时用 `--checksum md5:...` 补上,
|
|
455
|
+
旧 manifest 仍可 `verify`。见 [docs/migration-0.2.md](docs/migration-0.2.md)。
|
|
456
|
+
|
|
457
|
+
### 已知限制
|
|
458
|
+
|
|
459
|
+
- `manifest.json` 没有签名:`verify` 证明磁盘上的字节与 manifest 一致,不能证明
|
|
460
|
+
manifest 本身没被改写。补这一层用 `verify --against-remote`(会重新问源站),
|
|
461
|
+
细节见 [SECURITY.md](SECURITY.md)。
|
|
462
|
+
- arXiv 不公布校验和,验证降级为字节数,`verified` 是 `null` 而不是假的安心。
|
|
463
|
+
- 没有搜索与分页,目标得你自己给。
|
|
464
|
+
- Windows ARM 未测;macOS 只做尽力而为(在 CI 里,但不阻塞发布)。
|
|
465
|
+
|
|
466
|
+
### 更多
|
|
467
|
+
|
|
468
|
+
- 30 秒演示 [docs/demo.md](docs/demo.md) · 按症状排障 [docs/troubleshooting.md](docs/troubleshooting.md) · 接进流水线 [docs/integrations.md](docs/integrations.md)(Make、CI、aria2、Snakemake 插件、pooch 下载器)
|
|
469
|
+
- 加源、动传输、写测试的规矩见 [CONTRIBUTING.md](CONTRIBUTING.md)
|
|
470
|
+
|
|
471
|
+
### 许可
|
|
472
|
+
|
|
473
|
+
工具是 MIT。数据本身的许可见 manifest 的 `license` 字段;`unknown` 表示**转发前先
|
|
474
|
+
确认**,而 `-n` 会在花掉带宽之前就把它打出来。
|
|
382
475
|
|
|
383
476
|
## Citing
|
|
384
477
|
|
|
@@ -4,6 +4,8 @@
|
|
|
4
4
|
[](https://pypi.org/project/acadfetch/)
|
|
5
5
|
[](LICENSE)
|
|
6
6
|
|
|
7
|
+
**English** · [简体中文](#中文)
|
|
8
|
+
|
|
7
9
|
Batch-download files from academic data repositories — resumably, rate-limited, and
|
|
8
10
|
checked against the checksums the repository published.
|
|
9
11
|
|
|
@@ -306,9 +308,8 @@ honest way to test this code path.
|
|
|
306
308
|
## Getting involved
|
|
307
309
|
|
|
308
310
|
A [30-second tour](docs/demo.md) shows the whole loop, and
|
|
309
|
-
[docs/integrations.md](docs/integrations.md) has Make, CI, aria2, Snakemake
|
|
310
|
-
pooch downloader
|
|
311
|
-
recipes.
|
|
311
|
+
[docs/integrations.md](docs/integrations.md) has Make, CI, aria2, a Snakemake
|
|
312
|
+
storage plugin and a pooch downloader.
|
|
312
313
|
|
|
313
314
|
A nightly job downloads three fixed public records and verifies them, so a
|
|
314
315
|
rotted adapter opens an issue before a user reports one. It never blocks a
|
|
@@ -342,12 +343,104 @@ merge; the next good night closes the issue.
|
|
|
342
343
|
学术数据仓库的批量下载器:断点续传、全局限速、按仓库公布的校验和验证,并落
|
|
343
344
|
`manifest.json`(DOI、许可协议、每文件 SHA-256)供日后复现。
|
|
344
345
|
|
|
346
|
+
它不是搜索器、不抓付费墙、也不是引文管理器 —— 它只管"取数据"这一步,并让这一步
|
|
347
|
+
事后可证明。
|
|
348
|
+
|
|
349
|
+
### 安装
|
|
350
|
+
|
|
351
|
+
Python 3.10–3.15,零运行时依赖。
|
|
352
|
+
|
|
353
|
+
```bash
|
|
354
|
+
uvx acadfetch fetch 10.5281/zenodo.23612969 # 不安装,直接跑
|
|
355
|
+
```
|
|
356
|
+
|
|
357
|
+
```bash
|
|
358
|
+
pipx install acadfetch # 或者装到 PATH 上
|
|
359
|
+
```
|
|
360
|
+
|
|
361
|
+
### 快速开始
|
|
362
|
+
|
|
363
|
+
`10.5281/zenodo.23612969` 是真实记录,可以直接复制;其余 id 都是占位符。
|
|
364
|
+
|
|
345
365
|
```bash
|
|
346
|
-
acadfetch fetch 10.5281/zenodo.
|
|
347
|
-
acadfetch
|
|
366
|
+
acadfetch fetch 10.5281/zenodo.23612969 -n # 先看:落到哪、传多少、许可是什么
|
|
367
|
+
acadfetch fetch 10.5281/zenodo.23612969 -o ./data
|
|
368
|
+
acadfetch verify ./data/manifest.json # 事后复验;对不上就非零退出
|
|
348
369
|
```
|
|
349
370
|
|
|
350
|
-
|
|
371
|
+
目标也可以是 `arxiv:1706.03762`、任意 DOI,或任意 `https://` 直链。
|
|
372
|
+
|
|
373
|
+
> 直链成败由对方主机决定:figshare 的 CDN 在某些网络会回 403 拦截页(当年删掉
|
|
374
|
+
> figshare 适配器正是因为这个)。acadfetch 会明说这是拦截页、而不是把 HTML 存成
|
|
375
|
+
> PDF,并把下一条命令给你。
|
|
376
|
+
|
|
377
|
+
### 它替你保住的东西
|
|
378
|
+
|
|
379
|
+
| 情况 | 它怎么做 |
|
|
380
|
+
| --- | --- |
|
|
381
|
+
| 传到 80% 断了 | `.part` 保留,重跑用 `Range`/`If-Range` 接着传;服务器若无视 `Range`,旧字节先挪到一边,整份新文件到齐才替换 |
|
|
382
|
+
| 悄悄损坏 | 按仓库公布的 md5/sha256 校验,不符就重下;仍不符则 `verified: false` 并删掉坏字节 |
|
|
383
|
+
| "我到底下的是哪个版本" | `manifest.json` 记 DOI、许可、每文件 SHA-256、最终 URL、HTTP 状态与时间 |
|
|
384
|
+
| 被限流吃 403 | 全局令牌桶限速,遵守 `Retry-After`,拦截页按次数重试 |
|
|
385
|
+
|
|
386
|
+
默认 3 并发、2 请求/秒、4 次重试。Zenodo 和 arXiv 靠募捐运营,`--rate 1 -j 1`
|
|
387
|
+
多花你几分钟,对面少受点罪。
|
|
388
|
+
|
|
389
|
+
### 进流水线
|
|
390
|
+
|
|
391
|
+
```bash
|
|
392
|
+
acadfetch fetch 10.5281/zenodo.23612969 --json # 一个 JSON 文档给 stdout,进度走 stderr
|
|
393
|
+
acadfetch fetch 10.5281/zenodo.23612969 -w - # 输出 aria2 可用 URL 列表,不传字节
|
|
394
|
+
```
|
|
395
|
+
|
|
396
|
+
`--json` 的字段契约见 [docs/json-report.md](docs/json-report.md);`report` 是文档
|
|
397
|
+
版本号,只加字段、不改语义。
|
|
398
|
+
|
|
399
|
+
### 退出码
|
|
400
|
+
|
|
401
|
+
`0` 成功、`1` 失败、`2` 部分成功、`64` 用法错误、`130` 被中断(`.part` 已保留,
|
|
402
|
+
重跑即可续)。
|
|
403
|
+
|
|
404
|
+
### 令牌
|
|
405
|
+
|
|
406
|
+
`ZENODO_TOKEN` 从环境读取,**只发给 Zenodo**;跨主机跳转时剥掉,https 降级到
|
|
407
|
+
http 时也会剥掉并告警。manifest 只记变量名和后四位。
|
|
408
|
+
|
|
409
|
+
```bash
|
|
410
|
+
export ZENODO_TOKEN=...
|
|
411
|
+
acadfetch fetch zenodo:123456
|
|
412
|
+
```
|
|
413
|
+
|
|
414
|
+
### 支持的源
|
|
415
|
+
|
|
416
|
+
| 源 | 目标形式 | 校验和 |
|
|
417
|
+
| --- | --- | --- |
|
|
418
|
+
| Zenodo | DOI `10.5281/zenodo.N`、`zenodo:N`、记录 URL | md5 |
|
|
419
|
+
| arXiv | `arxiv:ID`、裸 ID、abs/pdf URL | 无(只比字节数) |
|
|
420
|
+
| doi.org | 任意 DOI,解析后再分派 | 视源而定 |
|
|
421
|
+
| 直链 | 任意 `http(s)` URL | 响应头里有就用 |
|
|
422
|
+
|
|
423
|
+
figshare 适配器在 0.2.0 已移除;源不公布校验和时用 `--checksum md5:...` 补上,
|
|
424
|
+
旧 manifest 仍可 `verify`。见 [docs/migration-0.2.md](docs/migration-0.2.md)。
|
|
425
|
+
|
|
426
|
+
### 已知限制
|
|
427
|
+
|
|
428
|
+
- `manifest.json` 没有签名:`verify` 证明磁盘上的字节与 manifest 一致,不能证明
|
|
429
|
+
manifest 本身没被改写。补这一层用 `verify --against-remote`(会重新问源站),
|
|
430
|
+
细节见 [SECURITY.md](SECURITY.md)。
|
|
431
|
+
- arXiv 不公布校验和,验证降级为字节数,`verified` 是 `null` 而不是假的安心。
|
|
432
|
+
- 没有搜索与分页,目标得你自己给。
|
|
433
|
+
- Windows ARM 未测;macOS 只做尽力而为(在 CI 里,但不阻塞发布)。
|
|
434
|
+
|
|
435
|
+
### 更多
|
|
436
|
+
|
|
437
|
+
- 30 秒演示 [docs/demo.md](docs/demo.md) · 按症状排障 [docs/troubleshooting.md](docs/troubleshooting.md) · 接进流水线 [docs/integrations.md](docs/integrations.md)(Make、CI、aria2、Snakemake 插件、pooch 下载器)
|
|
438
|
+
- 加源、动传输、写测试的规矩见 [CONTRIBUTING.md](CONTRIBUTING.md)
|
|
439
|
+
|
|
440
|
+
### 许可
|
|
441
|
+
|
|
442
|
+
工具是 MIT。数据本身的许可见 manifest 的 `license` 字段;`unknown` 表示**转发前先
|
|
443
|
+
确认**,而 `-n` 会在花掉带宽之前就把它打出来。
|
|
351
444
|
|
|
352
445
|
## Citing
|
|
353
446
|
|
|
@@ -325,6 +325,26 @@ class Client:
|
|
|
325
325
|
return {}
|
|
326
326
|
raise
|
|
327
327
|
|
|
328
|
+
def available(self, url: str) -> bool:
|
|
329
|
+
"""Whether the URL answers at all — the question a cache asks before fetching.
|
|
330
|
+
|
|
331
|
+
A HEAD is cheap, but ``head()`` folds 403/405/501 into "no headers", which
|
|
332
|
+
is indistinguishable from a 200 that sent none. So an empty answer is
|
|
333
|
+
settled with a one-byte range request instead of being read as success:
|
|
334
|
+
guessing wrong here makes a cache believe a file exists when it does not,
|
|
335
|
+
or refetch one that is fine.
|
|
336
|
+
"""
|
|
337
|
+
try:
|
|
338
|
+
if self.head(url):
|
|
339
|
+
return True
|
|
340
|
+
except HttpError:
|
|
341
|
+
return False
|
|
342
|
+
try:
|
|
343
|
+
with self.open(url, {"Range": "bytes=0-0"}, attempts=1):
|
|
344
|
+
return True
|
|
345
|
+
except HttpError:
|
|
346
|
+
return False
|
|
347
|
+
|
|
328
348
|
|
|
329
349
|
def _snippet(exc: urllib.error.HTTPError) -> str:
|
|
330
350
|
try:
|
|
@@ -32,7 +32,7 @@ from typing import Any, cast
|
|
|
32
32
|
from . import atomic, debug, naming, verify
|
|
33
33
|
from .client import Client
|
|
34
34
|
from .consts import CHUNK
|
|
35
|
-
from .errors import HttpError, LocalError, PartialError, TargetError
|
|
35
|
+
from .errors import AcFError, HttpError, LocalError, PartialError, TargetError
|
|
36
36
|
from .progress import Progress
|
|
37
37
|
from .sources.base import File, Record, Result
|
|
38
38
|
from .units import human
|
|
@@ -1006,3 +1006,36 @@ def check_total(record: Record, limit: int) -> None:
|
|
|
1006
1006
|
from .errors import SourceError
|
|
1007
1007
|
|
|
1008
1008
|
raise SourceError(f"record is {human(total)}, over --max-size ({human(limit)})")
|
|
1009
|
+
|
|
1010
|
+
|
|
1011
|
+
def fetch_one(
|
|
1012
|
+
client: Client,
|
|
1013
|
+
url: str,
|
|
1014
|
+
target: Path,
|
|
1015
|
+
*,
|
|
1016
|
+
retries: int = TRANSFER_ATTEMPTS,
|
|
1017
|
+
fsync: bool = True,
|
|
1018
|
+
) -> None:
|
|
1019
|
+
"""Download ``url`` to exactly ``target``, raising instead of returning a row.
|
|
1020
|
+
|
|
1021
|
+
The CLI reports a row per file because one bad file must not hide the rest; a
|
|
1022
|
+
library caller, by contrast, wants an exception and no partial result. The
|
|
1023
|
+
pooch downloader and the Snakemake plugin both ask this question, so they ask
|
|
1024
|
+
it here rather than each rebuilding the row-to-exception translation.
|
|
1025
|
+
|
|
1026
|
+
The name on disk is the caller's, not the repository's: if ``naming`` had to
|
|
1027
|
+
rewrite it, the produced file is moved back to what was asked for.
|
|
1028
|
+
"""
|
|
1029
|
+
target = Path(target)
|
|
1030
|
+
target.parent.mkdir(parents=True, exist_ok=True)
|
|
1031
|
+
result = fetch_file(
|
|
1032
|
+
client,
|
|
1033
|
+
File(url=url, name=target.name),
|
|
1034
|
+
Options(out=target.parent, concurrency=1, retries=retries, fsync=fsync),
|
|
1035
|
+
set(),
|
|
1036
|
+
)
|
|
1037
|
+
if result.state == "failed":
|
|
1038
|
+
raise AcFError(result.error or f"could not fetch {url}")
|
|
1039
|
+
produced = Path(result.path or target)
|
|
1040
|
+
if produced != target:
|
|
1041
|
+
atomic.replace(produced, target)
|
|
@@ -35,12 +35,10 @@ from collections.abc import Mapping
|
|
|
35
35
|
from pathlib import Path
|
|
36
36
|
from typing import Any
|
|
37
37
|
|
|
38
|
-
from . import atomic
|
|
39
38
|
from .client import Client
|
|
40
39
|
from .consts import DEFAULT_RATE, DEFAULT_RETRIES, DEFAULT_TIMEOUT
|
|
41
|
-
from .downloader import CHUNK,
|
|
42
|
-
from .errors import
|
|
43
|
-
from .sources.base import File
|
|
40
|
+
from .downloader import CHUNK, fetch_one
|
|
41
|
+
from .errors import TargetError
|
|
44
42
|
|
|
45
43
|
__all__ = ["Downloader", "downloader"]
|
|
46
44
|
|
|
@@ -100,46 +98,18 @@ class Downloader:
|
|
|
100
98
|
return None
|
|
101
99
|
|
|
102
100
|
target = Path(output_file)
|
|
103
|
-
|
|
104
|
-
result = fetch_file(
|
|
101
|
+
fetch_one(
|
|
105
102
|
self._client(),
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
retries=self.retries,
|
|
111
|
-
fsync=self.fsync,
|
|
112
|
-
),
|
|
113
|
-
set(),
|
|
103
|
+
url,
|
|
104
|
+
target,
|
|
105
|
+
retries=self.retries,
|
|
106
|
+
fsync=self.fsync,
|
|
114
107
|
)
|
|
115
|
-
if result.state == "failed":
|
|
116
|
-
# pooch does not catch this, so the sentence the CLI would print is
|
|
117
|
-
# the sentence the caller sees.
|
|
118
|
-
raise AcFError(result.error or f"could not fetch {url}")
|
|
119
|
-
produced = Path(result.path or target)
|
|
120
|
-
if produced != target:
|
|
121
|
-
# `naming` is allowed to change a name; pooch's own file must win.
|
|
122
|
-
atomic.replace(produced, target)
|
|
123
108
|
return None
|
|
124
109
|
|
|
125
110
|
def available(self, url: str) -> bool:
|
|
126
|
-
"""Whether the URL answers at all
|
|
127
|
-
|
|
128
|
-
``Client.head`` treats a 403/405/501 as "no headers", which is
|
|
129
|
-
indistinguishable from a 200 that sent none, so an empty answer falls
|
|
130
|
-
through to a one-byte range request rather than being read as success.
|
|
131
|
-
"""
|
|
132
|
-
client = self._client()
|
|
133
|
-
try:
|
|
134
|
-
if client.head(url):
|
|
135
|
-
return True
|
|
136
|
-
except HttpError:
|
|
137
|
-
return False
|
|
138
|
-
try:
|
|
139
|
-
with client.open(url, {"Range": "bytes=0-0"}, attempts=1):
|
|
140
|
-
return True
|
|
141
|
-
except HttpError:
|
|
142
|
-
return False
|
|
111
|
+
"""Whether the URL answers at all; see :meth:`acadfetch.client.Client.available`."""
|
|
112
|
+
return self._client().available(url)
|
|
143
113
|
|
|
144
114
|
def _stream(self, url: str, handle: Any) -> None:
|
|
145
115
|
client = self._client()
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: acadfetch
|
|
3
|
-
Version: 0.7.
|
|
3
|
+
Version: 0.7.2
|
|
4
4
|
Summary: Resumable, rate-limited, checksum-verified bulk downloads from academic data repositories
|
|
5
5
|
Author: young-ski
|
|
6
6
|
License-Expression: MIT
|
|
@@ -35,6 +35,8 @@ Dynamic: license-file
|
|
|
35
35
|
[](https://pypi.org/project/acadfetch/)
|
|
36
36
|
[](LICENSE)
|
|
37
37
|
|
|
38
|
+
**English** · [简体中文](#中文)
|
|
39
|
+
|
|
38
40
|
Batch-download files from academic data repositories — resumably, rate-limited, and
|
|
39
41
|
checked against the checksums the repository published.
|
|
40
42
|
|
|
@@ -337,9 +339,8 @@ honest way to test this code path.
|
|
|
337
339
|
## Getting involved
|
|
338
340
|
|
|
339
341
|
A [30-second tour](docs/demo.md) shows the whole loop, and
|
|
340
|
-
[docs/integrations.md](docs/integrations.md) has Make, CI, aria2, Snakemake
|
|
341
|
-
pooch downloader
|
|
342
|
-
recipes.
|
|
342
|
+
[docs/integrations.md](docs/integrations.md) has Make, CI, aria2, a Snakemake
|
|
343
|
+
storage plugin and a pooch downloader.
|
|
343
344
|
|
|
344
345
|
A nightly job downloads three fixed public records and verifies them, so a
|
|
345
346
|
rotted adapter opens an issue before a user reports one. It never blocks a
|
|
@@ -373,12 +374,104 @@ merge; the next good night closes the issue.
|
|
|
373
374
|
学术数据仓库的批量下载器:断点续传、全局限速、按仓库公布的校验和验证,并落
|
|
374
375
|
`manifest.json`(DOI、许可协议、每文件 SHA-256)供日后复现。
|
|
375
376
|
|
|
377
|
+
它不是搜索器、不抓付费墙、也不是引文管理器 —— 它只管"取数据"这一步,并让这一步
|
|
378
|
+
事后可证明。
|
|
379
|
+
|
|
380
|
+
### 安装
|
|
381
|
+
|
|
382
|
+
Python 3.10–3.15,零运行时依赖。
|
|
383
|
+
|
|
384
|
+
```bash
|
|
385
|
+
uvx acadfetch fetch 10.5281/zenodo.23612969 # 不安装,直接跑
|
|
386
|
+
```
|
|
387
|
+
|
|
388
|
+
```bash
|
|
389
|
+
pipx install acadfetch # 或者装到 PATH 上
|
|
390
|
+
```
|
|
391
|
+
|
|
392
|
+
### 快速开始
|
|
393
|
+
|
|
394
|
+
`10.5281/zenodo.23612969` 是真实记录,可以直接复制;其余 id 都是占位符。
|
|
395
|
+
|
|
376
396
|
```bash
|
|
377
|
-
acadfetch fetch 10.5281/zenodo.
|
|
378
|
-
acadfetch
|
|
397
|
+
acadfetch fetch 10.5281/zenodo.23612969 -n # 先看:落到哪、传多少、许可是什么
|
|
398
|
+
acadfetch fetch 10.5281/zenodo.23612969 -o ./data
|
|
399
|
+
acadfetch verify ./data/manifest.json # 事后复验;对不上就非零退出
|
|
379
400
|
```
|
|
380
401
|
|
|
381
|
-
|
|
402
|
+
目标也可以是 `arxiv:1706.03762`、任意 DOI,或任意 `https://` 直链。
|
|
403
|
+
|
|
404
|
+
> 直链成败由对方主机决定:figshare 的 CDN 在某些网络会回 403 拦截页(当年删掉
|
|
405
|
+
> figshare 适配器正是因为这个)。acadfetch 会明说这是拦截页、而不是把 HTML 存成
|
|
406
|
+
> PDF,并把下一条命令给你。
|
|
407
|
+
|
|
408
|
+
### 它替你保住的东西
|
|
409
|
+
|
|
410
|
+
| 情况 | 它怎么做 |
|
|
411
|
+
| --- | --- |
|
|
412
|
+
| 传到 80% 断了 | `.part` 保留,重跑用 `Range`/`If-Range` 接着传;服务器若无视 `Range`,旧字节先挪到一边,整份新文件到齐才替换 |
|
|
413
|
+
| 悄悄损坏 | 按仓库公布的 md5/sha256 校验,不符就重下;仍不符则 `verified: false` 并删掉坏字节 |
|
|
414
|
+
| "我到底下的是哪个版本" | `manifest.json` 记 DOI、许可、每文件 SHA-256、最终 URL、HTTP 状态与时间 |
|
|
415
|
+
| 被限流吃 403 | 全局令牌桶限速,遵守 `Retry-After`,拦截页按次数重试 |
|
|
416
|
+
|
|
417
|
+
默认 3 并发、2 请求/秒、4 次重试。Zenodo 和 arXiv 靠募捐运营,`--rate 1 -j 1`
|
|
418
|
+
多花你几分钟,对面少受点罪。
|
|
419
|
+
|
|
420
|
+
### 进流水线
|
|
421
|
+
|
|
422
|
+
```bash
|
|
423
|
+
acadfetch fetch 10.5281/zenodo.23612969 --json # 一个 JSON 文档给 stdout,进度走 stderr
|
|
424
|
+
acadfetch fetch 10.5281/zenodo.23612969 -w - # 输出 aria2 可用 URL 列表,不传字节
|
|
425
|
+
```
|
|
426
|
+
|
|
427
|
+
`--json` 的字段契约见 [docs/json-report.md](docs/json-report.md);`report` 是文档
|
|
428
|
+
版本号,只加字段、不改语义。
|
|
429
|
+
|
|
430
|
+
### 退出码
|
|
431
|
+
|
|
432
|
+
`0` 成功、`1` 失败、`2` 部分成功、`64` 用法错误、`130` 被中断(`.part` 已保留,
|
|
433
|
+
重跑即可续)。
|
|
434
|
+
|
|
435
|
+
### 令牌
|
|
436
|
+
|
|
437
|
+
`ZENODO_TOKEN` 从环境读取,**只发给 Zenodo**;跨主机跳转时剥掉,https 降级到
|
|
438
|
+
http 时也会剥掉并告警。manifest 只记变量名和后四位。
|
|
439
|
+
|
|
440
|
+
```bash
|
|
441
|
+
export ZENODO_TOKEN=...
|
|
442
|
+
acadfetch fetch zenodo:123456
|
|
443
|
+
```
|
|
444
|
+
|
|
445
|
+
### 支持的源
|
|
446
|
+
|
|
447
|
+
| 源 | 目标形式 | 校验和 |
|
|
448
|
+
| --- | --- | --- |
|
|
449
|
+
| Zenodo | DOI `10.5281/zenodo.N`、`zenodo:N`、记录 URL | md5 |
|
|
450
|
+
| arXiv | `arxiv:ID`、裸 ID、abs/pdf URL | 无(只比字节数) |
|
|
451
|
+
| doi.org | 任意 DOI,解析后再分派 | 视源而定 |
|
|
452
|
+
| 直链 | 任意 `http(s)` URL | 响应头里有就用 |
|
|
453
|
+
|
|
454
|
+
figshare 适配器在 0.2.0 已移除;源不公布校验和时用 `--checksum md5:...` 补上,
|
|
455
|
+
旧 manifest 仍可 `verify`。见 [docs/migration-0.2.md](docs/migration-0.2.md)。
|
|
456
|
+
|
|
457
|
+
### 已知限制
|
|
458
|
+
|
|
459
|
+
- `manifest.json` 没有签名:`verify` 证明磁盘上的字节与 manifest 一致,不能证明
|
|
460
|
+
manifest 本身没被改写。补这一层用 `verify --against-remote`(会重新问源站),
|
|
461
|
+
细节见 [SECURITY.md](SECURITY.md)。
|
|
462
|
+
- arXiv 不公布校验和,验证降级为字节数,`verified` 是 `null` 而不是假的安心。
|
|
463
|
+
- 没有搜索与分页,目标得你自己给。
|
|
464
|
+
- Windows ARM 未测;macOS 只做尽力而为(在 CI 里,但不阻塞发布)。
|
|
465
|
+
|
|
466
|
+
### 更多
|
|
467
|
+
|
|
468
|
+
- 30 秒演示 [docs/demo.md](docs/demo.md) · 按症状排障 [docs/troubleshooting.md](docs/troubleshooting.md) · 接进流水线 [docs/integrations.md](docs/integrations.md)(Make、CI、aria2、Snakemake 插件、pooch 下载器)
|
|
469
|
+
- 加源、动传输、写测试的规矩见 [CONTRIBUTING.md](CONTRIBUTING.md)
|
|
470
|
+
|
|
471
|
+
### 许可
|
|
472
|
+
|
|
473
|
+
工具是 MIT。数据本身的许可见 manifest 的 `license` 字段;`unknown` 表示**转发前先
|
|
474
|
+
确认**,而 `-n` 会在花掉带宽之前就把它打出来。
|
|
382
475
|
|
|
383
476
|
## Citing
|
|
384
477
|
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "acadfetch"
|
|
7
|
-
version = "0.7.
|
|
7
|
+
version = "0.7.2"
|
|
8
8
|
description = "Resumable, rate-limited, checksum-verified bulk downloads from academic data repositories"
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.10"
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|