acadfetch 0.7.2__tar.gz → 0.9.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {acadfetch-0.7.2/acadfetch.egg-info → acadfetch-0.9.0}/PKG-INFO +65 -2
- {acadfetch-0.7.2 → acadfetch-0.9.0}/README.md +64 -1
- {acadfetch-0.7.2 → acadfetch-0.9.0}/acadfetch/cli.py +415 -83
- {acadfetch-0.7.2 → acadfetch-0.9.0}/acadfetch/consts.py +1 -1
- {acadfetch-0.7.2 → acadfetch-0.9.0/acadfetch.egg-info}/PKG-INFO +65 -2
- {acadfetch-0.7.2 → acadfetch-0.9.0}/acadfetch.egg-info/SOURCES.txt +2 -0
- {acadfetch-0.7.2 → acadfetch-0.9.0}/pyproject.toml +1 -1
- acadfetch-0.9.0/tests/test_batch.py +262 -0
- {acadfetch-0.7.2 → acadfetch-0.9.0}/tests/test_json.py +16 -2
- acadfetch-0.9.0/tests/test_selection.py +320 -0
- {acadfetch-0.7.2 → acadfetch-0.9.0}/LICENSE +0 -0
- {acadfetch-0.7.2 → acadfetch-0.9.0}/MANIFEST.in +0 -0
- {acadfetch-0.7.2 → acadfetch-0.9.0}/acadfetch/__init__.py +0 -0
- {acadfetch-0.7.2 → acadfetch-0.9.0}/acadfetch/__main__.py +0 -0
- {acadfetch-0.7.2 → acadfetch-0.9.0}/acadfetch/atomic.py +0 -0
- {acadfetch-0.7.2 → acadfetch-0.9.0}/acadfetch/auth.py +0 -0
- {acadfetch-0.7.2 → acadfetch-0.9.0}/acadfetch/client.py +0 -0
- {acadfetch-0.7.2 → acadfetch-0.9.0}/acadfetch/debug.py +0 -0
- {acadfetch-0.7.2 → acadfetch-0.9.0}/acadfetch/downloader.py +0 -0
- {acadfetch-0.7.2 → acadfetch-0.9.0}/acadfetch/errors.py +0 -0
- {acadfetch-0.7.2 → acadfetch-0.9.0}/acadfetch/manifest.py +0 -0
- {acadfetch-0.7.2 → acadfetch-0.9.0}/acadfetch/naming.py +0 -0
- {acadfetch-0.7.2 → acadfetch-0.9.0}/acadfetch/pooch.py +0 -0
- {acadfetch-0.7.2 → acadfetch-0.9.0}/acadfetch/progress.py +0 -0
- {acadfetch-0.7.2 → acadfetch-0.9.0}/acadfetch/py.typed +0 -0
- {acadfetch-0.7.2 → acadfetch-0.9.0}/acadfetch/ratelimit.py +0 -0
- {acadfetch-0.7.2 → acadfetch-0.9.0}/acadfetch/remote.py +0 -0
- {acadfetch-0.7.2 → acadfetch-0.9.0}/acadfetch/sources/__init__.py +0 -0
- {acadfetch-0.7.2 → acadfetch-0.9.0}/acadfetch/sources/arxiv.py +0 -0
- {acadfetch-0.7.2 → acadfetch-0.9.0}/acadfetch/sources/base.py +0 -0
- {acadfetch-0.7.2 → acadfetch-0.9.0}/acadfetch/sources/direct.py +0 -0
- {acadfetch-0.7.2 → acadfetch-0.9.0}/acadfetch/sources/doi.py +0 -0
- {acadfetch-0.7.2 → acadfetch-0.9.0}/acadfetch/sources/registry.py +0 -0
- {acadfetch-0.7.2 → acadfetch-0.9.0}/acadfetch/sources/removed.py +0 -0
- {acadfetch-0.7.2 → acadfetch-0.9.0}/acadfetch/sources/zenodo.py +0 -0
- {acadfetch-0.7.2 → acadfetch-0.9.0}/acadfetch/units.py +0 -0
- {acadfetch-0.7.2 → acadfetch-0.9.0}/acadfetch/urls.py +0 -0
- {acadfetch-0.7.2 → acadfetch-0.9.0}/acadfetch/verify.py +0 -0
- {acadfetch-0.7.2 → acadfetch-0.9.0}/acadfetch.egg-info/dependency_links.txt +0 -0
- {acadfetch-0.7.2 → acadfetch-0.9.0}/acadfetch.egg-info/entry_points.txt +0 -0
- {acadfetch-0.7.2 → acadfetch-0.9.0}/acadfetch.egg-info/top_level.txt +0 -0
- {acadfetch-0.7.2 → acadfetch-0.9.0}/setup.cfg +0 -0
- {acadfetch-0.7.2 → acadfetch-0.9.0}/tests/conftest.py +0 -0
- {acadfetch-0.7.2 → acadfetch-0.9.0}/tests/fixtures/README.md +0 -0
- {acadfetch-0.7.2 → acadfetch-0.9.0}/tests/fixtures/ca.pem +0 -0
- {acadfetch-0.7.2 → acadfetch-0.9.0}/tests/fixtures/figshare_403_block.html +0 -0
- {acadfetch-0.7.2 → acadfetch-0.9.0}/tests/fixtures/figshare_403_forbidden.json +0 -0
- {acadfetch-0.7.2 → acadfetch-0.9.0}/tests/fixtures/figshare_article.json +0 -0
- {acadfetch-0.7.2 → acadfetch-0.9.0}/tests/fixtures/figshare_file.pdf +0 -0
- {acadfetch-0.7.2 → acadfetch-0.9.0}/tests/test_atomic.py +0 -0
- {acadfetch-0.7.2 → acadfetch-0.9.0}/tests/test_auth.py +0 -0
- {acadfetch-0.7.2 → acadfetch-0.9.0}/tests/test_checksum.py +0 -0
- {acadfetch-0.7.2 → acadfetch-0.9.0}/tests/test_citation.py +0 -0
- {acadfetch-0.7.2 → acadfetch-0.9.0}/tests/test_claims.py +0 -0
- {acadfetch-0.7.2 → acadfetch-0.9.0}/tests/test_cli.py +0 -0
- {acadfetch-0.7.2 → acadfetch-0.9.0}/tests/test_client.py +0 -0
- {acadfetch-0.7.2 → acadfetch-0.9.0}/tests/test_control_names.py +0 -0
- {acadfetch-0.7.2 → acadfetch-0.9.0}/tests/test_debug.py +0 -0
- {acadfetch-0.7.2 → acadfetch-0.9.0}/tests/test_disk.py +0 -0
- {acadfetch-0.7.2 → acadfetch-0.9.0}/tests/test_downloader.py +0 -0
- {acadfetch-0.7.2 → acadfetch-0.9.0}/tests/test_hash_once.py +0 -0
- {acadfetch-0.7.2 → acadfetch-0.9.0}/tests/test_identity.py +0 -0
- {acadfetch-0.7.2 → acadfetch-0.9.0}/tests/test_isolation.py +0 -0
- {acadfetch-0.7.2 → acadfetch-0.9.0}/tests/test_lock.py +0 -0
- {acadfetch-0.7.2 → acadfetch-0.9.0}/tests/test_manifest.py +0 -0
- {acadfetch-0.7.2 → acadfetch-0.9.0}/tests/test_naming.py +0 -0
- {acadfetch-0.7.2 → acadfetch-0.9.0}/tests/test_naming_race.py +0 -0
- {acadfetch-0.7.2 → acadfetch-0.9.0}/tests/test_network.py +0 -0
- {acadfetch-0.7.2 → acadfetch-0.9.0}/tests/test_packaging.py +0 -0
- {acadfetch-0.7.2 → acadfetch-0.9.0}/tests/test_pooch.py +0 -0
- {acadfetch-0.7.2 → acadfetch-0.9.0}/tests/test_progress.py +0 -0
- {acadfetch-0.7.2 → acadfetch-0.9.0}/tests/test_ratelimit.py +0 -0
- {acadfetch-0.7.2 → acadfetch-0.9.0}/tests/test_redirect.py +0 -0
- {acadfetch-0.7.2 → acadfetch-0.9.0}/tests/test_remote.py +0 -0
- {acadfetch-0.7.2 → acadfetch-0.9.0}/tests/test_removed.py +0 -0
- {acadfetch-0.7.2 → acadfetch-0.9.0}/tests/test_sources.py +0 -0
- {acadfetch-0.7.2 → acadfetch-0.9.0}/tests/test_urls.py +0 -0
- {acadfetch-0.7.2 → acadfetch-0.9.0}/tests/test_urls_glob.py +0 -0
- {acadfetch-0.7.2 → acadfetch-0.9.0}/tests/test_verify.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: acadfetch
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.9.0
|
|
4
4
|
Summary: Resumable, rate-limited, checksum-verified bulk downloads from academic data repositories
|
|
5
5
|
Author: young-ski
|
|
6
6
|
License-Expression: MIT
|
|
@@ -114,6 +114,23 @@ acadfetch fetch arxiv:1706.03762 # an arXiv id
|
|
|
114
114
|
acadfetch fetch https://ndownloader.figshare.com/files/70600233/report.pdf # any URL
|
|
115
115
|
```
|
|
116
116
|
|
|
117
|
+
Or a whole list, with the failures kept for the next run:
|
|
118
|
+
|
|
119
|
+
```bash
|
|
120
|
+
acadfetch batch identifiers.txt # one identifier per line
|
|
121
|
+
acadfetch batch identifiers.txt -o ./data --failed-out todo.txt
|
|
122
|
+
```
|
|
123
|
+
|
|
124
|
+
Blank lines and `#` comments are skipped, so a list can carry notes. Every
|
|
125
|
+
identifier still gets a directory of its own under `-o` (default
|
|
126
|
+
`acadfetch-out`); one bad line does not stop the rest, and the identifiers that
|
|
127
|
+
failed land in `failed.txt` (or `--failed-out`), verbatim and one per line — so
|
|
128
|
+
the retry is `acadfetch batch failed.txt`. The file is replaced even when
|
|
129
|
+
nothing failed, so a stale list never lies. Exit codes keep their meaning: `0`
|
|
130
|
+
all ok, `1` all failed, `2` partial, `130` interrupted, and an interrupt puts the
|
|
131
|
+
lines that were never attempted on the list too. Records are attempted one after
|
|
132
|
+
another; `-j` still applies inside a record.
|
|
133
|
+
|
|
117
134
|
The first two are answered by Zenodo and arXiv and work from any network those
|
|
118
135
|
are reachable on. The third is a direct URL, so the host decides: figshare's CDN
|
|
119
136
|
answers 403 with a WAF block page from some networks — the same behaviour that
|
|
@@ -122,6 +139,24 @@ nothing, rather than saving the block page as a PDF. If you hit that, the file
|
|
|
122
139
|
itself is fine; see the `403` section of
|
|
123
140
|
[docs/troubleshooting.md](docs/troubleshooting.md).
|
|
124
141
|
|
|
142
|
+
### Picking a few files out of many
|
|
143
|
+
|
|
144
|
+
A record with 70 files rarely wants all of them. `--list-files` prints the names
|
|
145
|
+
and transfers nothing; edit the list, then feed it back:
|
|
146
|
+
|
|
147
|
+
```bash
|
|
148
|
+
acadfetch fetch 10.5281/zenodo.19665762 --list-files > want.txt
|
|
149
|
+
acadfetch fetch 10.5281/zenodo.19665762 --only-file want.txt --failed-out failed.txt
|
|
150
|
+
acadfetch fetch 10.5281/zenodo.19665762 --only-file failed.txt # retry the misses
|
|
151
|
+
```
|
|
152
|
+
|
|
153
|
+
`--only-file` takes one exact name per line (blank lines and `#` comments are
|
|
154
|
+
skipped, `-` reads stdin, names are case-insensitive); it combines with `--only`
|
|
155
|
+
and is narrowed by `--glob`. `--failed-out` writes the selected names that could
|
|
156
|
+
not be fetched, verbatim and in selection order, replacing the file even when
|
|
157
|
+
nothing failed. A name that matches nothing is a failure, not a quietly smaller
|
|
158
|
+
download: it is named on stderr, written to the list, and the run exits `2`.
|
|
159
|
+
|
|
125
160
|
figshare was removed in 0.2.0: pass the article's download URL instead, and
|
|
126
161
|
[`--checksum`](docs/migration-0.2.md) keeps the verification the API used to
|
|
127
162
|
give you.
|
|
@@ -188,7 +223,8 @@ checksum matches, so an interrupted or failed run never leaves something that
|
|
|
188
223
|
it resumes. The one case where a run must start over is a server that ignores
|
|
189
224
|
`Range`; the bytes already on disk are then moved aside and kept until a whole
|
|
190
225
|
new copy has arrived, so a restart that dies early costs nothing. `acadfetch-out/`
|
|
191
|
-
is worth adding to a project's `.gitignore`; this repository already has it
|
|
226
|
+
is worth adding to a project's `.gitignore`; this repository already has it, and
|
|
227
|
+
names the default `failed.txt` there as well.
|
|
192
228
|
|
|
193
229
|
The default directory names the target, not only its filename — `data.csv` from
|
|
194
230
|
two hosts is two datasets, and they do not share a directory. A file already
|
|
@@ -405,6 +441,33 @@ acadfetch verify ./data/manifest.json # 事后复验;对不上就非
|
|
|
405
441
|
> figshare 适配器正是因为这个)。acadfetch 会明说这是拦截页、而不是把 HTML 存成
|
|
406
442
|
> PDF,并把下一条命令给你。
|
|
407
443
|
|
|
444
|
+
批量清单则一条一条来,失败的留下:
|
|
445
|
+
|
|
446
|
+
```bash
|
|
447
|
+
acadfetch batch identifiers.txt # 每行一个目标
|
|
448
|
+
acadfetch batch identifiers.txt -o ./data --failed-out todo.txt
|
|
449
|
+
```
|
|
450
|
+
|
|
451
|
+
空行与 `#` 注释会被跳过;每个目标仍在 `-o`(默认 `acadfetch-out`)下拿到自己
|
|
452
|
+
的目录,一行坏掉不影响其余。失败的目标原样写进 `failed.txt`(或
|
|
453
|
+
`--failed-out`),重跑就是 `acadfetch batch failed.txt`;没失败也会清空该文件,
|
|
454
|
+
不会留下上次的假清单。退出码含义不变:`0` 全成、`1` 全败、`2` 部分、`130`
|
|
455
|
+
被中断 —— 中断时还没轮到的行也会进清单。
|
|
456
|
+
|
|
457
|
+
单个记录里只挑几个文件时,用 `--list-files` 取名单(只打印名字、不传字节),再
|
|
458
|
+
用 `--only-file` 喂回去:
|
|
459
|
+
|
|
460
|
+
```bash
|
|
461
|
+
acadfetch fetch 10.5281/zenodo.19665762 --list-files > want.txt
|
|
462
|
+
acadfetch fetch 10.5281/zenodo.19665762 --only-file want.txt --failed-out failed.txt
|
|
463
|
+
acadfetch fetch 10.5281/zenodo.19665762 --only-file failed.txt # 重试漏掉的
|
|
464
|
+
```
|
|
465
|
+
|
|
466
|
+
名单一行一个文件名,空行与 `#` 注释跳过,`-` 读标准输入,名字不区分大小写;可与
|
|
467
|
+
`--only` 合并、再被 `--glob` 收窄。写错的名单算失败:stderr 会点名、会进清单、退出
|
|
468
|
+
码是 `2`,不会被悄悄跳过。注意 `batch` 的 `--failed-out` 记的是目标,`fetch` 的记的
|
|
469
|
+
是文件名。
|
|
470
|
+
|
|
408
471
|
### 它替你保住的东西
|
|
409
472
|
|
|
410
473
|
| 情况 | 它怎么做 |
|
|
@@ -83,6 +83,23 @@ acadfetch fetch arxiv:1706.03762 # an arXiv id
|
|
|
83
83
|
acadfetch fetch https://ndownloader.figshare.com/files/70600233/report.pdf # any URL
|
|
84
84
|
```
|
|
85
85
|
|
|
86
|
+
Or a whole list, with the failures kept for the next run:
|
|
87
|
+
|
|
88
|
+
```bash
|
|
89
|
+
acadfetch batch identifiers.txt # one identifier per line
|
|
90
|
+
acadfetch batch identifiers.txt -o ./data --failed-out todo.txt
|
|
91
|
+
```
|
|
92
|
+
|
|
93
|
+
Blank lines and `#` comments are skipped, so a list can carry notes. Every
|
|
94
|
+
identifier still gets a directory of its own under `-o` (default
|
|
95
|
+
`acadfetch-out`); one bad line does not stop the rest, and the identifiers that
|
|
96
|
+
failed land in `failed.txt` (or `--failed-out`), verbatim and one per line — so
|
|
97
|
+
the retry is `acadfetch batch failed.txt`. The file is replaced even when
|
|
98
|
+
nothing failed, so a stale list never lies. Exit codes keep their meaning: `0`
|
|
99
|
+
all ok, `1` all failed, `2` partial, `130` interrupted, and an interrupt puts the
|
|
100
|
+
lines that were never attempted on the list too. Records are attempted one after
|
|
101
|
+
another; `-j` still applies inside a record.
|
|
102
|
+
|
|
86
103
|
The first two are answered by Zenodo and arXiv and work from any network those
|
|
87
104
|
are reachable on. The third is a direct URL, so the host decides: figshare's CDN
|
|
88
105
|
answers 403 with a WAF block page from some networks — the same behaviour that
|
|
@@ -91,6 +108,24 @@ nothing, rather than saving the block page as a PDF. If you hit that, the file
|
|
|
91
108
|
itself is fine; see the `403` section of
|
|
92
109
|
[docs/troubleshooting.md](docs/troubleshooting.md).
|
|
93
110
|
|
|
111
|
+
### Picking a few files out of many
|
|
112
|
+
|
|
113
|
+
A record with 70 files rarely wants all of them. `--list-files` prints the names
|
|
114
|
+
and transfers nothing; edit the list, then feed it back:
|
|
115
|
+
|
|
116
|
+
```bash
|
|
117
|
+
acadfetch fetch 10.5281/zenodo.19665762 --list-files > want.txt
|
|
118
|
+
acadfetch fetch 10.5281/zenodo.19665762 --only-file want.txt --failed-out failed.txt
|
|
119
|
+
acadfetch fetch 10.5281/zenodo.19665762 --only-file failed.txt # retry the misses
|
|
120
|
+
```
|
|
121
|
+
|
|
122
|
+
`--only-file` takes one exact name per line (blank lines and `#` comments are
|
|
123
|
+
skipped, `-` reads stdin, names are case-insensitive); it combines with `--only`
|
|
124
|
+
and is narrowed by `--glob`. `--failed-out` writes the selected names that could
|
|
125
|
+
not be fetched, verbatim and in selection order, replacing the file even when
|
|
126
|
+
nothing failed. A name that matches nothing is a failure, not a quietly smaller
|
|
127
|
+
download: it is named on stderr, written to the list, and the run exits `2`.
|
|
128
|
+
|
|
94
129
|
figshare was removed in 0.2.0: pass the article's download URL instead, and
|
|
95
130
|
[`--checksum`](docs/migration-0.2.md) keeps the verification the API used to
|
|
96
131
|
give you.
|
|
@@ -157,7 +192,8 @@ checksum matches, so an interrupted or failed run never leaves something that
|
|
|
157
192
|
it resumes. The one case where a run must start over is a server that ignores
|
|
158
193
|
`Range`; the bytes already on disk are then moved aside and kept until a whole
|
|
159
194
|
new copy has arrived, so a restart that dies early costs nothing. `acadfetch-out/`
|
|
160
|
-
is worth adding to a project's `.gitignore`; this repository already has it
|
|
195
|
+
is worth adding to a project's `.gitignore`; this repository already has it, and
|
|
196
|
+
names the default `failed.txt` there as well.
|
|
161
197
|
|
|
162
198
|
The default directory names the target, not only its filename — `data.csv` from
|
|
163
199
|
two hosts is two datasets, and they do not share a directory. A file already
|
|
@@ -374,6 +410,33 @@ acadfetch verify ./data/manifest.json # 事后复验;对不上就非
|
|
|
374
410
|
> figshare 适配器正是因为这个)。acadfetch 会明说这是拦截页、而不是把 HTML 存成
|
|
375
411
|
> PDF,并把下一条命令给你。
|
|
376
412
|
|
|
413
|
+
批量清单则一条一条来,失败的留下:
|
|
414
|
+
|
|
415
|
+
```bash
|
|
416
|
+
acadfetch batch identifiers.txt # 每行一个目标
|
|
417
|
+
acadfetch batch identifiers.txt -o ./data --failed-out todo.txt
|
|
418
|
+
```
|
|
419
|
+
|
|
420
|
+
空行与 `#` 注释会被跳过;每个目标仍在 `-o`(默认 `acadfetch-out`)下拿到自己
|
|
421
|
+
的目录,一行坏掉不影响其余。失败的目标原样写进 `failed.txt`(或
|
|
422
|
+
`--failed-out`),重跑就是 `acadfetch batch failed.txt`;没失败也会清空该文件,
|
|
423
|
+
不会留下上次的假清单。退出码含义不变:`0` 全成、`1` 全败、`2` 部分、`130`
|
|
424
|
+
被中断 —— 中断时还没轮到的行也会进清单。
|
|
425
|
+
|
|
426
|
+
单个记录里只挑几个文件时,用 `--list-files` 取名单(只打印名字、不传字节),再
|
|
427
|
+
用 `--only-file` 喂回去:
|
|
428
|
+
|
|
429
|
+
```bash
|
|
430
|
+
acadfetch fetch 10.5281/zenodo.19665762 --list-files > want.txt
|
|
431
|
+
acadfetch fetch 10.5281/zenodo.19665762 --only-file want.txt --failed-out failed.txt
|
|
432
|
+
acadfetch fetch 10.5281/zenodo.19665762 --only-file failed.txt # 重试漏掉的
|
|
433
|
+
```
|
|
434
|
+
|
|
435
|
+
名单一行一个文件名,空行与 `#` 注释跳过,`-` 读标准输入,名字不区分大小写;可与
|
|
436
|
+
`--only` 合并、再被 `--glob` 收窄。写错的名单算失败:stderr 会点名、会进清单、退出
|
|
437
|
+
码是 `2`,不会被悄悄跳过。注意 `batch` 的 `--failed-out` 记的是目标,`fetch` 的记的
|
|
438
|
+
是文件名。
|
|
439
|
+
|
|
377
440
|
### 它替你保住的东西
|
|
378
441
|
|
|
379
442
|
| 情况 | 它怎么做 |
|