acadfetch 0.7.2__tar.gz → 0.9.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (79) hide show
  1. {acadfetch-0.7.2/acadfetch.egg-info → acadfetch-0.9.0}/PKG-INFO +65 -2
  2. {acadfetch-0.7.2 → acadfetch-0.9.0}/README.md +64 -1
  3. {acadfetch-0.7.2 → acadfetch-0.9.0}/acadfetch/cli.py +415 -83
  4. {acadfetch-0.7.2 → acadfetch-0.9.0}/acadfetch/consts.py +1 -1
  5. {acadfetch-0.7.2 → acadfetch-0.9.0/acadfetch.egg-info}/PKG-INFO +65 -2
  6. {acadfetch-0.7.2 → acadfetch-0.9.0}/acadfetch.egg-info/SOURCES.txt +2 -0
  7. {acadfetch-0.7.2 → acadfetch-0.9.0}/pyproject.toml +1 -1
  8. acadfetch-0.9.0/tests/test_batch.py +262 -0
  9. {acadfetch-0.7.2 → acadfetch-0.9.0}/tests/test_json.py +16 -2
  10. acadfetch-0.9.0/tests/test_selection.py +320 -0
  11. {acadfetch-0.7.2 → acadfetch-0.9.0}/LICENSE +0 -0
  12. {acadfetch-0.7.2 → acadfetch-0.9.0}/MANIFEST.in +0 -0
  13. {acadfetch-0.7.2 → acadfetch-0.9.0}/acadfetch/__init__.py +0 -0
  14. {acadfetch-0.7.2 → acadfetch-0.9.0}/acadfetch/__main__.py +0 -0
  15. {acadfetch-0.7.2 → acadfetch-0.9.0}/acadfetch/atomic.py +0 -0
  16. {acadfetch-0.7.2 → acadfetch-0.9.0}/acadfetch/auth.py +0 -0
  17. {acadfetch-0.7.2 → acadfetch-0.9.0}/acadfetch/client.py +0 -0
  18. {acadfetch-0.7.2 → acadfetch-0.9.0}/acadfetch/debug.py +0 -0
  19. {acadfetch-0.7.2 → acadfetch-0.9.0}/acadfetch/downloader.py +0 -0
  20. {acadfetch-0.7.2 → acadfetch-0.9.0}/acadfetch/errors.py +0 -0
  21. {acadfetch-0.7.2 → acadfetch-0.9.0}/acadfetch/manifest.py +0 -0
  22. {acadfetch-0.7.2 → acadfetch-0.9.0}/acadfetch/naming.py +0 -0
  23. {acadfetch-0.7.2 → acadfetch-0.9.0}/acadfetch/pooch.py +0 -0
  24. {acadfetch-0.7.2 → acadfetch-0.9.0}/acadfetch/progress.py +0 -0
  25. {acadfetch-0.7.2 → acadfetch-0.9.0}/acadfetch/py.typed +0 -0
  26. {acadfetch-0.7.2 → acadfetch-0.9.0}/acadfetch/ratelimit.py +0 -0
  27. {acadfetch-0.7.2 → acadfetch-0.9.0}/acadfetch/remote.py +0 -0
  28. {acadfetch-0.7.2 → acadfetch-0.9.0}/acadfetch/sources/__init__.py +0 -0
  29. {acadfetch-0.7.2 → acadfetch-0.9.0}/acadfetch/sources/arxiv.py +0 -0
  30. {acadfetch-0.7.2 → acadfetch-0.9.0}/acadfetch/sources/base.py +0 -0
  31. {acadfetch-0.7.2 → acadfetch-0.9.0}/acadfetch/sources/direct.py +0 -0
  32. {acadfetch-0.7.2 → acadfetch-0.9.0}/acadfetch/sources/doi.py +0 -0
  33. {acadfetch-0.7.2 → acadfetch-0.9.0}/acadfetch/sources/registry.py +0 -0
  34. {acadfetch-0.7.2 → acadfetch-0.9.0}/acadfetch/sources/removed.py +0 -0
  35. {acadfetch-0.7.2 → acadfetch-0.9.0}/acadfetch/sources/zenodo.py +0 -0
  36. {acadfetch-0.7.2 → acadfetch-0.9.0}/acadfetch/units.py +0 -0
  37. {acadfetch-0.7.2 → acadfetch-0.9.0}/acadfetch/urls.py +0 -0
  38. {acadfetch-0.7.2 → acadfetch-0.9.0}/acadfetch/verify.py +0 -0
  39. {acadfetch-0.7.2 → acadfetch-0.9.0}/acadfetch.egg-info/dependency_links.txt +0 -0
  40. {acadfetch-0.7.2 → acadfetch-0.9.0}/acadfetch.egg-info/entry_points.txt +0 -0
  41. {acadfetch-0.7.2 → acadfetch-0.9.0}/acadfetch.egg-info/top_level.txt +0 -0
  42. {acadfetch-0.7.2 → acadfetch-0.9.0}/setup.cfg +0 -0
  43. {acadfetch-0.7.2 → acadfetch-0.9.0}/tests/conftest.py +0 -0
  44. {acadfetch-0.7.2 → acadfetch-0.9.0}/tests/fixtures/README.md +0 -0
  45. {acadfetch-0.7.2 → acadfetch-0.9.0}/tests/fixtures/ca.pem +0 -0
  46. {acadfetch-0.7.2 → acadfetch-0.9.0}/tests/fixtures/figshare_403_block.html +0 -0
  47. {acadfetch-0.7.2 → acadfetch-0.9.0}/tests/fixtures/figshare_403_forbidden.json +0 -0
  48. {acadfetch-0.7.2 → acadfetch-0.9.0}/tests/fixtures/figshare_article.json +0 -0
  49. {acadfetch-0.7.2 → acadfetch-0.9.0}/tests/fixtures/figshare_file.pdf +0 -0
  50. {acadfetch-0.7.2 → acadfetch-0.9.0}/tests/test_atomic.py +0 -0
  51. {acadfetch-0.7.2 → acadfetch-0.9.0}/tests/test_auth.py +0 -0
  52. {acadfetch-0.7.2 → acadfetch-0.9.0}/tests/test_checksum.py +0 -0
  53. {acadfetch-0.7.2 → acadfetch-0.9.0}/tests/test_citation.py +0 -0
  54. {acadfetch-0.7.2 → acadfetch-0.9.0}/tests/test_claims.py +0 -0
  55. {acadfetch-0.7.2 → acadfetch-0.9.0}/tests/test_cli.py +0 -0
  56. {acadfetch-0.7.2 → acadfetch-0.9.0}/tests/test_client.py +0 -0
  57. {acadfetch-0.7.2 → acadfetch-0.9.0}/tests/test_control_names.py +0 -0
  58. {acadfetch-0.7.2 → acadfetch-0.9.0}/tests/test_debug.py +0 -0
  59. {acadfetch-0.7.2 → acadfetch-0.9.0}/tests/test_disk.py +0 -0
  60. {acadfetch-0.7.2 → acadfetch-0.9.0}/tests/test_downloader.py +0 -0
  61. {acadfetch-0.7.2 → acadfetch-0.9.0}/tests/test_hash_once.py +0 -0
  62. {acadfetch-0.7.2 → acadfetch-0.9.0}/tests/test_identity.py +0 -0
  63. {acadfetch-0.7.2 → acadfetch-0.9.0}/tests/test_isolation.py +0 -0
  64. {acadfetch-0.7.2 → acadfetch-0.9.0}/tests/test_lock.py +0 -0
  65. {acadfetch-0.7.2 → acadfetch-0.9.0}/tests/test_manifest.py +0 -0
  66. {acadfetch-0.7.2 → acadfetch-0.9.0}/tests/test_naming.py +0 -0
  67. {acadfetch-0.7.2 → acadfetch-0.9.0}/tests/test_naming_race.py +0 -0
  68. {acadfetch-0.7.2 → acadfetch-0.9.0}/tests/test_network.py +0 -0
  69. {acadfetch-0.7.2 → acadfetch-0.9.0}/tests/test_packaging.py +0 -0
  70. {acadfetch-0.7.2 → acadfetch-0.9.0}/tests/test_pooch.py +0 -0
  71. {acadfetch-0.7.2 → acadfetch-0.9.0}/tests/test_progress.py +0 -0
  72. {acadfetch-0.7.2 → acadfetch-0.9.0}/tests/test_ratelimit.py +0 -0
  73. {acadfetch-0.7.2 → acadfetch-0.9.0}/tests/test_redirect.py +0 -0
  74. {acadfetch-0.7.2 → acadfetch-0.9.0}/tests/test_remote.py +0 -0
  75. {acadfetch-0.7.2 → acadfetch-0.9.0}/tests/test_removed.py +0 -0
  76. {acadfetch-0.7.2 → acadfetch-0.9.0}/tests/test_sources.py +0 -0
  77. {acadfetch-0.7.2 → acadfetch-0.9.0}/tests/test_urls.py +0 -0
  78. {acadfetch-0.7.2 → acadfetch-0.9.0}/tests/test_urls_glob.py +0 -0
  79. {acadfetch-0.7.2 → acadfetch-0.9.0}/tests/test_verify.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: acadfetch
3
- Version: 0.7.2
3
+ Version: 0.9.0
4
4
  Summary: Resumable, rate-limited, checksum-verified bulk downloads from academic data repositories
5
5
  Author: young-ski
6
6
  License-Expression: MIT
@@ -114,6 +114,23 @@ acadfetch fetch arxiv:1706.03762 # an arXiv id
114
114
  acadfetch fetch https://ndownloader.figshare.com/files/70600233/report.pdf # any URL
115
115
  ```
116
116
 
117
+ Or a whole list, with the failures kept for the next run:
118
+
119
+ ```bash
120
+ acadfetch batch identifiers.txt # one identifier per line
121
+ acadfetch batch identifiers.txt -o ./data --failed-out todo.txt
122
+ ```
123
+
124
+ Blank lines and `#` comments are skipped, so a list can carry notes. Every
125
+ identifier still gets a directory of its own under `-o` (default
126
+ `acadfetch-out`); one bad line does not stop the rest, and the identifiers that
127
+ failed land in `failed.txt` (or `--failed-out`), verbatim and one per line — so
128
+ the retry is `acadfetch batch failed.txt`. The file is replaced even when
129
+ nothing failed, so a stale list never lies. Exit codes keep their meaning: `0`
130
+ all ok, `1` all failed, `2` partial, `130` interrupted, and an interrupt puts the
131
+ lines that were never attempted on the list too. Records are attempted one after
132
+ another; `-j` still applies inside a record.
133
+
117
134
  The first two are answered by Zenodo and arXiv and work from any network those
118
135
  are reachable on. The third is a direct URL, so the host decides: figshare's CDN
119
136
  answers 403 with a WAF block page from some networks — the same behaviour that
@@ -122,6 +139,24 @@ nothing, rather than saving the block page as a PDF. If you hit that, the file
122
139
  itself is fine; see the `403` section of
123
140
  [docs/troubleshooting.md](docs/troubleshooting.md).
124
141
 
142
+ ### Picking a few files out of many
143
+
144
+ A record with 70 files rarely wants all of them. `--list-files` prints the names
145
+ and transfers nothing; edit the list, then feed it back:
146
+
147
+ ```bash
148
+ acadfetch fetch 10.5281/zenodo.19665762 --list-files > want.txt
149
+ acadfetch fetch 10.5281/zenodo.19665762 --only-file want.txt --failed-out failed.txt
150
+ acadfetch fetch 10.5281/zenodo.19665762 --only-file failed.txt # retry the misses
151
+ ```
152
+
153
+ `--only-file` takes one exact name per line (blank lines and `#` comments are
154
+ skipped, `-` reads stdin, names are case-insensitive); it combines with `--only`
155
+ and is narrowed by `--glob`. `--failed-out` writes the selected names that could
156
+ not be fetched, verbatim and in selection order, replacing the file even when
157
+ nothing failed. A name that matches nothing is a failure, not a quietly smaller
158
+ download: it is named on stderr, written to the list, and the run exits `2`.
159
+
125
160
  figshare was removed in 0.2.0: pass the article's download URL instead, and
126
161
  [`--checksum`](docs/migration-0.2.md) keeps the verification the API used to
127
162
  give you.
@@ -188,7 +223,8 @@ checksum matches, so an interrupted or failed run never leaves something that
188
223
  it resumes. The one case where a run must start over is a server that ignores
189
224
  `Range`; the bytes already on disk are then moved aside and kept until a whole
190
225
  new copy has arrived, so a restart that dies early costs nothing. `acadfetch-out/`
191
- is worth adding to a project's `.gitignore`; this repository already has it.
226
+ is worth adding to a project's `.gitignore`; this repository already has it, and
227
+ names the default `failed.txt` there as well.
192
228
 
193
229
  The default directory names the target, not only its filename — `data.csv` from
194
230
  two hosts is two datasets, and they do not share a directory. A file already
@@ -405,6 +441,33 @@ acadfetch verify ./data/manifest.json # 事后复验;对不上就非
405
441
  > figshare 适配器正是因为这个)。acadfetch 会明说这是拦截页、而不是把 HTML 存成
406
442
  > PDF,并把下一条命令给你。
407
443
 
444
+ 批量清单则一条一条来,失败的留下:
445
+
446
+ ```bash
447
+ acadfetch batch identifiers.txt # 每行一个目标
448
+ acadfetch batch identifiers.txt -o ./data --failed-out todo.txt
449
+ ```
450
+
451
+ 空行与 `#` 注释会被跳过;每个目标仍在 `-o`(默认 `acadfetch-out`)下拿到自己
452
+ 的目录,一行坏掉不影响其余。失败的目标原样写进 `failed.txt`(或
453
+ `--failed-out`),重跑就是 `acadfetch batch failed.txt`;没失败也会清空该文件,
454
+ 不会留下上次的假清单。退出码含义不变:`0` 全成、`1` 全败、`2` 部分、`130`
455
+ 被中断 —— 中断时还没轮到的行也会进清单。
456
+
457
+ 单个记录里只挑几个文件时,用 `--list-files` 取名单(只打印名字、不传字节),再
458
+ 用 `--only-file` 喂回去:
459
+
460
+ ```bash
461
+ acadfetch fetch 10.5281/zenodo.19665762 --list-files > want.txt
462
+ acadfetch fetch 10.5281/zenodo.19665762 --only-file want.txt --failed-out failed.txt
463
+ acadfetch fetch 10.5281/zenodo.19665762 --only-file failed.txt # 重试漏掉的
464
+ ```
465
+
466
+ 名单一行一个文件名,空行与 `#` 注释跳过,`-` 读标准输入,名字不区分大小写;可与
467
+ `--only` 合并、再被 `--glob` 收窄。写错的名单算失败:stderr 会点名、会进清单、退出
468
+ 码是 `2`,不会被悄悄跳过。注意 `batch` 的 `--failed-out` 记的是目标,`fetch` 的记的
469
+ 是文件名。
470
+
408
471
  ### 它替你保住的东西
409
472
 
410
473
  | 情况 | 它怎么做 |
@@ -83,6 +83,23 @@ acadfetch fetch arxiv:1706.03762 # an arXiv id
83
83
  acadfetch fetch https://ndownloader.figshare.com/files/70600233/report.pdf # any URL
84
84
  ```
85
85
 
86
+ Or a whole list, with the failures kept for the next run:
87
+
88
+ ```bash
89
+ acadfetch batch identifiers.txt # one identifier per line
90
+ acadfetch batch identifiers.txt -o ./data --failed-out todo.txt
91
+ ```
92
+
93
+ Blank lines and `#` comments are skipped, so a list can carry notes. Every
94
+ identifier still gets a directory of its own under `-o` (default
95
+ `acadfetch-out`); one bad line does not stop the rest, and the identifiers that
96
+ failed land in `failed.txt` (or `--failed-out`), verbatim and one per line — so
97
+ the retry is `acadfetch batch failed.txt`. The file is replaced even when
98
+ nothing failed, so a stale list never lies. Exit codes keep their meaning: `0`
99
+ all ok, `1` all failed, `2` partial, `130` interrupted, and an interrupt puts the
100
+ lines that were never attempted on the list too. Records are attempted one after
101
+ another; `-j` still applies inside a record.
102
+
86
103
  The first two are answered by Zenodo and arXiv and work from any network those
87
104
  are reachable on. The third is a direct URL, so the host decides: figshare's CDN
88
105
  answers 403 with a WAF block page from some networks — the same behaviour that
@@ -91,6 +108,24 @@ nothing, rather than saving the block page as a PDF. If you hit that, the file
91
108
  itself is fine; see the `403` section of
92
109
  [docs/troubleshooting.md](docs/troubleshooting.md).
93
110
 
111
+ ### Picking a few files out of many
112
+
113
+ A record with 70 files rarely wants all of them. `--list-files` prints the names
114
+ and transfers nothing; edit the list, then feed it back:
115
+
116
+ ```bash
117
+ acadfetch fetch 10.5281/zenodo.19665762 --list-files > want.txt
118
+ acadfetch fetch 10.5281/zenodo.19665762 --only-file want.txt --failed-out failed.txt
119
+ acadfetch fetch 10.5281/zenodo.19665762 --only-file failed.txt # retry the misses
120
+ ```
121
+
122
+ `--only-file` takes one exact name per line (blank lines and `#` comments are
123
+ skipped, `-` reads stdin, names are case-insensitive); it combines with `--only`
124
+ and is narrowed by `--glob`. `--failed-out` writes the selected names that could
125
+ not be fetched, verbatim and in selection order, replacing the file even when
126
+ nothing failed. A name that matches nothing is a failure, not a quietly smaller
127
+ download: it is named on stderr, written to the list, and the run exits `2`.
128
+
94
129
  figshare was removed in 0.2.0: pass the article's download URL instead, and
95
130
  [`--checksum`](docs/migration-0.2.md) keeps the verification the API used to
96
131
  give you.
@@ -157,7 +192,8 @@ checksum matches, so an interrupted or failed run never leaves something that
157
192
  it resumes. The one case where a run must start over is a server that ignores
158
193
  `Range`; the bytes already on disk are then moved aside and kept until a whole
159
194
  new copy has arrived, so a restart that dies early costs nothing. `acadfetch-out/`
160
- is worth adding to a project's `.gitignore`; this repository already has it.
195
+ is worth adding to a project's `.gitignore`; this repository already has it, and
196
+ names the default `failed.txt` there as well.
161
197
 
162
198
  The default directory names the target, not only its filename — `data.csv` from
163
199
  two hosts is two datasets, and they do not share a directory. A file already
@@ -374,6 +410,33 @@ acadfetch verify ./data/manifest.json # 事后复验;对不上就非
374
410
  > figshare 适配器正是因为这个)。acadfetch 会明说这是拦截页、而不是把 HTML 存成
375
411
  > PDF,并把下一条命令给你。
376
412
 
413
+ 批量清单则一条一条来,失败的留下:
414
+
415
+ ```bash
416
+ acadfetch batch identifiers.txt # 每行一个目标
417
+ acadfetch batch identifiers.txt -o ./data --failed-out todo.txt
418
+ ```
419
+
420
+ 空行与 `#` 注释会被跳过;每个目标仍在 `-o`(默认 `acadfetch-out`)下拿到自己
421
+ 的目录,一行坏掉不影响其余。失败的目标原样写进 `failed.txt`(或
422
+ `--failed-out`),重跑就是 `acadfetch batch failed.txt`;没失败也会清空该文件,
423
+ 不会留下上次的假清单。退出码含义不变:`0` 全成、`1` 全败、`2` 部分、`130`
424
+ 被中断 —— 中断时还没轮到的行也会进清单。
425
+
426
+ 单个记录里只挑几个文件时,用 `--list-files` 取名单(只打印名字、不传字节),再
427
+ 用 `--only-file` 喂回去:
428
+
429
+ ```bash
430
+ acadfetch fetch 10.5281/zenodo.19665762 --list-files > want.txt
431
+ acadfetch fetch 10.5281/zenodo.19665762 --only-file want.txt --failed-out failed.txt
432
+ acadfetch fetch 10.5281/zenodo.19665762 --only-file failed.txt # 重试漏掉的
433
+ ```
434
+
435
+ 名单一行一个文件名,空行与 `#` 注释跳过,`-` 读标准输入,名字不区分大小写;可与
436
+ `--only` 合并、再被 `--glob` 收窄。写错的名单算失败:stderr 会点名、会进清单、退出
437
+ 码是 `2`,不会被悄悄跳过。注意 `batch` 的 `--failed-out` 记的是目标,`fetch` 的记的
438
+ 是文件名。
439
+
377
440
  ### 它替你保住的东西
378
441
 
379
442
  | 情况 | 它怎么做 |