acadfetch 0.1.1__tar.gz → 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- acadfetch-0.2.0/PKG-INFO +330 -0
- acadfetch-0.2.0/README.md +299 -0
- {acadfetch-0.1.1 → acadfetch-0.2.0}/acadfetch/auth.py +0 -1
- acadfetch-0.2.0/acadfetch/cli.py +699 -0
- {acadfetch-0.1.1 → acadfetch-0.2.0}/acadfetch/client.py +137 -15
- {acadfetch-0.1.1 → acadfetch-0.2.0}/acadfetch/consts.py +1 -1
- acadfetch-0.2.0/acadfetch/downloader.py +735 -0
- {acadfetch-0.1.1 → acadfetch-0.2.0}/acadfetch/errors.py +22 -2
- acadfetch-0.2.0/acadfetch/manifest.py +240 -0
- {acadfetch-0.1.1 → acadfetch-0.2.0}/acadfetch/sources/arxiv.py +39 -4
- {acadfetch-0.1.1 → acadfetch-0.2.0}/acadfetch/sources/base.py +17 -3
- {acadfetch-0.1.1 → acadfetch-0.2.0}/acadfetch/sources/direct.py +14 -9
- acadfetch-0.2.0/acadfetch/sources/doi.py +85 -0
- {acadfetch-0.1.1 → acadfetch-0.2.0}/acadfetch/sources/registry.py +9 -4
- acadfetch-0.2.0/acadfetch/sources/removed.py +42 -0
- acadfetch-0.2.0/acadfetch/urls.py +52 -0
- {acadfetch-0.1.1 → acadfetch-0.2.0}/acadfetch/verify.py +19 -0
- acadfetch-0.2.0/acadfetch.egg-info/PKG-INFO +330 -0
- {acadfetch-0.1.1 → acadfetch-0.2.0}/acadfetch.egg-info/SOURCES.txt +12 -1
- {acadfetch-0.1.1 → acadfetch-0.2.0}/pyproject.toml +1 -1
- acadfetch-0.2.0/tests/conftest.py +170 -0
- acadfetch-0.2.0/tests/fixtures/README.md +47 -0
- {acadfetch-0.1.1 → acadfetch-0.2.0}/tests/test_auth.py +2 -2
- acadfetch-0.2.0/tests/test_checksum.py +88 -0
- {acadfetch-0.1.1 → acadfetch-0.2.0}/tests/test_cli.py +85 -0
- {acadfetch-0.1.1 → acadfetch-0.2.0}/tests/test_client.py +45 -0
- acadfetch-0.2.0/tests/test_control_names.py +80 -0
- acadfetch-0.2.0/tests/test_disk.py +92 -0
- {acadfetch-0.1.1 → acadfetch-0.2.0}/tests/test_downloader.py +188 -141
- acadfetch-0.2.0/tests/test_identity.py +212 -0
- acadfetch-0.2.0/tests/test_isolation.py +167 -0
- acadfetch-0.2.0/tests/test_json.py +275 -0
- acadfetch-0.2.0/tests/test_manifest.py +356 -0
- acadfetch-0.2.0/tests/test_redirect.py +122 -0
- acadfetch-0.2.0/tests/test_removed.py +51 -0
- {acadfetch-0.1.1 → acadfetch-0.2.0}/tests/test_sources.py +94 -84
- acadfetch-0.2.0/tests/test_urls.py +46 -0
- {acadfetch-0.1.1 → acadfetch-0.2.0}/tests/test_verify.py +11 -0
- acadfetch-0.1.1/PKG-INFO +0 -237
- acadfetch-0.1.1/README.md +0 -206
- acadfetch-0.1.1/acadfetch/cli.py +0 -311
- acadfetch-0.1.1/acadfetch/downloader.py +0 -443
- acadfetch-0.1.1/acadfetch/manifest.py +0 -145
- acadfetch-0.1.1/acadfetch/sources/doi.py +0 -54
- acadfetch-0.1.1/acadfetch/sources/figshare.py +0 -86
- acadfetch-0.1.1/acadfetch.egg-info/PKG-INFO +0 -237
- acadfetch-0.1.1/tests/fixtures/README.md +0 -41
- acadfetch-0.1.1/tests/test_manifest.py +0 -85
- {acadfetch-0.1.1 → acadfetch-0.2.0}/LICENSE +0 -0
- {acadfetch-0.1.1 → acadfetch-0.2.0}/MANIFEST.in +0 -0
- {acadfetch-0.1.1 → acadfetch-0.2.0}/acadfetch/__init__.py +0 -0
- {acadfetch-0.1.1 → acadfetch-0.2.0}/acadfetch/__main__.py +0 -0
- {acadfetch-0.1.1 → acadfetch-0.2.0}/acadfetch/atomic.py +0 -0
- {acadfetch-0.1.1 → acadfetch-0.2.0}/acadfetch/naming.py +0 -0
- {acadfetch-0.1.1 → acadfetch-0.2.0}/acadfetch/progress.py +0 -0
- {acadfetch-0.1.1 → acadfetch-0.2.0}/acadfetch/py.typed +0 -0
- {acadfetch-0.1.1 → acadfetch-0.2.0}/acadfetch/ratelimit.py +0 -0
- {acadfetch-0.1.1 → acadfetch-0.2.0}/acadfetch/sources/__init__.py +0 -0
- {acadfetch-0.1.1 → acadfetch-0.2.0}/acadfetch/sources/zenodo.py +0 -0
- {acadfetch-0.1.1 → acadfetch-0.2.0}/acadfetch/units.py +0 -0
- {acadfetch-0.1.1 → acadfetch-0.2.0}/acadfetch.egg-info/dependency_links.txt +0 -0
- {acadfetch-0.1.1 → acadfetch-0.2.0}/acadfetch.egg-info/entry_points.txt +0 -0
- {acadfetch-0.1.1 → acadfetch-0.2.0}/acadfetch.egg-info/top_level.txt +0 -0
- {acadfetch-0.1.1 → acadfetch-0.2.0}/setup.cfg +0 -0
- {acadfetch-0.1.1 → acadfetch-0.2.0}/tests/fixtures/figshare_403_block.html +0 -0
- {acadfetch-0.1.1 → acadfetch-0.2.0}/tests/fixtures/figshare_403_forbidden.json +0 -0
- {acadfetch-0.1.1 → acadfetch-0.2.0}/tests/fixtures/figshare_article.json +0 -0
- {acadfetch-0.1.1 → acadfetch-0.2.0}/tests/fixtures/figshare_file.pdf +0 -0
- {acadfetch-0.1.1 → acadfetch-0.2.0}/tests/test_atomic.py +0 -0
- {acadfetch-0.1.1 → acadfetch-0.2.0}/tests/test_naming.py +0 -0
- {acadfetch-0.1.1 → acadfetch-0.2.0}/tests/test_packaging.py +0 -0
- {acadfetch-0.1.1 → acadfetch-0.2.0}/tests/test_progress.py +0 -0
- {acadfetch-0.1.1 → acadfetch-0.2.0}/tests/test_ratelimit.py +0 -0
acadfetch-0.2.0/PKG-INFO
ADDED
|
@@ -0,0 +1,330 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: acadfetch
|
|
3
|
+
Version: 0.2.0
|
|
4
|
+
Summary: Resumable, rate-limited, checksum-verified bulk downloads from academic data repositories
|
|
5
|
+
Author: young-ski
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/young-ski/acadfetch
|
|
8
|
+
Project-URL: Source, https://github.com/young-ski/acadfetch
|
|
9
|
+
Project-URL: Issues, https://github.com/young-ski/acadfetch/issues
|
|
10
|
+
Keywords: zenodo,figshare,arxiv,doi,reproducibility,download
|
|
11
|
+
Classifier: Development Status :: 4 - Beta
|
|
12
|
+
Classifier: Environment :: Console
|
|
13
|
+
Classifier: Intended Audience :: Science/Research
|
|
14
|
+
Classifier: Operating System :: MacOS
|
|
15
|
+
Classifier: Operating System :: Microsoft :: Windows
|
|
16
|
+
Classifier: Operating System :: POSIX :: Linux
|
|
17
|
+
Classifier: Programming Language :: Python :: 3
|
|
18
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
22
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
23
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
24
|
+
Classifier: Programming Language :: Python :: 3.15
|
|
25
|
+
Classifier: Topic :: Scientific/Engineering
|
|
26
|
+
Classifier: Typing :: Typed
|
|
27
|
+
Requires-Python: >=3.10
|
|
28
|
+
Description-Content-Type: text/markdown
|
|
29
|
+
License-File: LICENSE
|
|
30
|
+
Dynamic: license-file
|
|
31
|
+
|
|
32
|
+
# acadfetch
|
|
33
|
+
|
|
34
|
+
[](https://github.com/young-ski/acadfetch/actions/workflows/ci.yml)
|
|
35
|
+
[](https://pypi.org/project/acadfetch/)
|
|
36
|
+
[](LICENSE)
|
|
37
|
+
|
|
38
|
+
Batch-download files from academic data repositories — resumably, rate-limited, and
|
|
39
|
+
checked against the checksums the repository published.
|
|
40
|
+
|
|
41
|
+
```
|
|
42
|
+
$ acadfetch fetch 10.5281/zenodo.23612969 -o ./data
|
|
43
|
+
title : Dinler Tarihi Alanında Güncel Konular ve Akademik Değerlendirmeler
|
|
44
|
+
doi : 10.5281/zenodo.23612969
|
|
45
|
+
license : cc-by-4.0
|
|
46
|
+
files : 1, 1.6 MiB
|
|
47
|
+
landing : https://zenodo.org/records/23612969
|
|
48
|
+
Dinler Tarihi ... .pdf 1.2 MiB/1.6 MiB 312.4 KiB/s ETA 0:02
|
|
49
|
+
ok Dinler Tarihi ... .pdf 1.6 MiB (resumed at 512.0 KiB)
|
|
50
|
+
|
|
51
|
+
1/1 file(s) in ./data
|
|
52
|
+
```
|
|
53
|
+
|
|
54
|
+
**What it is not:** a search engine, a scraper for paywalled publishers, or a
|
|
55
|
+
citation manager. It is the boring part in the middle — the part that gets killed
|
|
56
|
+
by a flaky VPN at 80%, and the part that nobody writes down.
|
|
57
|
+
|
|
58
|
+
## Why another downloader
|
|
59
|
+
|
|
60
|
+
Because `curl -O` in a loop loses three things that matter for research data:
|
|
61
|
+
|
|
62
|
+
| Problem | What happens with a plain loop | What acadfetch does |
|
|
63
|
+
| --- | --- | --- |
|
|
64
|
+
| Connection drops at 80% | restart from zero | resumes with `Range` + `If-Range`, keeps the partial |
|
|
65
|
+
| Silent corruption | nobody notices until analysis breaks | verifies the repository's `md5`/`sha256`, re-downloads on mismatch |
|
|
66
|
+
| "Which version did I actually use?" | untraceable | writes `manifest.json` with DOI, license, per-file SHA-256 |
|
|
67
|
+
| Getting rate-limited into a 403 | hammering, then blocked | one global token bucket, honours `Retry-After` |
|
|
68
|
+
|
|
69
|
+
The manifest is the point. A dataset you cannot re-verify is a dataset you cannot
|
|
70
|
+
cite.
|
|
71
|
+
|
|
72
|
+
## Install
|
|
73
|
+
|
|
74
|
+
Python 3.10–3.15. No runtime dependencies.
|
|
75
|
+
|
|
76
|
+
Try it without installing anything:
|
|
77
|
+
|
|
78
|
+
```bash
|
|
79
|
+
uvx acadfetch fetch 10.5281/zenodo.23612969 # uv
|
|
80
|
+
```
|
|
81
|
+
|
|
82
|
+
Or keep it on the PATH:
|
|
83
|
+
|
|
84
|
+
```bash
|
|
85
|
+
pipx install acadfetch
|
|
86
|
+
```
|
|
87
|
+
|
|
88
|
+
From a checkout, for development:
|
|
89
|
+
|
|
90
|
+
```bash
|
|
91
|
+
git clone https://github.com/young-ski/acadfetch && cd acadfetch
|
|
92
|
+
python -m pip install -e . pytest && python -m pytest -q
|
|
93
|
+
```
|
|
94
|
+
|
|
95
|
+
## Supported platforms
|
|
96
|
+
|
|
97
|
+
| Platform | Status |
|
|
98
|
+
| --- | --- |
|
|
99
|
+
| Linux | tested — CI, every supported Python |
|
|
100
|
+
| Windows | tested — CI on 3.10, 3.13 and 3.15 |
|
|
101
|
+
| macOS | best effort — CI runs it on 3.12 and 3.15, a failure is a bug to fix but does not hold a release |
|
|
102
|
+
| Windows ARM | untested |
|
|
103
|
+
| Python 3.15 | tested — CI, on Linux and Windows |
|
|
104
|
+
|
|
105
|
+
## Usage
|
|
106
|
+
|
|
107
|
+
These work as typed:
|
|
108
|
+
|
|
109
|
+
```bash
|
|
110
|
+
acadfetch fetch 10.5281/zenodo.23612969 # a DOI
|
|
111
|
+
acadfetch fetch arxiv:1706.03762 # an arXiv id
|
|
112
|
+
acadfetch fetch https://ndownloader.figshare.com/files/70600233/report.pdf # any URL
|
|
113
|
+
```
|
|
114
|
+
|
|
115
|
+
figshare was removed in 0.2.0: pass the article's download URL instead, and
|
|
116
|
+
[`--checksum`](docs/migration-0.2.md) keeps the verification the API used to
|
|
117
|
+
give you.
|
|
118
|
+
|
|
119
|
+
The rest of the surface:
|
|
120
|
+
|
|
121
|
+
```bash
|
|
122
|
+
acadfetch fetch 10.5281/zenodo.23612969 -n # dry run: which dir, which files
|
|
123
|
+
acadfetch fetch 10.5281/zenodo.23612969 --only raw.tar.gz
|
|
124
|
+
acadfetch fetch 10.5281/zenodo.23612969 --max-size 2GB # refuse anything bigger
|
|
125
|
+
acadfetch fetch 10.5281/zenodo.23612969 -j 1 --rate 0.5 # be a good citizen
|
|
126
|
+
acadfetch manifest 10.5281/zenodo.23612969 -o m.json # metadata only, no bytes
|
|
127
|
+
acadfetch fetch zenodo:123456 --header "Authorization: Bearer $TOKEN" # restricted
|
|
128
|
+
|
|
129
|
+
acadfetch verify ./data/manifest.json # re-hash what you downloaded
|
|
130
|
+
acadfetch sources # adapters, with an example each
|
|
131
|
+
```
|
|
132
|
+
|
|
133
|
+
`10.5281/zenodo.123456`, `zenodo:123456` and `arxiv:2401.12345` are placeholders;
|
|
134
|
+
use any real id.
|
|
135
|
+
|
|
136
|
+
### In a pipeline
|
|
137
|
+
|
|
138
|
+
`--json` puts one report document on stdout and moves progress to stderr, so the
|
|
139
|
+
two never interleave:
|
|
140
|
+
|
|
141
|
+
```bash
|
|
142
|
+
acadfetch fetch zenodo:123456 --json | jq -r '.files[] | select(.state=="failed") | .name'
|
|
143
|
+
```
|
|
144
|
+
|
|
145
|
+
Exit codes are unchanged, and a failure before any transfer still produces a
|
|
146
|
+
document with `error` set. `report` is the document's schema version (now `2`):
|
|
147
|
+
fields are added, never removed or given a new meaning, and `state` may gain
|
|
148
|
+
values — treat an unknown one as non-terminal. `verify --json` reports per-file
|
|
149
|
+
`MISMATCH` rows the same way, and its `summary` counts `skipped` rows
|
|
150
|
+
separately. At the end of a failed transfer, `part` names the `.part` file that
|
|
151
|
+
was kept, so a retry can resume instead of starting over. A 403 that carries a
|
|
152
|
+
block page is retried; see [docs/troubleshooting.md](docs/troubleshooting.md).
|
|
153
|
+
|
|
154
|
+
Progress is one aggregate line for the whole run
|
|
155
|
+
(`3 file(s) 12.4/44.1 MiB 1.8 MiB/s ETA 0:17`), redrawn in place on a TTY.
|
|
156
|
+
When stdout is a pipe it becomes a new line at most every 5 s (one extra when a
|
|
157
|
+
file starts or stops), so a CI log stays greppable without becoming a wall. `-q`
|
|
158
|
+
turns it off.
|
|
159
|
+
|
|
160
|
+
Output defaults to `acadfetch-out/<source>-<id>` (a URL adds its host and a short
|
|
161
|
+
hash), and `-n` prints it before
|
|
162
|
+
you spend bandwidth. Files land in `<name>.part` and are renamed only after the
|
|
163
|
+
checksum matches, so an interrupted or failed run never leaves something that
|
|
164
|
+
*looks* finished -- and never throws away the bytes it already paid for: rerun and
|
|
165
|
+
it resumes. `acadfetch-out/` is worth adding to a project's `.gitignore`; this
|
|
166
|
+
repository already has it.
|
|
167
|
+
|
|
168
|
+
The default directory names the target, not only its filename — `data.csv` from
|
|
169
|
+
two hosts is two datasets, and they do not share a directory. A file already
|
|
170
|
+
there is reused when its checksum matches; with no checksum, the size plus one
|
|
171
|
+
more tie (an explicit `-o`, or a manifest in that directory that recorded the
|
|
172
|
+
same URL) is required, and `--force` skips the question entirely.
|
|
173
|
+
|
|
174
|
+
A DOI that resolves to a web page instead of a file is refused, naming the
|
|
175
|
+
page: fetching the landing page as if it were the dataset is how a repository's
|
|
176
|
+
front door ends up being cited as a data file.
|
|
177
|
+
|
|
178
|
+
A run that cannot fit is refused before the first byte is transferred, naming the shortfall:
|
|
179
|
+
`needs 41.2 GiB but 12.0 GiB is free`. What is already on disk is subtracted, so
|
|
180
|
+
a resume is judged on the remainder. Each finished file is flushed before its
|
|
181
|
+
rename (`--no-fsync` skips that; it is faster and weaker against a power cut).
|
|
182
|
+
|
|
183
|
+
Exit codes: `0` success, `1` failure, `2` partial, `64` bad usage, `130` interrupted
|
|
184
|
+
(rerun to resume).
|
|
185
|
+
|
|
186
|
+
## Tokens
|
|
187
|
+
|
|
188
|
+
`ZENODO_TOKEN` is read from the environment and sent only to Zenodo, so a token
|
|
189
|
+
never travels to a host that has no business seeing it -- including across a
|
|
190
|
+
redirect: a hop to another host (a storage CDN, a download mirror) goes out
|
|
191
|
+
without it, a hop from https to http drops it with a warning (`--allow-http`
|
|
192
|
+
keeps it), and a presigned URL gets no `Authorization` at all, because S3
|
|
193
|
+
rejects a request that carries both. Cookies and proxy credentials are dropped
|
|
194
|
+
on any cross-origin hop as well. `--header` still overrides, `--no-token`
|
|
195
|
+
ignores the environment, and the manifest records the variable name plus the
|
|
196
|
+
last four characters only:
|
|
197
|
+
|
|
198
|
+
```bash
|
|
199
|
+
export ZENODO_TOKEN=... # restricted records, higher rate limits
|
|
200
|
+
acadfetch fetch zenodo:123456 # prints: auth : token from $ZENODO_TOKEN (…9f2c)
|
|
201
|
+
```
|
|
202
|
+
|
|
203
|
+
## Verification and the manifest
|
|
204
|
+
|
|
205
|
+
Every completed run writes `manifest.json`:
|
|
206
|
+
|
|
207
|
+
```json
|
|
208
|
+
{
|
|
209
|
+
"schema": 3,
|
|
210
|
+
"doi": "10.5281/zenodo.123456",
|
|
211
|
+
"license": "cc-by-4.0",
|
|
212
|
+
"files": [
|
|
213
|
+
{ "name": "raw.tar.gz", "size": 8123, "checksum": "md5:9f2c…",
|
|
214
|
+
"sha256": "b41d…", "state": "downloaded", "selected": true,
|
|
215
|
+
"verified": true, "verified_reason": "matched-md5" },
|
|
216
|
+
{ "name": "notes.txt", "size": 12, "state": "not-selected", "selected": false,
|
|
217
|
+
"sha256": null, "verified": null, "verified_reason": "not-selected" }
|
|
218
|
+
]
|
|
219
|
+
}
|
|
220
|
+
```
|
|
221
|
+
|
|
222
|
+
The manifest describes **the record**, not the run: `--only` still writes a row
|
|
223
|
+
for every file, so `verify` can tell "not fetched" from "fetched and then lost",
|
|
224
|
+
and a successful `--only` run verifies green. Add `--require-all` when you want
|
|
225
|
+
the whole record accounted for. Each fetched row also records `final_url`,
|
|
226
|
+
`etag`, `http_status` and `retrieved_at`, and credential query parameters are
|
|
227
|
+
stripped from every URL (`--record-full-url` keeps them). Full field and state
|
|
228
|
+
reference: [docs/manifest.md](docs/manifest.md).
|
|
229
|
+
|
|
230
|
+
`name` is the file *on disk*, which can differ from the repository's name when two
|
|
231
|
+
files collide. `verified_reason` separates the two kinds of `verified: null`:
|
|
232
|
+
`no-checksum-published` (arXiv) from `unparseable-checksum` (a repository that
|
|
233
|
+
published something we cannot name an algorithm for). Schema 1 and 2 manifests
|
|
234
|
+
still read, and `verify` says so for schema 1, because their names were recorded
|
|
235
|
+
before duplicates were resolved.
|
|
236
|
+
|
|
237
|
+
`acadfetch verify` recomputes SHA-256 for each entry and exits non-zero on any
|
|
238
|
+
mismatch or missing file; rows the run never selected are reported `skipped`.
|
|
239
|
+
Commit the manifest next to your analysis; it is what makes the download
|
|
240
|
+
reproducible a year later.
|
|
241
|
+
|
|
242
|
+
Names are resolved against the manifest's own directory, and a name that points
|
|
243
|
+
outside it — `../sibling/file`, an absolute path, a symlink leading out — is
|
|
244
|
+
reported as `escaped` rather than quietly hashed, because a manifest is a
|
|
245
|
+
document someone can hand you. `--allow-escape` follows them; `--root` changes
|
|
246
|
+
what "inside" means.
|
|
247
|
+
|
|
248
|
+
When a source publishes no checksum (arXiv), verification degrades to a byte
|
|
249
|
+
count and `verified` is `null` rather than a false sense of safety.
|
|
250
|
+
|
|
251
|
+
## Supported sources
|
|
252
|
+
|
|
253
|
+
| Source | Target forms | Checksums |
|
|
254
|
+
| --- | --- | --- |
|
|
255
|
+
| Zenodo | DOI `10.5281/zenodo.N`, `zenodo:N`, record URL | md5 |
|
|
256
|
+
| arXiv | `arxiv:ID`, bare ID, abs/pdf URL | none (size only) |
|
|
257
|
+
| doi.org | any DOI, resolved then dispatched | depends |
|
|
258
|
+
| direct | any `http(s)` URL | headers, if present |
|
|
259
|
+
|
|
260
|
+
figshare (`figshare:N`, `10.6084/…`, article URLs) was removed in 0.2.0; it
|
|
261
|
+
answers with the conversion instead of a generic "cannot tell what this is".
|
|
262
|
+
See [docs/migration-0.2.md](docs/migration-0.2.md).
|
|
263
|
+
|
|
264
|
+
A CDN in front of a direct URL can answer 403 with a block page. Those are
|
|
265
|
+
retried twice with backoff, while a 403 carrying the application's own JSON is
|
|
266
|
+
terminal on the first answer; `--retry-403 N` changes the count.
|
|
267
|
+
|
|
268
|
+
## Politeness
|
|
269
|
+
|
|
270
|
+
Defaults: 3 parallel transfers, 2 requests/second, 4 retries with jittered
|
|
271
|
+
exponential backoff, `Retry-After` honoured up to 120 s. A block page is retried twice
|
|
272
|
+
(`--retry-403 N`); Zenodo and arXiv are donation-funded —
|
|
273
|
+
`--rate 1 -j 1` costs you minutes and costs them less.
|
|
274
|
+
|
|
275
|
+
Respect the license in the manifest. `--dry-run` prints it before you spend
|
|
276
|
+
bandwidth; `unknown` means *check before redistributing*.
|
|
277
|
+
|
|
278
|
+
## Adding a source
|
|
279
|
+
|
|
280
|
+
1. Read `acadfetch/sources/zenodo.py` — an adapter is `match(target) -> id | None`
|
|
281
|
+
plus `resolve(client, target) -> Record`.
|
|
282
|
+
2. Fill `File(checksum=...)` whenever the API publishes one.
|
|
283
|
+
3. Never hardcode rate limits; take the shared `Client`.
|
|
284
|
+
4. Add a `match` test and, if possible, a fixture of the real API payload.
|
|
285
|
+
|
|
286
|
+
`scripts/probe_resume.py` interrupts a live transfer and resumes it — the only
|
|
287
|
+
honest way to test this code path.
|
|
288
|
+
|
|
289
|
+
## Getting involved
|
|
290
|
+
|
|
291
|
+
- **Stuck on something** → [docs/troubleshooting.md](docs/troubleshooting.md) first; it is short and indexed by symptom
|
|
292
|
+
- **Usage questions** → [Discussions](https://github.com/young-ski/acadfetch/discussions)
|
|
293
|
+
- **Bugs** → the bug template; say whether `verify` reports `MISMATCH`
|
|
294
|
+
- **Security** (bad bytes, path escapes, token leaks) → [`SECURITY.md`](SECURITY.md), privately
|
|
295
|
+
- **Adapters, tests, docs** → PRs welcome, read [`CONTRIBUTING.md`](CONTRIBUTING.md) first
|
|
296
|
+
|
|
297
|
+
## Known limits
|
|
298
|
+
|
|
299
|
+
- figshare was removed in 0.2.0; existing figshare manifests still verify, and
|
|
300
|
+
new targets convert with one `--checksum` command ([docs/migration-0.2.md](docs/migration-0.2.md)).
|
|
301
|
+
- Tokens come from the environment only; no interactive `login` flow and no
|
|
302
|
+
per-file token refresh.
|
|
303
|
+
- No search/pagination; you bring the target.
|
|
304
|
+
- arXiv PDFs are fetched by constructed URL, so a withdrawn ID gives a 404 after
|
|
305
|
+
resolution.
|
|
306
|
+
- Windows ARM is untested.
|
|
307
|
+
- macOS is best effort: it is in CI but not release-blocking, and nobody has
|
|
308
|
+
run it by hand yet.
|
|
309
|
+
- A source that publishes neither a size nor a checksum (bare `direct` URLs
|
|
310
|
+
without a `Content-Length` on `HEAD`) can still be truncated silently by a
|
|
311
|
+
server that lies about `Content-Length`, because HTTP gives no in-band way to
|
|
312
|
+
notice the extra bytes.
|
|
313
|
+
|
|
314
|
+
## 中文
|
|
315
|
+
|
|
316
|
+
学术数据仓库的批量下载器:断点续传、全局限速、按仓库公布的校验和验证,并落
|
|
317
|
+
`manifest.json`(DOI、许可协议、每文件 SHA-256)供日后复现。
|
|
318
|
+
|
|
319
|
+
```bash
|
|
320
|
+
acadfetch fetch 10.5281/zenodo.123456 -o ./data
|
|
321
|
+
acadfetch verify ./data/manifest.json
|
|
322
|
+
```
|
|
323
|
+
|
|
324
|
+
MIT。
|
|
325
|
+
|
|
326
|
+
## Audit trail
|
|
327
|
+
|
|
328
|
+
How the 3.15 support claim was reached, including what it did **not** prove —
|
|
329
|
+
the free-threaded build, Windows ARM, and every place the probe ran short:
|
|
330
|
+
[docs/py315-windows-report.md](docs/py315-windows-report.md).
|
|
@@ -0,0 +1,299 @@
|
|
|
1
|
+
# acadfetch
|
|
2
|
+
|
|
3
|
+
[](https://github.com/young-ski/acadfetch/actions/workflows/ci.yml)
|
|
4
|
+
[](https://pypi.org/project/acadfetch/)
|
|
5
|
+
[](LICENSE)
|
|
6
|
+
|
|
7
|
+
Batch-download files from academic data repositories — resumably, rate-limited, and
|
|
8
|
+
checked against the checksums the repository published.
|
|
9
|
+
|
|
10
|
+
```
|
|
11
|
+
$ acadfetch fetch 10.5281/zenodo.23612969 -o ./data
|
|
12
|
+
title : Dinler Tarihi Alanında Güncel Konular ve Akademik Değerlendirmeler
|
|
13
|
+
doi : 10.5281/zenodo.23612969
|
|
14
|
+
license : cc-by-4.0
|
|
15
|
+
files : 1, 1.6 MiB
|
|
16
|
+
landing : https://zenodo.org/records/23612969
|
|
17
|
+
Dinler Tarihi ... .pdf 1.2 MiB/1.6 MiB 312.4 KiB/s ETA 0:02
|
|
18
|
+
ok Dinler Tarihi ... .pdf 1.6 MiB (resumed at 512.0 KiB)
|
|
19
|
+
|
|
20
|
+
1/1 file(s) in ./data
|
|
21
|
+
```
|
|
22
|
+
|
|
23
|
+
**What it is not:** a search engine, a scraper for paywalled publishers, or a
|
|
24
|
+
citation manager. It is the boring part in the middle — the part that gets killed
|
|
25
|
+
by a flaky VPN at 80%, and the part that nobody writes down.
|
|
26
|
+
|
|
27
|
+
## Why another downloader
|
|
28
|
+
|
|
29
|
+
Because `curl -O` in a loop loses three things that matter for research data:
|
|
30
|
+
|
|
31
|
+
| Problem | What happens with a plain loop | What acadfetch does |
|
|
32
|
+
| --- | --- | --- |
|
|
33
|
+
| Connection drops at 80% | restart from zero | resumes with `Range` + `If-Range`, keeps the partial |
|
|
34
|
+
| Silent corruption | nobody notices until analysis breaks | verifies the repository's `md5`/`sha256`, re-downloads on mismatch |
|
|
35
|
+
| "Which version did I actually use?" | untraceable | writes `manifest.json` with DOI, license, per-file SHA-256 |
|
|
36
|
+
| Getting rate-limited into a 403 | hammering, then blocked | one global token bucket, honours `Retry-After` |
|
|
37
|
+
|
|
38
|
+
The manifest is the point. A dataset you cannot re-verify is a dataset you cannot
|
|
39
|
+
cite.
|
|
40
|
+
|
|
41
|
+
## Install
|
|
42
|
+
|
|
43
|
+
Python 3.10–3.15. No runtime dependencies.
|
|
44
|
+
|
|
45
|
+
Try it without installing anything:
|
|
46
|
+
|
|
47
|
+
```bash
|
|
48
|
+
uvx acadfetch fetch 10.5281/zenodo.23612969 # uv
|
|
49
|
+
```
|
|
50
|
+
|
|
51
|
+
Or keep it on the PATH:
|
|
52
|
+
|
|
53
|
+
```bash
|
|
54
|
+
pipx install acadfetch
|
|
55
|
+
```
|
|
56
|
+
|
|
57
|
+
From a checkout, for development:
|
|
58
|
+
|
|
59
|
+
```bash
|
|
60
|
+
git clone https://github.com/young-ski/acadfetch && cd acadfetch
|
|
61
|
+
python -m pip install -e . pytest && python -m pytest -q
|
|
62
|
+
```
|
|
63
|
+
|
|
64
|
+
## Supported platforms
|
|
65
|
+
|
|
66
|
+
| Platform | Status |
|
|
67
|
+
| --- | --- |
|
|
68
|
+
| Linux | tested — CI, every supported Python |
|
|
69
|
+
| Windows | tested — CI on 3.10, 3.13 and 3.15 |
|
|
70
|
+
| macOS | best effort — CI runs it on 3.12 and 3.15, a failure is a bug to fix but does not hold a release |
|
|
71
|
+
| Windows ARM | untested |
|
|
72
|
+
| Python 3.15 | tested — CI, on Linux and Windows |
|
|
73
|
+
|
|
74
|
+
## Usage
|
|
75
|
+
|
|
76
|
+
These work as typed:
|
|
77
|
+
|
|
78
|
+
```bash
|
|
79
|
+
acadfetch fetch 10.5281/zenodo.23612969 # a DOI
|
|
80
|
+
acadfetch fetch arxiv:1706.03762 # an arXiv id
|
|
81
|
+
acadfetch fetch https://ndownloader.figshare.com/files/70600233/report.pdf # any URL
|
|
82
|
+
```
|
|
83
|
+
|
|
84
|
+
figshare was removed in 0.2.0: pass the article's download URL instead, and
|
|
85
|
+
[`--checksum`](docs/migration-0.2.md) keeps the verification the API used to
|
|
86
|
+
give you.
|
|
87
|
+
|
|
88
|
+
The rest of the surface:
|
|
89
|
+
|
|
90
|
+
```bash
|
|
91
|
+
acadfetch fetch 10.5281/zenodo.23612969 -n # dry run: which dir, which files
|
|
92
|
+
acadfetch fetch 10.5281/zenodo.23612969 --only raw.tar.gz
|
|
93
|
+
acadfetch fetch 10.5281/zenodo.23612969 --max-size 2GB # refuse anything bigger
|
|
94
|
+
acadfetch fetch 10.5281/zenodo.23612969 -j 1 --rate 0.5 # be a good citizen
|
|
95
|
+
acadfetch manifest 10.5281/zenodo.23612969 -o m.json # metadata only, no bytes
|
|
96
|
+
acadfetch fetch zenodo:123456 --header "Authorization: Bearer $TOKEN" # restricted
|
|
97
|
+
|
|
98
|
+
acadfetch verify ./data/manifest.json # re-hash what you downloaded
|
|
99
|
+
acadfetch sources # adapters, with an example each
|
|
100
|
+
```
|
|
101
|
+
|
|
102
|
+
`10.5281/zenodo.123456`, `zenodo:123456` and `arxiv:2401.12345` are placeholders;
|
|
103
|
+
use any real id.
|
|
104
|
+
|
|
105
|
+
### In a pipeline
|
|
106
|
+
|
|
107
|
+
`--json` puts one report document on stdout and moves progress to stderr, so the
|
|
108
|
+
two never interleave:
|
|
109
|
+
|
|
110
|
+
```bash
|
|
111
|
+
acadfetch fetch zenodo:123456 --json | jq -r '.files[] | select(.state=="failed") | .name'
|
|
112
|
+
```
|
|
113
|
+
|
|
114
|
+
Exit codes are unchanged, and a failure before any transfer still produces a
|
|
115
|
+
document with `error` set. `report` is the document's schema version (now `2`):
|
|
116
|
+
fields are added, never removed or given a new meaning, and `state` may gain
|
|
117
|
+
values — treat an unknown one as non-terminal. `verify --json` reports per-file
|
|
118
|
+
`MISMATCH` rows the same way, and its `summary` counts `skipped` rows
|
|
119
|
+
separately. At the end of a failed transfer, `part` names the `.part` file that
|
|
120
|
+
was kept, so a retry can resume instead of starting over. A 403 that carries a
|
|
121
|
+
block page is retried; see [docs/troubleshooting.md](docs/troubleshooting.md).
|
|
122
|
+
|
|
123
|
+
Progress is one aggregate line for the whole run
|
|
124
|
+
(`3 file(s) 12.4/44.1 MiB 1.8 MiB/s ETA 0:17`), redrawn in place on a TTY.
|
|
125
|
+
When stdout is a pipe it becomes a new line at most every 5 s (one extra when a
|
|
126
|
+
file starts or stops), so a CI log stays greppable without becoming a wall. `-q`
|
|
127
|
+
turns it off.
|
|
128
|
+
|
|
129
|
+
Output defaults to `acadfetch-out/<source>-<id>` (a URL adds its host and a short
|
|
130
|
+
hash), and `-n` prints it before
|
|
131
|
+
you spend bandwidth. Files land in `<name>.part` and are renamed only after the
|
|
132
|
+
checksum matches, so an interrupted or failed run never leaves something that
|
|
133
|
+
*looks* finished -- and never throws away the bytes it already paid for: rerun and
|
|
134
|
+
it resumes. `acadfetch-out/` is worth adding to a project's `.gitignore`; this
|
|
135
|
+
repository already has it.
|
|
136
|
+
|
|
137
|
+
The default directory names the target, not only its filename — `data.csv` from
|
|
138
|
+
two hosts is two datasets, and they do not share a directory. A file already
|
|
139
|
+
there is reused when its checksum matches; with no checksum, the size plus one
|
|
140
|
+
more tie (an explicit `-o`, or a manifest in that directory that recorded the
|
|
141
|
+
same URL) is required, and `--force` skips the question entirely.
|
|
142
|
+
|
|
143
|
+
A DOI that resolves to a web page instead of a file is refused, naming the
|
|
144
|
+
page: fetching the landing page as if it were the dataset is how a repository's
|
|
145
|
+
front door ends up being cited as a data file.
|
|
146
|
+
|
|
147
|
+
A run that cannot fit is refused before the first byte is transferred, naming the shortfall:
|
|
148
|
+
`needs 41.2 GiB but 12.0 GiB is free`. What is already on disk is subtracted, so
|
|
149
|
+
a resume is judged on the remainder. Each finished file is flushed before its
|
|
150
|
+
rename (`--no-fsync` skips that; it is faster and weaker against a power cut).
|
|
151
|
+
|
|
152
|
+
Exit codes: `0` success, `1` failure, `2` partial, `64` bad usage, `130` interrupted
|
|
153
|
+
(rerun to resume).
|
|
154
|
+
|
|
155
|
+
## Tokens
|
|
156
|
+
|
|
157
|
+
`ZENODO_TOKEN` is read from the environment and sent only to Zenodo, so a token
|
|
158
|
+
never travels to a host that has no business seeing it -- including across a
|
|
159
|
+
redirect: a hop to another host (a storage CDN, a download mirror) goes out
|
|
160
|
+
without it, a hop from https to http drops it with a warning (`--allow-http`
|
|
161
|
+
keeps it), and a presigned URL gets no `Authorization` at all, because S3
|
|
162
|
+
rejects a request that carries both. Cookies and proxy credentials are dropped
|
|
163
|
+
on any cross-origin hop as well. `--header` still overrides, `--no-token`
|
|
164
|
+
ignores the environment, and the manifest records the variable name plus the
|
|
165
|
+
last four characters only:
|
|
166
|
+
|
|
167
|
+
```bash
|
|
168
|
+
export ZENODO_TOKEN=... # restricted records, higher rate limits
|
|
169
|
+
acadfetch fetch zenodo:123456 # prints: auth : token from $ZENODO_TOKEN (…9f2c)
|
|
170
|
+
```
|
|
171
|
+
|
|
172
|
+
## Verification and the manifest
|
|
173
|
+
|
|
174
|
+
Every completed run writes `manifest.json`:
|
|
175
|
+
|
|
176
|
+
```json
|
|
177
|
+
{
|
|
178
|
+
"schema": 3,
|
|
179
|
+
"doi": "10.5281/zenodo.123456",
|
|
180
|
+
"license": "cc-by-4.0",
|
|
181
|
+
"files": [
|
|
182
|
+
{ "name": "raw.tar.gz", "size": 8123, "checksum": "md5:9f2c…",
|
|
183
|
+
"sha256": "b41d…", "state": "downloaded", "selected": true,
|
|
184
|
+
"verified": true, "verified_reason": "matched-md5" },
|
|
185
|
+
{ "name": "notes.txt", "size": 12, "state": "not-selected", "selected": false,
|
|
186
|
+
"sha256": null, "verified": null, "verified_reason": "not-selected" }
|
|
187
|
+
]
|
|
188
|
+
}
|
|
189
|
+
```
|
|
190
|
+
|
|
191
|
+
The manifest describes **the record**, not the run: `--only` still writes a row
|
|
192
|
+
for every file, so `verify` can tell "not fetched" from "fetched and then lost",
|
|
193
|
+
and a successful `--only` run verifies green. Add `--require-all` when you want
|
|
194
|
+
the whole record accounted for. Each fetched row also records `final_url`,
|
|
195
|
+
`etag`, `http_status` and `retrieved_at`, and credential query parameters are
|
|
196
|
+
stripped from every URL (`--record-full-url` keeps them). Full field and state
|
|
197
|
+
reference: [docs/manifest.md](docs/manifest.md).
|
|
198
|
+
|
|
199
|
+
`name` is the file *on disk*, which can differ from the repository's name when two
|
|
200
|
+
files collide. `verified_reason` separates the two kinds of `verified: null`:
|
|
201
|
+
`no-checksum-published` (arXiv) from `unparseable-checksum` (a repository that
|
|
202
|
+
published something we cannot name an algorithm for). Schema 1 and 2 manifests
|
|
203
|
+
still read, and `verify` says so for schema 1, because their names were recorded
|
|
204
|
+
before duplicates were resolved.
|
|
205
|
+
|
|
206
|
+
`acadfetch verify` recomputes SHA-256 for each entry and exits non-zero on any
|
|
207
|
+
mismatch or missing file; rows the run never selected are reported `skipped`.
|
|
208
|
+
Commit the manifest next to your analysis; it is what makes the download
|
|
209
|
+
reproducible a year later.
|
|
210
|
+
|
|
211
|
+
Names are resolved against the manifest's own directory, and a name that points
|
|
212
|
+
outside it — `../sibling/file`, an absolute path, a symlink leading out — is
|
|
213
|
+
reported as `escaped` rather than quietly hashed, because a manifest is a
|
|
214
|
+
document someone can hand you. `--allow-escape` follows them; `--root` changes
|
|
215
|
+
what "inside" means.
|
|
216
|
+
|
|
217
|
+
When a source publishes no checksum (arXiv), verification degrades to a byte
|
|
218
|
+
count and `verified` is `null` rather than a false sense of safety.
|
|
219
|
+
|
|
220
|
+
## Supported sources
|
|
221
|
+
|
|
222
|
+
| Source | Target forms | Checksums |
|
|
223
|
+
| --- | --- | --- |
|
|
224
|
+
| Zenodo | DOI `10.5281/zenodo.N`, `zenodo:N`, record URL | md5 |
|
|
225
|
+
| arXiv | `arxiv:ID`, bare ID, abs/pdf URL | none (size only) |
|
|
226
|
+
| doi.org | any DOI, resolved then dispatched | depends |
|
|
227
|
+
| direct | any `http(s)` URL | headers, if present |
|
|
228
|
+
|
|
229
|
+
figshare (`figshare:N`, `10.6084/…`, article URLs) was removed in 0.2.0; it
|
|
230
|
+
answers with the conversion instead of a generic "cannot tell what this is".
|
|
231
|
+
See [docs/migration-0.2.md](docs/migration-0.2.md).
|
|
232
|
+
|
|
233
|
+
A CDN in front of a direct URL can answer 403 with a block page. Those are
|
|
234
|
+
retried twice with backoff, while a 403 carrying the application's own JSON is
|
|
235
|
+
terminal on the first answer; `--retry-403 N` changes the count.
|
|
236
|
+
|
|
237
|
+
## Politeness
|
|
238
|
+
|
|
239
|
+
Defaults: 3 parallel transfers, 2 requests/second, 4 retries with jittered
|
|
240
|
+
exponential backoff, `Retry-After` honoured up to 120 s. A block page is retried twice
|
|
241
|
+
(`--retry-403 N`); Zenodo and arXiv are donation-funded —
|
|
242
|
+
`--rate 1 -j 1` costs you minutes and costs them less.
|
|
243
|
+
|
|
244
|
+
Respect the license in the manifest. `--dry-run` prints it before you spend
|
|
245
|
+
bandwidth; `unknown` means *check before redistributing*.
|
|
246
|
+
|
|
247
|
+
## Adding a source
|
|
248
|
+
|
|
249
|
+
1. Read `acadfetch/sources/zenodo.py` — an adapter is `match(target) -> id | None`
|
|
250
|
+
plus `resolve(client, target) -> Record`.
|
|
251
|
+
2. Fill `File(checksum=...)` whenever the API publishes one.
|
|
252
|
+
3. Never hardcode rate limits; take the shared `Client`.
|
|
253
|
+
4. Add a `match` test and, if possible, a fixture of the real API payload.
|
|
254
|
+
|
|
255
|
+
`scripts/probe_resume.py` interrupts a live transfer and resumes it — the only
|
|
256
|
+
honest way to test this code path.
|
|
257
|
+
|
|
258
|
+
## Getting involved
|
|
259
|
+
|
|
260
|
+
- **Stuck on something** → [docs/troubleshooting.md](docs/troubleshooting.md) first; it is short and indexed by symptom
|
|
261
|
+
- **Usage questions** → [Discussions](https://github.com/young-ski/acadfetch/discussions)
|
|
262
|
+
- **Bugs** → the bug template; say whether `verify` reports `MISMATCH`
|
|
263
|
+
- **Security** (bad bytes, path escapes, token leaks) → [`SECURITY.md`](SECURITY.md), privately
|
|
264
|
+
- **Adapters, tests, docs** → PRs welcome, read [`CONTRIBUTING.md`](CONTRIBUTING.md) first
|
|
265
|
+
|
|
266
|
+
## Known limits
|
|
267
|
+
|
|
268
|
+
- figshare was removed in 0.2.0; existing figshare manifests still verify, and
|
|
269
|
+
new targets convert with one `--checksum` command ([docs/migration-0.2.md](docs/migration-0.2.md)).
|
|
270
|
+
- Tokens come from the environment only; no interactive `login` flow and no
|
|
271
|
+
per-file token refresh.
|
|
272
|
+
- No search/pagination; you bring the target.
|
|
273
|
+
- arXiv PDFs are fetched by constructed URL, so a withdrawn ID gives a 404 after
|
|
274
|
+
resolution.
|
|
275
|
+
- Windows ARM is untested.
|
|
276
|
+
- macOS is best effort: it is in CI but not release-blocking, and nobody has
|
|
277
|
+
run it by hand yet.
|
|
278
|
+
- A source that publishes neither a size nor a checksum (bare `direct` URLs
|
|
279
|
+
without a `Content-Length` on `HEAD`) can still be truncated silently by a
|
|
280
|
+
server that lies about `Content-Length`, because HTTP gives no in-band way to
|
|
281
|
+
notice the extra bytes.
|
|
282
|
+
|
|
283
|
+
## 中文
|
|
284
|
+
|
|
285
|
+
学术数据仓库的批量下载器:断点续传、全局限速、按仓库公布的校验和验证,并落
|
|
286
|
+
`manifest.json`(DOI、许可协议、每文件 SHA-256)供日后复现。
|
|
287
|
+
|
|
288
|
+
```bash
|
|
289
|
+
acadfetch fetch 10.5281/zenodo.123456 -o ./data
|
|
290
|
+
acadfetch verify ./data/manifest.json
|
|
291
|
+
```
|
|
292
|
+
|
|
293
|
+
MIT。
|
|
294
|
+
|
|
295
|
+
## Audit trail
|
|
296
|
+
|
|
297
|
+
How the 3.15 support claim was reached, including what it did **not** prove —
|
|
298
|
+
the free-threaded build, Windows ARM, and every place the probe ran short:
|
|
299
|
+
[docs/py315-windows-report.md](docs/py315-windows-report.md).
|