acadfetch 0.1.1__tar.gz → 0.2.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (73) hide show
  1. acadfetch-0.2.0/PKG-INFO +330 -0
  2. acadfetch-0.2.0/README.md +299 -0
  3. {acadfetch-0.1.1 → acadfetch-0.2.0}/acadfetch/auth.py +0 -1
  4. acadfetch-0.2.0/acadfetch/cli.py +699 -0
  5. {acadfetch-0.1.1 → acadfetch-0.2.0}/acadfetch/client.py +137 -15
  6. {acadfetch-0.1.1 → acadfetch-0.2.0}/acadfetch/consts.py +1 -1
  7. acadfetch-0.2.0/acadfetch/downloader.py +735 -0
  8. {acadfetch-0.1.1 → acadfetch-0.2.0}/acadfetch/errors.py +22 -2
  9. acadfetch-0.2.0/acadfetch/manifest.py +240 -0
  10. {acadfetch-0.1.1 → acadfetch-0.2.0}/acadfetch/sources/arxiv.py +39 -4
  11. {acadfetch-0.1.1 → acadfetch-0.2.0}/acadfetch/sources/base.py +17 -3
  12. {acadfetch-0.1.1 → acadfetch-0.2.0}/acadfetch/sources/direct.py +14 -9
  13. acadfetch-0.2.0/acadfetch/sources/doi.py +85 -0
  14. {acadfetch-0.1.1 → acadfetch-0.2.0}/acadfetch/sources/registry.py +9 -4
  15. acadfetch-0.2.0/acadfetch/sources/removed.py +42 -0
  16. acadfetch-0.2.0/acadfetch/urls.py +52 -0
  17. {acadfetch-0.1.1 → acadfetch-0.2.0}/acadfetch/verify.py +19 -0
  18. acadfetch-0.2.0/acadfetch.egg-info/PKG-INFO +330 -0
  19. {acadfetch-0.1.1 → acadfetch-0.2.0}/acadfetch.egg-info/SOURCES.txt +12 -1
  20. {acadfetch-0.1.1 → acadfetch-0.2.0}/pyproject.toml +1 -1
  21. acadfetch-0.2.0/tests/conftest.py +170 -0
  22. acadfetch-0.2.0/tests/fixtures/README.md +47 -0
  23. {acadfetch-0.1.1 → acadfetch-0.2.0}/tests/test_auth.py +2 -2
  24. acadfetch-0.2.0/tests/test_checksum.py +88 -0
  25. {acadfetch-0.1.1 → acadfetch-0.2.0}/tests/test_cli.py +85 -0
  26. {acadfetch-0.1.1 → acadfetch-0.2.0}/tests/test_client.py +45 -0
  27. acadfetch-0.2.0/tests/test_control_names.py +80 -0
  28. acadfetch-0.2.0/tests/test_disk.py +92 -0
  29. {acadfetch-0.1.1 → acadfetch-0.2.0}/tests/test_downloader.py +188 -141
  30. acadfetch-0.2.0/tests/test_identity.py +212 -0
  31. acadfetch-0.2.0/tests/test_isolation.py +167 -0
  32. acadfetch-0.2.0/tests/test_json.py +275 -0
  33. acadfetch-0.2.0/tests/test_manifest.py +356 -0
  34. acadfetch-0.2.0/tests/test_redirect.py +122 -0
  35. acadfetch-0.2.0/tests/test_removed.py +51 -0
  36. {acadfetch-0.1.1 → acadfetch-0.2.0}/tests/test_sources.py +94 -84
  37. acadfetch-0.2.0/tests/test_urls.py +46 -0
  38. {acadfetch-0.1.1 → acadfetch-0.2.0}/tests/test_verify.py +11 -0
  39. acadfetch-0.1.1/PKG-INFO +0 -237
  40. acadfetch-0.1.1/README.md +0 -206
  41. acadfetch-0.1.1/acadfetch/cli.py +0 -311
  42. acadfetch-0.1.1/acadfetch/downloader.py +0 -443
  43. acadfetch-0.1.1/acadfetch/manifest.py +0 -145
  44. acadfetch-0.1.1/acadfetch/sources/doi.py +0 -54
  45. acadfetch-0.1.1/acadfetch/sources/figshare.py +0 -86
  46. acadfetch-0.1.1/acadfetch.egg-info/PKG-INFO +0 -237
  47. acadfetch-0.1.1/tests/fixtures/README.md +0 -41
  48. acadfetch-0.1.1/tests/test_manifest.py +0 -85
  49. {acadfetch-0.1.1 → acadfetch-0.2.0}/LICENSE +0 -0
  50. {acadfetch-0.1.1 → acadfetch-0.2.0}/MANIFEST.in +0 -0
  51. {acadfetch-0.1.1 → acadfetch-0.2.0}/acadfetch/__init__.py +0 -0
  52. {acadfetch-0.1.1 → acadfetch-0.2.0}/acadfetch/__main__.py +0 -0
  53. {acadfetch-0.1.1 → acadfetch-0.2.0}/acadfetch/atomic.py +0 -0
  54. {acadfetch-0.1.1 → acadfetch-0.2.0}/acadfetch/naming.py +0 -0
  55. {acadfetch-0.1.1 → acadfetch-0.2.0}/acadfetch/progress.py +0 -0
  56. {acadfetch-0.1.1 → acadfetch-0.2.0}/acadfetch/py.typed +0 -0
  57. {acadfetch-0.1.1 → acadfetch-0.2.0}/acadfetch/ratelimit.py +0 -0
  58. {acadfetch-0.1.1 → acadfetch-0.2.0}/acadfetch/sources/__init__.py +0 -0
  59. {acadfetch-0.1.1 → acadfetch-0.2.0}/acadfetch/sources/zenodo.py +0 -0
  60. {acadfetch-0.1.1 → acadfetch-0.2.0}/acadfetch/units.py +0 -0
  61. {acadfetch-0.1.1 → acadfetch-0.2.0}/acadfetch.egg-info/dependency_links.txt +0 -0
  62. {acadfetch-0.1.1 → acadfetch-0.2.0}/acadfetch.egg-info/entry_points.txt +0 -0
  63. {acadfetch-0.1.1 → acadfetch-0.2.0}/acadfetch.egg-info/top_level.txt +0 -0
  64. {acadfetch-0.1.1 → acadfetch-0.2.0}/setup.cfg +0 -0
  65. {acadfetch-0.1.1 → acadfetch-0.2.0}/tests/fixtures/figshare_403_block.html +0 -0
  66. {acadfetch-0.1.1 → acadfetch-0.2.0}/tests/fixtures/figshare_403_forbidden.json +0 -0
  67. {acadfetch-0.1.1 → acadfetch-0.2.0}/tests/fixtures/figshare_article.json +0 -0
  68. {acadfetch-0.1.1 → acadfetch-0.2.0}/tests/fixtures/figshare_file.pdf +0 -0
  69. {acadfetch-0.1.1 → acadfetch-0.2.0}/tests/test_atomic.py +0 -0
  70. {acadfetch-0.1.1 → acadfetch-0.2.0}/tests/test_naming.py +0 -0
  71. {acadfetch-0.1.1 → acadfetch-0.2.0}/tests/test_packaging.py +0 -0
  72. {acadfetch-0.1.1 → acadfetch-0.2.0}/tests/test_progress.py +0 -0
  73. {acadfetch-0.1.1 → acadfetch-0.2.0}/tests/test_ratelimit.py +0 -0
@@ -0,0 +1,330 @@
1
+ Metadata-Version: 2.4
2
+ Name: acadfetch
3
+ Version: 0.2.0
4
+ Summary: Resumable, rate-limited, checksum-verified bulk downloads from academic data repositories
5
+ Author: young-ski
6
+ License-Expression: MIT
7
+ Project-URL: Homepage, https://github.com/young-ski/acadfetch
8
+ Project-URL: Source, https://github.com/young-ski/acadfetch
9
+ Project-URL: Issues, https://github.com/young-ski/acadfetch/issues
10
+ Keywords: zenodo,figshare,arxiv,doi,reproducibility,download
11
+ Classifier: Development Status :: 4 - Beta
12
+ Classifier: Environment :: Console
13
+ Classifier: Intended Audience :: Science/Research
14
+ Classifier: Operating System :: MacOS
15
+ Classifier: Operating System :: Microsoft :: Windows
16
+ Classifier: Operating System :: POSIX :: Linux
17
+ Classifier: Programming Language :: Python :: 3
18
+ Classifier: Programming Language :: Python :: 3 :: Only
19
+ Classifier: Programming Language :: Python :: 3.10
20
+ Classifier: Programming Language :: Python :: 3.11
21
+ Classifier: Programming Language :: Python :: 3.12
22
+ Classifier: Programming Language :: Python :: 3.13
23
+ Classifier: Programming Language :: Python :: 3.14
24
+ Classifier: Programming Language :: Python :: 3.15
25
+ Classifier: Topic :: Scientific/Engineering
26
+ Classifier: Typing :: Typed
27
+ Requires-Python: >=3.10
28
+ Description-Content-Type: text/markdown
29
+ License-File: LICENSE
30
+ Dynamic: license-file
31
+
32
+ # acadfetch
33
+
34
+ [![CI](https://github.com/young-ski/acadfetch/actions/workflows/ci.yml/badge.svg)](https://github.com/young-ski/acadfetch/actions/workflows/ci.yml)
35
+ [![PyPI](https://img.shields.io/pypi/v/acadfetch.svg)](https://pypi.org/project/acadfetch/)
36
+ [![License: MIT](https://img.shields.io/badge/license-MIT-blue.svg)](LICENSE)
37
+
38
+ Batch-download files from academic data repositories — resumably, rate-limited, and
39
+ checked against the checksums the repository published.
40
+
41
+ ```
42
+ $ acadfetch fetch 10.5281/zenodo.23612969 -o ./data
43
+ title : Dinler Tarihi Alanında Güncel Konular ve Akademik Değerlendirmeler
44
+ doi : 10.5281/zenodo.23612969
45
+ license : cc-by-4.0
46
+ files : 1, 1.6 MiB
47
+ landing : https://zenodo.org/records/23612969
48
+ Dinler Tarihi ... .pdf 1.2 MiB/1.6 MiB 312.4 KiB/s ETA 0:02
49
+ ok Dinler Tarihi ... .pdf 1.6 MiB (resumed at 512.0 KiB)
50
+
51
+ 1/1 file(s) in ./data
52
+ ```
53
+
54
+ **What it is not:** a search engine, a scraper for paywalled publishers, or a
55
+ citation manager. It is the boring part in the middle — the part that gets killed
56
+ by a flaky VPN at 80%, and the part that nobody writes down.
57
+
58
+ ## Why another downloader
59
+
60
+ Because `curl -O` in a loop loses three things that matter for research data:
61
+
62
+ | Problem | What happens with a plain loop | What acadfetch does |
63
+ | --- | --- | --- |
64
+ | Connection drops at 80% | restart from zero | resumes with `Range` + `If-Range`, keeps the partial |
65
+ | Silent corruption | nobody notices until analysis breaks | verifies the repository's `md5`/`sha256`, re-downloads on mismatch |
66
+ | "Which version did I actually use?" | untraceable | writes `manifest.json` with DOI, license, per-file SHA-256 |
67
+ | Getting rate-limited into a 403 | hammering, then blocked | one global token bucket, honours `Retry-After` |
68
+
69
+ The manifest is the point. A dataset you cannot re-verify is a dataset you cannot
70
+ cite.
71
+
72
+ ## Install
73
+
74
+ Python 3.10–3.15. No runtime dependencies.
75
+
76
+ Try it without installing anything:
77
+
78
+ ```bash
79
+ uvx acadfetch fetch 10.5281/zenodo.23612969 # uv
80
+ ```
81
+
82
+ Or keep it on the PATH:
83
+
84
+ ```bash
85
+ pipx install acadfetch
86
+ ```
87
+
88
+ From a checkout, for development:
89
+
90
+ ```bash
91
+ git clone https://github.com/young-ski/acadfetch && cd acadfetch
92
+ python -m pip install -e . pytest && python -m pytest -q
93
+ ```
94
+
95
+ ## Supported platforms
96
+
97
+ | Platform | Status |
98
+ | --- | --- |
99
+ | Linux | tested — CI, every supported Python |
100
+ | Windows | tested — CI on 3.10, 3.13 and 3.15 |
101
+ | macOS | best effort — CI runs it on 3.12 and 3.15, a failure is a bug to fix but does not hold a release |
102
+ | Windows ARM | untested |
103
+ | Python 3.15 | tested — CI, on Linux and Windows |
104
+
105
+ ## Usage
106
+
107
+ These work as typed:
108
+
109
+ ```bash
110
+ acadfetch fetch 10.5281/zenodo.23612969 # a DOI
111
+ acadfetch fetch arxiv:1706.03762 # an arXiv id
112
+ acadfetch fetch https://ndownloader.figshare.com/files/70600233/report.pdf # any URL
113
+ ```
114
+
115
+ figshare was removed in 0.2.0: pass the article's download URL instead, and
116
+ [`--checksum`](docs/migration-0.2.md) keeps the verification the API used to
117
+ give you.
118
+
119
+ The rest of the surface:
120
+
121
+ ```bash
122
+ acadfetch fetch 10.5281/zenodo.23612969 -n # dry run: which dir, which files
123
+ acadfetch fetch 10.5281/zenodo.23612969 --only raw.tar.gz
124
+ acadfetch fetch 10.5281/zenodo.23612969 --max-size 2GB # refuse anything bigger
125
+ acadfetch fetch 10.5281/zenodo.23612969 -j 1 --rate 0.5 # be a good citizen
126
+ acadfetch manifest 10.5281/zenodo.23612969 -o m.json # metadata only, no bytes
127
+ acadfetch fetch zenodo:123456 --header "Authorization: Bearer $TOKEN" # restricted
128
+
129
+ acadfetch verify ./data/manifest.json # re-hash what you downloaded
130
+ acadfetch sources # adapters, with an example each
131
+ ```
132
+
133
+ `10.5281/zenodo.123456`, `zenodo:123456` and `arxiv:2401.12345` are placeholders;
134
+ use any real id.
135
+
136
+ ### In a pipeline
137
+
138
+ `--json` puts one report document on stdout and moves progress to stderr, so the
139
+ two never interleave:
140
+
141
+ ```bash
142
+ acadfetch fetch zenodo:123456 --json | jq -r '.files[] | select(.state=="failed") | .name'
143
+ ```
144
+
145
+ Exit codes are unchanged, and a failure before any transfer still produces a
146
+ document with `error` set. `report` is the document's schema version (now `2`):
147
+ fields are added, never removed or given a new meaning, and `state` may gain
148
+ values — treat an unknown one as non-terminal. `verify --json` reports per-file
149
+ `MISMATCH` rows the same way, and its `summary` counts `skipped` rows
150
+ separately. At the end of a failed transfer, `part` names the `.part` file that
151
+ was kept, so a retry can resume instead of starting over. A 403 that carries a
152
+ block page is retried; see [docs/troubleshooting.md](docs/troubleshooting.md).
153
+
154
+ Progress is one aggregate line for the whole run
155
+ (`3 file(s) 12.4/44.1 MiB 1.8 MiB/s ETA 0:17`), redrawn in place on a TTY.
156
+ When stdout is a pipe it becomes a new line at most every 5 s (one extra when a
157
+ file starts or stops), so a CI log stays greppable without becoming a wall. `-q`
158
+ turns it off.
159
+
160
+ Output defaults to `acadfetch-out/<source>-<id>` (a URL adds its host and a short
161
+ hash), and `-n` prints it before
162
+ you spend bandwidth. Files land in `<name>.part` and are renamed only after the
163
+ checksum matches, so an interrupted or failed run never leaves something that
164
+ *looks* finished -- and never throws away the bytes it already paid for: rerun and
165
+ it resumes. `acadfetch-out/` is worth adding to a project's `.gitignore`; this
166
+ repository already has it.
167
+
168
+ The default directory names the target, not only its filename — `data.csv` from
169
+ two hosts is two datasets, and they do not share a directory. A file already
170
+ there is reused when its checksum matches; with no checksum, the size plus one
171
+ more tie (an explicit `-o`, or a manifest in that directory that recorded the
172
+ same URL) is required, and `--force` skips the question entirely.
173
+
174
+ A DOI that resolves to a web page instead of a file is refused, naming the
175
+ page: fetching the landing page as if it were the dataset is how a repository's
176
+ front door ends up being cited as a data file.
177
+
178
+ A run that cannot fit is refused before the first byte is transferred, naming the shortfall:
179
+ `needs 41.2 GiB but 12.0 GiB is free`. What is already on disk is subtracted, so
180
+ a resume is judged on the remainder. Each finished file is flushed before its
181
+ rename (`--no-fsync` skips that; it is faster and weaker against a power cut).
182
+
183
+ Exit codes: `0` success, `1` failure, `2` partial, `64` bad usage, `130` interrupted
184
+ (rerun to resume).
185
+
186
+ ## Tokens
187
+
188
+ `ZENODO_TOKEN` is read from the environment and sent only to Zenodo, so a token
189
+ never travels to a host that has no business seeing it -- including across a
190
+ redirect: a hop to another host (a storage CDN, a download mirror) goes out
191
+ without it, a hop from https to http drops it with a warning (`--allow-http`
192
+ keeps it), and a presigned URL gets no `Authorization` at all, because S3
193
+ rejects a request that carries both. Cookies and proxy credentials are dropped
194
+ on any cross-origin hop as well. `--header` still overrides, `--no-token`
195
+ ignores the environment, and the manifest records the variable name plus the
196
+ last four characters only:
197
+
198
+ ```bash
199
+ export ZENODO_TOKEN=... # restricted records, higher rate limits
200
+ acadfetch fetch zenodo:123456 # prints: auth : token from $ZENODO_TOKEN (…9f2c)
201
+ ```
202
+
203
+ ## Verification and the manifest
204
+
205
+ Every completed run writes `manifest.json`:
206
+
207
+ ```json
208
+ {
209
+ "schema": 3,
210
+ "doi": "10.5281/zenodo.123456",
211
+ "license": "cc-by-4.0",
212
+ "files": [
213
+ { "name": "raw.tar.gz", "size": 8123, "checksum": "md5:9f2c…",
214
+ "sha256": "b41d…", "state": "downloaded", "selected": true,
215
+ "verified": true, "verified_reason": "matched-md5" },
216
+ { "name": "notes.txt", "size": 12, "state": "not-selected", "selected": false,
217
+ "sha256": null, "verified": null, "verified_reason": "not-selected" }
218
+ ]
219
+ }
220
+ ```
221
+
222
+ The manifest describes **the record**, not the run: `--only` still writes a row
223
+ for every file, so `verify` can tell "not fetched" from "fetched and then lost",
224
+ and a successful `--only` run verifies green. Add `--require-all` when you want
225
+ the whole record accounted for. Each fetched row also records `final_url`,
226
+ `etag`, `http_status` and `retrieved_at`, and credential query parameters are
227
+ stripped from every URL (`--record-full-url` keeps them). Full field and state
228
+ reference: [docs/manifest.md](docs/manifest.md).
229
+
230
+ `name` is the file *on disk*, which can differ from the repository's name when two
231
+ files collide. `verified_reason` separates the two kinds of `verified: null`:
232
+ `no-checksum-published` (arXiv) from `unparseable-checksum` (a repository that
233
+ published something we cannot name an algorithm for). Schema 1 and 2 manifests
234
+ still read, and `verify` says so for schema 1, because their names were recorded
235
+ before duplicates were resolved.
236
+
237
+ `acadfetch verify` recomputes SHA-256 for each entry and exits non-zero on any
238
+ mismatch or missing file; rows the run never selected are reported `skipped`.
239
+ Commit the manifest next to your analysis; it is what makes the download
240
+ reproducible a year later.
241
+
242
+ Names are resolved against the manifest's own directory, and a name that points
243
+ outside it — `../sibling/file`, an absolute path, a symlink leading out — is
244
+ reported as `escaped` rather than quietly hashed, because a manifest is a
245
+ document someone can hand you. `--allow-escape` follows them; `--root` changes
246
+ what "inside" means.
247
+
248
+ When a source publishes no checksum (arXiv), verification degrades to a byte
249
+ count and `verified` is `null` rather than a false sense of safety.
250
+
251
+ ## Supported sources
252
+
253
+ | Source | Target forms | Checksums |
254
+ | --- | --- | --- |
255
+ | Zenodo | DOI `10.5281/zenodo.N`, `zenodo:N`, record URL | md5 |
256
+ | arXiv | `arxiv:ID`, bare ID, abs/pdf URL | none (size only) |
257
+ | doi.org | any DOI, resolved then dispatched | depends |
258
+ | direct | any `http(s)` URL | headers, if present |
259
+
260
+ figshare (`figshare:N`, `10.6084/…`, article URLs) was removed in 0.2.0; it
261
+ answers with the conversion instead of a generic "cannot tell what this is".
262
+ See [docs/migration-0.2.md](docs/migration-0.2.md).
263
+
264
+ A CDN in front of a direct URL can answer 403 with a block page. Those are
265
+ retried twice with backoff, while a 403 carrying the application's own JSON is
266
+ terminal on the first answer; `--retry-403 N` changes the count.
267
+
268
+ ## Politeness
269
+
270
+ Defaults: 3 parallel transfers, 2 requests/second, 4 retries with jittered
271
+ exponential backoff, `Retry-After` honoured up to 120 s. A block page is retried twice
272
+ (`--retry-403 N`); Zenodo and arXiv are donation-funded —
273
+ `--rate 1 -j 1` costs you minutes and costs them less.
274
+
275
+ Respect the license in the manifest. `--dry-run` prints it before you spend
276
+ bandwidth; `unknown` means *check before redistributing*.
277
+
278
+ ## Adding a source
279
+
280
+ 1. Read `acadfetch/sources/zenodo.py` — an adapter is `match(target) -> id | None`
281
+ plus `resolve(client, target) -> Record`.
282
+ 2. Fill `File(checksum=...)` whenever the API publishes one.
283
+ 3. Never hardcode rate limits; take the shared `Client`.
284
+ 4. Add a `match` test and, if possible, a fixture of the real API payload.
285
+
286
+ `scripts/probe_resume.py` interrupts a live transfer and resumes it — the only
287
+ honest way to test this code path.
288
+
289
+ ## Getting involved
290
+
291
+ - **Stuck on something** → [docs/troubleshooting.md](docs/troubleshooting.md) first; it is short and indexed by symptom
292
+ - **Usage questions** → [Discussions](https://github.com/young-ski/acadfetch/discussions)
293
+ - **Bugs** → the bug template; say whether `verify` reports `MISMATCH`
294
+ - **Security** (bad bytes, path escapes, token leaks) → [`SECURITY.md`](SECURITY.md), privately
295
+ - **Adapters, tests, docs** → PRs welcome, read [`CONTRIBUTING.md`](CONTRIBUTING.md) first
296
+
297
+ ## Known limits
298
+
299
+ - figshare was removed in 0.2.0; existing figshare manifests still verify, and
300
+ new targets convert with one `--checksum` command ([docs/migration-0.2.md](docs/migration-0.2.md)).
301
+ - Tokens come from the environment only; no interactive `login` flow and no
302
+ per-file token refresh.
303
+ - No search/pagination; you bring the target.
304
+ - arXiv PDFs are fetched by constructed URL, so a withdrawn ID gives a 404 after
305
+ resolution.
306
+ - Windows ARM is untested.
307
+ - macOS is best effort: it is in CI but not release-blocking, and nobody has
308
+ run it by hand yet.
309
+ - A source that publishes neither a size nor a checksum (bare `direct` URLs
310
+ without a `Content-Length` on `HEAD`) can still be truncated silently by a
311
+ server that lies about `Content-Length`, because HTTP gives no in-band way to
312
+ notice the extra bytes.
313
+
314
+ ## 中文
315
+
316
+ 学术数据仓库的批量下载器:断点续传、全局限速、按仓库公布的校验和验证,并落
317
+ `manifest.json`(DOI、许可协议、每文件 SHA-256)供日后复现。
318
+
319
+ ```bash
320
+ acadfetch fetch 10.5281/zenodo.123456 -o ./data
321
+ acadfetch verify ./data/manifest.json
322
+ ```
323
+
324
+ MIT。
325
+
326
+ ## Audit trail
327
+
328
+ How the 3.15 support claim was reached, including what it did **not** prove —
329
+ the free-threaded build, Windows ARM, and every place the probe ran short:
330
+ [docs/py315-windows-report.md](docs/py315-windows-report.md).
@@ -0,0 +1,299 @@
1
+ # acadfetch
2
+
3
+ [![CI](https://github.com/young-ski/acadfetch/actions/workflows/ci.yml/badge.svg)](https://github.com/young-ski/acadfetch/actions/workflows/ci.yml)
4
+ [![PyPI](https://img.shields.io/pypi/v/acadfetch.svg)](https://pypi.org/project/acadfetch/)
5
+ [![License: MIT](https://img.shields.io/badge/license-MIT-blue.svg)](LICENSE)
6
+
7
+ Batch-download files from academic data repositories — resumably, rate-limited, and
8
+ checked against the checksums the repository published.
9
+
10
+ ```
11
+ $ acadfetch fetch 10.5281/zenodo.23612969 -o ./data
12
+ title : Dinler Tarihi Alanında Güncel Konular ve Akademik Değerlendirmeler
13
+ doi : 10.5281/zenodo.23612969
14
+ license : cc-by-4.0
15
+ files : 1, 1.6 MiB
16
+ landing : https://zenodo.org/records/23612969
17
+ Dinler Tarihi ... .pdf 1.2 MiB/1.6 MiB 312.4 KiB/s ETA 0:02
18
+ ok Dinler Tarihi ... .pdf 1.6 MiB (resumed at 512.0 KiB)
19
+
20
+ 1/1 file(s) in ./data
21
+ ```
22
+
23
+ **What it is not:** a search engine, a scraper for paywalled publishers, or a
24
+ citation manager. It is the boring part in the middle — the part that gets killed
25
+ by a flaky VPN at 80%, and the part that nobody writes down.
26
+
27
+ ## Why another downloader
28
+
29
+ Because `curl -O` in a loop loses three things that matter for research data:
30
+
31
+ | Problem | What happens with a plain loop | What acadfetch does |
32
+ | --- | --- | --- |
33
+ | Connection drops at 80% | restart from zero | resumes with `Range` + `If-Range`, keeps the partial |
34
+ | Silent corruption | nobody notices until analysis breaks | verifies the repository's `md5`/`sha256`, re-downloads on mismatch |
35
+ | "Which version did I actually use?" | untraceable | writes `manifest.json` with DOI, license, per-file SHA-256 |
36
+ | Getting rate-limited into a 403 | hammering, then blocked | one global token bucket, honours `Retry-After` |
37
+
38
+ The manifest is the point. A dataset you cannot re-verify is a dataset you cannot
39
+ cite.
40
+
41
+ ## Install
42
+
43
+ Python 3.10–3.15. No runtime dependencies.
44
+
45
+ Try it without installing anything:
46
+
47
+ ```bash
48
+ uvx acadfetch fetch 10.5281/zenodo.23612969 # uv
49
+ ```
50
+
51
+ Or keep it on the PATH:
52
+
53
+ ```bash
54
+ pipx install acadfetch
55
+ ```
56
+
57
+ From a checkout, for development:
58
+
59
+ ```bash
60
+ git clone https://github.com/young-ski/acadfetch && cd acadfetch
61
+ python -m pip install -e . pytest && python -m pytest -q
62
+ ```
63
+
64
+ ## Supported platforms
65
+
66
+ | Platform | Status |
67
+ | --- | --- |
68
+ | Linux | tested — CI, every supported Python |
69
+ | Windows | tested — CI on 3.10, 3.13 and 3.15 |
70
+ | macOS | best effort — CI runs it on 3.12 and 3.15, a failure is a bug to fix but does not hold a release |
71
+ | Windows ARM | untested |
72
+ | Python 3.15 | tested — CI, on Linux and Windows |
73
+
74
+ ## Usage
75
+
76
+ These work as typed:
77
+
78
+ ```bash
79
+ acadfetch fetch 10.5281/zenodo.23612969 # a DOI
80
+ acadfetch fetch arxiv:1706.03762 # an arXiv id
81
+ acadfetch fetch https://ndownloader.figshare.com/files/70600233/report.pdf # any URL
82
+ ```
83
+
84
+ figshare was removed in 0.2.0: pass the article's download URL instead, and
85
+ [`--checksum`](docs/migration-0.2.md) keeps the verification the API used to
86
+ give you.
87
+
88
+ The rest of the surface:
89
+
90
+ ```bash
91
+ acadfetch fetch 10.5281/zenodo.23612969 -n # dry run: which dir, which files
92
+ acadfetch fetch 10.5281/zenodo.23612969 --only raw.tar.gz
93
+ acadfetch fetch 10.5281/zenodo.23612969 --max-size 2GB # refuse anything bigger
94
+ acadfetch fetch 10.5281/zenodo.23612969 -j 1 --rate 0.5 # be a good citizen
95
+ acadfetch manifest 10.5281/zenodo.23612969 -o m.json # metadata only, no bytes
96
+ acadfetch fetch zenodo:123456 --header "Authorization: Bearer $TOKEN" # restricted
97
+
98
+ acadfetch verify ./data/manifest.json # re-hash what you downloaded
99
+ acadfetch sources # adapters, with an example each
100
+ ```
101
+
102
+ `10.5281/zenodo.123456`, `zenodo:123456` and `arxiv:2401.12345` are placeholders;
103
+ use any real id.
104
+
105
+ ### In a pipeline
106
+
107
+ `--json` puts one report document on stdout and moves progress to stderr, so the
108
+ two never interleave:
109
+
110
+ ```bash
111
+ acadfetch fetch zenodo:123456 --json | jq -r '.files[] | select(.state=="failed") | .name'
112
+ ```
113
+
114
+ Exit codes are unchanged, and a failure before any transfer still produces a
115
+ document with `error` set. `report` is the document's schema version (now `2`):
116
+ fields are added, never removed or given a new meaning, and `state` may gain
117
+ values — treat an unknown one as non-terminal. `verify --json` reports per-file
118
+ `MISMATCH` rows the same way, and its `summary` counts `skipped` rows
119
+ separately. At the end of a failed transfer, `part` names the `.part` file that
120
+ was kept, so a retry can resume instead of starting over. A 403 that carries a
121
+ block page is retried; see [docs/troubleshooting.md](docs/troubleshooting.md).
122
+
123
+ Progress is one aggregate line for the whole run
124
+ (`3 file(s) 12.4/44.1 MiB 1.8 MiB/s ETA 0:17`), redrawn in place on a TTY.
125
+ When stdout is a pipe it becomes a new line at most every 5 s (one extra when a
126
+ file starts or stops), so a CI log stays greppable without becoming a wall. `-q`
127
+ turns it off.
128
+
129
+ Output defaults to `acadfetch-out/<source>-<id>` (a URL adds its host and a short
130
+ hash), and `-n` prints it before
131
+ you spend bandwidth. Files land in `<name>.part` and are renamed only after the
132
+ checksum matches, so an interrupted or failed run never leaves something that
133
+ *looks* finished -- and never throws away the bytes it already paid for: rerun and
134
+ it resumes. `acadfetch-out/` is worth adding to a project's `.gitignore`; this
135
+ repository already has it.
136
+
137
+ The default directory names the target, not only its filename — `data.csv` from
138
+ two hosts is two datasets, and they do not share a directory. A file already
139
+ there is reused when its checksum matches; with no checksum, the size plus one
140
+ more tie (an explicit `-o`, or a manifest in that directory that recorded the
141
+ same URL) is required, and `--force` skips the question entirely.
142
+
143
+ A DOI that resolves to a web page instead of a file is refused, naming the
144
+ page: fetching the landing page as if it were the dataset is how a repository's
145
+ front door ends up being cited as a data file.
146
+
147
+ A run that cannot fit is refused before the first byte is transferred, naming the shortfall:
148
+ `needs 41.2 GiB but 12.0 GiB is free`. What is already on disk is subtracted, so
149
+ a resume is judged on the remainder. Each finished file is flushed before its
150
+ rename (`--no-fsync` skips that; it is faster and weaker against a power cut).
151
+
152
+ Exit codes: `0` success, `1` failure, `2` partial, `64` bad usage, `130` interrupted
153
+ (rerun to resume).
154
+
155
+ ## Tokens
156
+
157
+ `ZENODO_TOKEN` is read from the environment and sent only to Zenodo, so a token
158
+ never travels to a host that has no business seeing it -- including across a
159
+ redirect: a hop to another host (a storage CDN, a download mirror) goes out
160
+ without it, a hop from https to http drops it with a warning (`--allow-http`
161
+ keeps it), and a presigned URL gets no `Authorization` at all, because S3
162
+ rejects a request that carries both. Cookies and proxy credentials are dropped
163
+ on any cross-origin hop as well. `--header` still overrides, `--no-token`
164
+ ignores the environment, and the manifest records the variable name plus the
165
+ last four characters only:
166
+
167
+ ```bash
168
+ export ZENODO_TOKEN=... # restricted records, higher rate limits
169
+ acadfetch fetch zenodo:123456 # prints: auth : token from $ZENODO_TOKEN (…9f2c)
170
+ ```
171
+
172
+ ## Verification and the manifest
173
+
174
+ Every completed run writes `manifest.json`:
175
+
176
+ ```json
177
+ {
178
+ "schema": 3,
179
+ "doi": "10.5281/zenodo.123456",
180
+ "license": "cc-by-4.0",
181
+ "files": [
182
+ { "name": "raw.tar.gz", "size": 8123, "checksum": "md5:9f2c…",
183
+ "sha256": "b41d…", "state": "downloaded", "selected": true,
184
+ "verified": true, "verified_reason": "matched-md5" },
185
+ { "name": "notes.txt", "size": 12, "state": "not-selected", "selected": false,
186
+ "sha256": null, "verified": null, "verified_reason": "not-selected" }
187
+ ]
188
+ }
189
+ ```
190
+
191
+ The manifest describes **the record**, not the run: `--only` still writes a row
192
+ for every file, so `verify` can tell "not fetched" from "fetched and then lost",
193
+ and a successful `--only` run verifies green. Add `--require-all` when you want
194
+ the whole record accounted for. Each fetched row also records `final_url`,
195
+ `etag`, `http_status` and `retrieved_at`, and credential query parameters are
196
+ stripped from every URL (`--record-full-url` keeps them). Full field and state
197
+ reference: [docs/manifest.md](docs/manifest.md).
198
+
199
+ `name` is the file *on disk*, which can differ from the repository's name when two
200
+ files collide. `verified_reason` separates the two kinds of `verified: null`:
201
+ `no-checksum-published` (arXiv) from `unparseable-checksum` (a repository that
202
+ published something we cannot name an algorithm for). Schema 1 and 2 manifests
203
+ still read, and `verify` says so for schema 1, because their names were recorded
204
+ before duplicates were resolved.
205
+
206
+ `acadfetch verify` recomputes SHA-256 for each entry and exits non-zero on any
207
+ mismatch or missing file; rows the run never selected are reported `skipped`.
208
+ Commit the manifest next to your analysis; it is what makes the download
209
+ reproducible a year later.
210
+
211
+ Names are resolved against the manifest's own directory, and a name that points
212
+ outside it — `../sibling/file`, an absolute path, a symlink leading out — is
213
+ reported as `escaped` rather than quietly hashed, because a manifest is a
214
+ document someone can hand you. `--allow-escape` follows them; `--root` changes
215
+ what "inside" means.
216
+
217
+ When a source publishes no checksum (arXiv), verification degrades to a byte
218
+ count and `verified` is `null` rather than a false sense of safety.
219
+
220
+ ## Supported sources
221
+
222
+ | Source | Target forms | Checksums |
223
+ | --- | --- | --- |
224
+ | Zenodo | DOI `10.5281/zenodo.N`, `zenodo:N`, record URL | md5 |
225
+ | arXiv | `arxiv:ID`, bare ID, abs/pdf URL | none (size only) |
226
+ | doi.org | any DOI, resolved then dispatched | depends |
227
+ | direct | any `http(s)` URL | headers, if present |
228
+
229
+ figshare (`figshare:N`, `10.6084/…`, article URLs) was removed in 0.2.0; it
230
+ answers with the conversion instead of a generic "cannot tell what this is".
231
+ See [docs/migration-0.2.md](docs/migration-0.2.md).
232
+
233
+ A CDN in front of a direct URL can answer 403 with a block page. Those are
234
+ retried twice with backoff, while a 403 carrying the application's own JSON is
235
+ terminal on the first answer; `--retry-403 N` changes the count.
236
+
237
+ ## Politeness
238
+
239
+ Defaults: 3 parallel transfers, 2 requests/second, 4 retries with jittered
240
+ exponential backoff, `Retry-After` honoured up to 120 s. A block page is retried twice
241
+ (`--retry-403 N`); Zenodo and arXiv are donation-funded —
242
+ `--rate 1 -j 1` costs you minutes and costs them less.
243
+
244
+ Respect the license in the manifest. `--dry-run` prints it before you spend
245
+ bandwidth; `unknown` means *check before redistributing*.
246
+
247
+ ## Adding a source
248
+
249
+ 1. Read `acadfetch/sources/zenodo.py` — an adapter is `match(target) -> id | None`
250
+ plus `resolve(client, target) -> Record`.
251
+ 2. Fill `File(checksum=...)` whenever the API publishes one.
252
+ 3. Never hardcode rate limits; take the shared `Client`.
253
+ 4. Add a `match` test and, if possible, a fixture of the real API payload.
254
+
255
+ `scripts/probe_resume.py` interrupts a live transfer and resumes it — the only
256
+ honest way to test this code path.
257
+
258
+ ## Getting involved
259
+
260
+ - **Stuck on something** → [docs/troubleshooting.md](docs/troubleshooting.md) first; it is short and indexed by symptom
261
+ - **Usage questions** → [Discussions](https://github.com/young-ski/acadfetch/discussions)
262
+ - **Bugs** → the bug template; say whether `verify` reports `MISMATCH`
263
+ - **Security** (bad bytes, path escapes, token leaks) → [`SECURITY.md`](SECURITY.md), privately
264
+ - **Adapters, tests, docs** → PRs welcome, read [`CONTRIBUTING.md`](CONTRIBUTING.md) first
265
+
266
+ ## Known limits
267
+
268
+ - figshare was removed in 0.2.0; existing figshare manifests still verify, and
269
+ new targets convert with one `--checksum` command ([docs/migration-0.2.md](docs/migration-0.2.md)).
270
+ - Tokens come from the environment only; no interactive `login` flow and no
271
+ per-file token refresh.
272
+ - No search/pagination; you bring the target.
273
+ - arXiv PDFs are fetched by constructed URL, so a withdrawn ID gives a 404 after
274
+ resolution.
275
+ - Windows ARM is untested.
276
+ - macOS is best effort: it is in CI but not release-blocking, and nobody has
277
+ run it by hand yet.
278
+ - A source that publishes neither a size nor a checksum (bare `direct` URLs
279
+ without a `Content-Length` on `HEAD`) can still be truncated silently by a
280
+ server that lies about `Content-Length`, because HTTP gives no in-band way to
281
+ notice the extra bytes.
282
+
283
+ ## 中文
284
+
285
+ 学术数据仓库的批量下载器:断点续传、全局限速、按仓库公布的校验和验证,并落
286
+ `manifest.json`(DOI、许可协议、每文件 SHA-256)供日后复现。
287
+
288
+ ```bash
289
+ acadfetch fetch 10.5281/zenodo.123456 -o ./data
290
+ acadfetch verify ./data/manifest.json
291
+ ```
292
+
293
+ MIT。
294
+
295
+ ## Audit trail
296
+
297
+ How the 3.15 support claim was reached, including what it did **not** prove —
298
+ the free-threaded build, Windows ARM, and every place the probe ran short:
299
+ [docs/py315-windows-report.md](docs/py315-windows-report.md).
@@ -15,7 +15,6 @@ from dataclasses import dataclass
15
15
  # source name -> variable that holds its token
16
16
  TOKEN_ENV = {
17
17
  "zenodo": "ZENODO_TOKEN",
18
- "figshare": "FIGSHARE_TOKEN",
19
18
  }
20
19
 
21
20