acadfetch 0.6.3__tar.gz → 0.7.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {acadfetch-0.6.3/acadfetch.egg-info → acadfetch-0.7.0}/PKG-INFO +23 -9
- {acadfetch-0.6.3 → acadfetch-0.7.0}/README.md +22 -8
- {acadfetch-0.6.3 → acadfetch-0.7.0}/acadfetch/cli.py +15 -2
- {acadfetch-0.6.3 → acadfetch-0.7.0}/acadfetch/consts.py +1 -1
- {acadfetch-0.6.3 → acadfetch-0.7.0}/acadfetch/downloader.py +132 -57
- {acadfetch-0.6.3 → acadfetch-0.7.0}/acadfetch/manifest.py +4 -4
- acadfetch-0.7.0/acadfetch/pooch.py +157 -0
- {acadfetch-0.6.3 → acadfetch-0.7.0}/acadfetch/progress.py +62 -19
- {acadfetch-0.6.3 → acadfetch-0.7.0}/acadfetch/remote.py +5 -0
- {acadfetch-0.6.3 → acadfetch-0.7.0}/acadfetch/verify.py +12 -3
- {acadfetch-0.6.3 → acadfetch-0.7.0/acadfetch.egg-info}/PKG-INFO +23 -9
- {acadfetch-0.6.3 → acadfetch-0.7.0}/acadfetch.egg-info/SOURCES.txt +3 -0
- {acadfetch-0.6.3 → acadfetch-0.7.0}/pyproject.toml +1 -1
- acadfetch-0.7.0/tests/test_claims.py +174 -0
- {acadfetch-0.6.3 → acadfetch-0.7.0}/tests/test_cli.py +9 -0
- {acadfetch-0.6.3 → acadfetch-0.7.0}/tests/test_downloader.py +74 -0
- {acadfetch-0.6.3 → acadfetch-0.7.0}/tests/test_json.py +19 -0
- acadfetch-0.7.0/tests/test_pooch.py +149 -0
- {acadfetch-0.6.3 → acadfetch-0.7.0}/tests/test_progress.py +30 -2
- {acadfetch-0.6.3 → acadfetch-0.7.0}/tests/test_remote.py +26 -0
- {acadfetch-0.6.3 → acadfetch-0.7.0}/LICENSE +0 -0
- {acadfetch-0.6.3 → acadfetch-0.7.0}/MANIFEST.in +0 -0
- {acadfetch-0.6.3 → acadfetch-0.7.0}/acadfetch/__init__.py +0 -0
- {acadfetch-0.6.3 → acadfetch-0.7.0}/acadfetch/__main__.py +0 -0
- {acadfetch-0.6.3 → acadfetch-0.7.0}/acadfetch/atomic.py +0 -0
- {acadfetch-0.6.3 → acadfetch-0.7.0}/acadfetch/auth.py +0 -0
- {acadfetch-0.6.3 → acadfetch-0.7.0}/acadfetch/client.py +0 -0
- {acadfetch-0.6.3 → acadfetch-0.7.0}/acadfetch/debug.py +0 -0
- {acadfetch-0.6.3 → acadfetch-0.7.0}/acadfetch/errors.py +0 -0
- {acadfetch-0.6.3 → acadfetch-0.7.0}/acadfetch/naming.py +0 -0
- {acadfetch-0.6.3 → acadfetch-0.7.0}/acadfetch/py.typed +0 -0
- {acadfetch-0.6.3 → acadfetch-0.7.0}/acadfetch/ratelimit.py +0 -0
- {acadfetch-0.6.3 → acadfetch-0.7.0}/acadfetch/sources/__init__.py +0 -0
- {acadfetch-0.6.3 → acadfetch-0.7.0}/acadfetch/sources/arxiv.py +0 -0
- {acadfetch-0.6.3 → acadfetch-0.7.0}/acadfetch/sources/base.py +0 -0
- {acadfetch-0.6.3 → acadfetch-0.7.0}/acadfetch/sources/direct.py +0 -0
- {acadfetch-0.6.3 → acadfetch-0.7.0}/acadfetch/sources/doi.py +0 -0
- {acadfetch-0.6.3 → acadfetch-0.7.0}/acadfetch/sources/registry.py +0 -0
- {acadfetch-0.6.3 → acadfetch-0.7.0}/acadfetch/sources/removed.py +0 -0
- {acadfetch-0.6.3 → acadfetch-0.7.0}/acadfetch/sources/zenodo.py +0 -0
- {acadfetch-0.6.3 → acadfetch-0.7.0}/acadfetch/units.py +0 -0
- {acadfetch-0.6.3 → acadfetch-0.7.0}/acadfetch/urls.py +0 -0
- {acadfetch-0.6.3 → acadfetch-0.7.0}/acadfetch.egg-info/dependency_links.txt +0 -0
- {acadfetch-0.6.3 → acadfetch-0.7.0}/acadfetch.egg-info/entry_points.txt +0 -0
- {acadfetch-0.6.3 → acadfetch-0.7.0}/acadfetch.egg-info/top_level.txt +0 -0
- {acadfetch-0.6.3 → acadfetch-0.7.0}/setup.cfg +0 -0
- {acadfetch-0.6.3 → acadfetch-0.7.0}/tests/conftest.py +0 -0
- {acadfetch-0.6.3 → acadfetch-0.7.0}/tests/fixtures/README.md +0 -0
- {acadfetch-0.6.3 → acadfetch-0.7.0}/tests/fixtures/ca.pem +0 -0
- {acadfetch-0.6.3 → acadfetch-0.7.0}/tests/fixtures/figshare_403_block.html +0 -0
- {acadfetch-0.6.3 → acadfetch-0.7.0}/tests/fixtures/figshare_403_forbidden.json +0 -0
- {acadfetch-0.6.3 → acadfetch-0.7.0}/tests/fixtures/figshare_article.json +0 -0
- {acadfetch-0.6.3 → acadfetch-0.7.0}/tests/fixtures/figshare_file.pdf +0 -0
- {acadfetch-0.6.3 → acadfetch-0.7.0}/tests/test_atomic.py +0 -0
- {acadfetch-0.6.3 → acadfetch-0.7.0}/tests/test_auth.py +0 -0
- {acadfetch-0.6.3 → acadfetch-0.7.0}/tests/test_checksum.py +0 -0
- {acadfetch-0.6.3 → acadfetch-0.7.0}/tests/test_citation.py +0 -0
- {acadfetch-0.6.3 → acadfetch-0.7.0}/tests/test_client.py +0 -0
- {acadfetch-0.6.3 → acadfetch-0.7.0}/tests/test_control_names.py +0 -0
- {acadfetch-0.6.3 → acadfetch-0.7.0}/tests/test_debug.py +0 -0
- {acadfetch-0.6.3 → acadfetch-0.7.0}/tests/test_disk.py +0 -0
- {acadfetch-0.6.3 → acadfetch-0.7.0}/tests/test_hash_once.py +0 -0
- {acadfetch-0.6.3 → acadfetch-0.7.0}/tests/test_identity.py +0 -0
- {acadfetch-0.6.3 → acadfetch-0.7.0}/tests/test_isolation.py +0 -0
- {acadfetch-0.6.3 → acadfetch-0.7.0}/tests/test_lock.py +0 -0
- {acadfetch-0.6.3 → acadfetch-0.7.0}/tests/test_manifest.py +0 -0
- {acadfetch-0.6.3 → acadfetch-0.7.0}/tests/test_naming.py +0 -0
- {acadfetch-0.6.3 → acadfetch-0.7.0}/tests/test_naming_race.py +0 -0
- {acadfetch-0.6.3 → acadfetch-0.7.0}/tests/test_network.py +0 -0
- {acadfetch-0.6.3 → acadfetch-0.7.0}/tests/test_packaging.py +0 -0
- {acadfetch-0.6.3 → acadfetch-0.7.0}/tests/test_ratelimit.py +0 -0
- {acadfetch-0.6.3 → acadfetch-0.7.0}/tests/test_redirect.py +0 -0
- {acadfetch-0.6.3 → acadfetch-0.7.0}/tests/test_removed.py +0 -0
- {acadfetch-0.6.3 → acadfetch-0.7.0}/tests/test_sources.py +0 -0
- {acadfetch-0.6.3 → acadfetch-0.7.0}/tests/test_urls.py +0 -0
- {acadfetch-0.6.3 → acadfetch-0.7.0}/tests/test_urls_glob.py +0 -0
- {acadfetch-0.6.3 → acadfetch-0.7.0}/tests/test_verify.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: acadfetch
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.7.0
|
|
4
4
|
Summary: Resumable, rate-limited, checksum-verified bulk downloads from academic data repositories
|
|
5
5
|
Author: young-ski
|
|
6
6
|
License-Expression: MIT
|
|
@@ -104,7 +104,7 @@ python -m pip install -e . pytest mypy && python -m pytest -q
|
|
|
104
104
|
|
|
105
105
|
## Usage
|
|
106
106
|
|
|
107
|
-
|
|
107
|
+
One target per run:
|
|
108
108
|
|
|
109
109
|
```bash
|
|
110
110
|
acadfetch fetch 10.5281/zenodo.23612969 # a DOI
|
|
@@ -112,6 +112,14 @@ acadfetch fetch arxiv:1706.03762 # an arXiv id
|
|
|
112
112
|
acadfetch fetch https://ndownloader.figshare.com/files/70600233/report.pdf # any URL
|
|
113
113
|
```
|
|
114
114
|
|
|
115
|
+
The first two are answered by Zenodo and arXiv and work from any network those
|
|
116
|
+
are reachable on. The third is a direct URL, so the host decides: figshare's CDN
|
|
117
|
+
answers 403 with a WAF block page from some networks — the same behaviour that
|
|
118
|
+
made its *adapter* not worth keeping in 0.2.0. acadfetch says so and keeps
|
|
119
|
+
nothing, rather than saving the block page as a PDF. If you hit that, the file
|
|
120
|
+
itself is fine; see the `403` section of
|
|
121
|
+
[docs/troubleshooting.md](docs/troubleshooting.md).
|
|
122
|
+
|
|
115
123
|
figshare was removed in 0.2.0: pass the article's download URL instead, and
|
|
116
124
|
[`--checksum`](docs/migration-0.2.md) keeps the verification the API used to
|
|
117
125
|
give you.
|
|
@@ -123,10 +131,13 @@ acadfetch fetch 10.5281/zenodo.23612969 -n # dry run: which dir, which files
|
|
|
123
131
|
acadfetch fetch 10.5281/zenodo.23612969 --only raw.tar.gz
|
|
124
132
|
acadfetch fetch 10.5281/zenodo.23612969 --max-size 2GB # refuse anything bigger
|
|
125
133
|
acadfetch fetch 10.5281/zenodo.23612969 -j 1 --rate 0.5 # be a good citizen
|
|
134
|
+
acadfetch fetch 10.5281/zenodo.23612969 --retries 2 --retry-403 0 # give up sooner
|
|
135
|
+
acadfetch fetch 10.5281/zenodo.23612969 --manifest-out build/m.json # a second copy
|
|
126
136
|
acadfetch manifest 10.5281/zenodo.23612969 -o m.json # metadata only, no bytes
|
|
127
137
|
acadfetch fetch zenodo:123456 --header "Authorization: Bearer $TOKEN" # restricted
|
|
128
138
|
|
|
129
139
|
acadfetch verify ./data/manifest.json # re-hash what you downloaded
|
|
140
|
+
acadfetch --version # what is installed
|
|
130
141
|
acadfetch sources # adapters, with an example each
|
|
131
142
|
```
|
|
132
143
|
|
|
@@ -164,16 +175,18 @@ block page is retried; see [docs/troubleshooting.md](docs/troubleshooting.md).
|
|
|
164
175
|
Progress is one aggregate line for the whole run
|
|
165
176
|
(`3 file(s) 12.4/44.1 MiB 1.8 MiB/s ETA 0:17`), redrawn in place on a TTY.
|
|
166
177
|
When stdout is a pipe it becomes a new line at most every 5 s (one extra when a
|
|
167
|
-
file starts or stops), so a CI log stays greppable without becoming a wall.
|
|
168
|
-
turns it off.
|
|
178
|
+
file starts or stops), so a CI log stays greppable without becoming a wall.
|
|
179
|
+
`-q/--quiet` turns it off.
|
|
169
180
|
|
|
170
|
-
|
|
181
|
+
`-o/--out` defaults to `acadfetch-out/<source>-<id>` (a URL adds its host and a short
|
|
171
182
|
hash), and `-n` prints it before
|
|
172
183
|
you spend bandwidth. Files land in `<name>.part` and are renamed only after the
|
|
173
184
|
checksum matches, so an interrupted or failed run never leaves something that
|
|
174
185
|
*looks* finished -- and never throws away the bytes it already paid for: rerun and
|
|
175
|
-
it resumes.
|
|
176
|
-
|
|
186
|
+
it resumes. The one case where a run must start over is a server that ignores
|
|
187
|
+
`Range`; the bytes already on disk are then moved aside and kept until a whole
|
|
188
|
+
new copy has arrived, so a restart that dies early costs nothing. `acadfetch-out/`
|
|
189
|
+
is worth adding to a project's `.gitignore`; this repository already has it.
|
|
177
190
|
|
|
178
191
|
The default directory names the target, not only its filename — `data.csv` from
|
|
179
192
|
two hosts is two datasets, and they do not share a directory. A file already
|
|
@@ -302,7 +315,7 @@ so the trace can be pasted into an issue.
|
|
|
302
315
|
|
|
303
316
|
## Politeness
|
|
304
317
|
|
|
305
|
-
Defaults: 3 parallel transfers, 2 requests/second, 4 retries with jittered
|
|
318
|
+
Defaults: 3 parallel transfers (`-j/--concurrency`), 2 requests/second, 4 retries with jittered
|
|
306
319
|
exponential backoff, `Retry-After` honoured up to 120 s. A block page is retried twice
|
|
307
320
|
(`--retry-403 N`); Zenodo and arXiv are donation-funded —
|
|
308
321
|
`--rate 1 -j 1` costs you minutes and costs them less.
|
|
@@ -324,7 +337,8 @@ honest way to test this code path.
|
|
|
324
337
|
## Getting involved
|
|
325
338
|
|
|
326
339
|
A [30-second tour](docs/demo.md) shows the whole loop, and
|
|
327
|
-
[docs/integrations.md](docs/integrations.md) has Make, CI, aria2 and
|
|
340
|
+
[docs/integrations.md](docs/integrations.md) has Make, CI, aria2, Snakemake and a
|
|
341
|
+
pooch downloader
|
|
328
342
|
recipes.
|
|
329
343
|
|
|
330
344
|
A nightly job downloads three fixed public records and verifies them, so a
|
|
@@ -73,7 +73,7 @@ python -m pip install -e . pytest mypy && python -m pytest -q
|
|
|
73
73
|
|
|
74
74
|
## Usage
|
|
75
75
|
|
|
76
|
-
|
|
76
|
+
One target per run:
|
|
77
77
|
|
|
78
78
|
```bash
|
|
79
79
|
acadfetch fetch 10.5281/zenodo.23612969 # a DOI
|
|
@@ -81,6 +81,14 @@ acadfetch fetch arxiv:1706.03762 # an arXiv id
|
|
|
81
81
|
acadfetch fetch https://ndownloader.figshare.com/files/70600233/report.pdf # any URL
|
|
82
82
|
```
|
|
83
83
|
|
|
84
|
+
The first two are answered by Zenodo and arXiv and work from any network those
|
|
85
|
+
are reachable on. The third is a direct URL, so the host decides: figshare's CDN
|
|
86
|
+
answers 403 with a WAF block page from some networks — the same behaviour that
|
|
87
|
+
made its *adapter* not worth keeping in 0.2.0. acadfetch says so and keeps
|
|
88
|
+
nothing, rather than saving the block page as a PDF. If you hit that, the file
|
|
89
|
+
itself is fine; see the `403` section of
|
|
90
|
+
[docs/troubleshooting.md](docs/troubleshooting.md).
|
|
91
|
+
|
|
84
92
|
figshare was removed in 0.2.0: pass the article's download URL instead, and
|
|
85
93
|
[`--checksum`](docs/migration-0.2.md) keeps the verification the API used to
|
|
86
94
|
give you.
|
|
@@ -92,10 +100,13 @@ acadfetch fetch 10.5281/zenodo.23612969 -n # dry run: which dir, which files
|
|
|
92
100
|
acadfetch fetch 10.5281/zenodo.23612969 --only raw.tar.gz
|
|
93
101
|
acadfetch fetch 10.5281/zenodo.23612969 --max-size 2GB # refuse anything bigger
|
|
94
102
|
acadfetch fetch 10.5281/zenodo.23612969 -j 1 --rate 0.5 # be a good citizen
|
|
103
|
+
acadfetch fetch 10.5281/zenodo.23612969 --retries 2 --retry-403 0 # give up sooner
|
|
104
|
+
acadfetch fetch 10.5281/zenodo.23612969 --manifest-out build/m.json # a second copy
|
|
95
105
|
acadfetch manifest 10.5281/zenodo.23612969 -o m.json # metadata only, no bytes
|
|
96
106
|
acadfetch fetch zenodo:123456 --header "Authorization: Bearer $TOKEN" # restricted
|
|
97
107
|
|
|
98
108
|
acadfetch verify ./data/manifest.json # re-hash what you downloaded
|
|
109
|
+
acadfetch --version # what is installed
|
|
99
110
|
acadfetch sources # adapters, with an example each
|
|
100
111
|
```
|
|
101
112
|
|
|
@@ -133,16 +144,18 @@ block page is retried; see [docs/troubleshooting.md](docs/troubleshooting.md).
|
|
|
133
144
|
Progress is one aggregate line for the whole run
|
|
134
145
|
(`3 file(s) 12.4/44.1 MiB 1.8 MiB/s ETA 0:17`), redrawn in place on a TTY.
|
|
135
146
|
When stdout is a pipe it becomes a new line at most every 5 s (one extra when a
|
|
136
|
-
file starts or stops), so a CI log stays greppable without becoming a wall.
|
|
137
|
-
turns it off.
|
|
147
|
+
file starts or stops), so a CI log stays greppable without becoming a wall.
|
|
148
|
+
`-q/--quiet` turns it off.
|
|
138
149
|
|
|
139
|
-
|
|
150
|
+
`-o/--out` defaults to `acadfetch-out/<source>-<id>` (a URL adds its host and a short
|
|
140
151
|
hash), and `-n` prints it before
|
|
141
152
|
you spend bandwidth. Files land in `<name>.part` and are renamed only after the
|
|
142
153
|
checksum matches, so an interrupted or failed run never leaves something that
|
|
143
154
|
*looks* finished -- and never throws away the bytes it already paid for: rerun and
|
|
144
|
-
it resumes.
|
|
145
|
-
|
|
155
|
+
it resumes. The one case where a run must start over is a server that ignores
|
|
156
|
+
`Range`; the bytes already on disk are then moved aside and kept until a whole
|
|
157
|
+
new copy has arrived, so a restart that dies early costs nothing. `acadfetch-out/`
|
|
158
|
+
is worth adding to a project's `.gitignore`; this repository already has it.
|
|
146
159
|
|
|
147
160
|
The default directory names the target, not only its filename — `data.csv` from
|
|
148
161
|
two hosts is two datasets, and they do not share a directory. A file already
|
|
@@ -271,7 +284,7 @@ so the trace can be pasted into an issue.
|
|
|
271
284
|
|
|
272
285
|
## Politeness
|
|
273
286
|
|
|
274
|
-
Defaults: 3 parallel transfers, 2 requests/second, 4 retries with jittered
|
|
287
|
+
Defaults: 3 parallel transfers (`-j/--concurrency`), 2 requests/second, 4 retries with jittered
|
|
275
288
|
exponential backoff, `Retry-After` honoured up to 120 s. A block page is retried twice
|
|
276
289
|
(`--retry-403 N`); Zenodo and arXiv are donation-funded —
|
|
277
290
|
`--rate 1 -j 1` costs you minutes and costs them less.
|
|
@@ -293,7 +306,8 @@ honest way to test this code path.
|
|
|
293
306
|
## Getting involved
|
|
294
307
|
|
|
295
308
|
A [30-second tour](docs/demo.md) shows the whole loop, and
|
|
296
|
-
[docs/integrations.md](docs/integrations.md) has Make, CI, aria2 and
|
|
309
|
+
[docs/integrations.md](docs/integrations.md) has Make, CI, aria2, Snakemake and a
|
|
310
|
+
pooch downloader
|
|
297
311
|
recipes.
|
|
298
312
|
|
|
299
313
|
A nightly job downloads three fixed public records and verifies them, so a
|
|
@@ -256,7 +256,7 @@ def _file_row(row, redact: bool = True) -> dict: # noqa: ANN001
|
|
|
256
256
|
"http_status": row.http_status,
|
|
257
257
|
"final_url": hide(row.final_url) if row.final_url else None,
|
|
258
258
|
"verified": row.verified,
|
|
259
|
-
"verified_reason": verify_reason(row.file.checksum, row.verified),
|
|
259
|
+
"verified_reason": verify_reason(row.file.checksum, row.verified, row.state),
|
|
260
260
|
"resumed_from": row.resumed_from,
|
|
261
261
|
"part": Path(row.part).name if row.part else None,
|
|
262
262
|
"error": row.error,
|
|
@@ -427,7 +427,10 @@ def cmd_fetch(args: argparse.Namespace) -> int:
|
|
|
427
427
|
+ ("" if disk["ok"] else " NOT ENOUGH")
|
|
428
428
|
)
|
|
429
429
|
for file in chosen:
|
|
430
|
-
|
|
430
|
+
# "?" in a size column reads as a bug; arXiv genuinely publishes no
|
|
431
|
+
# size, so say that instead of printing a question mark.
|
|
432
|
+
size = human(file.size) if file.size is not None else "size not published"
|
|
433
|
+
print(f" would fetch {naming.sanitize(file.name)} ({size})")
|
|
431
434
|
return EXIT_OK
|
|
432
435
|
|
|
433
436
|
try:
|
|
@@ -519,6 +522,16 @@ def cmd_verify(args: argparse.Namespace) -> int:
|
|
|
519
522
|
rows = manifest.verify_manifest(manifest_path, root, allow_escape=args.allow_escape, data=data)
|
|
520
523
|
remote: dict[str, tuple[str, str]] = {}
|
|
521
524
|
if args.against_remote:
|
|
525
|
+
if data.get("url_redacted"):
|
|
526
|
+
# A presigned URL is stripped on the way into the manifest, so the
|
|
527
|
+
# recorded URL cannot be resolved again. Say why instead of leaving
|
|
528
|
+
# a page of remote_unavailable to be puzzled over.
|
|
529
|
+
print(
|
|
530
|
+
"note: this manifest had credential query parameters stripped, so a "
|
|
531
|
+
"row whose URL cannot be resolved again reports remote_unavailable "
|
|
532
|
+
"(--record-full-url keeps the original)",
|
|
533
|
+
file=sys.stderr,
|
|
534
|
+
)
|
|
522
535
|
published_now, reason = remote_check.published(_client(args), data)
|
|
523
536
|
entries = [e for e in data.get("files") or [] if isinstance(e, dict)]
|
|
524
537
|
# verify_manifest walks the same list in order; the rows do not carry
|
|
@@ -153,9 +153,29 @@ def _kept_bytes(part: Path) -> int:
|
|
|
153
153
|
return 0
|
|
154
154
|
|
|
155
155
|
|
|
156
|
-
def
|
|
156
|
+
def _room_for(part: Path, need: int) -> bool:
|
|
157
|
+
"""Whether keeping the old bytes while a restarting copy runs still fits."""
|
|
158
|
+
try:
|
|
159
|
+
return shutil.disk_usage(part.parent).free >= need
|
|
160
|
+
except OSError:
|
|
161
|
+
return False
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
def _drop(*paths: Path) -> bool:
|
|
165
|
+
"""Remove files, and say whether they are all gone.
|
|
166
|
+
|
|
167
|
+
``atomic.unlink`` swallows a sharing violation on purpose: losing a race with
|
|
168
|
+
a reader is usually harmless, and a cleanup step must not become a new way to
|
|
169
|
+
fail. Where the deletion is load-bearing -- a mismatched part that must not
|
|
170
|
+
be resumed onto, a part that is already over ``--max-size`` -- the caller has
|
|
171
|
+
to know it did not happen, or the next attempt silently resumes onto bytes it
|
|
172
|
+
was told to forget.
|
|
173
|
+
"""
|
|
174
|
+
kept = False
|
|
157
175
|
for path in paths:
|
|
158
176
|
atomic.unlink(path)
|
|
177
|
+
kept = kept or path.exists()
|
|
178
|
+
return not kept
|
|
159
179
|
|
|
160
180
|
|
|
161
181
|
def _content_length(headers, url: str) -> int | None:
|
|
@@ -340,6 +360,17 @@ def _transfer(client: Client, file: File, part: Path, meta_path: Path, opts: Opt
|
|
|
340
360
|
else:
|
|
341
361
|
mode, offset = "ab", have
|
|
342
362
|
|
|
363
|
+
# A 200 to a Range request means the server ignored it, so this attempt
|
|
364
|
+
# restarts from zero. The bytes already on disk are still the best copy
|
|
365
|
+
# of the file there is until a new one completes, so a restart is written
|
|
366
|
+
# beside them and swapped in only once it is whole: opening the part with
|
|
367
|
+
# "wb" threw away 6 MiB in order to save 1, which is the opposite of what
|
|
368
|
+
# "never throws away the bytes it already paid for" promises. Without
|
|
369
|
+
# room for both copies, the old behaviour returns.
|
|
370
|
+
sink, aside = part, None
|
|
371
|
+
if whole and have and _room_for(part, have):
|
|
372
|
+
aside = sink = part.with_name(part.name + ".new")
|
|
373
|
+
|
|
343
374
|
# Hash while the bytes go by instead of reading the finished file again.
|
|
344
375
|
# A resumed transfer has to be fed what is already on disk first.
|
|
345
376
|
parsed = verify.parse(file.checksum)
|
|
@@ -352,58 +383,73 @@ def _transfer(client: Client, file: File, part: Path, meta_path: Path, opts: Opt
|
|
|
352
383
|
for old in iter(lambda: existing.read(1 << 20), b""):
|
|
353
384
|
for hasher in hashers.values():
|
|
354
385
|
hasher.update(old)
|
|
355
|
-
_write_meta(
|
|
356
|
-
meta_path,
|
|
357
|
-
{
|
|
358
|
-
"url": file.url,
|
|
359
|
-
"etag": resp.headers.get("ETag"),
|
|
360
|
-
"last_modified": resp.headers.get("Last-Modified"),
|
|
361
|
-
"size": file.size,
|
|
362
|
-
},
|
|
363
|
-
)
|
|
364
|
-
if declared is not None and file.size is not None:
|
|
365
|
-
expected = file.size - offset
|
|
366
|
-
if declared != expected:
|
|
367
|
-
raise HttpError(
|
|
368
|
-
0,
|
|
369
|
-
file.url,
|
|
370
|
-
f"Content-Length {declared} disagrees with the published size "
|
|
371
|
-
f"({expected} bytes still expected)",
|
|
372
|
-
)
|
|
373
|
-
# Reads poll with a short socket timeout so an interrupt is noticed
|
|
374
|
-
# within a second instead of after the whole 60 s socket timeout, and
|
|
375
|
-
# so an idle connection fails on its own budget. Closing the response
|
|
376
|
-
# from another thread additionally kicks a read that is already
|
|
377
|
-
# blocked.
|
|
378
|
-
with _LIVE_LOCK:
|
|
379
|
-
_LIVE.add(resp)
|
|
380
386
|
try:
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
387
|
+
_write_meta(
|
|
388
|
+
meta_path,
|
|
389
|
+
{
|
|
390
|
+
"url": file.url,
|
|
391
|
+
"etag": resp.headers.get("ETag"),
|
|
392
|
+
"last_modified": resp.headers.get("Last-Modified"),
|
|
393
|
+
"size": file.size,
|
|
394
|
+
},
|
|
395
|
+
)
|
|
396
|
+
if declared is not None and file.size is not None:
|
|
397
|
+
expected = file.size - offset
|
|
398
|
+
if declared != expected:
|
|
399
|
+
raise HttpError(
|
|
400
|
+
0,
|
|
401
|
+
file.url,
|
|
402
|
+
f"Content-Length {declared} disagrees with the published size "
|
|
403
|
+
f"({expected} bytes still expected)",
|
|
404
|
+
)
|
|
405
|
+
# Reads poll with a short socket timeout so an interrupt is noticed
|
|
406
|
+
# within a second instead of after the whole 60 s socket timeout, and
|
|
407
|
+
# so an idle connection fails on its own budget. Closing the response
|
|
408
|
+
# from another thread additionally kicks a read that is already
|
|
409
|
+
# blocked.
|
|
403
410
|
with _LIVE_LOCK:
|
|
404
|
-
_LIVE.
|
|
405
|
-
|
|
406
|
-
|
|
411
|
+
_LIVE.add(resp)
|
|
412
|
+
try:
|
|
413
|
+
with sink.open(mode) as fh:
|
|
414
|
+
for chunk in _chunks(resp, client.timeout, file.url):
|
|
415
|
+
fh.write(chunk)
|
|
416
|
+
for hasher in hashers.values():
|
|
417
|
+
hasher.update(chunk)
|
|
418
|
+
written += len(chunk)
|
|
419
|
+
if opts.progress:
|
|
420
|
+
opts.progress.advance(len(chunk))
|
|
421
|
+
if opts.max_bytes and offset + written > opts.max_bytes:
|
|
422
|
+
raise HttpError(
|
|
423
|
+
0,
|
|
424
|
+
file.url,
|
|
425
|
+
f"refused locally: over --max-size ({human(opts.max_bytes)})",
|
|
426
|
+
reason="max-size",
|
|
427
|
+
)
|
|
428
|
+
if opts.fsync:
|
|
429
|
+
# The rename below is atomic, but only against crashes that
|
|
430
|
+
# leave the page cache intact. This is what makes the bytes
|
|
431
|
+
# survive a power cut before they get a name.
|
|
432
|
+
fh.flush()
|
|
433
|
+
os.fsync(fh.fileno())
|
|
434
|
+
finally:
|
|
435
|
+
with _LIVE_LOCK:
|
|
436
|
+
_LIVE.discard(resp)
|
|
437
|
+
if declared is not None and declared != written:
|
|
438
|
+
raise HttpError(0, file.url, f"read {written} bytes, response declared {declared}")
|
|
439
|
+
if aside is not None:
|
|
440
|
+
# A whole body, agreeing with the declared length: only now does
|
|
441
|
+
# the restart replace what was already there.
|
|
442
|
+
atomic.replace(sink, part)
|
|
443
|
+
except BaseException:
|
|
444
|
+
if aside is not None:
|
|
445
|
+
# Keep the bytes that were there, and the sidecar that described
|
|
446
|
+
# them rather than this attempt's validator.
|
|
447
|
+
atomic.unlink(aside)
|
|
448
|
+
if meta:
|
|
449
|
+
_write_meta(meta_path, meta)
|
|
450
|
+
else:
|
|
451
|
+
atomic.unlink(meta_path)
|
|
452
|
+
raise
|
|
407
453
|
return Transfer(
|
|
408
454
|
written=written,
|
|
409
455
|
final_url=resp.geturl() or file.url,
|
|
@@ -560,6 +606,8 @@ def _reserved(files) -> set[str]: # noqa: ANN001
|
|
|
560
606
|
for file in files:
|
|
561
607
|
name = naming.sanitize(file.name).lower()
|
|
562
608
|
reserved.add(name + ".part")
|
|
609
|
+
# A restart beside a kept part (a server that ignored Range) writes here.
|
|
610
|
+
reserved.add(name + ".part.new")
|
|
563
611
|
reserved.add(name + META_SUFFIX)
|
|
564
612
|
return reserved
|
|
565
613
|
|
|
@@ -675,12 +723,19 @@ def _attempt(
|
|
|
675
723
|
_drop(part, meta_path)
|
|
676
724
|
if exc.reason == "max-size":
|
|
677
725
|
# Do not leave an over-limit part for the next run to
|
|
678
|
-
# resume straight back into the same wall.
|
|
679
|
-
|
|
726
|
+
# resume straight back into the same wall. Saying so when the
|
|
727
|
+
# deletion fails is the difference between a rerun that works
|
|
728
|
+
# and one that fails the same way with no explanation.
|
|
729
|
+
kept = not _drop(part, meta_path)
|
|
680
730
|
return Result(
|
|
681
731
|
file=file,
|
|
682
732
|
state="failed",
|
|
683
|
-
error=str(exc)
|
|
733
|
+
error=str(exc)
|
|
734
|
+
+ (
|
|
735
|
+
f" (the over-limit {part.name} could not be deleted; remove it by hand)"
|
|
736
|
+
if kept
|
|
737
|
+
else ""
|
|
738
|
+
),
|
|
684
739
|
http_status=exc.status,
|
|
685
740
|
retrieved_at=_now(),
|
|
686
741
|
)
|
|
@@ -695,7 +750,12 @@ def _attempt(
|
|
|
695
750
|
return Result(file=file, state="failed", error="interrupted")
|
|
696
751
|
if attempt < opts.retries:
|
|
697
752
|
time.sleep(min(2**attempt, 10) * (0.7 + random.random() * 0.6))
|
|
698
|
-
|
|
753
|
+
# ``_transfer`` re-reads the part size itself, so the offset this
|
|
754
|
+
# attempt resumes from needs no bookkeeping here. ``resumed_from``
|
|
755
|
+
# is deliberately *not* updated: it answers "how much was on disk
|
|
756
|
+
# when this run started", which a retry does not change. Rewriting
|
|
757
|
+
# it made a fresh download report ``(resumed at 1.6 MiB)``. See
|
|
758
|
+
# docs/json-report.md.
|
|
699
759
|
else:
|
|
700
760
|
assert last is not None # the loop only ends after a failure
|
|
701
761
|
return Result(
|
|
@@ -712,7 +772,10 @@ def _attempt(
|
|
|
712
772
|
return Result(
|
|
713
773
|
file=file,
|
|
714
774
|
state="failed",
|
|
715
|
-
|
|
775
|
+
# A terminal 403 breaks out of the loop above, so it lands here
|
|
776
|
+
# rather than on the retry-exhausted path below. Without the hint
|
|
777
|
+
# that made it the one error a user could not act on.
|
|
778
|
+
error=f"{last}{_hint(last)}",
|
|
716
779
|
http_status=getattr(last, "status", None),
|
|
717
780
|
retrieved_at=_now(),
|
|
718
781
|
)
|
|
@@ -729,7 +792,19 @@ def _attempt(
|
|
|
729
792
|
|
|
730
793
|
verdict = _verdict(file, info.digests if info else None)
|
|
731
794
|
if verdict is False:
|
|
732
|
-
_drop(part, meta_path)
|
|
795
|
+
if not _drop(part, meta_path):
|
|
796
|
+
# Re-downloading would splice onto the bytes just proven wrong.
|
|
797
|
+
return Result(
|
|
798
|
+
file=file,
|
|
799
|
+
state="failed",
|
|
800
|
+
error=(
|
|
801
|
+
f"checksum mismatch, and the bad {part.name} could not be deleted; "
|
|
802
|
+
f"close whatever is reading it and retry"
|
|
803
|
+
),
|
|
804
|
+
verified=False,
|
|
805
|
+
bytes=_kept_bytes(part),
|
|
806
|
+
retrieved_at=_now(),
|
|
807
|
+
)
|
|
733
808
|
try:
|
|
734
809
|
info = _transfer(client, file, part, meta_path, opts)
|
|
735
810
|
verdict = _verdict(file, info.digests if info else None)
|
|
@@ -119,10 +119,10 @@ def _entry( # noqa: ANN001
|
|
|
119
119
|
"http_status": result.http_status if result else None,
|
|
120
120
|
"retrieved_at": result.retrieved_at if result else None,
|
|
121
121
|
# PR-2 adds final_url / etag / http_status / retrieved_at here.
|
|
122
|
-
"verified_reason": (
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
122
|
+
"verified_reason": verify.reason(
|
|
123
|
+
f.checksum,
|
|
124
|
+
result.verified if result else None,
|
|
125
|
+
result.state if result else None,
|
|
126
126
|
),
|
|
127
127
|
}
|
|
128
128
|
if state == "not-selected":
|
|
@@ -0,0 +1,157 @@
|
|
|
1
|
+
"""A pooch downloader that borrows acadfetch's HTTP client.
|
|
2
|
+
|
|
3
|
+
``pooch`` already caches, verifies and moves a file into place. What it does not
|
|
4
|
+
do is survive a link that stalls at 80%: its own downloader issues a flat GET,
|
|
5
|
+
and a drop means the whole transfer again. This adds the transport only — every
|
|
6
|
+
request goes through :class:`acadfetch.client.Client`, so it honours
|
|
7
|
+
``Retry-After``, retries a 5xx or a WAF block page, keeps a ``.part`` and resumes
|
|
8
|
+
it with ``Range``/``If-Range`` inside the attempt, and drops a credential header
|
|
9
|
+
when a redirect leaves the origin.
|
|
10
|
+
|
|
11
|
+
.. code-block:: python
|
|
12
|
+
|
|
13
|
+
import pooch
|
|
14
|
+
from acadfetch.pooch import downloader
|
|
15
|
+
|
|
16
|
+
path = pooch.retrieve(url, known_hash="md5:…", downloader=downloader)
|
|
17
|
+
|
|
18
|
+
``pooch`` still does the hashing and the move, so this cannot make verification
|
|
19
|
+
weaker — only the arrival more likely. It is a plain callable rather than a
|
|
20
|
+
subclass because a callable is the whole contract ``pooch`` asks for.
|
|
21
|
+
|
|
22
|
+
Two limits worth knowing, both from how ``pooch`` drives a downloader:
|
|
23
|
+
|
|
24
|
+
* it hands each attempt a **fresh temporary file**, so a partial is resumed
|
|
25
|
+
within one call and not across calls — ``pooch`` owns the retry loop between
|
|
26
|
+
them;
|
|
27
|
+
* ``pooch``'s ``progressbar=True`` is implemented inside its own downloader and
|
|
28
|
+
is not passed here, so a transfer through this one is silent. The CLI's
|
|
29
|
+
progress line is a separate thing.
|
|
30
|
+
"""
|
|
31
|
+
|
|
32
|
+
from __future__ import annotations
|
|
33
|
+
|
|
34
|
+
from collections.abc import Mapping
|
|
35
|
+
from pathlib import Path
|
|
36
|
+
from typing import Any
|
|
37
|
+
|
|
38
|
+
from . import atomic
|
|
39
|
+
from .client import Client
|
|
40
|
+
from .consts import DEFAULT_RATE, DEFAULT_RETRIES, DEFAULT_TIMEOUT
|
|
41
|
+
from .downloader import CHUNK, Options, fetch_file
|
|
42
|
+
from .errors import AcFError, HttpError, TargetError
|
|
43
|
+
from .sources.base import File
|
|
44
|
+
|
|
45
|
+
__all__ = ["Downloader", "downloader"]
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
class Downloader:
|
|
49
|
+
"""The callable ``pooch`` wants, configured like the CLI.
|
|
50
|
+
|
|
51
|
+
Every argument maps to a flag of the same name, so a pipeline that already
|
|
52
|
+
tuned the CLI does not have to learn a second vocabulary.
|
|
53
|
+
"""
|
|
54
|
+
|
|
55
|
+
def __init__(
|
|
56
|
+
self,
|
|
57
|
+
*,
|
|
58
|
+
rate: float = DEFAULT_RATE,
|
|
59
|
+
retries: int = DEFAULT_RETRIES,
|
|
60
|
+
timeout: float = DEFAULT_TIMEOUT,
|
|
61
|
+
headers: Mapping[str, str] | None = None,
|
|
62
|
+
block_retries: int = 2,
|
|
63
|
+
allow_http: bool = False,
|
|
64
|
+
fsync: bool = True,
|
|
65
|
+
) -> None:
|
|
66
|
+
self.rate = rate
|
|
67
|
+
self.retries = retries
|
|
68
|
+
self.timeout = timeout
|
|
69
|
+
self.headers = dict(headers or {})
|
|
70
|
+
self.block_retries = block_retries
|
|
71
|
+
self.allow_http = allow_http
|
|
72
|
+
self.fsync = fsync
|
|
73
|
+
|
|
74
|
+
def _client(self) -> Client:
|
|
75
|
+
return Client(
|
|
76
|
+
rate=self.rate,
|
|
77
|
+
retries=self.retries,
|
|
78
|
+
timeout=self.timeout,
|
|
79
|
+
headers=self.headers,
|
|
80
|
+
block_retries=self.block_retries,
|
|
81
|
+
allow_http=self.allow_http,
|
|
82
|
+
)
|
|
83
|
+
|
|
84
|
+
# -- the two things pooch asks for -------------------------------------
|
|
85
|
+
def __call__(
|
|
86
|
+
self,
|
|
87
|
+
url: str,
|
|
88
|
+
output_file: Any = None,
|
|
89
|
+
pooch: Any = None,
|
|
90
|
+
check_only: bool = False,
|
|
91
|
+
) -> bool | None:
|
|
92
|
+
if check_only:
|
|
93
|
+
return self.available(url)
|
|
94
|
+
if output_file is None:
|
|
95
|
+
raise TargetError("pooch asked for a download but passed no output file")
|
|
96
|
+
if hasattr(output_file, "write"):
|
|
97
|
+
# A caller-supplied handle: pooch owns the file, so there is nothing
|
|
98
|
+
# to rename and nothing worth resuming onto.
|
|
99
|
+
self._stream(url, output_file)
|
|
100
|
+
return None
|
|
101
|
+
|
|
102
|
+
target = Path(output_file)
|
|
103
|
+
target.parent.mkdir(parents=True, exist_ok=True)
|
|
104
|
+
result = fetch_file(
|
|
105
|
+
self._client(),
|
|
106
|
+
File(url=url, name=target.name),
|
|
107
|
+
Options(
|
|
108
|
+
out=target.parent,
|
|
109
|
+
concurrency=1,
|
|
110
|
+
retries=self.retries,
|
|
111
|
+
fsync=self.fsync,
|
|
112
|
+
),
|
|
113
|
+
set(),
|
|
114
|
+
)
|
|
115
|
+
if result.state == "failed":
|
|
116
|
+
# pooch does not catch this, so the sentence the CLI would print is
|
|
117
|
+
# the sentence the caller sees.
|
|
118
|
+
raise AcFError(result.error or f"could not fetch {url}")
|
|
119
|
+
produced = Path(result.path or target)
|
|
120
|
+
if produced != target:
|
|
121
|
+
# `naming` is allowed to change a name; pooch's own file must win.
|
|
122
|
+
atomic.replace(produced, target)
|
|
123
|
+
return None
|
|
124
|
+
|
|
125
|
+
def available(self, url: str) -> bool:
|
|
126
|
+
"""Whether the URL answers at all — HEAD first, one ranged byte after.
|
|
127
|
+
|
|
128
|
+
``Client.head`` treats a 403/405/501 as "no headers", which is
|
|
129
|
+
indistinguishable from a 200 that sent none, so an empty answer falls
|
|
130
|
+
through to a one-byte range request rather than being read as success.
|
|
131
|
+
"""
|
|
132
|
+
client = self._client()
|
|
133
|
+
try:
|
|
134
|
+
if client.head(url):
|
|
135
|
+
return True
|
|
136
|
+
except HttpError:
|
|
137
|
+
return False
|
|
138
|
+
try:
|
|
139
|
+
with client.open(url, {"Range": "bytes=0-0"}, attempts=1):
|
|
140
|
+
return True
|
|
141
|
+
except HttpError:
|
|
142
|
+
return False
|
|
143
|
+
|
|
144
|
+
def _stream(self, url: str, handle: Any) -> None:
|
|
145
|
+
client = self._client()
|
|
146
|
+
with client.open(url, attempts=self.retries + 1) as resp:
|
|
147
|
+
reader = getattr(resp, "read1", None) or resp.read
|
|
148
|
+
while True:
|
|
149
|
+
chunk = reader(CHUNK)
|
|
150
|
+
if not chunk:
|
|
151
|
+
break
|
|
152
|
+
handle.write(chunk)
|
|
153
|
+
handle.flush()
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
#: The zero-configuration instance: ``pooch.retrieve(..., downloader=downloader)``.
|
|
157
|
+
downloader = Downloader()
|