acadfetch 0.6.2__tar.gz → 0.6.4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {acadfetch-0.6.2/acadfetch.egg-info → acadfetch-0.6.4}/PKG-INFO +21 -8
- {acadfetch-0.6.2 → acadfetch-0.6.4}/README.md +20 -7
- {acadfetch-0.6.2 → acadfetch-0.6.4}/acadfetch/cli.py +15 -2
- {acadfetch-0.6.2 → acadfetch-0.6.4}/acadfetch/consts.py +1 -1
- {acadfetch-0.6.2 → acadfetch-0.6.4}/acadfetch/downloader.py +223 -61
- {acadfetch-0.6.2 → acadfetch-0.6.4}/acadfetch/manifest.py +4 -4
- {acadfetch-0.6.2 → acadfetch-0.6.4}/acadfetch/progress.py +62 -19
- {acadfetch-0.6.2 → acadfetch-0.6.4}/acadfetch/remote.py +5 -0
- {acadfetch-0.6.2 → acadfetch-0.6.4}/acadfetch/verify.py +12 -3
- {acadfetch-0.6.2 → acadfetch-0.6.4/acadfetch.egg-info}/PKG-INFO +21 -8
- {acadfetch-0.6.2 → acadfetch-0.6.4}/acadfetch.egg-info/SOURCES.txt +2 -0
- {acadfetch-0.6.2 → acadfetch-0.6.4}/pyproject.toml +1 -1
- acadfetch-0.6.4/tests/test_claims.py +174 -0
- {acadfetch-0.6.2 → acadfetch-0.6.4}/tests/test_cli.py +9 -0
- {acadfetch-0.6.2 → acadfetch-0.6.4}/tests/test_downloader.py +74 -0
- {acadfetch-0.6.2 → acadfetch-0.6.4}/tests/test_json.py +19 -0
- acadfetch-0.6.4/tests/test_lock.py +146 -0
- {acadfetch-0.6.2 → acadfetch-0.6.4}/tests/test_progress.py +30 -2
- {acadfetch-0.6.2 → acadfetch-0.6.4}/tests/test_remote.py +26 -0
- {acadfetch-0.6.2 → acadfetch-0.6.4}/LICENSE +0 -0
- {acadfetch-0.6.2 → acadfetch-0.6.4}/MANIFEST.in +0 -0
- {acadfetch-0.6.2 → acadfetch-0.6.4}/acadfetch/__init__.py +0 -0
- {acadfetch-0.6.2 → acadfetch-0.6.4}/acadfetch/__main__.py +0 -0
- {acadfetch-0.6.2 → acadfetch-0.6.4}/acadfetch/atomic.py +0 -0
- {acadfetch-0.6.2 → acadfetch-0.6.4}/acadfetch/auth.py +0 -0
- {acadfetch-0.6.2 → acadfetch-0.6.4}/acadfetch/client.py +0 -0
- {acadfetch-0.6.2 → acadfetch-0.6.4}/acadfetch/debug.py +0 -0
- {acadfetch-0.6.2 → acadfetch-0.6.4}/acadfetch/errors.py +0 -0
- {acadfetch-0.6.2 → acadfetch-0.6.4}/acadfetch/naming.py +0 -0
- {acadfetch-0.6.2 → acadfetch-0.6.4}/acadfetch/py.typed +0 -0
- {acadfetch-0.6.2 → acadfetch-0.6.4}/acadfetch/ratelimit.py +0 -0
- {acadfetch-0.6.2 → acadfetch-0.6.4}/acadfetch/sources/__init__.py +0 -0
- {acadfetch-0.6.2 → acadfetch-0.6.4}/acadfetch/sources/arxiv.py +0 -0
- {acadfetch-0.6.2 → acadfetch-0.6.4}/acadfetch/sources/base.py +0 -0
- {acadfetch-0.6.2 → acadfetch-0.6.4}/acadfetch/sources/direct.py +0 -0
- {acadfetch-0.6.2 → acadfetch-0.6.4}/acadfetch/sources/doi.py +0 -0
- {acadfetch-0.6.2 → acadfetch-0.6.4}/acadfetch/sources/registry.py +0 -0
- {acadfetch-0.6.2 → acadfetch-0.6.4}/acadfetch/sources/removed.py +0 -0
- {acadfetch-0.6.2 → acadfetch-0.6.4}/acadfetch/sources/zenodo.py +0 -0
- {acadfetch-0.6.2 → acadfetch-0.6.4}/acadfetch/units.py +0 -0
- {acadfetch-0.6.2 → acadfetch-0.6.4}/acadfetch/urls.py +0 -0
- {acadfetch-0.6.2 → acadfetch-0.6.4}/acadfetch.egg-info/dependency_links.txt +0 -0
- {acadfetch-0.6.2 → acadfetch-0.6.4}/acadfetch.egg-info/entry_points.txt +0 -0
- {acadfetch-0.6.2 → acadfetch-0.6.4}/acadfetch.egg-info/top_level.txt +0 -0
- {acadfetch-0.6.2 → acadfetch-0.6.4}/setup.cfg +0 -0
- {acadfetch-0.6.2 → acadfetch-0.6.4}/tests/conftest.py +0 -0
- {acadfetch-0.6.2 → acadfetch-0.6.4}/tests/fixtures/README.md +0 -0
- {acadfetch-0.6.2 → acadfetch-0.6.4}/tests/fixtures/ca.pem +0 -0
- {acadfetch-0.6.2 → acadfetch-0.6.4}/tests/fixtures/figshare_403_block.html +0 -0
- {acadfetch-0.6.2 → acadfetch-0.6.4}/tests/fixtures/figshare_403_forbidden.json +0 -0
- {acadfetch-0.6.2 → acadfetch-0.6.4}/tests/fixtures/figshare_article.json +0 -0
- {acadfetch-0.6.2 → acadfetch-0.6.4}/tests/fixtures/figshare_file.pdf +0 -0
- {acadfetch-0.6.2 → acadfetch-0.6.4}/tests/test_atomic.py +0 -0
- {acadfetch-0.6.2 → acadfetch-0.6.4}/tests/test_auth.py +0 -0
- {acadfetch-0.6.2 → acadfetch-0.6.4}/tests/test_checksum.py +0 -0
- {acadfetch-0.6.2 → acadfetch-0.6.4}/tests/test_citation.py +0 -0
- {acadfetch-0.6.2 → acadfetch-0.6.4}/tests/test_client.py +0 -0
- {acadfetch-0.6.2 → acadfetch-0.6.4}/tests/test_control_names.py +0 -0
- {acadfetch-0.6.2 → acadfetch-0.6.4}/tests/test_debug.py +0 -0
- {acadfetch-0.6.2 → acadfetch-0.6.4}/tests/test_disk.py +0 -0
- {acadfetch-0.6.2 → acadfetch-0.6.4}/tests/test_hash_once.py +0 -0
- {acadfetch-0.6.2 → acadfetch-0.6.4}/tests/test_identity.py +0 -0
- {acadfetch-0.6.2 → acadfetch-0.6.4}/tests/test_isolation.py +0 -0
- {acadfetch-0.6.2 → acadfetch-0.6.4}/tests/test_manifest.py +0 -0
- {acadfetch-0.6.2 → acadfetch-0.6.4}/tests/test_naming.py +0 -0
- {acadfetch-0.6.2 → acadfetch-0.6.4}/tests/test_naming_race.py +0 -0
- {acadfetch-0.6.2 → acadfetch-0.6.4}/tests/test_network.py +0 -0
- {acadfetch-0.6.2 → acadfetch-0.6.4}/tests/test_packaging.py +0 -0
- {acadfetch-0.6.2 → acadfetch-0.6.4}/tests/test_ratelimit.py +0 -0
- {acadfetch-0.6.2 → acadfetch-0.6.4}/tests/test_redirect.py +0 -0
- {acadfetch-0.6.2 → acadfetch-0.6.4}/tests/test_removed.py +0 -0
- {acadfetch-0.6.2 → acadfetch-0.6.4}/tests/test_sources.py +0 -0
- {acadfetch-0.6.2 → acadfetch-0.6.4}/tests/test_urls.py +0 -0
- {acadfetch-0.6.2 → acadfetch-0.6.4}/tests/test_urls_glob.py +0 -0
- {acadfetch-0.6.2 → acadfetch-0.6.4}/tests/test_verify.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: acadfetch
|
|
3
|
-
Version: 0.6.
|
|
3
|
+
Version: 0.6.4
|
|
4
4
|
Summary: Resumable, rate-limited, checksum-verified bulk downloads from academic data repositories
|
|
5
5
|
Author: young-ski
|
|
6
6
|
License-Expression: MIT
|
|
@@ -104,7 +104,7 @@ python -m pip install -e . pytest mypy && python -m pytest -q
|
|
|
104
104
|
|
|
105
105
|
## Usage
|
|
106
106
|
|
|
107
|
-
|
|
107
|
+
One target per run:
|
|
108
108
|
|
|
109
109
|
```bash
|
|
110
110
|
acadfetch fetch 10.5281/zenodo.23612969 # a DOI
|
|
@@ -112,6 +112,14 @@ acadfetch fetch arxiv:1706.03762 # an arXiv id
|
|
|
112
112
|
acadfetch fetch https://ndownloader.figshare.com/files/70600233/report.pdf # any URL
|
|
113
113
|
```
|
|
114
114
|
|
|
115
|
+
The first two are answered by Zenodo and arXiv and work from any network those
|
|
116
|
+
are reachable on. The third is a direct URL, so the host decides: figshare's CDN
|
|
117
|
+
answers 403 with a WAF block page from some networks — the same behaviour that
|
|
118
|
+
made its *adapter* not worth keeping in 0.2.0. acadfetch says so and keeps
|
|
119
|
+
nothing, rather than saving the block page as a PDF. If you hit that, the file
|
|
120
|
+
itself is fine; see the `403` section of
|
|
121
|
+
[docs/troubleshooting.md](docs/troubleshooting.md).
|
|
122
|
+
|
|
115
123
|
figshare was removed in 0.2.0: pass the article's download URL instead, and
|
|
116
124
|
[`--checksum`](docs/migration-0.2.md) keeps the verification the API used to
|
|
117
125
|
give you.
|
|
@@ -123,10 +131,13 @@ acadfetch fetch 10.5281/zenodo.23612969 -n # dry run: which dir, which files
|
|
|
123
131
|
acadfetch fetch 10.5281/zenodo.23612969 --only raw.tar.gz
|
|
124
132
|
acadfetch fetch 10.5281/zenodo.23612969 --max-size 2GB # refuse anything bigger
|
|
125
133
|
acadfetch fetch 10.5281/zenodo.23612969 -j 1 --rate 0.5 # be a good citizen
|
|
134
|
+
acadfetch fetch 10.5281/zenodo.23612969 --retries 2 --retry-403 0 # give up sooner
|
|
135
|
+
acadfetch fetch 10.5281/zenodo.23612969 --manifest-out build/m.json # a second copy
|
|
126
136
|
acadfetch manifest 10.5281/zenodo.23612969 -o m.json # metadata only, no bytes
|
|
127
137
|
acadfetch fetch zenodo:123456 --header "Authorization: Bearer $TOKEN" # restricted
|
|
128
138
|
|
|
129
139
|
acadfetch verify ./data/manifest.json # re-hash what you downloaded
|
|
140
|
+
acadfetch --version # what is installed
|
|
130
141
|
acadfetch sources # adapters, with an example each
|
|
131
142
|
```
|
|
132
143
|
|
|
@@ -164,16 +175,18 @@ block page is retried; see [docs/troubleshooting.md](docs/troubleshooting.md).
|
|
|
164
175
|
Progress is one aggregate line for the whole run
|
|
165
176
|
(`3 file(s) 12.4/44.1 MiB 1.8 MiB/s ETA 0:17`), redrawn in place on a TTY.
|
|
166
177
|
When stdout is a pipe it becomes a new line at most every 5 s (one extra when a
|
|
167
|
-
file starts or stops), so a CI log stays greppable without becoming a wall.
|
|
168
|
-
turns it off.
|
|
178
|
+
file starts or stops), so a CI log stays greppable without becoming a wall.
|
|
179
|
+
`-q/--quiet` turns it off.
|
|
169
180
|
|
|
170
|
-
|
|
181
|
+
`-o/--out` defaults to `acadfetch-out/<source>-<id>` (a URL adds its host and a short
|
|
171
182
|
hash), and `-n` prints it before
|
|
172
183
|
you spend bandwidth. Files land in `<name>.part` and are renamed only after the
|
|
173
184
|
checksum matches, so an interrupted or failed run never leaves something that
|
|
174
185
|
*looks* finished -- and never throws away the bytes it already paid for: rerun and
|
|
175
|
-
it resumes.
|
|
176
|
-
|
|
186
|
+
it resumes. The one case where a run must start over is a server that ignores
|
|
187
|
+
`Range`; the bytes already on disk are then moved aside and kept until a whole
|
|
188
|
+
new copy has arrived, so a restart that dies early costs nothing. `acadfetch-out/`
|
|
189
|
+
is worth adding to a project's `.gitignore`; this repository already has it.
|
|
177
190
|
|
|
178
191
|
The default directory names the target, not only its filename — `data.csv` from
|
|
179
192
|
two hosts is two datasets, and they do not share a directory. A file already
|
|
@@ -302,7 +315,7 @@ so the trace can be pasted into an issue.
|
|
|
302
315
|
|
|
303
316
|
## Politeness
|
|
304
317
|
|
|
305
|
-
Defaults: 3 parallel transfers, 2 requests/second, 4 retries with jittered
|
|
318
|
+
Defaults: 3 parallel transfers (`-j/--concurrency`), 2 requests/second, 4 retries with jittered
|
|
306
319
|
exponential backoff, `Retry-After` honoured up to 120 s. A block page is retried twice
|
|
307
320
|
(`--retry-403 N`); Zenodo and arXiv are donation-funded —
|
|
308
321
|
`--rate 1 -j 1` costs you minutes and costs them less.
|
|
@@ -73,7 +73,7 @@ python -m pip install -e . pytest mypy && python -m pytest -q
|
|
|
73
73
|
|
|
74
74
|
## Usage
|
|
75
75
|
|
|
76
|
-
|
|
76
|
+
One target per run:
|
|
77
77
|
|
|
78
78
|
```bash
|
|
79
79
|
acadfetch fetch 10.5281/zenodo.23612969 # a DOI
|
|
@@ -81,6 +81,14 @@ acadfetch fetch arxiv:1706.03762 # an arXiv id
|
|
|
81
81
|
acadfetch fetch https://ndownloader.figshare.com/files/70600233/report.pdf # any URL
|
|
82
82
|
```
|
|
83
83
|
|
|
84
|
+
The first two are answered by Zenodo and arXiv and work from any network those
|
|
85
|
+
are reachable on. The third is a direct URL, so the host decides: figshare's CDN
|
|
86
|
+
answers 403 with a WAF block page from some networks — the same behaviour that
|
|
87
|
+
made its *adapter* not worth keeping in 0.2.0. acadfetch says so and keeps
|
|
88
|
+
nothing, rather than saving the block page as a PDF. If you hit that, the file
|
|
89
|
+
itself is fine; see the `403` section of
|
|
90
|
+
[docs/troubleshooting.md](docs/troubleshooting.md).
|
|
91
|
+
|
|
84
92
|
figshare was removed in 0.2.0: pass the article's download URL instead, and
|
|
85
93
|
[`--checksum`](docs/migration-0.2.md) keeps the verification the API used to
|
|
86
94
|
give you.
|
|
@@ -92,10 +100,13 @@ acadfetch fetch 10.5281/zenodo.23612969 -n # dry run: which dir, which files
|
|
|
92
100
|
acadfetch fetch 10.5281/zenodo.23612969 --only raw.tar.gz
|
|
93
101
|
acadfetch fetch 10.5281/zenodo.23612969 --max-size 2GB # refuse anything bigger
|
|
94
102
|
acadfetch fetch 10.5281/zenodo.23612969 -j 1 --rate 0.5 # be a good citizen
|
|
103
|
+
acadfetch fetch 10.5281/zenodo.23612969 --retries 2 --retry-403 0 # give up sooner
|
|
104
|
+
acadfetch fetch 10.5281/zenodo.23612969 --manifest-out build/m.json # a second copy
|
|
95
105
|
acadfetch manifest 10.5281/zenodo.23612969 -o m.json # metadata only, no bytes
|
|
96
106
|
acadfetch fetch zenodo:123456 --header "Authorization: Bearer $TOKEN" # restricted
|
|
97
107
|
|
|
98
108
|
acadfetch verify ./data/manifest.json # re-hash what you downloaded
|
|
109
|
+
acadfetch --version # what is installed
|
|
99
110
|
acadfetch sources # adapters, with an example each
|
|
100
111
|
```
|
|
101
112
|
|
|
@@ -133,16 +144,18 @@ block page is retried; see [docs/troubleshooting.md](docs/troubleshooting.md).
|
|
|
133
144
|
Progress is one aggregate line for the whole run
|
|
134
145
|
(`3 file(s) 12.4/44.1 MiB 1.8 MiB/s ETA 0:17`), redrawn in place on a TTY.
|
|
135
146
|
When stdout is a pipe it becomes a new line at most every 5 s (one extra when a
|
|
136
|
-
file starts or stops), so a CI log stays greppable without becoming a wall.
|
|
137
|
-
turns it off.
|
|
147
|
+
file starts or stops), so a CI log stays greppable without becoming a wall.
|
|
148
|
+
`-q/--quiet` turns it off.
|
|
138
149
|
|
|
139
|
-
|
|
150
|
+
`-o/--out` defaults to `acadfetch-out/<source>-<id>` (a URL adds its host and a short
|
|
140
151
|
hash), and `-n` prints it before
|
|
141
152
|
you spend bandwidth. Files land in `<name>.part` and are renamed only after the
|
|
142
153
|
checksum matches, so an interrupted or failed run never leaves something that
|
|
143
154
|
*looks* finished -- and never throws away the bytes it already paid for: rerun and
|
|
144
|
-
it resumes.
|
|
145
|
-
|
|
155
|
+
it resumes. The one case where a run must start over is a server that ignores
|
|
156
|
+
`Range`; the bytes already on disk are then moved aside and kept until a whole
|
|
157
|
+
new copy has arrived, so a restart that dies early costs nothing. `acadfetch-out/`
|
|
158
|
+
is worth adding to a project's `.gitignore`; this repository already has it.
|
|
146
159
|
|
|
147
160
|
The default directory names the target, not only its filename — `data.csv` from
|
|
148
161
|
two hosts is two datasets, and they do not share a directory. A file already
|
|
@@ -271,7 +284,7 @@ so the trace can be pasted into an issue.
|
|
|
271
284
|
|
|
272
285
|
## Politeness
|
|
273
286
|
|
|
274
|
-
Defaults: 3 parallel transfers, 2 requests/second, 4 retries with jittered
|
|
287
|
+
Defaults: 3 parallel transfers (`-j/--concurrency`), 2 requests/second, 4 retries with jittered
|
|
275
288
|
exponential backoff, `Retry-After` honoured up to 120 s. A block page is retried twice
|
|
276
289
|
(`--retry-403 N`); Zenodo and arXiv are donation-funded —
|
|
277
290
|
`--rate 1 -j 1` costs you minutes and costs them less.
|
|
@@ -256,7 +256,7 @@ def _file_row(row, redact: bool = True) -> dict: # noqa: ANN001
|
|
|
256
256
|
"http_status": row.http_status,
|
|
257
257
|
"final_url": hide(row.final_url) if row.final_url else None,
|
|
258
258
|
"verified": row.verified,
|
|
259
|
-
"verified_reason": verify_reason(row.file.checksum, row.verified),
|
|
259
|
+
"verified_reason": verify_reason(row.file.checksum, row.verified, row.state),
|
|
260
260
|
"resumed_from": row.resumed_from,
|
|
261
261
|
"part": Path(row.part).name if row.part else None,
|
|
262
262
|
"error": row.error,
|
|
@@ -427,7 +427,10 @@ def cmd_fetch(args: argparse.Namespace) -> int:
|
|
|
427
427
|
+ ("" if disk["ok"] else " NOT ENOUGH")
|
|
428
428
|
)
|
|
429
429
|
for file in chosen:
|
|
430
|
-
|
|
430
|
+
# "?" in a size column reads as a bug; arXiv genuinely publishes no
|
|
431
|
+
# size, so say that instead of printing a question mark.
|
|
432
|
+
size = human(file.size) if file.size is not None else "size not published"
|
|
433
|
+
print(f" would fetch {naming.sanitize(file.name)} ({size})")
|
|
431
434
|
return EXIT_OK
|
|
432
435
|
|
|
433
436
|
try:
|
|
@@ -519,6 +522,16 @@ def cmd_verify(args: argparse.Namespace) -> int:
|
|
|
519
522
|
rows = manifest.verify_manifest(manifest_path, root, allow_escape=args.allow_escape, data=data)
|
|
520
523
|
remote: dict[str, tuple[str, str]] = {}
|
|
521
524
|
if args.against_remote:
|
|
525
|
+
if data.get("url_redacted"):
|
|
526
|
+
# A presigned URL is stripped on the way into the manifest, so the
|
|
527
|
+
# recorded URL cannot be resolved again. Say why instead of leaving
|
|
528
|
+
# a page of remote_unavailable to be puzzled over.
|
|
529
|
+
print(
|
|
530
|
+
"note: this manifest had credential query parameters stripped, so a "
|
|
531
|
+
"row whose URL cannot be resolved again reports remote_unavailable "
|
|
532
|
+
"(--record-full-url keeps the original)",
|
|
533
|
+
file=sys.stderr,
|
|
534
|
+
)
|
|
522
535
|
published_now, reason = remote_check.published(_client(args), data)
|
|
523
536
|
entries = [e for e in data.get("files") or [] if isinstance(e, dict)]
|
|
524
537
|
# verify_manifest walks the same list in order; the rows do not carry
|
|
@@ -27,7 +27,7 @@ from dataclasses import dataclass, field
|
|
|
27
27
|
from datetime import datetime, timezone
|
|
28
28
|
from http.client import HTTPException
|
|
29
29
|
from pathlib import Path
|
|
30
|
-
from typing import cast
|
|
30
|
+
from typing import Any, cast
|
|
31
31
|
|
|
32
32
|
from . import atomic, debug, naming, verify
|
|
33
33
|
from .client import Client
|
|
@@ -73,6 +73,7 @@ _BINARY_EXT = frozenset(
|
|
|
73
73
|
}
|
|
74
74
|
)
|
|
75
75
|
META_SUFFIX = ".part.meta.json"
|
|
76
|
+
LOCK_NAME = ".acadfetch.lock"
|
|
76
77
|
|
|
77
78
|
|
|
78
79
|
@dataclass(frozen=True)
|
|
@@ -152,9 +153,29 @@ def _kept_bytes(part: Path) -> int:
|
|
|
152
153
|
return 0
|
|
153
154
|
|
|
154
155
|
|
|
155
|
-
def
|
|
156
|
+
def _room_for(part: Path, need: int) -> bool:
|
|
157
|
+
"""Whether keeping the old bytes while a restarting copy runs still fits."""
|
|
158
|
+
try:
|
|
159
|
+
return shutil.disk_usage(part.parent).free >= need
|
|
160
|
+
except OSError:
|
|
161
|
+
return False
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
def _drop(*paths: Path) -> bool:
|
|
165
|
+
"""Remove files, and say whether they are all gone.
|
|
166
|
+
|
|
167
|
+
``atomic.unlink`` swallows a sharing violation on purpose: losing a race with
|
|
168
|
+
a reader is usually harmless, and a cleanup step must not become a new way to
|
|
169
|
+
fail. Where the deletion is load-bearing -- a mismatched part that must not
|
|
170
|
+
be resumed onto, a part that is already over ``--max-size`` -- the caller has
|
|
171
|
+
to know it did not happen, or the next attempt silently resumes onto bytes it
|
|
172
|
+
was told to forget.
|
|
173
|
+
"""
|
|
174
|
+
kept = False
|
|
156
175
|
for path in paths:
|
|
157
176
|
atomic.unlink(path)
|
|
177
|
+
kept = kept or path.exists()
|
|
178
|
+
return not kept
|
|
158
179
|
|
|
159
180
|
|
|
160
181
|
def _content_length(headers, url: str) -> int | None:
|
|
@@ -339,6 +360,17 @@ def _transfer(client: Client, file: File, part: Path, meta_path: Path, opts: Opt
|
|
|
339
360
|
else:
|
|
340
361
|
mode, offset = "ab", have
|
|
341
362
|
|
|
363
|
+
# A 200 to a Range request means the server ignored it, so this attempt
|
|
364
|
+
# restarts from zero. The bytes already on disk are still the best copy
|
|
365
|
+
# of the file there is until a new one completes, so a restart is written
|
|
366
|
+
# beside them and swapped in only once it is whole: opening the part with
|
|
367
|
+
# "wb" threw away 6 MiB in order to save 1, which is the opposite of what
|
|
368
|
+
# "never throws away the bytes it already paid for" promises. Without
|
|
369
|
+
# room for both copies, the old behaviour returns.
|
|
370
|
+
sink, aside = part, None
|
|
371
|
+
if whole and have and _room_for(part, have):
|
|
372
|
+
aside = sink = part.with_name(part.name + ".new")
|
|
373
|
+
|
|
342
374
|
# Hash while the bytes go by instead of reading the finished file again.
|
|
343
375
|
# A resumed transfer has to be fed what is already on disk first.
|
|
344
376
|
parsed = verify.parse(file.checksum)
|
|
@@ -351,58 +383,73 @@ def _transfer(client: Client, file: File, part: Path, meta_path: Path, opts: Opt
|
|
|
351
383
|
for old in iter(lambda: existing.read(1 << 20), b""):
|
|
352
384
|
for hasher in hashers.values():
|
|
353
385
|
hasher.update(old)
|
|
354
|
-
_write_meta(
|
|
355
|
-
meta_path,
|
|
356
|
-
{
|
|
357
|
-
"url": file.url,
|
|
358
|
-
"etag": resp.headers.get("ETag"),
|
|
359
|
-
"last_modified": resp.headers.get("Last-Modified"),
|
|
360
|
-
"size": file.size,
|
|
361
|
-
},
|
|
362
|
-
)
|
|
363
|
-
if declared is not None and file.size is not None:
|
|
364
|
-
expected = file.size - offset
|
|
365
|
-
if declared != expected:
|
|
366
|
-
raise HttpError(
|
|
367
|
-
0,
|
|
368
|
-
file.url,
|
|
369
|
-
f"Content-Length {declared} disagrees with the published size "
|
|
370
|
-
f"({expected} bytes still expected)",
|
|
371
|
-
)
|
|
372
|
-
# Reads poll with a short socket timeout so an interrupt is noticed
|
|
373
|
-
# within a second instead of after the whole 60 s socket timeout, and
|
|
374
|
-
# so an idle connection fails on its own budget. Closing the response
|
|
375
|
-
# from another thread additionally kicks a read that is already
|
|
376
|
-
# blocked.
|
|
377
|
-
with _LIVE_LOCK:
|
|
378
|
-
_LIVE.add(resp)
|
|
379
386
|
try:
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
387
|
+
_write_meta(
|
|
388
|
+
meta_path,
|
|
389
|
+
{
|
|
390
|
+
"url": file.url,
|
|
391
|
+
"etag": resp.headers.get("ETag"),
|
|
392
|
+
"last_modified": resp.headers.get("Last-Modified"),
|
|
393
|
+
"size": file.size,
|
|
394
|
+
},
|
|
395
|
+
)
|
|
396
|
+
if declared is not None and file.size is not None:
|
|
397
|
+
expected = file.size - offset
|
|
398
|
+
if declared != expected:
|
|
399
|
+
raise HttpError(
|
|
400
|
+
0,
|
|
401
|
+
file.url,
|
|
402
|
+
f"Content-Length {declared} disagrees with the published size "
|
|
403
|
+
f"({expected} bytes still expected)",
|
|
404
|
+
)
|
|
405
|
+
# Reads poll with a short socket timeout so an interrupt is noticed
|
|
406
|
+
# within a second instead of after the whole 60 s socket timeout, and
|
|
407
|
+
# so an idle connection fails on its own budget. Closing the response
|
|
408
|
+
# from another thread additionally kicks a read that is already
|
|
409
|
+
# blocked.
|
|
402
410
|
with _LIVE_LOCK:
|
|
403
|
-
_LIVE.
|
|
404
|
-
|
|
405
|
-
|
|
411
|
+
_LIVE.add(resp)
|
|
412
|
+
try:
|
|
413
|
+
with sink.open(mode) as fh:
|
|
414
|
+
for chunk in _chunks(resp, client.timeout, file.url):
|
|
415
|
+
fh.write(chunk)
|
|
416
|
+
for hasher in hashers.values():
|
|
417
|
+
hasher.update(chunk)
|
|
418
|
+
written += len(chunk)
|
|
419
|
+
if opts.progress:
|
|
420
|
+
opts.progress.advance(len(chunk))
|
|
421
|
+
if opts.max_bytes and offset + written > opts.max_bytes:
|
|
422
|
+
raise HttpError(
|
|
423
|
+
0,
|
|
424
|
+
file.url,
|
|
425
|
+
f"refused locally: over --max-size ({human(opts.max_bytes)})",
|
|
426
|
+
reason="max-size",
|
|
427
|
+
)
|
|
428
|
+
if opts.fsync:
|
|
429
|
+
# The rename below is atomic, but only against crashes that
|
|
430
|
+
# leave the page cache intact. This is what makes the bytes
|
|
431
|
+
# survive a power cut before they get a name.
|
|
432
|
+
fh.flush()
|
|
433
|
+
os.fsync(fh.fileno())
|
|
434
|
+
finally:
|
|
435
|
+
with _LIVE_LOCK:
|
|
436
|
+
_LIVE.discard(resp)
|
|
437
|
+
if declared is not None and declared != written:
|
|
438
|
+
raise HttpError(0, file.url, f"read {written} bytes, response declared {declared}")
|
|
439
|
+
if aside is not None:
|
|
440
|
+
# A whole body, agreeing with the declared length: only now does
|
|
441
|
+
# the restart replace what was already there.
|
|
442
|
+
atomic.replace(sink, part)
|
|
443
|
+
except BaseException:
|
|
444
|
+
if aside is not None:
|
|
445
|
+
# Keep the bytes that were there, and the sidecar that described
|
|
446
|
+
# them rather than this attempt's validator.
|
|
447
|
+
atomic.unlink(aside)
|
|
448
|
+
if meta:
|
|
449
|
+
_write_meta(meta_path, meta)
|
|
450
|
+
else:
|
|
451
|
+
atomic.unlink(meta_path)
|
|
452
|
+
raise
|
|
406
453
|
return Transfer(
|
|
407
454
|
written=written,
|
|
408
455
|
final_url=resp.geturl() or file.url,
|
|
@@ -471,6 +518,82 @@ def check_disk(
|
|
|
471
518
|
return (free, needed)
|
|
472
519
|
|
|
473
520
|
|
|
521
|
+
def _pid_alive(pid: int) -> bool:
|
|
522
|
+
"""Is ``pid`` still running, without disturbing it?
|
|
523
|
+
|
|
524
|
+
``os.kill(pid, 0)`` is the POSIX idiom, but on Windows it sends a console
|
|
525
|
+
CTRL_C: probing our own pid interrupts the process asking the question.
|
|
526
|
+
"""
|
|
527
|
+
if os.name == "nt":
|
|
528
|
+
import ctypes
|
|
529
|
+
|
|
530
|
+
# The attribute does not exist off Windows, and both ruff and mypy
|
|
531
|
+
# check this file on Linux; a typed lookup keeps both quiet.
|
|
532
|
+
windll: Any = getattr(ctypes, "windll") # noqa: B009
|
|
533
|
+
kernel32 = windll.kernel32
|
|
534
|
+
handle = kernel32.OpenProcess(0x1000, False, pid) # query limited info
|
|
535
|
+
if not handle:
|
|
536
|
+
return False
|
|
537
|
+
try:
|
|
538
|
+
code = ctypes.c_ulong()
|
|
539
|
+
if not kernel32.GetExitCodeProcess(handle, ctypes.byref(code)):
|
|
540
|
+
return False
|
|
541
|
+
return code.value == 259 # STILL_ACTIVE
|
|
542
|
+
finally:
|
|
543
|
+
kernel32.CloseHandle(handle)
|
|
544
|
+
try:
|
|
545
|
+
os.kill(pid, 0)
|
|
546
|
+
except ProcessLookupError:
|
|
547
|
+
return False
|
|
548
|
+
except PermissionError: # exists, owned by someone else
|
|
549
|
+
return True
|
|
550
|
+
except OSError:
|
|
551
|
+
return False
|
|
552
|
+
return True
|
|
553
|
+
|
|
554
|
+
|
|
555
|
+
def _take_lock(out: Path) -> Path:
|
|
556
|
+
"""Refuse to share the directory with a live concurrent run.
|
|
557
|
+
|
|
558
|
+
Two runs into one ``-o`` interleave writes into the same ``.part`` and the
|
|
559
|
+
survivor is a file neither of them wrote. The lock records the owner; a dead
|
|
560
|
+
owner, or one we cannot read, is taken over.
|
|
561
|
+
"""
|
|
562
|
+
lock = out / LOCK_NAME
|
|
563
|
+
for _ in range(2):
|
|
564
|
+
try:
|
|
565
|
+
fd = os.open(lock, os.O_CREAT | os.O_EXCL | os.O_WRONLY)
|
|
566
|
+
except FileExistsError:
|
|
567
|
+
owner = 0
|
|
568
|
+
try:
|
|
569
|
+
owner = int(json.loads(lock.read_text("utf-8")).get("pid") or 0)
|
|
570
|
+
except (OSError, ValueError):
|
|
571
|
+
owner = 0
|
|
572
|
+
# A live owner is another run, even when it is this pid: two
|
|
573
|
+
# fetches into one directory in one process clash just as badly.
|
|
574
|
+
# Only a dead or unreadable owner is stale.
|
|
575
|
+
if owner and _pid_alive(owner):
|
|
576
|
+
raise TargetError(
|
|
577
|
+
f"{out} is being written by pid {owner}; wait for it, or "
|
|
578
|
+
f"delete {lock} if that process is gone"
|
|
579
|
+
) from None
|
|
580
|
+
atomic.unlink(lock)
|
|
581
|
+
continue
|
|
582
|
+
with os.fdopen(fd, "w", encoding="utf-8") as fh:
|
|
583
|
+
json.dump({"pid": os.getpid(), "started": _now()}, fh)
|
|
584
|
+
return lock
|
|
585
|
+
raise TargetError(f"could not take the lock in {out}; is another run active?")
|
|
586
|
+
|
|
587
|
+
|
|
588
|
+
def _release_lock(lock: Path) -> None:
|
|
589
|
+
try:
|
|
590
|
+
if int(json.loads(lock.read_text("utf-8")).get("pid") or 0) != os.getpid():
|
|
591
|
+
return
|
|
592
|
+
except (OSError, ValueError):
|
|
593
|
+
pass
|
|
594
|
+
atomic.unlink(lock)
|
|
595
|
+
|
|
596
|
+
|
|
474
597
|
def _reserved(files) -> set[str]: # noqa: ANN001
|
|
475
598
|
"""Names the tool itself will write, so a repository file cannot take them.
|
|
476
599
|
|
|
@@ -479,10 +602,12 @@ def _reserved(files) -> set[str]: # noqa: ANN001
|
|
|
479
602
|
Reserving the derived names is cheaper than teaching every later step to
|
|
480
603
|
dodge them.
|
|
481
604
|
"""
|
|
482
|
-
reserved = {"manifest.json", "manifest.json.tmp"}
|
|
605
|
+
reserved = {"manifest.json", "manifest.json.tmp", LOCK_NAME}
|
|
483
606
|
for file in files:
|
|
484
607
|
name = naming.sanitize(file.name).lower()
|
|
485
608
|
reserved.add(name + ".part")
|
|
609
|
+
# A restart beside a kept part (a server that ignored Range) writes here.
|
|
610
|
+
reserved.add(name + ".part.new")
|
|
486
611
|
reserved.add(name + META_SUFFIX)
|
|
487
612
|
return reserved
|
|
488
613
|
|
|
@@ -598,12 +723,19 @@ def _attempt(
|
|
|
598
723
|
_drop(part, meta_path)
|
|
599
724
|
if exc.reason == "max-size":
|
|
600
725
|
# Do not leave an over-limit part for the next run to
|
|
601
|
-
# resume straight back into the same wall.
|
|
602
|
-
|
|
726
|
+
# resume straight back into the same wall. Saying so when the
|
|
727
|
+
# deletion fails is the difference between a rerun that works
|
|
728
|
+
# and one that fails the same way with no explanation.
|
|
729
|
+
kept = not _drop(part, meta_path)
|
|
603
730
|
return Result(
|
|
604
731
|
file=file,
|
|
605
732
|
state="failed",
|
|
606
|
-
error=str(exc)
|
|
733
|
+
error=str(exc)
|
|
734
|
+
+ (
|
|
735
|
+
f" (the over-limit {part.name} could not be deleted; remove it by hand)"
|
|
736
|
+
if kept
|
|
737
|
+
else ""
|
|
738
|
+
),
|
|
607
739
|
http_status=exc.status,
|
|
608
740
|
retrieved_at=_now(),
|
|
609
741
|
)
|
|
@@ -618,7 +750,12 @@ def _attempt(
|
|
|
618
750
|
return Result(file=file, state="failed", error="interrupted")
|
|
619
751
|
if attempt < opts.retries:
|
|
620
752
|
time.sleep(min(2**attempt, 10) * (0.7 + random.random() * 0.6))
|
|
621
|
-
|
|
753
|
+
# ``_transfer`` re-reads the part size itself, so the offset this
|
|
754
|
+
# attempt resumes from needs no bookkeeping here. ``resumed_from``
|
|
755
|
+
# is deliberately *not* updated: it answers "how much was on disk
|
|
756
|
+
# when this run started", which a retry does not change. Rewriting
|
|
757
|
+
# it made a fresh download report ``(resumed at 1.6 MiB)``. See
|
|
758
|
+
# docs/json-report.md.
|
|
622
759
|
else:
|
|
623
760
|
assert last is not None # the loop only ends after a failure
|
|
624
761
|
return Result(
|
|
@@ -635,7 +772,10 @@ def _attempt(
|
|
|
635
772
|
return Result(
|
|
636
773
|
file=file,
|
|
637
774
|
state="failed",
|
|
638
|
-
|
|
775
|
+
# A terminal 403 breaks out of the loop above, so it lands here
|
|
776
|
+
# rather than on the retry-exhausted path below. Without the hint
|
|
777
|
+
# that made it the one error a user could not act on.
|
|
778
|
+
error=f"{last}{_hint(last)}",
|
|
639
779
|
http_status=getattr(last, "status", None),
|
|
640
780
|
retrieved_at=_now(),
|
|
641
781
|
)
|
|
@@ -652,7 +792,19 @@ def _attempt(
|
|
|
652
792
|
|
|
653
793
|
verdict = _verdict(file, info.digests if info else None)
|
|
654
794
|
if verdict is False:
|
|
655
|
-
_drop(part, meta_path)
|
|
795
|
+
if not _drop(part, meta_path):
|
|
796
|
+
# Re-downloading would splice onto the bytes just proven wrong.
|
|
797
|
+
return Result(
|
|
798
|
+
file=file,
|
|
799
|
+
state="failed",
|
|
800
|
+
error=(
|
|
801
|
+
f"checksum mismatch, and the bad {part.name} could not be deleted; "
|
|
802
|
+
f"close whatever is reading it and retry"
|
|
803
|
+
),
|
|
804
|
+
verified=False,
|
|
805
|
+
bytes=_kept_bytes(part),
|
|
806
|
+
retrieved_at=_now(),
|
|
807
|
+
)
|
|
656
808
|
try:
|
|
657
809
|
info = _transfer(client, file, part, meta_path, opts)
|
|
658
810
|
verdict = _verdict(file, info.digests if info else None)
|
|
@@ -736,15 +888,24 @@ def _fetch_record(
|
|
|
736
888
|
|
|
737
889
|
if opts.max_bytes:
|
|
738
890
|
check_total(record, opts.max_bytes)
|
|
891
|
+
lock: Path | None = None
|
|
739
892
|
if not opts.dry_run:
|
|
740
893
|
check_disk(files, opts.out)
|
|
894
|
+
lock = _take_lock(opts.out)
|
|
741
895
|
opts.trusted = _trusted_urls(opts.out / "manifest.json")
|
|
742
896
|
|
|
743
897
|
taken = _reserved(files)
|
|
744
|
-
lock = threading.Lock()
|
|
745
898
|
results: list[Result] = []
|
|
746
899
|
opts.interrupted = False
|
|
747
900
|
|
|
901
|
+
try:
|
|
902
|
+
return _drain(files, record, opts, client, taken, results)
|
|
903
|
+
finally:
|
|
904
|
+
if lock is not None:
|
|
905
|
+
_release_lock(lock)
|
|
906
|
+
|
|
907
|
+
|
|
908
|
+
def _drain(files, record, opts, client, taken, results): # noqa: ANN001, ANN202
|
|
748
909
|
if opts.concurrency <= 1:
|
|
749
910
|
for file in files:
|
|
750
911
|
try:
|
|
@@ -767,9 +928,10 @@ def _fetch_record(
|
|
|
767
928
|
window = max(opts.concurrency * 2, 1)
|
|
768
929
|
queue = iter(files)
|
|
769
930
|
futures = {}
|
|
931
|
+
name_lock = threading.Lock()
|
|
770
932
|
|
|
771
933
|
def submit(file: File) -> None: # noqa: ANN001
|
|
772
|
-
with
|
|
934
|
+
with name_lock:
|
|
773
935
|
futures[pool.submit(fetch_file, client, file, opts, taken)] = file
|
|
774
936
|
|
|
775
937
|
try:
|
|
@@ -119,10 +119,10 @@ def _entry( # noqa: ANN001
|
|
|
119
119
|
"http_status": result.http_status if result else None,
|
|
120
120
|
"retrieved_at": result.retrieved_at if result else None,
|
|
121
121
|
# PR-2 adds final_url / etag / http_status / retrieved_at here.
|
|
122
|
-
"verified_reason": (
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
122
|
+
"verified_reason": verify.reason(
|
|
123
|
+
f.checksum,
|
|
124
|
+
result.verified if result else None,
|
|
125
|
+
result.state if result else None,
|
|
126
126
|
),
|
|
127
127
|
}
|
|
128
128
|
if state == "not-selected":
|