acadfetch 0.6.3__tar.gz → 0.6.4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {acadfetch-0.6.3/acadfetch.egg-info → acadfetch-0.6.4}/PKG-INFO +21 -8
- {acadfetch-0.6.3 → acadfetch-0.6.4}/README.md +20 -7
- {acadfetch-0.6.3 → acadfetch-0.6.4}/acadfetch/cli.py +15 -2
- {acadfetch-0.6.3 → acadfetch-0.6.4}/acadfetch/consts.py +1 -1
- {acadfetch-0.6.3 → acadfetch-0.6.4}/acadfetch/downloader.py +132 -57
- {acadfetch-0.6.3 → acadfetch-0.6.4}/acadfetch/manifest.py +4 -4
- {acadfetch-0.6.3 → acadfetch-0.6.4}/acadfetch/progress.py +62 -19
- {acadfetch-0.6.3 → acadfetch-0.6.4}/acadfetch/remote.py +5 -0
- {acadfetch-0.6.3 → acadfetch-0.6.4}/acadfetch/verify.py +12 -3
- {acadfetch-0.6.3 → acadfetch-0.6.4/acadfetch.egg-info}/PKG-INFO +21 -8
- {acadfetch-0.6.3 → acadfetch-0.6.4}/acadfetch.egg-info/SOURCES.txt +1 -0
- {acadfetch-0.6.3 → acadfetch-0.6.4}/pyproject.toml +1 -1
- acadfetch-0.6.4/tests/test_claims.py +174 -0
- {acadfetch-0.6.3 → acadfetch-0.6.4}/tests/test_cli.py +9 -0
- {acadfetch-0.6.3 → acadfetch-0.6.4}/tests/test_downloader.py +74 -0
- {acadfetch-0.6.3 → acadfetch-0.6.4}/tests/test_json.py +19 -0
- {acadfetch-0.6.3 → acadfetch-0.6.4}/tests/test_progress.py +30 -2
- {acadfetch-0.6.3 → acadfetch-0.6.4}/tests/test_remote.py +26 -0
- {acadfetch-0.6.3 → acadfetch-0.6.4}/LICENSE +0 -0
- {acadfetch-0.6.3 → acadfetch-0.6.4}/MANIFEST.in +0 -0
- {acadfetch-0.6.3 → acadfetch-0.6.4}/acadfetch/__init__.py +0 -0
- {acadfetch-0.6.3 → acadfetch-0.6.4}/acadfetch/__main__.py +0 -0
- {acadfetch-0.6.3 → acadfetch-0.6.4}/acadfetch/atomic.py +0 -0
- {acadfetch-0.6.3 → acadfetch-0.6.4}/acadfetch/auth.py +0 -0
- {acadfetch-0.6.3 → acadfetch-0.6.4}/acadfetch/client.py +0 -0
- {acadfetch-0.6.3 → acadfetch-0.6.4}/acadfetch/debug.py +0 -0
- {acadfetch-0.6.3 → acadfetch-0.6.4}/acadfetch/errors.py +0 -0
- {acadfetch-0.6.3 → acadfetch-0.6.4}/acadfetch/naming.py +0 -0
- {acadfetch-0.6.3 → acadfetch-0.6.4}/acadfetch/py.typed +0 -0
- {acadfetch-0.6.3 → acadfetch-0.6.4}/acadfetch/ratelimit.py +0 -0
- {acadfetch-0.6.3 → acadfetch-0.6.4}/acadfetch/sources/__init__.py +0 -0
- {acadfetch-0.6.3 → acadfetch-0.6.4}/acadfetch/sources/arxiv.py +0 -0
- {acadfetch-0.6.3 → acadfetch-0.6.4}/acadfetch/sources/base.py +0 -0
- {acadfetch-0.6.3 → acadfetch-0.6.4}/acadfetch/sources/direct.py +0 -0
- {acadfetch-0.6.3 → acadfetch-0.6.4}/acadfetch/sources/doi.py +0 -0
- {acadfetch-0.6.3 → acadfetch-0.6.4}/acadfetch/sources/registry.py +0 -0
- {acadfetch-0.6.3 → acadfetch-0.6.4}/acadfetch/sources/removed.py +0 -0
- {acadfetch-0.6.3 → acadfetch-0.6.4}/acadfetch/sources/zenodo.py +0 -0
- {acadfetch-0.6.3 → acadfetch-0.6.4}/acadfetch/units.py +0 -0
- {acadfetch-0.6.3 → acadfetch-0.6.4}/acadfetch/urls.py +0 -0
- {acadfetch-0.6.3 → acadfetch-0.6.4}/acadfetch.egg-info/dependency_links.txt +0 -0
- {acadfetch-0.6.3 → acadfetch-0.6.4}/acadfetch.egg-info/entry_points.txt +0 -0
- {acadfetch-0.6.3 → acadfetch-0.6.4}/acadfetch.egg-info/top_level.txt +0 -0
- {acadfetch-0.6.3 → acadfetch-0.6.4}/setup.cfg +0 -0
- {acadfetch-0.6.3 → acadfetch-0.6.4}/tests/conftest.py +0 -0
- {acadfetch-0.6.3 → acadfetch-0.6.4}/tests/fixtures/README.md +0 -0
- {acadfetch-0.6.3 → acadfetch-0.6.4}/tests/fixtures/ca.pem +0 -0
- {acadfetch-0.6.3 → acadfetch-0.6.4}/tests/fixtures/figshare_403_block.html +0 -0
- {acadfetch-0.6.3 → acadfetch-0.6.4}/tests/fixtures/figshare_403_forbidden.json +0 -0
- {acadfetch-0.6.3 → acadfetch-0.6.4}/tests/fixtures/figshare_article.json +0 -0
- {acadfetch-0.6.3 → acadfetch-0.6.4}/tests/fixtures/figshare_file.pdf +0 -0
- {acadfetch-0.6.3 → acadfetch-0.6.4}/tests/test_atomic.py +0 -0
- {acadfetch-0.6.3 → acadfetch-0.6.4}/tests/test_auth.py +0 -0
- {acadfetch-0.6.3 → acadfetch-0.6.4}/tests/test_checksum.py +0 -0
- {acadfetch-0.6.3 → acadfetch-0.6.4}/tests/test_citation.py +0 -0
- {acadfetch-0.6.3 → acadfetch-0.6.4}/tests/test_client.py +0 -0
- {acadfetch-0.6.3 → acadfetch-0.6.4}/tests/test_control_names.py +0 -0
- {acadfetch-0.6.3 → acadfetch-0.6.4}/tests/test_debug.py +0 -0
- {acadfetch-0.6.3 → acadfetch-0.6.4}/tests/test_disk.py +0 -0
- {acadfetch-0.6.3 → acadfetch-0.6.4}/tests/test_hash_once.py +0 -0
- {acadfetch-0.6.3 → acadfetch-0.6.4}/tests/test_identity.py +0 -0
- {acadfetch-0.6.3 → acadfetch-0.6.4}/tests/test_isolation.py +0 -0
- {acadfetch-0.6.3 → acadfetch-0.6.4}/tests/test_lock.py +0 -0
- {acadfetch-0.6.3 → acadfetch-0.6.4}/tests/test_manifest.py +0 -0
- {acadfetch-0.6.3 → acadfetch-0.6.4}/tests/test_naming.py +0 -0
- {acadfetch-0.6.3 → acadfetch-0.6.4}/tests/test_naming_race.py +0 -0
- {acadfetch-0.6.3 → acadfetch-0.6.4}/tests/test_network.py +0 -0
- {acadfetch-0.6.3 → acadfetch-0.6.4}/tests/test_packaging.py +0 -0
- {acadfetch-0.6.3 → acadfetch-0.6.4}/tests/test_ratelimit.py +0 -0
- {acadfetch-0.6.3 → acadfetch-0.6.4}/tests/test_redirect.py +0 -0
- {acadfetch-0.6.3 → acadfetch-0.6.4}/tests/test_removed.py +0 -0
- {acadfetch-0.6.3 → acadfetch-0.6.4}/tests/test_sources.py +0 -0
- {acadfetch-0.6.3 → acadfetch-0.6.4}/tests/test_urls.py +0 -0
- {acadfetch-0.6.3 → acadfetch-0.6.4}/tests/test_urls_glob.py +0 -0
- {acadfetch-0.6.3 → acadfetch-0.6.4}/tests/test_verify.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: acadfetch
|
|
3
|
-
Version: 0.6.
|
|
3
|
+
Version: 0.6.4
|
|
4
4
|
Summary: Resumable, rate-limited, checksum-verified bulk downloads from academic data repositories
|
|
5
5
|
Author: young-ski
|
|
6
6
|
License-Expression: MIT
|
|
@@ -104,7 +104,7 @@ python -m pip install -e . pytest mypy && python -m pytest -q
|
|
|
104
104
|
|
|
105
105
|
## Usage
|
|
106
106
|
|
|
107
|
-
|
|
107
|
+
One target per run:
|
|
108
108
|
|
|
109
109
|
```bash
|
|
110
110
|
acadfetch fetch 10.5281/zenodo.23612969 # a DOI
|
|
@@ -112,6 +112,14 @@ acadfetch fetch arxiv:1706.03762 # an arXiv id
|
|
|
112
112
|
acadfetch fetch https://ndownloader.figshare.com/files/70600233/report.pdf # any URL
|
|
113
113
|
```
|
|
114
114
|
|
|
115
|
+
The first two are answered by Zenodo and arXiv and work from any network those
|
|
116
|
+
are reachable on. The third is a direct URL, so the host decides: figshare's CDN
|
|
117
|
+
answers 403 with a WAF block page from some networks — the same behaviour that
|
|
118
|
+
made its *adapter* not worth keeping in 0.2.0. acadfetch says so and keeps
|
|
119
|
+
nothing, rather than saving the block page as a PDF. If you hit that, the file
|
|
120
|
+
itself is fine; see the `403` section of
|
|
121
|
+
[docs/troubleshooting.md](docs/troubleshooting.md).
|
|
122
|
+
|
|
115
123
|
figshare was removed in 0.2.0: pass the article's download URL instead, and
|
|
116
124
|
[`--checksum`](docs/migration-0.2.md) keeps the verification the API used to
|
|
117
125
|
give you.
|
|
@@ -123,10 +131,13 @@ acadfetch fetch 10.5281/zenodo.23612969 -n # dry run: which dir, which files
|
|
|
123
131
|
acadfetch fetch 10.5281/zenodo.23612969 --only raw.tar.gz
|
|
124
132
|
acadfetch fetch 10.5281/zenodo.23612969 --max-size 2GB # refuse anything bigger
|
|
125
133
|
acadfetch fetch 10.5281/zenodo.23612969 -j 1 --rate 0.5 # be a good citizen
|
|
134
|
+
acadfetch fetch 10.5281/zenodo.23612969 --retries 2 --retry-403 0 # give up sooner
|
|
135
|
+
acadfetch fetch 10.5281/zenodo.23612969 --manifest-out build/m.json # a second copy
|
|
126
136
|
acadfetch manifest 10.5281/zenodo.23612969 -o m.json # metadata only, no bytes
|
|
127
137
|
acadfetch fetch zenodo:123456 --header "Authorization: Bearer $TOKEN" # restricted
|
|
128
138
|
|
|
129
139
|
acadfetch verify ./data/manifest.json # re-hash what you downloaded
|
|
140
|
+
acadfetch --version # what is installed
|
|
130
141
|
acadfetch sources # adapters, with an example each
|
|
131
142
|
```
|
|
132
143
|
|
|
@@ -164,16 +175,18 @@ block page is retried; see [docs/troubleshooting.md](docs/troubleshooting.md).
|
|
|
164
175
|
Progress is one aggregate line for the whole run
|
|
165
176
|
(`3 file(s) 12.4/44.1 MiB 1.8 MiB/s ETA 0:17`), redrawn in place on a TTY.
|
|
166
177
|
When stdout is a pipe it becomes a new line at most every 5 s (one extra when a
|
|
167
|
-
file starts or stops), so a CI log stays greppable without becoming a wall.
|
|
168
|
-
turns it off.
|
|
178
|
+
file starts or stops), so a CI log stays greppable without becoming a wall.
|
|
179
|
+
`-q/--quiet` turns it off.
|
|
169
180
|
|
|
170
|
-
|
|
181
|
+
`-o/--out` defaults to `acadfetch-out/<source>-<id>` (a URL adds its host and a short
|
|
171
182
|
hash), and `-n` prints it before
|
|
172
183
|
you spend bandwidth. Files land in `<name>.part` and are renamed only after the
|
|
173
184
|
checksum matches, so an interrupted or failed run never leaves something that
|
|
174
185
|
*looks* finished -- and never throws away the bytes it already paid for: rerun and
|
|
175
|
-
it resumes.
|
|
176
|
-
|
|
186
|
+
it resumes. The one case where a run must start over is a server that ignores
|
|
187
|
+
`Range`; the bytes already on disk are then moved aside and kept until a whole
|
|
188
|
+
new copy has arrived, so a restart that dies early costs nothing. `acadfetch-out/`
|
|
189
|
+
is worth adding to a project's `.gitignore`; this repository already has it.
|
|
177
190
|
|
|
178
191
|
The default directory names the target, not only its filename — `data.csv` from
|
|
179
192
|
two hosts is two datasets, and they do not share a directory. A file already
|
|
@@ -302,7 +315,7 @@ so the trace can be pasted into an issue.
|
|
|
302
315
|
|
|
303
316
|
## Politeness
|
|
304
317
|
|
|
305
|
-
Defaults: 3 parallel transfers, 2 requests/second, 4 retries with jittered
|
|
318
|
+
Defaults: 3 parallel transfers (`-j/--concurrency`), 2 requests/second, 4 retries with jittered
|
|
306
319
|
exponential backoff, `Retry-After` honoured up to 120 s. A block page is retried twice
|
|
307
320
|
(`--retry-403 N`); Zenodo and arXiv are donation-funded —
|
|
308
321
|
`--rate 1 -j 1` costs you minutes and costs them less.
|
|
@@ -73,7 +73,7 @@ python -m pip install -e . pytest mypy && python -m pytest -q
|
|
|
73
73
|
|
|
74
74
|
## Usage
|
|
75
75
|
|
|
76
|
-
|
|
76
|
+
One target per run:
|
|
77
77
|
|
|
78
78
|
```bash
|
|
79
79
|
acadfetch fetch 10.5281/zenodo.23612969 # a DOI
|
|
@@ -81,6 +81,14 @@ acadfetch fetch arxiv:1706.03762 # an arXiv id
|
|
|
81
81
|
acadfetch fetch https://ndownloader.figshare.com/files/70600233/report.pdf # any URL
|
|
82
82
|
```
|
|
83
83
|
|
|
84
|
+
The first two are answered by Zenodo and arXiv and work from any network those
|
|
85
|
+
are reachable on. The third is a direct URL, so the host decides: figshare's CDN
|
|
86
|
+
answers 403 with a WAF block page from some networks — the same behaviour that
|
|
87
|
+
made its *adapter* not worth keeping in 0.2.0. acadfetch says so and keeps
|
|
88
|
+
nothing, rather than saving the block page as a PDF. If you hit that, the file
|
|
89
|
+
itself is fine; see the `403` section of
|
|
90
|
+
[docs/troubleshooting.md](docs/troubleshooting.md).
|
|
91
|
+
|
|
84
92
|
figshare was removed in 0.2.0: pass the article's download URL instead, and
|
|
85
93
|
[`--checksum`](docs/migration-0.2.md) keeps the verification the API used to
|
|
86
94
|
give you.
|
|
@@ -92,10 +100,13 @@ acadfetch fetch 10.5281/zenodo.23612969 -n # dry run: which dir, which files
|
|
|
92
100
|
acadfetch fetch 10.5281/zenodo.23612969 --only raw.tar.gz
|
|
93
101
|
acadfetch fetch 10.5281/zenodo.23612969 --max-size 2GB # refuse anything bigger
|
|
94
102
|
acadfetch fetch 10.5281/zenodo.23612969 -j 1 --rate 0.5 # be a good citizen
|
|
103
|
+
acadfetch fetch 10.5281/zenodo.23612969 --retries 2 --retry-403 0 # give up sooner
|
|
104
|
+
acadfetch fetch 10.5281/zenodo.23612969 --manifest-out build/m.json # a second copy
|
|
95
105
|
acadfetch manifest 10.5281/zenodo.23612969 -o m.json # metadata only, no bytes
|
|
96
106
|
acadfetch fetch zenodo:123456 --header "Authorization: Bearer $TOKEN" # restricted
|
|
97
107
|
|
|
98
108
|
acadfetch verify ./data/manifest.json # re-hash what you downloaded
|
|
109
|
+
acadfetch --version # what is installed
|
|
99
110
|
acadfetch sources # adapters, with an example each
|
|
100
111
|
```
|
|
101
112
|
|
|
@@ -133,16 +144,18 @@ block page is retried; see [docs/troubleshooting.md](docs/troubleshooting.md).
|
|
|
133
144
|
Progress is one aggregate line for the whole run
|
|
134
145
|
(`3 file(s) 12.4/44.1 MiB 1.8 MiB/s ETA 0:17`), redrawn in place on a TTY.
|
|
135
146
|
When stdout is a pipe it becomes a new line at most every 5 s (one extra when a
|
|
136
|
-
file starts or stops), so a CI log stays greppable without becoming a wall.
|
|
137
|
-
turns it off.
|
|
147
|
+
file starts or stops), so a CI log stays greppable without becoming a wall.
|
|
148
|
+
`-q/--quiet` turns it off.
|
|
138
149
|
|
|
139
|
-
|
|
150
|
+
`-o/--out` defaults to `acadfetch-out/<source>-<id>` (a URL adds its host and a short
|
|
140
151
|
hash), and `-n` prints it before
|
|
141
152
|
you spend bandwidth. Files land in `<name>.part` and are renamed only after the
|
|
142
153
|
checksum matches, so an interrupted or failed run never leaves something that
|
|
143
154
|
*looks* finished -- and never throws away the bytes it already paid for: rerun and
|
|
144
|
-
it resumes.
|
|
145
|
-
|
|
155
|
+
it resumes. The one case where a run must start over is a server that ignores
|
|
156
|
+
`Range`; the bytes already on disk are then moved aside and kept until a whole
|
|
157
|
+
new copy has arrived, so a restart that dies early costs nothing. `acadfetch-out/`
|
|
158
|
+
is worth adding to a project's `.gitignore`; this repository already has it.
|
|
146
159
|
|
|
147
160
|
The default directory names the target, not only its filename — `data.csv` from
|
|
148
161
|
two hosts is two datasets, and they do not share a directory. A file already
|
|
@@ -271,7 +284,7 @@ so the trace can be pasted into an issue.
|
|
|
271
284
|
|
|
272
285
|
## Politeness
|
|
273
286
|
|
|
274
|
-
Defaults: 3 parallel transfers, 2 requests/second, 4 retries with jittered
|
|
287
|
+
Defaults: 3 parallel transfers (`-j/--concurrency`), 2 requests/second, 4 retries with jittered
|
|
275
288
|
exponential backoff, `Retry-After` honoured up to 120 s. A block page is retried twice
|
|
276
289
|
(`--retry-403 N`); Zenodo and arXiv are donation-funded —
|
|
277
290
|
`--rate 1 -j 1` costs you minutes and costs them less.
|
|
@@ -256,7 +256,7 @@ def _file_row(row, redact: bool = True) -> dict: # noqa: ANN001
|
|
|
256
256
|
"http_status": row.http_status,
|
|
257
257
|
"final_url": hide(row.final_url) if row.final_url else None,
|
|
258
258
|
"verified": row.verified,
|
|
259
|
-
"verified_reason": verify_reason(row.file.checksum, row.verified),
|
|
259
|
+
"verified_reason": verify_reason(row.file.checksum, row.verified, row.state),
|
|
260
260
|
"resumed_from": row.resumed_from,
|
|
261
261
|
"part": Path(row.part).name if row.part else None,
|
|
262
262
|
"error": row.error,
|
|
@@ -427,7 +427,10 @@ def cmd_fetch(args: argparse.Namespace) -> int:
|
|
|
427
427
|
+ ("" if disk["ok"] else " NOT ENOUGH")
|
|
428
428
|
)
|
|
429
429
|
for file in chosen:
|
|
430
|
-
|
|
430
|
+
# "?" in a size column reads as a bug; arXiv genuinely publishes no
|
|
431
|
+
# size, so say that instead of printing a question mark.
|
|
432
|
+
size = human(file.size) if file.size is not None else "size not published"
|
|
433
|
+
print(f" would fetch {naming.sanitize(file.name)} ({size})")
|
|
431
434
|
return EXIT_OK
|
|
432
435
|
|
|
433
436
|
try:
|
|
@@ -519,6 +522,16 @@ def cmd_verify(args: argparse.Namespace) -> int:
|
|
|
519
522
|
rows = manifest.verify_manifest(manifest_path, root, allow_escape=args.allow_escape, data=data)
|
|
520
523
|
remote: dict[str, tuple[str, str]] = {}
|
|
521
524
|
if args.against_remote:
|
|
525
|
+
if data.get("url_redacted"):
|
|
526
|
+
# A presigned URL is stripped on the way into the manifest, so the
|
|
527
|
+
# recorded URL cannot be resolved again. Say why instead of leaving
|
|
528
|
+
# a page of remote_unavailable to be puzzled over.
|
|
529
|
+
print(
|
|
530
|
+
"note: this manifest had credential query parameters stripped, so a "
|
|
531
|
+
"row whose URL cannot be resolved again reports remote_unavailable "
|
|
532
|
+
"(--record-full-url keeps the original)",
|
|
533
|
+
file=sys.stderr,
|
|
534
|
+
)
|
|
522
535
|
published_now, reason = remote_check.published(_client(args), data)
|
|
523
536
|
entries = [e for e in data.get("files") or [] if isinstance(e, dict)]
|
|
524
537
|
# verify_manifest walks the same list in order; the rows do not carry
|
|
@@ -153,9 +153,29 @@ def _kept_bytes(part: Path) -> int:
|
|
|
153
153
|
return 0
|
|
154
154
|
|
|
155
155
|
|
|
156
|
-
def
|
|
156
|
+
def _room_for(part: Path, need: int) -> bool:
|
|
157
|
+
"""Whether keeping the old bytes while a restarting copy runs still fits."""
|
|
158
|
+
try:
|
|
159
|
+
return shutil.disk_usage(part.parent).free >= need
|
|
160
|
+
except OSError:
|
|
161
|
+
return False
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
def _drop(*paths: Path) -> bool:
|
|
165
|
+
"""Remove files, and say whether they are all gone.
|
|
166
|
+
|
|
167
|
+
``atomic.unlink`` swallows a sharing violation on purpose: losing a race with
|
|
168
|
+
a reader is usually harmless, and a cleanup step must not become a new way to
|
|
169
|
+
fail. Where the deletion is load-bearing -- a mismatched part that must not
|
|
170
|
+
be resumed onto, a part that is already over ``--max-size`` -- the caller has
|
|
171
|
+
to know it did not happen, or the next attempt silently resumes onto bytes it
|
|
172
|
+
was told to forget.
|
|
173
|
+
"""
|
|
174
|
+
kept = False
|
|
157
175
|
for path in paths:
|
|
158
176
|
atomic.unlink(path)
|
|
177
|
+
kept = kept or path.exists()
|
|
178
|
+
return not kept
|
|
159
179
|
|
|
160
180
|
|
|
161
181
|
def _content_length(headers, url: str) -> int | None:
|
|
@@ -340,6 +360,17 @@ def _transfer(client: Client, file: File, part: Path, meta_path: Path, opts: Opt
|
|
|
340
360
|
else:
|
|
341
361
|
mode, offset = "ab", have
|
|
342
362
|
|
|
363
|
+
# A 200 to a Range request means the server ignored it, so this attempt
|
|
364
|
+
# restarts from zero. The bytes already on disk are still the best copy
|
|
365
|
+
# of the file there is until a new one completes, so a restart is written
|
|
366
|
+
# beside them and swapped in only once it is whole: opening the part with
|
|
367
|
+
# "wb" threw away 6 MiB in order to save 1, which is the opposite of what
|
|
368
|
+
# "never throws away the bytes it already paid for" promises. Without
|
|
369
|
+
# room for both copies, the old behaviour returns.
|
|
370
|
+
sink, aside = part, None
|
|
371
|
+
if whole and have and _room_for(part, have):
|
|
372
|
+
aside = sink = part.with_name(part.name + ".new")
|
|
373
|
+
|
|
343
374
|
# Hash while the bytes go by instead of reading the finished file again.
|
|
344
375
|
# A resumed transfer has to be fed what is already on disk first.
|
|
345
376
|
parsed = verify.parse(file.checksum)
|
|
@@ -352,58 +383,73 @@ def _transfer(client: Client, file: File, part: Path, meta_path: Path, opts: Opt
|
|
|
352
383
|
for old in iter(lambda: existing.read(1 << 20), b""):
|
|
353
384
|
for hasher in hashers.values():
|
|
354
385
|
hasher.update(old)
|
|
355
|
-
_write_meta(
|
|
356
|
-
meta_path,
|
|
357
|
-
{
|
|
358
|
-
"url": file.url,
|
|
359
|
-
"etag": resp.headers.get("ETag"),
|
|
360
|
-
"last_modified": resp.headers.get("Last-Modified"),
|
|
361
|
-
"size": file.size,
|
|
362
|
-
},
|
|
363
|
-
)
|
|
364
|
-
if declared is not None and file.size is not None:
|
|
365
|
-
expected = file.size - offset
|
|
366
|
-
if declared != expected:
|
|
367
|
-
raise HttpError(
|
|
368
|
-
0,
|
|
369
|
-
file.url,
|
|
370
|
-
f"Content-Length {declared} disagrees with the published size "
|
|
371
|
-
f"({expected} bytes still expected)",
|
|
372
|
-
)
|
|
373
|
-
# Reads poll with a short socket timeout so an interrupt is noticed
|
|
374
|
-
# within a second instead of after the whole 60 s socket timeout, and
|
|
375
|
-
# so an idle connection fails on its own budget. Closing the response
|
|
376
|
-
# from another thread additionally kicks a read that is already
|
|
377
|
-
# blocked.
|
|
378
|
-
with _LIVE_LOCK:
|
|
379
|
-
_LIVE.add(resp)
|
|
380
386
|
try:
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
387
|
+
_write_meta(
|
|
388
|
+
meta_path,
|
|
389
|
+
{
|
|
390
|
+
"url": file.url,
|
|
391
|
+
"etag": resp.headers.get("ETag"),
|
|
392
|
+
"last_modified": resp.headers.get("Last-Modified"),
|
|
393
|
+
"size": file.size,
|
|
394
|
+
},
|
|
395
|
+
)
|
|
396
|
+
if declared is not None and file.size is not None:
|
|
397
|
+
expected = file.size - offset
|
|
398
|
+
if declared != expected:
|
|
399
|
+
raise HttpError(
|
|
400
|
+
0,
|
|
401
|
+
file.url,
|
|
402
|
+
f"Content-Length {declared} disagrees with the published size "
|
|
403
|
+
f"({expected} bytes still expected)",
|
|
404
|
+
)
|
|
405
|
+
# Reads poll with a short socket timeout so an interrupt is noticed
|
|
406
|
+
# within a second instead of after the whole 60 s socket timeout, and
|
|
407
|
+
# so an idle connection fails on its own budget. Closing the response
|
|
408
|
+
# from another thread additionally kicks a read that is already
|
|
409
|
+
# blocked.
|
|
403
410
|
with _LIVE_LOCK:
|
|
404
|
-
_LIVE.
|
|
405
|
-
|
|
406
|
-
|
|
411
|
+
_LIVE.add(resp)
|
|
412
|
+
try:
|
|
413
|
+
with sink.open(mode) as fh:
|
|
414
|
+
for chunk in _chunks(resp, client.timeout, file.url):
|
|
415
|
+
fh.write(chunk)
|
|
416
|
+
for hasher in hashers.values():
|
|
417
|
+
hasher.update(chunk)
|
|
418
|
+
written += len(chunk)
|
|
419
|
+
if opts.progress:
|
|
420
|
+
opts.progress.advance(len(chunk))
|
|
421
|
+
if opts.max_bytes and offset + written > opts.max_bytes:
|
|
422
|
+
raise HttpError(
|
|
423
|
+
0,
|
|
424
|
+
file.url,
|
|
425
|
+
f"refused locally: over --max-size ({human(opts.max_bytes)})",
|
|
426
|
+
reason="max-size",
|
|
427
|
+
)
|
|
428
|
+
if opts.fsync:
|
|
429
|
+
# The rename below is atomic, but only against crashes that
|
|
430
|
+
# leave the page cache intact. This is what makes the bytes
|
|
431
|
+
# survive a power cut before they get a name.
|
|
432
|
+
fh.flush()
|
|
433
|
+
os.fsync(fh.fileno())
|
|
434
|
+
finally:
|
|
435
|
+
with _LIVE_LOCK:
|
|
436
|
+
_LIVE.discard(resp)
|
|
437
|
+
if declared is not None and declared != written:
|
|
438
|
+
raise HttpError(0, file.url, f"read {written} bytes, response declared {declared}")
|
|
439
|
+
if aside is not None:
|
|
440
|
+
# A whole body, agreeing with the declared length: only now does
|
|
441
|
+
# the restart replace what was already there.
|
|
442
|
+
atomic.replace(sink, part)
|
|
443
|
+
except BaseException:
|
|
444
|
+
if aside is not None:
|
|
445
|
+
# Keep the bytes that were there, and the sidecar that described
|
|
446
|
+
# them rather than this attempt's validator.
|
|
447
|
+
atomic.unlink(aside)
|
|
448
|
+
if meta:
|
|
449
|
+
_write_meta(meta_path, meta)
|
|
450
|
+
else:
|
|
451
|
+
atomic.unlink(meta_path)
|
|
452
|
+
raise
|
|
407
453
|
return Transfer(
|
|
408
454
|
written=written,
|
|
409
455
|
final_url=resp.geturl() or file.url,
|
|
@@ -560,6 +606,8 @@ def _reserved(files) -> set[str]: # noqa: ANN001
|
|
|
560
606
|
for file in files:
|
|
561
607
|
name = naming.sanitize(file.name).lower()
|
|
562
608
|
reserved.add(name + ".part")
|
|
609
|
+
# A restart beside a kept part (a server that ignored Range) writes here.
|
|
610
|
+
reserved.add(name + ".part.new")
|
|
563
611
|
reserved.add(name + META_SUFFIX)
|
|
564
612
|
return reserved
|
|
565
613
|
|
|
@@ -675,12 +723,19 @@ def _attempt(
|
|
|
675
723
|
_drop(part, meta_path)
|
|
676
724
|
if exc.reason == "max-size":
|
|
677
725
|
# Do not leave an over-limit part for the next run to
|
|
678
|
-
# resume straight back into the same wall.
|
|
679
|
-
|
|
726
|
+
# resume straight back into the same wall. Saying so when the
|
|
727
|
+
# deletion fails is the difference between a rerun that works
|
|
728
|
+
# and one that fails the same way with no explanation.
|
|
729
|
+
kept = not _drop(part, meta_path)
|
|
680
730
|
return Result(
|
|
681
731
|
file=file,
|
|
682
732
|
state="failed",
|
|
683
|
-
error=str(exc)
|
|
733
|
+
error=str(exc)
|
|
734
|
+
+ (
|
|
735
|
+
f" (the over-limit {part.name} could not be deleted; remove it by hand)"
|
|
736
|
+
if kept
|
|
737
|
+
else ""
|
|
738
|
+
),
|
|
684
739
|
http_status=exc.status,
|
|
685
740
|
retrieved_at=_now(),
|
|
686
741
|
)
|
|
@@ -695,7 +750,12 @@ def _attempt(
|
|
|
695
750
|
return Result(file=file, state="failed", error="interrupted")
|
|
696
751
|
if attempt < opts.retries:
|
|
697
752
|
time.sleep(min(2**attempt, 10) * (0.7 + random.random() * 0.6))
|
|
698
|
-
|
|
753
|
+
# ``_transfer`` re-reads the part size itself, so the offset this
|
|
754
|
+
# attempt resumes from needs no bookkeeping here. ``resumed_from``
|
|
755
|
+
# is deliberately *not* updated: it answers "how much was on disk
|
|
756
|
+
# when this run started", which a retry does not change. Rewriting
|
|
757
|
+
# it made a fresh download report ``(resumed at 1.6 MiB)``. See
|
|
758
|
+
# docs/json-report.md.
|
|
699
759
|
else:
|
|
700
760
|
assert last is not None # the loop only ends after a failure
|
|
701
761
|
return Result(
|
|
@@ -712,7 +772,10 @@ def _attempt(
|
|
|
712
772
|
return Result(
|
|
713
773
|
file=file,
|
|
714
774
|
state="failed",
|
|
715
|
-
|
|
775
|
+
# A terminal 403 breaks out of the loop above, so it lands here
|
|
776
|
+
# rather than on the retry-exhausted path below. Without the hint
|
|
777
|
+
# that made it the one error a user could not act on.
|
|
778
|
+
error=f"{last}{_hint(last)}",
|
|
716
779
|
http_status=getattr(last, "status", None),
|
|
717
780
|
retrieved_at=_now(),
|
|
718
781
|
)
|
|
@@ -729,7 +792,19 @@ def _attempt(
|
|
|
729
792
|
|
|
730
793
|
verdict = _verdict(file, info.digests if info else None)
|
|
731
794
|
if verdict is False:
|
|
732
|
-
_drop(part, meta_path)
|
|
795
|
+
if not _drop(part, meta_path):
|
|
796
|
+
# Re-downloading would splice onto the bytes just proven wrong.
|
|
797
|
+
return Result(
|
|
798
|
+
file=file,
|
|
799
|
+
state="failed",
|
|
800
|
+
error=(
|
|
801
|
+
f"checksum mismatch, and the bad {part.name} could not be deleted; "
|
|
802
|
+
f"close whatever is reading it and retry"
|
|
803
|
+
),
|
|
804
|
+
verified=False,
|
|
805
|
+
bytes=_kept_bytes(part),
|
|
806
|
+
retrieved_at=_now(),
|
|
807
|
+
)
|
|
733
808
|
try:
|
|
734
809
|
info = _transfer(client, file, part, meta_path, opts)
|
|
735
810
|
verdict = _verdict(file, info.digests if info else None)
|
|
@@ -119,10 +119,10 @@ def _entry( # noqa: ANN001
|
|
|
119
119
|
"http_status": result.http_status if result else None,
|
|
120
120
|
"retrieved_at": result.retrieved_at if result else None,
|
|
121
121
|
# PR-2 adds final_url / etag / http_status / retrieved_at here.
|
|
122
|
-
"verified_reason": (
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
122
|
+
"verified_reason": verify.reason(
|
|
123
|
+
f.checksum,
|
|
124
|
+
result.verified if result else None,
|
|
125
|
+
result.state if result else None,
|
|
126
126
|
),
|
|
127
127
|
}
|
|
128
128
|
if state == "not-selected":
|
|
@@ -8,24 +8,50 @@ whole lines so the log stays greppable.
|
|
|
8
8
|
|
|
9
9
|
from __future__ import annotations
|
|
10
10
|
|
|
11
|
+
import math
|
|
12
|
+
import shutil
|
|
11
13
|
import sys
|
|
12
14
|
import threading
|
|
13
15
|
import time
|
|
14
|
-
|
|
16
|
+
import unicodedata
|
|
15
17
|
from collections.abc import Callable
|
|
16
18
|
from typing import TextIO
|
|
17
19
|
|
|
18
20
|
from .units import duration, human
|
|
19
21
|
|
|
20
|
-
|
|
22
|
+
#: Time constant of the speed estimate. Long enough that one slow chunk does not
|
|
23
|
+
#: move the ETA by minutes, short enough to follow a link that really did change.
|
|
24
|
+
SMOOTHING_SECONDS = 5.0
|
|
21
25
|
TTY_INTERVAL = 0.15
|
|
22
26
|
PIPE_INTERVAL = 5.0
|
|
23
27
|
MAX_WIDTH = 120
|
|
28
|
+
#: Below this a two-part line has no room to say anything useful.
|
|
29
|
+
MIN_WIDTH = 40
|
|
24
30
|
NAME_WIDTH = 40
|
|
25
31
|
|
|
26
32
|
|
|
27
|
-
def
|
|
28
|
-
return
|
|
33
|
+
def _char_width(char: str) -> int:
|
|
34
|
+
return 2 if unicodedata.east_asian_width(char) in ("W", "F") else 1
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def _width(text: str) -> int:
|
|
38
|
+
"""Terminal columns a string occupies; CJK glyphs are two, not one."""
|
|
39
|
+
return sum(_char_width(char) for char in text)
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def _clip(text: str, limit: int) -> str:
|
|
43
|
+
"""Cut to ``limit`` columns, counting wide characters, and mark the cut."""
|
|
44
|
+
if _width(text) <= limit:
|
|
45
|
+
return text
|
|
46
|
+
kept = []
|
|
47
|
+
used = 0
|
|
48
|
+
for char in text:
|
|
49
|
+
step = _char_width(char)
|
|
50
|
+
if used + step > limit - 1:
|
|
51
|
+
break
|
|
52
|
+
kept.append(char)
|
|
53
|
+
used += step
|
|
54
|
+
return "".join(kept) + "…"
|
|
29
55
|
|
|
30
56
|
|
|
31
57
|
class Progress:
|
|
@@ -35,16 +61,29 @@ class Progress:
|
|
|
35
61
|
enabled: bool = True,
|
|
36
62
|
is_tty: bool | None = None,
|
|
37
63
|
clock: Callable[[], float] = time.monotonic,
|
|
64
|
+
width: int | None = None,
|
|
38
65
|
) -> None:
|
|
39
66
|
self.stream = stream if stream is not None else sys.stdout
|
|
40
67
|
self.enabled = enabled
|
|
41
68
|
self.is_tty = (self.stream.isatty() if is_tty is None else is_tty) and enabled
|
|
42
69
|
self._clock = clock
|
|
43
70
|
self._lock = threading.RLock() # _render() re-enters under _maybe_emit
|
|
71
|
+
# A wrapped line is worse than a shorter one: the carriage return only
|
|
72
|
+
# clears the last visual row, so the redraw smears. Ask the terminal.
|
|
73
|
+
if width is not None:
|
|
74
|
+
self.width = max(MIN_WIDTH, min(width, MAX_WIDTH))
|
|
75
|
+
elif self.is_tty:
|
|
76
|
+
columns = shutil.get_terminal_size((MAX_WIDTH, 24)).columns
|
|
77
|
+
self.width = max(MIN_WIDTH, min(columns, MAX_WIDTH))
|
|
78
|
+
else:
|
|
79
|
+
# A pipe has no viewport to wrap in; 120 is the readability cap.
|
|
80
|
+
self.width = MAX_WIDTH
|
|
44
81
|
self._active: dict[str, int] = {}
|
|
45
82
|
self._done = 0
|
|
46
83
|
self._total = 0
|
|
47
|
-
self.
|
|
84
|
+
self._rate = 0.0
|
|
85
|
+
self._last_at: float | None = None
|
|
86
|
+
self._last_done = 0
|
|
48
87
|
self._last_emit = 0.0
|
|
49
88
|
self._printed = 0
|
|
50
89
|
self._last_count = 0
|
|
@@ -78,20 +117,25 @@ class Progress:
|
|
|
78
117
|
|
|
79
118
|
# -- rendering ---------------------------------------------------------
|
|
80
119
|
def _sample(self) -> None:
|
|
120
|
+
"""Fold the bytes since the last sample into an exponential average.
|
|
121
|
+
|
|
122
|
+
The old estimate divided the window's byte delta by its time span, so a
|
|
123
|
+
single slow chunk moved the rate by a factor of two and the ETA by
|
|
124
|
+
minutes -- observed jumping from 4:55 to 0:28 inside one run. An average
|
|
125
|
+
with a fixed time constant settles instead.
|
|
126
|
+
"""
|
|
81
127
|
now = self._clock()
|
|
82
|
-
self.
|
|
83
|
-
|
|
84
|
-
|
|
128
|
+
if self._last_at is not None:
|
|
129
|
+
dt = now - self._last_at
|
|
130
|
+
if dt > 0:
|
|
131
|
+
instant = max(0.0, (self._done - self._last_done) / dt)
|
|
132
|
+
alpha = 1.0 - math.exp(-dt / SMOOTHING_SECONDS)
|
|
133
|
+
self._rate += alpha * (instant - self._rate)
|
|
134
|
+
self._last_at, self._last_done = now, self._done
|
|
85
135
|
self._dirty = True
|
|
86
136
|
|
|
87
137
|
def _speed(self) -> float:
|
|
88
|
-
|
|
89
|
-
return 0.0
|
|
90
|
-
(t0, b0), (t1, b1) = self._samples[0], self._samples[-1]
|
|
91
|
-
span = t1 - t0
|
|
92
|
-
if span < 0.2:
|
|
93
|
-
return 0.0
|
|
94
|
-
return max(0.0, (b1 - b0) / span)
|
|
138
|
+
return self._rate
|
|
95
139
|
|
|
96
140
|
def _eta(self, speed: float) -> float | None:
|
|
97
141
|
if not self._total or speed <= 0:
|
|
@@ -116,8 +160,7 @@ class Progress:
|
|
|
116
160
|
eta = self._eta(speed)
|
|
117
161
|
if eta is not None:
|
|
118
162
|
parts.append(f"ETA {duration(eta)}")
|
|
119
|
-
|
|
120
|
-
return line[:MAX_WIDTH]
|
|
163
|
+
return _clip(" ".join(parts), self.width)
|
|
121
164
|
|
|
122
165
|
def _maybe_emit(self, force: bool = False) -> None:
|
|
123
166
|
if not self.enabled:
|
|
@@ -137,9 +180,9 @@ class Progress:
|
|
|
137
180
|
def _write(self, line: str, newline: bool) -> None:
|
|
138
181
|
try:
|
|
139
182
|
if self.is_tty:
|
|
140
|
-
pad = max(0, self._printed -
|
|
183
|
+
pad = max(0, self._printed - _width(line))
|
|
141
184
|
self.stream.write("\r" + line + " " * pad)
|
|
142
|
-
self._printed =
|
|
185
|
+
self._printed = _width(line)
|
|
143
186
|
else:
|
|
144
187
|
self.stream.write(line + ("\n" if newline else ""))
|
|
145
188
|
self.stream.flush()
|
|
@@ -68,6 +68,11 @@ def judge(
|
|
|
68
68
|
here, there = _normalise(local), _normalise(remote)
|
|
69
69
|
if here is None or there is None:
|
|
70
70
|
return NO_CHECKSUM, "a checksum is not parseable"
|
|
71
|
+
if here[0] != there[0]:
|
|
72
|
+
# Comparing an md5 with a sha256 proves nothing either way. Reporting a
|
|
73
|
+
# mismatch here would call a legitimate algorithm change tampering, and
|
|
74
|
+
# this status is the one that makes the whole check worth running.
|
|
75
|
+
return NO_CHECKSUM, f"the source publishes {there[0]} now, the row recorded {here[0]}"
|
|
71
76
|
return (
|
|
72
77
|
(MATCH, "") if here == there else (MISMATCH, f"published {there[0]}:{there[1][:12]}… now")
|
|
73
78
|
)
|
|
@@ -70,20 +70,29 @@ def describe(checksum: str | None) -> str:
|
|
|
70
70
|
return f"{parsed[0]}:{parsed[1][:12]}…"
|
|
71
71
|
|
|
72
72
|
|
|
73
|
-
def reason(checksum: str | None, verdict: bool | None) -> str:
|
|
73
|
+
def reason(checksum: str | None, verdict: bool | None, state: str | None = None) -> str:
|
|
74
74
|
"""A stable, machine-readable note beside ``verified``.
|
|
75
75
|
|
|
76
76
|
``verified: null`` on its own cannot tell "the repository published nothing"
|
|
77
77
|
from "it published something we could not parse", and those are different
|
|
78
78
|
promises to a reader.
|
|
79
|
+
|
|
80
|
+
``state`` adds the one thing a checksum cannot describe: a transfer that
|
|
81
|
+
never produced bytes to check. The manifest and the ``--json`` report both
|
|
82
|
+
call this, so one run cannot describe the same file two ways.
|
|
79
83
|
"""
|
|
84
|
+
if verdict is False:
|
|
85
|
+
# A failed row whose bytes were proven wrong is more useful as
|
|
86
|
+
# ``mismatch`` than as ``transfer-failed``: it names the cause, not only
|
|
87
|
+
# the symptom.
|
|
88
|
+
return "mismatch"
|
|
89
|
+
if state == "failed":
|
|
90
|
+
return "transfer-failed"
|
|
80
91
|
if not checksum:
|
|
81
92
|
return "no-checksum-published"
|
|
82
93
|
parsed = parse(checksum)
|
|
83
94
|
if parsed is None:
|
|
84
95
|
return "unparseable-checksum"
|
|
85
|
-
if verdict is False:
|
|
86
|
-
return "mismatch"
|
|
87
96
|
if verdict is True:
|
|
88
97
|
return f"matched-{parsed[0]}"
|
|
89
98
|
return "unchecked"
|