acadfetch 0.6.3__tar.gz → 0.6.4__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (75) hide show
  1. {acadfetch-0.6.3/acadfetch.egg-info → acadfetch-0.6.4}/PKG-INFO +21 -8
  2. {acadfetch-0.6.3 → acadfetch-0.6.4}/README.md +20 -7
  3. {acadfetch-0.6.3 → acadfetch-0.6.4}/acadfetch/cli.py +15 -2
  4. {acadfetch-0.6.3 → acadfetch-0.6.4}/acadfetch/consts.py +1 -1
  5. {acadfetch-0.6.3 → acadfetch-0.6.4}/acadfetch/downloader.py +132 -57
  6. {acadfetch-0.6.3 → acadfetch-0.6.4}/acadfetch/manifest.py +4 -4
  7. {acadfetch-0.6.3 → acadfetch-0.6.4}/acadfetch/progress.py +62 -19
  8. {acadfetch-0.6.3 → acadfetch-0.6.4}/acadfetch/remote.py +5 -0
  9. {acadfetch-0.6.3 → acadfetch-0.6.4}/acadfetch/verify.py +12 -3
  10. {acadfetch-0.6.3 → acadfetch-0.6.4/acadfetch.egg-info}/PKG-INFO +21 -8
  11. {acadfetch-0.6.3 → acadfetch-0.6.4}/acadfetch.egg-info/SOURCES.txt +1 -0
  12. {acadfetch-0.6.3 → acadfetch-0.6.4}/pyproject.toml +1 -1
  13. acadfetch-0.6.4/tests/test_claims.py +174 -0
  14. {acadfetch-0.6.3 → acadfetch-0.6.4}/tests/test_cli.py +9 -0
  15. {acadfetch-0.6.3 → acadfetch-0.6.4}/tests/test_downloader.py +74 -0
  16. {acadfetch-0.6.3 → acadfetch-0.6.4}/tests/test_json.py +19 -0
  17. {acadfetch-0.6.3 → acadfetch-0.6.4}/tests/test_progress.py +30 -2
  18. {acadfetch-0.6.3 → acadfetch-0.6.4}/tests/test_remote.py +26 -0
  19. {acadfetch-0.6.3 → acadfetch-0.6.4}/LICENSE +0 -0
  20. {acadfetch-0.6.3 → acadfetch-0.6.4}/MANIFEST.in +0 -0
  21. {acadfetch-0.6.3 → acadfetch-0.6.4}/acadfetch/__init__.py +0 -0
  22. {acadfetch-0.6.3 → acadfetch-0.6.4}/acadfetch/__main__.py +0 -0
  23. {acadfetch-0.6.3 → acadfetch-0.6.4}/acadfetch/atomic.py +0 -0
  24. {acadfetch-0.6.3 → acadfetch-0.6.4}/acadfetch/auth.py +0 -0
  25. {acadfetch-0.6.3 → acadfetch-0.6.4}/acadfetch/client.py +0 -0
  26. {acadfetch-0.6.3 → acadfetch-0.6.4}/acadfetch/debug.py +0 -0
  27. {acadfetch-0.6.3 → acadfetch-0.6.4}/acadfetch/errors.py +0 -0
  28. {acadfetch-0.6.3 → acadfetch-0.6.4}/acadfetch/naming.py +0 -0
  29. {acadfetch-0.6.3 → acadfetch-0.6.4}/acadfetch/py.typed +0 -0
  30. {acadfetch-0.6.3 → acadfetch-0.6.4}/acadfetch/ratelimit.py +0 -0
  31. {acadfetch-0.6.3 → acadfetch-0.6.4}/acadfetch/sources/__init__.py +0 -0
  32. {acadfetch-0.6.3 → acadfetch-0.6.4}/acadfetch/sources/arxiv.py +0 -0
  33. {acadfetch-0.6.3 → acadfetch-0.6.4}/acadfetch/sources/base.py +0 -0
  34. {acadfetch-0.6.3 → acadfetch-0.6.4}/acadfetch/sources/direct.py +0 -0
  35. {acadfetch-0.6.3 → acadfetch-0.6.4}/acadfetch/sources/doi.py +0 -0
  36. {acadfetch-0.6.3 → acadfetch-0.6.4}/acadfetch/sources/registry.py +0 -0
  37. {acadfetch-0.6.3 → acadfetch-0.6.4}/acadfetch/sources/removed.py +0 -0
  38. {acadfetch-0.6.3 → acadfetch-0.6.4}/acadfetch/sources/zenodo.py +0 -0
  39. {acadfetch-0.6.3 → acadfetch-0.6.4}/acadfetch/units.py +0 -0
  40. {acadfetch-0.6.3 → acadfetch-0.6.4}/acadfetch/urls.py +0 -0
  41. {acadfetch-0.6.3 → acadfetch-0.6.4}/acadfetch.egg-info/dependency_links.txt +0 -0
  42. {acadfetch-0.6.3 → acadfetch-0.6.4}/acadfetch.egg-info/entry_points.txt +0 -0
  43. {acadfetch-0.6.3 → acadfetch-0.6.4}/acadfetch.egg-info/top_level.txt +0 -0
  44. {acadfetch-0.6.3 → acadfetch-0.6.4}/setup.cfg +0 -0
  45. {acadfetch-0.6.3 → acadfetch-0.6.4}/tests/conftest.py +0 -0
  46. {acadfetch-0.6.3 → acadfetch-0.6.4}/tests/fixtures/README.md +0 -0
  47. {acadfetch-0.6.3 → acadfetch-0.6.4}/tests/fixtures/ca.pem +0 -0
  48. {acadfetch-0.6.3 → acadfetch-0.6.4}/tests/fixtures/figshare_403_block.html +0 -0
  49. {acadfetch-0.6.3 → acadfetch-0.6.4}/tests/fixtures/figshare_403_forbidden.json +0 -0
  50. {acadfetch-0.6.3 → acadfetch-0.6.4}/tests/fixtures/figshare_article.json +0 -0
  51. {acadfetch-0.6.3 → acadfetch-0.6.4}/tests/fixtures/figshare_file.pdf +0 -0
  52. {acadfetch-0.6.3 → acadfetch-0.6.4}/tests/test_atomic.py +0 -0
  53. {acadfetch-0.6.3 → acadfetch-0.6.4}/tests/test_auth.py +0 -0
  54. {acadfetch-0.6.3 → acadfetch-0.6.4}/tests/test_checksum.py +0 -0
  55. {acadfetch-0.6.3 → acadfetch-0.6.4}/tests/test_citation.py +0 -0
  56. {acadfetch-0.6.3 → acadfetch-0.6.4}/tests/test_client.py +0 -0
  57. {acadfetch-0.6.3 → acadfetch-0.6.4}/tests/test_control_names.py +0 -0
  58. {acadfetch-0.6.3 → acadfetch-0.6.4}/tests/test_debug.py +0 -0
  59. {acadfetch-0.6.3 → acadfetch-0.6.4}/tests/test_disk.py +0 -0
  60. {acadfetch-0.6.3 → acadfetch-0.6.4}/tests/test_hash_once.py +0 -0
  61. {acadfetch-0.6.3 → acadfetch-0.6.4}/tests/test_identity.py +0 -0
  62. {acadfetch-0.6.3 → acadfetch-0.6.4}/tests/test_isolation.py +0 -0
  63. {acadfetch-0.6.3 → acadfetch-0.6.4}/tests/test_lock.py +0 -0
  64. {acadfetch-0.6.3 → acadfetch-0.6.4}/tests/test_manifest.py +0 -0
  65. {acadfetch-0.6.3 → acadfetch-0.6.4}/tests/test_naming.py +0 -0
  66. {acadfetch-0.6.3 → acadfetch-0.6.4}/tests/test_naming_race.py +0 -0
  67. {acadfetch-0.6.3 → acadfetch-0.6.4}/tests/test_network.py +0 -0
  68. {acadfetch-0.6.3 → acadfetch-0.6.4}/tests/test_packaging.py +0 -0
  69. {acadfetch-0.6.3 → acadfetch-0.6.4}/tests/test_ratelimit.py +0 -0
  70. {acadfetch-0.6.3 → acadfetch-0.6.4}/tests/test_redirect.py +0 -0
  71. {acadfetch-0.6.3 → acadfetch-0.6.4}/tests/test_removed.py +0 -0
  72. {acadfetch-0.6.3 → acadfetch-0.6.4}/tests/test_sources.py +0 -0
  73. {acadfetch-0.6.3 → acadfetch-0.6.4}/tests/test_urls.py +0 -0
  74. {acadfetch-0.6.3 → acadfetch-0.6.4}/tests/test_urls_glob.py +0 -0
  75. {acadfetch-0.6.3 → acadfetch-0.6.4}/tests/test_verify.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: acadfetch
3
- Version: 0.6.3
3
+ Version: 0.6.4
4
4
  Summary: Resumable, rate-limited, checksum-verified bulk downloads from academic data repositories
5
5
  Author: young-ski
6
6
  License-Expression: MIT
@@ -104,7 +104,7 @@ python -m pip install -e . pytest mypy && python -m pytest -q
104
104
 
105
105
  ## Usage
106
106
 
107
- These work as typed:
107
+ One target per run:
108
108
 
109
109
  ```bash
110
110
  acadfetch fetch 10.5281/zenodo.23612969 # a DOI
@@ -112,6 +112,14 @@ acadfetch fetch arxiv:1706.03762 # an arXiv id
112
112
  acadfetch fetch https://ndownloader.figshare.com/files/70600233/report.pdf # any URL
113
113
  ```
114
114
 
115
+ The first two are answered by Zenodo and arXiv and work from any network those
116
+ are reachable on. The third is a direct URL, so the host decides: figshare's CDN
117
+ answers 403 with a WAF block page from some networks — the same behaviour that
118
+ made its *adapter* not worth keeping in 0.2.0. acadfetch says so and keeps
119
+ nothing, rather than saving the block page as a PDF. If you hit that, the file
120
+ itself is fine; see the `403` section of
121
+ [docs/troubleshooting.md](docs/troubleshooting.md).
122
+
115
123
  figshare was removed in 0.2.0: pass the article's download URL instead, and
116
124
  [`--checksum`](docs/migration-0.2.md) keeps the verification the API used to
117
125
  give you.
@@ -123,10 +131,13 @@ acadfetch fetch 10.5281/zenodo.23612969 -n # dry run: which dir, which files
123
131
  acadfetch fetch 10.5281/zenodo.23612969 --only raw.tar.gz
124
132
  acadfetch fetch 10.5281/zenodo.23612969 --max-size 2GB # refuse anything bigger
125
133
  acadfetch fetch 10.5281/zenodo.23612969 -j 1 --rate 0.5 # be a good citizen
134
+ acadfetch fetch 10.5281/zenodo.23612969 --retries 2 --retry-403 0 # give up sooner
135
+ acadfetch fetch 10.5281/zenodo.23612969 --manifest-out build/m.json # a second copy
126
136
  acadfetch manifest 10.5281/zenodo.23612969 -o m.json # metadata only, no bytes
127
137
  acadfetch fetch zenodo:123456 --header "Authorization: Bearer $TOKEN" # restricted
128
138
 
129
139
  acadfetch verify ./data/manifest.json # re-hash what you downloaded
140
+ acadfetch --version # what is installed
130
141
  acadfetch sources # adapters, with an example each
131
142
  ```
132
143
 
@@ -164,16 +175,18 @@ block page is retried; see [docs/troubleshooting.md](docs/troubleshooting.md).
164
175
  Progress is one aggregate line for the whole run
165
176
  (`3 file(s) 12.4/44.1 MiB 1.8 MiB/s ETA 0:17`), redrawn in place on a TTY.
166
177
  When stdout is a pipe it becomes a new line at most every 5 s (one extra when a
167
- file starts or stops), so a CI log stays greppable without becoming a wall. `-q`
168
- turns it off.
178
+ file starts or stops), so a CI log stays greppable without becoming a wall.
179
+ `-q/--quiet` turns it off.
169
180
 
170
- Output defaults to `acadfetch-out/<source>-<id>` (a URL adds its host and a short
181
+ `-o/--out` defaults to `acadfetch-out/<source>-<id>` (a URL adds its host and a short
171
182
  hash), and `-n` prints it before
172
183
  you spend bandwidth. Files land in `<name>.part` and are renamed only after the
173
184
  checksum matches, so an interrupted or failed run never leaves something that
174
185
  *looks* finished -- and never throws away the bytes it already paid for: rerun and
175
- it resumes. `acadfetch-out/` is worth adding to a project's `.gitignore`; this
176
- repository already has it.
186
+ it resumes. The one case where a run must start over is a server that ignores
187
+ `Range`; the bytes already on disk are then moved aside and kept until a whole
188
+ new copy has arrived, so a restart that dies early costs nothing. `acadfetch-out/`
189
+ is worth adding to a project's `.gitignore`; this repository already has it.
177
190
 
178
191
  The default directory names the target, not only its filename — `data.csv` from
179
192
  two hosts is two datasets, and they do not share a directory. A file already
@@ -302,7 +315,7 @@ so the trace can be pasted into an issue.
302
315
 
303
316
  ## Politeness
304
317
 
305
- Defaults: 3 parallel transfers, 2 requests/second, 4 retries with jittered
318
+ Defaults: 3 parallel transfers (`-j/--concurrency`), 2 requests/second, 4 retries with jittered
306
319
  exponential backoff, `Retry-After` honoured up to 120 s. A block page is retried twice
307
320
  (`--retry-403 N`); Zenodo and arXiv are donation-funded —
308
321
  `--rate 1 -j 1` costs you minutes and costs them less.
@@ -73,7 +73,7 @@ python -m pip install -e . pytest mypy && python -m pytest -q
73
73
 
74
74
  ## Usage
75
75
 
76
- These work as typed:
76
+ One target per run:
77
77
 
78
78
  ```bash
79
79
  acadfetch fetch 10.5281/zenodo.23612969 # a DOI
@@ -81,6 +81,14 @@ acadfetch fetch arxiv:1706.03762 # an arXiv id
81
81
  acadfetch fetch https://ndownloader.figshare.com/files/70600233/report.pdf # any URL
82
82
  ```
83
83
 
84
+ The first two are answered by Zenodo and arXiv and work from any network those
85
+ are reachable on. The third is a direct URL, so the host decides: figshare's CDN
86
+ answers 403 with a WAF block page from some networks — the same behaviour that
87
+ made its *adapter* not worth keeping in 0.2.0. acadfetch says so and keeps
88
+ nothing, rather than saving the block page as a PDF. If you hit that, the file
89
+ itself is fine; see the `403` section of
90
+ [docs/troubleshooting.md](docs/troubleshooting.md).
91
+
84
92
  figshare was removed in 0.2.0: pass the article's download URL instead, and
85
93
  [`--checksum`](docs/migration-0.2.md) keeps the verification the API used to
86
94
  give you.
@@ -92,10 +100,13 @@ acadfetch fetch 10.5281/zenodo.23612969 -n # dry run: which dir, which files
92
100
  acadfetch fetch 10.5281/zenodo.23612969 --only raw.tar.gz
93
101
  acadfetch fetch 10.5281/zenodo.23612969 --max-size 2GB # refuse anything bigger
94
102
  acadfetch fetch 10.5281/zenodo.23612969 -j 1 --rate 0.5 # be a good citizen
103
+ acadfetch fetch 10.5281/zenodo.23612969 --retries 2 --retry-403 0 # give up sooner
104
+ acadfetch fetch 10.5281/zenodo.23612969 --manifest-out build/m.json # a second copy
95
105
  acadfetch manifest 10.5281/zenodo.23612969 -o m.json # metadata only, no bytes
96
106
  acadfetch fetch zenodo:123456 --header "Authorization: Bearer $TOKEN" # restricted
97
107
 
98
108
  acadfetch verify ./data/manifest.json # re-hash what you downloaded
109
+ acadfetch --version # what is installed
99
110
  acadfetch sources # adapters, with an example each
100
111
  ```
101
112
 
@@ -133,16 +144,18 @@ block page is retried; see [docs/troubleshooting.md](docs/troubleshooting.md).
133
144
  Progress is one aggregate line for the whole run
134
145
  (`3 file(s) 12.4/44.1 MiB 1.8 MiB/s ETA 0:17`), redrawn in place on a TTY.
135
146
  When stdout is a pipe it becomes a new line at most every 5 s (one extra when a
136
- file starts or stops), so a CI log stays greppable without becoming a wall. `-q`
137
- turns it off.
147
+ file starts or stops), so a CI log stays greppable without becoming a wall.
148
+ `-q/--quiet` turns it off.
138
149
 
139
- Output defaults to `acadfetch-out/<source>-<id>` (a URL adds its host and a short
150
+ `-o/--out` defaults to `acadfetch-out/<source>-<id>` (a URL adds its host and a short
140
151
  hash), and `-n` prints it before
141
152
  you spend bandwidth. Files land in `<name>.part` and are renamed only after the
142
153
  checksum matches, so an interrupted or failed run never leaves something that
143
154
  *looks* finished -- and never throws away the bytes it already paid for: rerun and
144
- it resumes. `acadfetch-out/` is worth adding to a project's `.gitignore`; this
145
- repository already has it.
155
+ it resumes. The one case where a run must start over is a server that ignores
156
+ `Range`; the bytes already on disk are then moved aside and kept until a whole
157
+ new copy has arrived, so a restart that dies early costs nothing. `acadfetch-out/`
158
+ is worth adding to a project's `.gitignore`; this repository already has it.
146
159
 
147
160
  The default directory names the target, not only its filename — `data.csv` from
148
161
  two hosts is two datasets, and they do not share a directory. A file already
@@ -271,7 +284,7 @@ so the trace can be pasted into an issue.
271
284
 
272
285
  ## Politeness
273
286
 
274
- Defaults: 3 parallel transfers, 2 requests/second, 4 retries with jittered
287
+ Defaults: 3 parallel transfers (`-j/--concurrency`), 2 requests/second, 4 retries with jittered
275
288
  exponential backoff, `Retry-After` honoured up to 120 s. A block page is retried twice
276
289
  (`--retry-403 N`); Zenodo and arXiv are donation-funded —
277
290
  `--rate 1 -j 1` costs you minutes and costs them less.
@@ -256,7 +256,7 @@ def _file_row(row, redact: bool = True) -> dict: # noqa: ANN001
256
256
  "http_status": row.http_status,
257
257
  "final_url": hide(row.final_url) if row.final_url else None,
258
258
  "verified": row.verified,
259
- "verified_reason": verify_reason(row.file.checksum, row.verified),
259
+ "verified_reason": verify_reason(row.file.checksum, row.verified, row.state),
260
260
  "resumed_from": row.resumed_from,
261
261
  "part": Path(row.part).name if row.part else None,
262
262
  "error": row.error,
@@ -427,7 +427,10 @@ def cmd_fetch(args: argparse.Namespace) -> int:
427
427
  + ("" if disk["ok"] else " NOT ENOUGH")
428
428
  )
429
429
  for file in chosen:
430
- print(f" would fetch {naming.sanitize(file.name)} ({human(file.size)})")
430
+ # "?" in a size column reads as a bug; arXiv genuinely publishes no
431
+ # size, so say that instead of printing a question mark.
432
+ size = human(file.size) if file.size is not None else "size not published"
433
+ print(f" would fetch {naming.sanitize(file.name)} ({size})")
431
434
  return EXIT_OK
432
435
 
433
436
  try:
@@ -519,6 +522,16 @@ def cmd_verify(args: argparse.Namespace) -> int:
519
522
  rows = manifest.verify_manifest(manifest_path, root, allow_escape=args.allow_escape, data=data)
520
523
  remote: dict[str, tuple[str, str]] = {}
521
524
  if args.against_remote:
525
+ if data.get("url_redacted"):
526
+ # A presigned URL is stripped on the way into the manifest, so the
527
+ # recorded URL cannot be resolved again. Say why instead of leaving
528
+ # a page of remote_unavailable to be puzzled over.
529
+ print(
530
+ "note: this manifest had credential query parameters stripped, so a "
531
+ "row whose URL cannot be resolved again reports remote_unavailable "
532
+ "(--record-full-url keeps the original)",
533
+ file=sys.stderr,
534
+ )
522
535
  published_now, reason = remote_check.published(_client(args), data)
523
536
  entries = [e for e in data.get("files") or [] if isinstance(e, dict)]
524
537
  # verify_manifest walks the same list in order; the rows do not carry
@@ -1,6 +1,6 @@
1
1
  """Package-wide constants and version."""
2
2
 
3
- __version__ = "0.6.3"
3
+ __version__ = "0.6.4"
4
4
 
5
5
  USER_AGENT = f"acadfetch/{__version__} (+https://github.com/young-ski/acadfetch)"
6
6
 
@@ -153,9 +153,29 @@ def _kept_bytes(part: Path) -> int:
153
153
  return 0
154
154
 
155
155
 
156
- def _drop(*paths: Path) -> None:
156
+ def _room_for(part: Path, need: int) -> bool:
157
+ """Whether keeping the old bytes while a restarting copy runs still fits."""
158
+ try:
159
+ return shutil.disk_usage(part.parent).free >= need
160
+ except OSError:
161
+ return False
162
+
163
+
164
+ def _drop(*paths: Path) -> bool:
165
+ """Remove files, and say whether they are all gone.
166
+
167
+ ``atomic.unlink`` swallows a sharing violation on purpose: losing a race with
168
+ a reader is usually harmless, and a cleanup step must not become a new way to
169
+ fail. Where the deletion is load-bearing -- a mismatched part that must not
170
+ be resumed onto, a part that is already over ``--max-size`` -- the caller has
171
+ to know it did not happen, or the next attempt silently resumes onto bytes it
172
+ was told to forget.
173
+ """
174
+ kept = False
157
175
  for path in paths:
158
176
  atomic.unlink(path)
177
+ kept = kept or path.exists()
178
+ return not kept
159
179
 
160
180
 
161
181
  def _content_length(headers, url: str) -> int | None:
@@ -340,6 +360,17 @@ def _transfer(client: Client, file: File, part: Path, meta_path: Path, opts: Opt
340
360
  else:
341
361
  mode, offset = "ab", have
342
362
 
363
+ # A 200 to a Range request means the server ignored it, so this attempt
364
+ # restarts from zero. The bytes already on disk are still the best copy
365
+ # of the file there is until a new one completes, so a restart is written
366
+ # beside them and swapped in only once it is whole: opening the part with
367
+ # "wb" threw away 6 MiB in order to save 1, which is the opposite of what
368
+ # "never throws away the bytes it already paid for" promises. Without
369
+ # room for both copies, the old behaviour returns.
370
+ sink, aside = part, None
371
+ if whole and have and _room_for(part, have):
372
+ aside = sink = part.with_name(part.name + ".new")
373
+
343
374
  # Hash while the bytes go by instead of reading the finished file again.
344
375
  # A resumed transfer has to be fed what is already on disk first.
345
376
  parsed = verify.parse(file.checksum)
@@ -352,58 +383,73 @@ def _transfer(client: Client, file: File, part: Path, meta_path: Path, opts: Opt
352
383
  for old in iter(lambda: existing.read(1 << 20), b""):
353
384
  for hasher in hashers.values():
354
385
  hasher.update(old)
355
- _write_meta(
356
- meta_path,
357
- {
358
- "url": file.url,
359
- "etag": resp.headers.get("ETag"),
360
- "last_modified": resp.headers.get("Last-Modified"),
361
- "size": file.size,
362
- },
363
- )
364
- if declared is not None and file.size is not None:
365
- expected = file.size - offset
366
- if declared != expected:
367
- raise HttpError(
368
- 0,
369
- file.url,
370
- f"Content-Length {declared} disagrees with the published size "
371
- f"({expected} bytes still expected)",
372
- )
373
- # Reads poll with a short socket timeout so an interrupt is noticed
374
- # within a second instead of after the whole 60 s socket timeout, and
375
- # so an idle connection fails on its own budget. Closing the response
376
- # from another thread additionally kicks a read that is already
377
- # blocked.
378
- with _LIVE_LOCK:
379
- _LIVE.add(resp)
380
386
  try:
381
- with part.open(mode) as fh:
382
- for chunk in _chunks(resp, client.timeout, file.url):
383
- fh.write(chunk)
384
- for hasher in hashers.values():
385
- hasher.update(chunk)
386
- written += len(chunk)
387
- if opts.progress:
388
- opts.progress.advance(len(chunk))
389
- if opts.max_bytes and offset + written > opts.max_bytes:
390
- raise HttpError(
391
- 0,
392
- file.url,
393
- f"refused locally: over --max-size ({human(opts.max_bytes)})",
394
- reason="max-size",
395
- )
396
- if opts.fsync:
397
- # The rename below is atomic, but only against crashes that
398
- # leave the page cache intact. This is what makes the bytes
399
- # survive a power cut before they get a name.
400
- fh.flush()
401
- os.fsync(fh.fileno())
402
- finally:
387
+ _write_meta(
388
+ meta_path,
389
+ {
390
+ "url": file.url,
391
+ "etag": resp.headers.get("ETag"),
392
+ "last_modified": resp.headers.get("Last-Modified"),
393
+ "size": file.size,
394
+ },
395
+ )
396
+ if declared is not None and file.size is not None:
397
+ expected = file.size - offset
398
+ if declared != expected:
399
+ raise HttpError(
400
+ 0,
401
+ file.url,
402
+ f"Content-Length {declared} disagrees with the published size "
403
+ f"({expected} bytes still expected)",
404
+ )
405
+ # Reads poll with a short socket timeout so an interrupt is noticed
406
+ # within a second instead of after the whole 60 s socket timeout, and
407
+ # so an idle connection fails on its own budget. Closing the response
408
+ # from another thread additionally kicks a read that is already
409
+ # blocked.
403
410
  with _LIVE_LOCK:
404
- _LIVE.discard(resp)
405
- if declared is not None and declared != written:
406
- raise HttpError(0, file.url, f"read {written} bytes, response declared {declared}")
411
+ _LIVE.add(resp)
412
+ try:
413
+ with sink.open(mode) as fh:
414
+ for chunk in _chunks(resp, client.timeout, file.url):
415
+ fh.write(chunk)
416
+ for hasher in hashers.values():
417
+ hasher.update(chunk)
418
+ written += len(chunk)
419
+ if opts.progress:
420
+ opts.progress.advance(len(chunk))
421
+ if opts.max_bytes and offset + written > opts.max_bytes:
422
+ raise HttpError(
423
+ 0,
424
+ file.url,
425
+ f"refused locally: over --max-size ({human(opts.max_bytes)})",
426
+ reason="max-size",
427
+ )
428
+ if opts.fsync:
429
+ # The rename below is atomic, but only against crashes that
430
+ # leave the page cache intact. This is what makes the bytes
431
+ # survive a power cut before they get a name.
432
+ fh.flush()
433
+ os.fsync(fh.fileno())
434
+ finally:
435
+ with _LIVE_LOCK:
436
+ _LIVE.discard(resp)
437
+ if declared is not None and declared != written:
438
+ raise HttpError(0, file.url, f"read {written} bytes, response declared {declared}")
439
+ if aside is not None:
440
+ # A whole body, agreeing with the declared length: only now does
441
+ # the restart replace what was already there.
442
+ atomic.replace(sink, part)
443
+ except BaseException:
444
+ if aside is not None:
445
+ # Keep the bytes that were there, and the sidecar that described
446
+ # them rather than this attempt's validator.
447
+ atomic.unlink(aside)
448
+ if meta:
449
+ _write_meta(meta_path, meta)
450
+ else:
451
+ atomic.unlink(meta_path)
452
+ raise
407
453
  return Transfer(
408
454
  written=written,
409
455
  final_url=resp.geturl() or file.url,
@@ -560,6 +606,8 @@ def _reserved(files) -> set[str]: # noqa: ANN001
560
606
  for file in files:
561
607
  name = naming.sanitize(file.name).lower()
562
608
  reserved.add(name + ".part")
609
+ # A restart beside a kept part (a server that ignored Range) writes here.
610
+ reserved.add(name + ".part.new")
563
611
  reserved.add(name + META_SUFFIX)
564
612
  return reserved
565
613
 
@@ -675,12 +723,19 @@ def _attempt(
675
723
  _drop(part, meta_path)
676
724
  if exc.reason == "max-size":
677
725
  # Do not leave an over-limit part for the next run to
678
- # resume straight back into the same wall.
679
- _drop(part, meta_path)
726
+ # resume straight back into the same wall. Saying so when the
727
+ # deletion fails is the difference between a rerun that works
728
+ # and one that fails the same way with no explanation.
729
+ kept = not _drop(part, meta_path)
680
730
  return Result(
681
731
  file=file,
682
732
  state="failed",
683
- error=str(exc),
733
+ error=str(exc)
734
+ + (
735
+ f" (the over-limit {part.name} could not be deleted; remove it by hand)"
736
+ if kept
737
+ else ""
738
+ ),
684
739
  http_status=exc.status,
685
740
  retrieved_at=_now(),
686
741
  )
@@ -695,7 +750,12 @@ def _attempt(
695
750
  return Result(file=file, state="failed", error="interrupted")
696
751
  if attempt < opts.retries:
697
752
  time.sleep(min(2**attempt, 10) * (0.7 + random.random() * 0.6))
698
- resumed_from = part.stat().st_size if part.exists() else 0
753
+ # ``_transfer`` re-reads the part size itself, so the offset this
754
+ # attempt resumes from needs no bookkeeping here. ``resumed_from``
755
+ # is deliberately *not* updated: it answers "how much was on disk
756
+ # when this run started", which a retry does not change. Rewriting
757
+ # it made a fresh download report ``(resumed at 1.6 MiB)``. See
758
+ # docs/json-report.md.
699
759
  else:
700
760
  assert last is not None # the loop only ends after a failure
701
761
  return Result(
@@ -712,7 +772,10 @@ def _attempt(
712
772
  return Result(
713
773
  file=file,
714
774
  state="failed",
715
- error=str(last),
775
+ # A terminal 403 breaks out of the loop above, so it lands here
776
+ # rather than on the retry-exhausted path below. Without the hint
777
+ # that made it the one error a user could not act on.
778
+ error=f"{last}{_hint(last)}",
716
779
  http_status=getattr(last, "status", None),
717
780
  retrieved_at=_now(),
718
781
  )
@@ -729,7 +792,19 @@ def _attempt(
729
792
 
730
793
  verdict = _verdict(file, info.digests if info else None)
731
794
  if verdict is False:
732
- _drop(part, meta_path)
795
+ if not _drop(part, meta_path):
796
+ # Re-downloading would splice onto the bytes just proven wrong.
797
+ return Result(
798
+ file=file,
799
+ state="failed",
800
+ error=(
801
+ f"checksum mismatch, and the bad {part.name} could not be deleted; "
802
+ f"close whatever is reading it and retry"
803
+ ),
804
+ verified=False,
805
+ bytes=_kept_bytes(part),
806
+ retrieved_at=_now(),
807
+ )
733
808
  try:
734
809
  info = _transfer(client, file, part, meta_path, opts)
735
810
  verdict = _verdict(file, info.digests if info else None)
@@ -119,10 +119,10 @@ def _entry( # noqa: ANN001
119
119
  "http_status": result.http_status if result else None,
120
120
  "retrieved_at": result.retrieved_at if result else None,
121
121
  # PR-2 adds final_url / etag / http_status / retrieved_at here.
122
- "verified_reason": (
123
- "transfer-failed"
124
- if result and result.state == "failed"
125
- else verify.reason(f.checksum, result.verified if result else None)
122
+ "verified_reason": verify.reason(
123
+ f.checksum,
124
+ result.verified if result else None,
125
+ result.state if result else None,
126
126
  ),
127
127
  }
128
128
  if state == "not-selected":
@@ -8,24 +8,50 @@ whole lines so the log stays greppable.
8
8
 
9
9
  from __future__ import annotations
10
10
 
11
+ import math
12
+ import shutil
11
13
  import sys
12
14
  import threading
13
15
  import time
14
- from collections import deque
16
+ import unicodedata
15
17
  from collections.abc import Callable
16
18
  from typing import TextIO
17
19
 
18
20
  from .units import duration, human
19
21
 
20
- SMOOTHING_SECONDS = 1.0
22
+ #: Time constant of the speed estimate. Long enough that one slow chunk does not
23
+ #: move the ETA by minutes, short enough to follow a link that really did change.
24
+ SMOOTHING_SECONDS = 5.0
21
25
  TTY_INTERVAL = 0.15
22
26
  PIPE_INTERVAL = 5.0
23
27
  MAX_WIDTH = 120
28
+ #: Below this a two-part line has no room to say anything useful.
29
+ MIN_WIDTH = 40
24
30
  NAME_WIDTH = 40
25
31
 
26
32
 
27
- def _clip(text: str, width: int) -> str:
28
- return text if len(text) <= width else text[: width - 1] + "…"
33
+ def _char_width(char: str) -> int:
34
+ return 2 if unicodedata.east_asian_width(char) in ("W", "F") else 1
35
+
36
+
37
+ def _width(text: str) -> int:
38
+ """Terminal columns a string occupies; CJK glyphs are two, not one."""
39
+ return sum(_char_width(char) for char in text)
40
+
41
+
42
+ def _clip(text: str, limit: int) -> str:
43
+ """Cut to ``limit`` columns, counting wide characters, and mark the cut."""
44
+ if _width(text) <= limit:
45
+ return text
46
+ kept = []
47
+ used = 0
48
+ for char in text:
49
+ step = _char_width(char)
50
+ if used + step > limit - 1:
51
+ break
52
+ kept.append(char)
53
+ used += step
54
+ return "".join(kept) + "…"
29
55
 
30
56
 
31
57
  class Progress:
@@ -35,16 +61,29 @@ class Progress:
35
61
  enabled: bool = True,
36
62
  is_tty: bool | None = None,
37
63
  clock: Callable[[], float] = time.monotonic,
64
+ width: int | None = None,
38
65
  ) -> None:
39
66
  self.stream = stream if stream is not None else sys.stdout
40
67
  self.enabled = enabled
41
68
  self.is_tty = (self.stream.isatty() if is_tty is None else is_tty) and enabled
42
69
  self._clock = clock
43
70
  self._lock = threading.RLock() # _render() re-enters under _maybe_emit
71
+ # A wrapped line is worse than a shorter one: the carriage return only
72
+ # clears the last visual row, so the redraw smears. Ask the terminal.
73
+ if width is not None:
74
+ self.width = max(MIN_WIDTH, min(width, MAX_WIDTH))
75
+ elif self.is_tty:
76
+ columns = shutil.get_terminal_size((MAX_WIDTH, 24)).columns
77
+ self.width = max(MIN_WIDTH, min(columns, MAX_WIDTH))
78
+ else:
79
+ # A pipe has no viewport to wrap in; 120 is the readability cap.
80
+ self.width = MAX_WIDTH
44
81
  self._active: dict[str, int] = {}
45
82
  self._done = 0
46
83
  self._total = 0
47
- self._samples: deque[tuple[float, int]] = deque()
84
+ self._rate = 0.0
85
+ self._last_at: float | None = None
86
+ self._last_done = 0
48
87
  self._last_emit = 0.0
49
88
  self._printed = 0
50
89
  self._last_count = 0
@@ -78,20 +117,25 @@ class Progress:
78
117
 
79
118
  # -- rendering ---------------------------------------------------------
80
119
  def _sample(self) -> None:
120
+ """Fold the bytes since the last sample into an exponential average.
121
+
122
+ The old estimate divided the window's byte delta by its time span, so a
123
+ single slow chunk moved the rate by a factor of two and the ETA by
124
+ minutes -- observed jumping from 4:55 to 0:28 inside one run. An average
125
+ with a fixed time constant settles instead.
126
+ """
81
127
  now = self._clock()
82
- self._samples.append((now, self._done))
83
- while len(self._samples) > 2 and now - self._samples[0][0] > SMOOTHING_SECONDS * 4:
84
- self._samples.popleft()
128
+ if self._last_at is not None:
129
+ dt = now - self._last_at
130
+ if dt > 0:
131
+ instant = max(0.0, (self._done - self._last_done) / dt)
132
+ alpha = 1.0 - math.exp(-dt / SMOOTHING_SECONDS)
133
+ self._rate += alpha * (instant - self._rate)
134
+ self._last_at, self._last_done = now, self._done
85
135
  self._dirty = True
86
136
 
87
137
  def _speed(self) -> float:
88
- if len(self._samples) < 2:
89
- return 0.0
90
- (t0, b0), (t1, b1) = self._samples[0], self._samples[-1]
91
- span = t1 - t0
92
- if span < 0.2:
93
- return 0.0
94
- return max(0.0, (b1 - b0) / span)
138
+ return self._rate
95
139
 
96
140
  def _eta(self, speed: float) -> float | None:
97
141
  if not self._total or speed <= 0:
@@ -116,8 +160,7 @@ class Progress:
116
160
  eta = self._eta(speed)
117
161
  if eta is not None:
118
162
  parts.append(f"ETA {duration(eta)}")
119
- line = " ".join(parts)
120
- return line[:MAX_WIDTH]
163
+ return _clip(" ".join(parts), self.width)
121
164
 
122
165
  def _maybe_emit(self, force: bool = False) -> None:
123
166
  if not self.enabled:
@@ -137,9 +180,9 @@ class Progress:
137
180
  def _write(self, line: str, newline: bool) -> None:
138
181
  try:
139
182
  if self.is_tty:
140
- pad = max(0, self._printed - len(line))
183
+ pad = max(0, self._printed - _width(line))
141
184
  self.stream.write("\r" + line + " " * pad)
142
- self._printed = len(line)
185
+ self._printed = _width(line)
143
186
  else:
144
187
  self.stream.write(line + ("\n" if newline else ""))
145
188
  self.stream.flush()
@@ -68,6 +68,11 @@ def judge(
68
68
  here, there = _normalise(local), _normalise(remote)
69
69
  if here is None or there is None:
70
70
  return NO_CHECKSUM, "a checksum is not parseable"
71
+ if here[0] != there[0]:
72
+ # Comparing an md5 with a sha256 proves nothing either way. Reporting a
73
+ # mismatch here would call a legitimate algorithm change tampering, and
74
+ # this status is the one that makes the whole check worth running.
75
+ return NO_CHECKSUM, f"the source publishes {there[0]} now, the row recorded {here[0]}"
71
76
  return (
72
77
  (MATCH, "") if here == there else (MISMATCH, f"published {there[0]}:{there[1][:12]}… now")
73
78
  )
@@ -70,20 +70,29 @@ def describe(checksum: str | None) -> str:
70
70
  return f"{parsed[0]}:{parsed[1][:12]}…"
71
71
 
72
72
 
73
- def reason(checksum: str | None, verdict: bool | None) -> str:
73
+ def reason(checksum: str | None, verdict: bool | None, state: str | None = None) -> str:
74
74
  """A stable, machine-readable note beside ``verified``.
75
75
 
76
76
  ``verified: null`` on its own cannot tell "the repository published nothing"
77
77
  from "it published something we could not parse", and those are different
78
78
  promises to a reader.
79
+
80
+ ``state`` adds the one thing a checksum cannot describe: a transfer that
81
+ never produced bytes to check. The manifest and the ``--json`` report both
82
+ call this, so one run cannot describe the same file two ways.
79
83
  """
84
+ if verdict is False:
85
+ # A failed row whose bytes were proven wrong is more useful as
86
+ # ``mismatch`` than as ``transfer-failed``: it names the cause, not only
87
+ # the symptom.
88
+ return "mismatch"
89
+ if state == "failed":
90
+ return "transfer-failed"
80
91
  if not checksum:
81
92
  return "no-checksum-published"
82
93
  parsed = parse(checksum)
83
94
  if parsed is None:
84
95
  return "unparseable-checksum"
85
- if verdict is False:
86
- return "mismatch"
87
96
  if verdict is True:
88
97
  return f"matched-{parsed[0]}"
89
98
  return "unchecked"