acadfetch 0.6.2__tar.gz → 0.6.4__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (75) hide show
  1. {acadfetch-0.6.2/acadfetch.egg-info → acadfetch-0.6.4}/PKG-INFO +21 -8
  2. {acadfetch-0.6.2 → acadfetch-0.6.4}/README.md +20 -7
  3. {acadfetch-0.6.2 → acadfetch-0.6.4}/acadfetch/cli.py +15 -2
  4. {acadfetch-0.6.2 → acadfetch-0.6.4}/acadfetch/consts.py +1 -1
  5. {acadfetch-0.6.2 → acadfetch-0.6.4}/acadfetch/downloader.py +223 -61
  6. {acadfetch-0.6.2 → acadfetch-0.6.4}/acadfetch/manifest.py +4 -4
  7. {acadfetch-0.6.2 → acadfetch-0.6.4}/acadfetch/progress.py +62 -19
  8. {acadfetch-0.6.2 → acadfetch-0.6.4}/acadfetch/remote.py +5 -0
  9. {acadfetch-0.6.2 → acadfetch-0.6.4}/acadfetch/verify.py +12 -3
  10. {acadfetch-0.6.2 → acadfetch-0.6.4/acadfetch.egg-info}/PKG-INFO +21 -8
  11. {acadfetch-0.6.2 → acadfetch-0.6.4}/acadfetch.egg-info/SOURCES.txt +2 -0
  12. {acadfetch-0.6.2 → acadfetch-0.6.4}/pyproject.toml +1 -1
  13. acadfetch-0.6.4/tests/test_claims.py +174 -0
  14. {acadfetch-0.6.2 → acadfetch-0.6.4}/tests/test_cli.py +9 -0
  15. {acadfetch-0.6.2 → acadfetch-0.6.4}/tests/test_downloader.py +74 -0
  16. {acadfetch-0.6.2 → acadfetch-0.6.4}/tests/test_json.py +19 -0
  17. acadfetch-0.6.4/tests/test_lock.py +146 -0
  18. {acadfetch-0.6.2 → acadfetch-0.6.4}/tests/test_progress.py +30 -2
  19. {acadfetch-0.6.2 → acadfetch-0.6.4}/tests/test_remote.py +26 -0
  20. {acadfetch-0.6.2 → acadfetch-0.6.4}/LICENSE +0 -0
  21. {acadfetch-0.6.2 → acadfetch-0.6.4}/MANIFEST.in +0 -0
  22. {acadfetch-0.6.2 → acadfetch-0.6.4}/acadfetch/__init__.py +0 -0
  23. {acadfetch-0.6.2 → acadfetch-0.6.4}/acadfetch/__main__.py +0 -0
  24. {acadfetch-0.6.2 → acadfetch-0.6.4}/acadfetch/atomic.py +0 -0
  25. {acadfetch-0.6.2 → acadfetch-0.6.4}/acadfetch/auth.py +0 -0
  26. {acadfetch-0.6.2 → acadfetch-0.6.4}/acadfetch/client.py +0 -0
  27. {acadfetch-0.6.2 → acadfetch-0.6.4}/acadfetch/debug.py +0 -0
  28. {acadfetch-0.6.2 → acadfetch-0.6.4}/acadfetch/errors.py +0 -0
  29. {acadfetch-0.6.2 → acadfetch-0.6.4}/acadfetch/naming.py +0 -0
  30. {acadfetch-0.6.2 → acadfetch-0.6.4}/acadfetch/py.typed +0 -0
  31. {acadfetch-0.6.2 → acadfetch-0.6.4}/acadfetch/ratelimit.py +0 -0
  32. {acadfetch-0.6.2 → acadfetch-0.6.4}/acadfetch/sources/__init__.py +0 -0
  33. {acadfetch-0.6.2 → acadfetch-0.6.4}/acadfetch/sources/arxiv.py +0 -0
  34. {acadfetch-0.6.2 → acadfetch-0.6.4}/acadfetch/sources/base.py +0 -0
  35. {acadfetch-0.6.2 → acadfetch-0.6.4}/acadfetch/sources/direct.py +0 -0
  36. {acadfetch-0.6.2 → acadfetch-0.6.4}/acadfetch/sources/doi.py +0 -0
  37. {acadfetch-0.6.2 → acadfetch-0.6.4}/acadfetch/sources/registry.py +0 -0
  38. {acadfetch-0.6.2 → acadfetch-0.6.4}/acadfetch/sources/removed.py +0 -0
  39. {acadfetch-0.6.2 → acadfetch-0.6.4}/acadfetch/sources/zenodo.py +0 -0
  40. {acadfetch-0.6.2 → acadfetch-0.6.4}/acadfetch/units.py +0 -0
  41. {acadfetch-0.6.2 → acadfetch-0.6.4}/acadfetch/urls.py +0 -0
  42. {acadfetch-0.6.2 → acadfetch-0.6.4}/acadfetch.egg-info/dependency_links.txt +0 -0
  43. {acadfetch-0.6.2 → acadfetch-0.6.4}/acadfetch.egg-info/entry_points.txt +0 -0
  44. {acadfetch-0.6.2 → acadfetch-0.6.4}/acadfetch.egg-info/top_level.txt +0 -0
  45. {acadfetch-0.6.2 → acadfetch-0.6.4}/setup.cfg +0 -0
  46. {acadfetch-0.6.2 → acadfetch-0.6.4}/tests/conftest.py +0 -0
  47. {acadfetch-0.6.2 → acadfetch-0.6.4}/tests/fixtures/README.md +0 -0
  48. {acadfetch-0.6.2 → acadfetch-0.6.4}/tests/fixtures/ca.pem +0 -0
  49. {acadfetch-0.6.2 → acadfetch-0.6.4}/tests/fixtures/figshare_403_block.html +0 -0
  50. {acadfetch-0.6.2 → acadfetch-0.6.4}/tests/fixtures/figshare_403_forbidden.json +0 -0
  51. {acadfetch-0.6.2 → acadfetch-0.6.4}/tests/fixtures/figshare_article.json +0 -0
  52. {acadfetch-0.6.2 → acadfetch-0.6.4}/tests/fixtures/figshare_file.pdf +0 -0
  53. {acadfetch-0.6.2 → acadfetch-0.6.4}/tests/test_atomic.py +0 -0
  54. {acadfetch-0.6.2 → acadfetch-0.6.4}/tests/test_auth.py +0 -0
  55. {acadfetch-0.6.2 → acadfetch-0.6.4}/tests/test_checksum.py +0 -0
  56. {acadfetch-0.6.2 → acadfetch-0.6.4}/tests/test_citation.py +0 -0
  57. {acadfetch-0.6.2 → acadfetch-0.6.4}/tests/test_client.py +0 -0
  58. {acadfetch-0.6.2 → acadfetch-0.6.4}/tests/test_control_names.py +0 -0
  59. {acadfetch-0.6.2 → acadfetch-0.6.4}/tests/test_debug.py +0 -0
  60. {acadfetch-0.6.2 → acadfetch-0.6.4}/tests/test_disk.py +0 -0
  61. {acadfetch-0.6.2 → acadfetch-0.6.4}/tests/test_hash_once.py +0 -0
  62. {acadfetch-0.6.2 → acadfetch-0.6.4}/tests/test_identity.py +0 -0
  63. {acadfetch-0.6.2 → acadfetch-0.6.4}/tests/test_isolation.py +0 -0
  64. {acadfetch-0.6.2 → acadfetch-0.6.4}/tests/test_manifest.py +0 -0
  65. {acadfetch-0.6.2 → acadfetch-0.6.4}/tests/test_naming.py +0 -0
  66. {acadfetch-0.6.2 → acadfetch-0.6.4}/tests/test_naming_race.py +0 -0
  67. {acadfetch-0.6.2 → acadfetch-0.6.4}/tests/test_network.py +0 -0
  68. {acadfetch-0.6.2 → acadfetch-0.6.4}/tests/test_packaging.py +0 -0
  69. {acadfetch-0.6.2 → acadfetch-0.6.4}/tests/test_ratelimit.py +0 -0
  70. {acadfetch-0.6.2 → acadfetch-0.6.4}/tests/test_redirect.py +0 -0
  71. {acadfetch-0.6.2 → acadfetch-0.6.4}/tests/test_removed.py +0 -0
  72. {acadfetch-0.6.2 → acadfetch-0.6.4}/tests/test_sources.py +0 -0
  73. {acadfetch-0.6.2 → acadfetch-0.6.4}/tests/test_urls.py +0 -0
  74. {acadfetch-0.6.2 → acadfetch-0.6.4}/tests/test_urls_glob.py +0 -0
  75. {acadfetch-0.6.2 → acadfetch-0.6.4}/tests/test_verify.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: acadfetch
3
- Version: 0.6.2
3
+ Version: 0.6.4
4
4
  Summary: Resumable, rate-limited, checksum-verified bulk downloads from academic data repositories
5
5
  Author: young-ski
6
6
  License-Expression: MIT
@@ -104,7 +104,7 @@ python -m pip install -e . pytest mypy && python -m pytest -q
104
104
 
105
105
  ## Usage
106
106
 
107
- These work as typed:
107
+ One target per run:
108
108
 
109
109
  ```bash
110
110
  acadfetch fetch 10.5281/zenodo.23612969 # a DOI
@@ -112,6 +112,14 @@ acadfetch fetch arxiv:1706.03762 # an arXiv id
112
112
  acadfetch fetch https://ndownloader.figshare.com/files/70600233/report.pdf # any URL
113
113
  ```
114
114
 
115
+ The first two are answered by Zenodo and arXiv and work from any network those
116
+ are reachable on. The third is a direct URL, so the host decides: figshare's CDN
117
+ answers 403 with a WAF block page from some networks — the same behaviour that
118
+ made its *adapter* not worth keeping in 0.2.0. acadfetch says so and keeps
119
+ nothing, rather than saving the block page as a PDF. If you hit that, the file
120
+ itself is fine; see the `403` section of
121
+ [docs/troubleshooting.md](docs/troubleshooting.md).
122
+
115
123
  figshare was removed in 0.2.0: pass the article's download URL instead, and
116
124
  [`--checksum`](docs/migration-0.2.md) keeps the verification the API used to
117
125
  give you.
@@ -123,10 +131,13 @@ acadfetch fetch 10.5281/zenodo.23612969 -n # dry run: which dir, which files
123
131
  acadfetch fetch 10.5281/zenodo.23612969 --only raw.tar.gz
124
132
  acadfetch fetch 10.5281/zenodo.23612969 --max-size 2GB # refuse anything bigger
125
133
  acadfetch fetch 10.5281/zenodo.23612969 -j 1 --rate 0.5 # be a good citizen
134
+ acadfetch fetch 10.5281/zenodo.23612969 --retries 2 --retry-403 0 # give up sooner
135
+ acadfetch fetch 10.5281/zenodo.23612969 --manifest-out build/m.json # a second copy
126
136
  acadfetch manifest 10.5281/zenodo.23612969 -o m.json # metadata only, no bytes
127
137
  acadfetch fetch zenodo:123456 --header "Authorization: Bearer $TOKEN" # restricted
128
138
 
129
139
  acadfetch verify ./data/manifest.json # re-hash what you downloaded
140
+ acadfetch --version # what is installed
130
141
  acadfetch sources # adapters, with an example each
131
142
  ```
132
143
 
@@ -164,16 +175,18 @@ block page is retried; see [docs/troubleshooting.md](docs/troubleshooting.md).
164
175
  Progress is one aggregate line for the whole run
165
176
  (`3 file(s) 12.4/44.1 MiB 1.8 MiB/s ETA 0:17`), redrawn in place on a TTY.
166
177
  When stdout is a pipe it becomes a new line at most every 5 s (one extra when a
167
- file starts or stops), so a CI log stays greppable without becoming a wall. `-q`
168
- turns it off.
178
+ file starts or stops), so a CI log stays greppable without becoming a wall.
179
+ `-q/--quiet` turns it off.
169
180
 
170
- Output defaults to `acadfetch-out/<source>-<id>` (a URL adds its host and a short
181
+ `-o/--out` defaults to `acadfetch-out/<source>-<id>` (a URL adds its host and a short
171
182
  hash), and `-n` prints it before
172
183
  you spend bandwidth. Files land in `<name>.part` and are renamed only after the
173
184
  checksum matches, so an interrupted or failed run never leaves something that
174
185
  *looks* finished -- and never throws away the bytes it already paid for: rerun and
175
- it resumes. `acadfetch-out/` is worth adding to a project's `.gitignore`; this
176
- repository already has it.
186
+ it resumes. The one case where a run must start over is a server that ignores
187
+ `Range`; the bytes already on disk are then moved aside and kept until a whole
188
+ new copy has arrived, so a restart that dies early costs nothing. `acadfetch-out/`
189
+ is worth adding to a project's `.gitignore`; this repository already has it.
177
190
 
178
191
  The default directory names the target, not only its filename — `data.csv` from
179
192
  two hosts is two datasets, and they do not share a directory. A file already
@@ -302,7 +315,7 @@ so the trace can be pasted into an issue.
302
315
 
303
316
  ## Politeness
304
317
 
305
- Defaults: 3 parallel transfers, 2 requests/second, 4 retries with jittered
318
+ Defaults: 3 parallel transfers (`-j/--concurrency`), 2 requests/second, 4 retries with jittered
306
319
  exponential backoff, `Retry-After` honoured up to 120 s. A block page is retried twice
307
320
  (`--retry-403 N`); Zenodo and arXiv are donation-funded —
308
321
  `--rate 1 -j 1` costs you minutes and costs them less.
@@ -73,7 +73,7 @@ python -m pip install -e . pytest mypy && python -m pytest -q
73
73
 
74
74
  ## Usage
75
75
 
76
- These work as typed:
76
+ One target per run:
77
77
 
78
78
  ```bash
79
79
  acadfetch fetch 10.5281/zenodo.23612969 # a DOI
@@ -81,6 +81,14 @@ acadfetch fetch arxiv:1706.03762 # an arXiv id
81
81
  acadfetch fetch https://ndownloader.figshare.com/files/70600233/report.pdf # any URL
82
82
  ```
83
83
 
84
+ The first two are answered by Zenodo and arXiv and work from any network those
85
+ are reachable on. The third is a direct URL, so the host decides: figshare's CDN
86
+ answers 403 with a WAF block page from some networks — the same behaviour that
87
+ made its *adapter* not worth keeping in 0.2.0. acadfetch says so and keeps
88
+ nothing, rather than saving the block page as a PDF. If you hit that, the file
89
+ itself is fine; see the `403` section of
90
+ [docs/troubleshooting.md](docs/troubleshooting.md).
91
+
84
92
  figshare was removed in 0.2.0: pass the article's download URL instead, and
85
93
  [`--checksum`](docs/migration-0.2.md) keeps the verification the API used to
86
94
  give you.
@@ -92,10 +100,13 @@ acadfetch fetch 10.5281/zenodo.23612969 -n # dry run: which dir, which files
92
100
  acadfetch fetch 10.5281/zenodo.23612969 --only raw.tar.gz
93
101
  acadfetch fetch 10.5281/zenodo.23612969 --max-size 2GB # refuse anything bigger
94
102
  acadfetch fetch 10.5281/zenodo.23612969 -j 1 --rate 0.5 # be a good citizen
103
+ acadfetch fetch 10.5281/zenodo.23612969 --retries 2 --retry-403 0 # give up sooner
104
+ acadfetch fetch 10.5281/zenodo.23612969 --manifest-out build/m.json # a second copy
95
105
  acadfetch manifest 10.5281/zenodo.23612969 -o m.json # metadata only, no bytes
96
106
  acadfetch fetch zenodo:123456 --header "Authorization: Bearer $TOKEN" # restricted
97
107
 
98
108
  acadfetch verify ./data/manifest.json # re-hash what you downloaded
109
+ acadfetch --version # what is installed
99
110
  acadfetch sources # adapters, with an example each
100
111
  ```
101
112
 
@@ -133,16 +144,18 @@ block page is retried; see [docs/troubleshooting.md](docs/troubleshooting.md).
133
144
  Progress is one aggregate line for the whole run
134
145
  (`3 file(s) 12.4/44.1 MiB 1.8 MiB/s ETA 0:17`), redrawn in place on a TTY.
135
146
  When stdout is a pipe it becomes a new line at most every 5 s (one extra when a
136
- file starts or stops), so a CI log stays greppable without becoming a wall. `-q`
137
- turns it off.
147
+ file starts or stops), so a CI log stays greppable without becoming a wall.
148
+ `-q/--quiet` turns it off.
138
149
 
139
- Output defaults to `acadfetch-out/<source>-<id>` (a URL adds its host and a short
150
+ `-o/--out` defaults to `acadfetch-out/<source>-<id>` (a URL adds its host and a short
140
151
  hash), and `-n` prints it before
141
152
  you spend bandwidth. Files land in `<name>.part` and are renamed only after the
142
153
  checksum matches, so an interrupted or failed run never leaves something that
143
154
  *looks* finished -- and never throws away the bytes it already paid for: rerun and
144
- it resumes. `acadfetch-out/` is worth adding to a project's `.gitignore`; this
145
- repository already has it.
155
+ it resumes. The one case where a run must start over is a server that ignores
156
+ `Range`; the bytes already on disk are then moved aside and kept until a whole
157
+ new copy has arrived, so a restart that dies early costs nothing. `acadfetch-out/`
158
+ is worth adding to a project's `.gitignore`; this repository already has it.
146
159
 
147
160
  The default directory names the target, not only its filename — `data.csv` from
148
161
  two hosts is two datasets, and they do not share a directory. A file already
@@ -271,7 +284,7 @@ so the trace can be pasted into an issue.
271
284
 
272
285
  ## Politeness
273
286
 
274
- Defaults: 3 parallel transfers, 2 requests/second, 4 retries with jittered
287
+ Defaults: 3 parallel transfers (`-j/--concurrency`), 2 requests/second, 4 retries with jittered
275
288
  exponential backoff, `Retry-After` honoured up to 120 s. A block page is retried twice
276
289
  (`--retry-403 N`); Zenodo and arXiv are donation-funded —
277
290
  `--rate 1 -j 1` costs you minutes and costs them less.
@@ -256,7 +256,7 @@ def _file_row(row, redact: bool = True) -> dict: # noqa: ANN001
256
256
  "http_status": row.http_status,
257
257
  "final_url": hide(row.final_url) if row.final_url else None,
258
258
  "verified": row.verified,
259
- "verified_reason": verify_reason(row.file.checksum, row.verified),
259
+ "verified_reason": verify_reason(row.file.checksum, row.verified, row.state),
260
260
  "resumed_from": row.resumed_from,
261
261
  "part": Path(row.part).name if row.part else None,
262
262
  "error": row.error,
@@ -427,7 +427,10 @@ def cmd_fetch(args: argparse.Namespace) -> int:
427
427
  + ("" if disk["ok"] else " NOT ENOUGH")
428
428
  )
429
429
  for file in chosen:
430
- print(f" would fetch {naming.sanitize(file.name)} ({human(file.size)})")
430
+ # "?" in a size column reads as a bug; arXiv genuinely publishes no
431
+ # size, so say that instead of printing a question mark.
432
+ size = human(file.size) if file.size is not None else "size not published"
433
+ print(f" would fetch {naming.sanitize(file.name)} ({size})")
431
434
  return EXIT_OK
432
435
 
433
436
  try:
@@ -519,6 +522,16 @@ def cmd_verify(args: argparse.Namespace) -> int:
519
522
  rows = manifest.verify_manifest(manifest_path, root, allow_escape=args.allow_escape, data=data)
520
523
  remote: dict[str, tuple[str, str]] = {}
521
524
  if args.against_remote:
525
+ if data.get("url_redacted"):
526
+ # A presigned URL is stripped on the way into the manifest, so the
527
+ # recorded URL cannot be resolved again. Say why instead of leaving
528
+ # a page of remote_unavailable to be puzzled over.
529
+ print(
530
+ "note: this manifest had credential query parameters stripped, so a "
531
+ "row whose URL cannot be resolved again reports remote_unavailable "
532
+ "(--record-full-url keeps the original)",
533
+ file=sys.stderr,
534
+ )
522
535
  published_now, reason = remote_check.published(_client(args), data)
523
536
  entries = [e for e in data.get("files") or [] if isinstance(e, dict)]
524
537
  # verify_manifest walks the same list in order; the rows do not carry
@@ -1,6 +1,6 @@
1
1
  """Package-wide constants and version."""
2
2
 
3
- __version__ = "0.6.2"
3
+ __version__ = "0.6.4"
4
4
 
5
5
  USER_AGENT = f"acadfetch/{__version__} (+https://github.com/young-ski/acadfetch)"
6
6
 
@@ -27,7 +27,7 @@ from dataclasses import dataclass, field
27
27
  from datetime import datetime, timezone
28
28
  from http.client import HTTPException
29
29
  from pathlib import Path
30
- from typing import cast
30
+ from typing import Any, cast
31
31
 
32
32
  from . import atomic, debug, naming, verify
33
33
  from .client import Client
@@ -73,6 +73,7 @@ _BINARY_EXT = frozenset(
73
73
  }
74
74
  )
75
75
  META_SUFFIX = ".part.meta.json"
76
+ LOCK_NAME = ".acadfetch.lock"
76
77
 
77
78
 
78
79
  @dataclass(frozen=True)
@@ -152,9 +153,29 @@ def _kept_bytes(part: Path) -> int:
152
153
  return 0
153
154
 
154
155
 
155
- def _drop(*paths: Path) -> None:
156
+ def _room_for(part: Path, need: int) -> bool:
157
+ """Whether keeping the old bytes while a restarting copy runs still fits."""
158
+ try:
159
+ return shutil.disk_usage(part.parent).free >= need
160
+ except OSError:
161
+ return False
162
+
163
+
164
+ def _drop(*paths: Path) -> bool:
165
+ """Remove files, and say whether they are all gone.
166
+
167
+ ``atomic.unlink`` swallows a sharing violation on purpose: losing a race with
168
+ a reader is usually harmless, and a cleanup step must not become a new way to
169
+ fail. Where the deletion is load-bearing -- a mismatched part that must not
170
+ be resumed onto, a part that is already over ``--max-size`` -- the caller has
171
+ to know it did not happen, or the next attempt silently resumes onto bytes it
172
+ was told to forget.
173
+ """
174
+ kept = False
156
175
  for path in paths:
157
176
  atomic.unlink(path)
177
+ kept = kept or path.exists()
178
+ return not kept
158
179
 
159
180
 
160
181
  def _content_length(headers, url: str) -> int | None:
@@ -339,6 +360,17 @@ def _transfer(client: Client, file: File, part: Path, meta_path: Path, opts: Opt
339
360
  else:
340
361
  mode, offset = "ab", have
341
362
 
363
+ # A 200 to a Range request means the server ignored it, so this attempt
364
+ # restarts from zero. The bytes already on disk are still the best copy
365
+ # of the file there is until a new one completes, so a restart is written
366
+ # beside them and swapped in only once it is whole: opening the part with
367
+ # "wb" threw away 6 MiB in order to save 1, which is the opposite of what
368
+ # "never throws away the bytes it already paid for" promises. Without
369
+ # room for both copies, the old behaviour returns.
370
+ sink, aside = part, None
371
+ if whole and have and _room_for(part, have):
372
+ aside = sink = part.with_name(part.name + ".new")
373
+
342
374
  # Hash while the bytes go by instead of reading the finished file again.
343
375
  # A resumed transfer has to be fed what is already on disk first.
344
376
  parsed = verify.parse(file.checksum)
@@ -351,58 +383,73 @@ def _transfer(client: Client, file: File, part: Path, meta_path: Path, opts: Opt
351
383
  for old in iter(lambda: existing.read(1 << 20), b""):
352
384
  for hasher in hashers.values():
353
385
  hasher.update(old)
354
- _write_meta(
355
- meta_path,
356
- {
357
- "url": file.url,
358
- "etag": resp.headers.get("ETag"),
359
- "last_modified": resp.headers.get("Last-Modified"),
360
- "size": file.size,
361
- },
362
- )
363
- if declared is not None and file.size is not None:
364
- expected = file.size - offset
365
- if declared != expected:
366
- raise HttpError(
367
- 0,
368
- file.url,
369
- f"Content-Length {declared} disagrees with the published size "
370
- f"({expected} bytes still expected)",
371
- )
372
- # Reads poll with a short socket timeout so an interrupt is noticed
373
- # within a second instead of after the whole 60 s socket timeout, and
374
- # so an idle connection fails on its own budget. Closing the response
375
- # from another thread additionally kicks a read that is already
376
- # blocked.
377
- with _LIVE_LOCK:
378
- _LIVE.add(resp)
379
386
  try:
380
- with part.open(mode) as fh:
381
- for chunk in _chunks(resp, client.timeout, file.url):
382
- fh.write(chunk)
383
- for hasher in hashers.values():
384
- hasher.update(chunk)
385
- written += len(chunk)
386
- if opts.progress:
387
- opts.progress.advance(len(chunk))
388
- if opts.max_bytes and offset + written > opts.max_bytes:
389
- raise HttpError(
390
- 0,
391
- file.url,
392
- f"refused locally: over --max-size ({human(opts.max_bytes)})",
393
- reason="max-size",
394
- )
395
- if opts.fsync:
396
- # The rename below is atomic, but only against crashes that
397
- # leave the page cache intact. This is what makes the bytes
398
- # survive a power cut before they get a name.
399
- fh.flush()
400
- os.fsync(fh.fileno())
401
- finally:
387
+ _write_meta(
388
+ meta_path,
389
+ {
390
+ "url": file.url,
391
+ "etag": resp.headers.get("ETag"),
392
+ "last_modified": resp.headers.get("Last-Modified"),
393
+ "size": file.size,
394
+ },
395
+ )
396
+ if declared is not None and file.size is not None:
397
+ expected = file.size - offset
398
+ if declared != expected:
399
+ raise HttpError(
400
+ 0,
401
+ file.url,
402
+ f"Content-Length {declared} disagrees with the published size "
403
+ f"({expected} bytes still expected)",
404
+ )
405
+ # Reads poll with a short socket timeout so an interrupt is noticed
406
+ # within a second instead of after the whole 60 s socket timeout, and
407
+ # so an idle connection fails on its own budget. Closing the response
408
+ # from another thread additionally kicks a read that is already
409
+ # blocked.
402
410
  with _LIVE_LOCK:
403
- _LIVE.discard(resp)
404
- if declared is not None and declared != written:
405
- raise HttpError(0, file.url, f"read {written} bytes, response declared {declared}")
411
+ _LIVE.add(resp)
412
+ try:
413
+ with sink.open(mode) as fh:
414
+ for chunk in _chunks(resp, client.timeout, file.url):
415
+ fh.write(chunk)
416
+ for hasher in hashers.values():
417
+ hasher.update(chunk)
418
+ written += len(chunk)
419
+ if opts.progress:
420
+ opts.progress.advance(len(chunk))
421
+ if opts.max_bytes and offset + written > opts.max_bytes:
422
+ raise HttpError(
423
+ 0,
424
+ file.url,
425
+ f"refused locally: over --max-size ({human(opts.max_bytes)})",
426
+ reason="max-size",
427
+ )
428
+ if opts.fsync:
429
+ # The rename below is atomic, but only against crashes that
430
+ # leave the page cache intact. This is what makes the bytes
431
+ # survive a power cut before they get a name.
432
+ fh.flush()
433
+ os.fsync(fh.fileno())
434
+ finally:
435
+ with _LIVE_LOCK:
436
+ _LIVE.discard(resp)
437
+ if declared is not None and declared != written:
438
+ raise HttpError(0, file.url, f"read {written} bytes, response declared {declared}")
439
+ if aside is not None:
440
+ # A whole body, agreeing with the declared length: only now does
441
+ # the restart replace what was already there.
442
+ atomic.replace(sink, part)
443
+ except BaseException:
444
+ if aside is not None:
445
+ # Keep the bytes that were there, and the sidecar that described
446
+ # them rather than this attempt's validator.
447
+ atomic.unlink(aside)
448
+ if meta:
449
+ _write_meta(meta_path, meta)
450
+ else:
451
+ atomic.unlink(meta_path)
452
+ raise
406
453
  return Transfer(
407
454
  written=written,
408
455
  final_url=resp.geturl() or file.url,
@@ -471,6 +518,82 @@ def check_disk(
471
518
  return (free, needed)
472
519
 
473
520
 
521
+ def _pid_alive(pid: int) -> bool:
522
+ """Is ``pid`` still running, without disturbing it?
523
+
524
+ ``os.kill(pid, 0)`` is the POSIX idiom, but on Windows it sends a console
525
+ CTRL_C: probing our own pid interrupts the process asking the question.
526
+ """
527
+ if os.name == "nt":
528
+ import ctypes
529
+
530
+ # The attribute does not exist off Windows, and both ruff and mypy
531
+ # check this file on Linux; a typed lookup keeps both quiet.
532
+ windll: Any = getattr(ctypes, "windll") # noqa: B009
533
+ kernel32 = windll.kernel32
534
+ handle = kernel32.OpenProcess(0x1000, False, pid) # query limited info
535
+ if not handle:
536
+ return False
537
+ try:
538
+ code = ctypes.c_ulong()
539
+ if not kernel32.GetExitCodeProcess(handle, ctypes.byref(code)):
540
+ return False
541
+ return code.value == 259 # STILL_ACTIVE
542
+ finally:
543
+ kernel32.CloseHandle(handle)
544
+ try:
545
+ os.kill(pid, 0)
546
+ except ProcessLookupError:
547
+ return False
548
+ except PermissionError: # exists, owned by someone else
549
+ return True
550
+ except OSError:
551
+ return False
552
+ return True
553
+
554
+
555
+ def _take_lock(out: Path) -> Path:
556
+ """Refuse to share the directory with a live concurrent run.
557
+
558
+ Two runs into one ``-o`` interleave writes into the same ``.part`` and the
559
+ survivor is a file neither of them wrote. The lock records the owner; a dead
560
+ owner, or one we cannot read, is taken over.
561
+ """
562
+ lock = out / LOCK_NAME
563
+ for _ in range(2):
564
+ try:
565
+ fd = os.open(lock, os.O_CREAT | os.O_EXCL | os.O_WRONLY)
566
+ except FileExistsError:
567
+ owner = 0
568
+ try:
569
+ owner = int(json.loads(lock.read_text("utf-8")).get("pid") or 0)
570
+ except (OSError, ValueError):
571
+ owner = 0
572
+ # A live owner is another run, even when it is this pid: two
573
+ # fetches into one directory in one process clash just as badly.
574
+ # Only a dead or unreadable owner is stale.
575
+ if owner and _pid_alive(owner):
576
+ raise TargetError(
577
+ f"{out} is being written by pid {owner}; wait for it, or "
578
+ f"delete {lock} if that process is gone"
579
+ ) from None
580
+ atomic.unlink(lock)
581
+ continue
582
+ with os.fdopen(fd, "w", encoding="utf-8") as fh:
583
+ json.dump({"pid": os.getpid(), "started": _now()}, fh)
584
+ return lock
585
+ raise TargetError(f"could not take the lock in {out}; is another run active?")
586
+
587
+
588
+ def _release_lock(lock: Path) -> None:
589
+ try:
590
+ if int(json.loads(lock.read_text("utf-8")).get("pid") or 0) != os.getpid():
591
+ return
592
+ except (OSError, ValueError):
593
+ pass
594
+ atomic.unlink(lock)
595
+
596
+
474
597
  def _reserved(files) -> set[str]: # noqa: ANN001
475
598
  """Names the tool itself will write, so a repository file cannot take them.
476
599
 
@@ -479,10 +602,12 @@ def _reserved(files) -> set[str]: # noqa: ANN001
479
602
  Reserving the derived names is cheaper than teaching every later step to
480
603
  dodge them.
481
604
  """
482
- reserved = {"manifest.json", "manifest.json.tmp"}
605
+ reserved = {"manifest.json", "manifest.json.tmp", LOCK_NAME}
483
606
  for file in files:
484
607
  name = naming.sanitize(file.name).lower()
485
608
  reserved.add(name + ".part")
609
+ # A restart beside a kept part (a server that ignored Range) writes here.
610
+ reserved.add(name + ".part.new")
486
611
  reserved.add(name + META_SUFFIX)
487
612
  return reserved
488
613
 
@@ -598,12 +723,19 @@ def _attempt(
598
723
  _drop(part, meta_path)
599
724
  if exc.reason == "max-size":
600
725
  # Do not leave an over-limit part for the next run to
601
- # resume straight back into the same wall.
602
- _drop(part, meta_path)
726
+ # resume straight back into the same wall. Saying so when the
727
+ # deletion fails is the difference between a rerun that works
728
+ # and one that fails the same way with no explanation.
729
+ kept = not _drop(part, meta_path)
603
730
  return Result(
604
731
  file=file,
605
732
  state="failed",
606
- error=str(exc),
733
+ error=str(exc)
734
+ + (
735
+ f" (the over-limit {part.name} could not be deleted; remove it by hand)"
736
+ if kept
737
+ else ""
738
+ ),
607
739
  http_status=exc.status,
608
740
  retrieved_at=_now(),
609
741
  )
@@ -618,7 +750,12 @@ def _attempt(
618
750
  return Result(file=file, state="failed", error="interrupted")
619
751
  if attempt < opts.retries:
620
752
  time.sleep(min(2**attempt, 10) * (0.7 + random.random() * 0.6))
621
- resumed_from = part.stat().st_size if part.exists() else 0
753
+ # ``_transfer`` re-reads the part size itself, so the offset this
754
+ # attempt resumes from needs no bookkeeping here. ``resumed_from``
755
+ # is deliberately *not* updated: it answers "how much was on disk
756
+ # when this run started", which a retry does not change. Rewriting
757
+ # it made a fresh download report ``(resumed at 1.6 MiB)``. See
758
+ # docs/json-report.md.
622
759
  else:
623
760
  assert last is not None # the loop only ends after a failure
624
761
  return Result(
@@ -635,7 +772,10 @@ def _attempt(
635
772
  return Result(
636
773
  file=file,
637
774
  state="failed",
638
- error=str(last),
775
+ # A terminal 403 breaks out of the loop above, so it lands here
776
+ # rather than on the retry-exhausted path below. Without the hint
777
+ # that made it the one error a user could not act on.
778
+ error=f"{last}{_hint(last)}",
639
779
  http_status=getattr(last, "status", None),
640
780
  retrieved_at=_now(),
641
781
  )
@@ -652,7 +792,19 @@ def _attempt(
652
792
 
653
793
  verdict = _verdict(file, info.digests if info else None)
654
794
  if verdict is False:
655
- _drop(part, meta_path)
795
+ if not _drop(part, meta_path):
796
+ # Re-downloading would splice onto the bytes just proven wrong.
797
+ return Result(
798
+ file=file,
799
+ state="failed",
800
+ error=(
801
+ f"checksum mismatch, and the bad {part.name} could not be deleted; "
802
+ f"close whatever is reading it and retry"
803
+ ),
804
+ verified=False,
805
+ bytes=_kept_bytes(part),
806
+ retrieved_at=_now(),
807
+ )
656
808
  try:
657
809
  info = _transfer(client, file, part, meta_path, opts)
658
810
  verdict = _verdict(file, info.digests if info else None)
@@ -736,15 +888,24 @@ def _fetch_record(
736
888
 
737
889
  if opts.max_bytes:
738
890
  check_total(record, opts.max_bytes)
891
+ lock: Path | None = None
739
892
  if not opts.dry_run:
740
893
  check_disk(files, opts.out)
894
+ lock = _take_lock(opts.out)
741
895
  opts.trusted = _trusted_urls(opts.out / "manifest.json")
742
896
 
743
897
  taken = _reserved(files)
744
- lock = threading.Lock()
745
898
  results: list[Result] = []
746
899
  opts.interrupted = False
747
900
 
901
+ try:
902
+ return _drain(files, record, opts, client, taken, results)
903
+ finally:
904
+ if lock is not None:
905
+ _release_lock(lock)
906
+
907
+
908
+ def _drain(files, record, opts, client, taken, results): # noqa: ANN001, ANN202
748
909
  if opts.concurrency <= 1:
749
910
  for file in files:
750
911
  try:
@@ -767,9 +928,10 @@ def _fetch_record(
767
928
  window = max(opts.concurrency * 2, 1)
768
929
  queue = iter(files)
769
930
  futures = {}
931
+ name_lock = threading.Lock()
770
932
 
771
933
  def submit(file: File) -> None: # noqa: ANN001
772
- with lock:
934
+ with name_lock:
773
935
  futures[pool.submit(fetch_file, client, file, opts, taken)] = file
774
936
 
775
937
  try:
@@ -119,10 +119,10 @@ def _entry( # noqa: ANN001
119
119
  "http_status": result.http_status if result else None,
120
120
  "retrieved_at": result.retrieved_at if result else None,
121
121
  # PR-2 adds final_url / etag / http_status / retrieved_at here.
122
- "verified_reason": (
123
- "transfer-failed"
124
- if result and result.state == "failed"
125
- else verify.reason(f.checksum, result.verified if result else None)
122
+ "verified_reason": verify.reason(
123
+ f.checksum,
124
+ result.verified if result else None,
125
+ result.state if result else None,
126
126
  ),
127
127
  }
128
128
  if state == "not-selected":