acadfetch 0.6.3__tar.gz → 0.7.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (77) hide show
  1. {acadfetch-0.6.3/acadfetch.egg-info → acadfetch-0.7.0}/PKG-INFO +23 -9
  2. {acadfetch-0.6.3 → acadfetch-0.7.0}/README.md +22 -8
  3. {acadfetch-0.6.3 → acadfetch-0.7.0}/acadfetch/cli.py +15 -2
  4. {acadfetch-0.6.3 → acadfetch-0.7.0}/acadfetch/consts.py +1 -1
  5. {acadfetch-0.6.3 → acadfetch-0.7.0}/acadfetch/downloader.py +132 -57
  6. {acadfetch-0.6.3 → acadfetch-0.7.0}/acadfetch/manifest.py +4 -4
  7. acadfetch-0.7.0/acadfetch/pooch.py +157 -0
  8. {acadfetch-0.6.3 → acadfetch-0.7.0}/acadfetch/progress.py +62 -19
  9. {acadfetch-0.6.3 → acadfetch-0.7.0}/acadfetch/remote.py +5 -0
  10. {acadfetch-0.6.3 → acadfetch-0.7.0}/acadfetch/verify.py +12 -3
  11. {acadfetch-0.6.3 → acadfetch-0.7.0/acadfetch.egg-info}/PKG-INFO +23 -9
  12. {acadfetch-0.6.3 → acadfetch-0.7.0}/acadfetch.egg-info/SOURCES.txt +3 -0
  13. {acadfetch-0.6.3 → acadfetch-0.7.0}/pyproject.toml +1 -1
  14. acadfetch-0.7.0/tests/test_claims.py +174 -0
  15. {acadfetch-0.6.3 → acadfetch-0.7.0}/tests/test_cli.py +9 -0
  16. {acadfetch-0.6.3 → acadfetch-0.7.0}/tests/test_downloader.py +74 -0
  17. {acadfetch-0.6.3 → acadfetch-0.7.0}/tests/test_json.py +19 -0
  18. acadfetch-0.7.0/tests/test_pooch.py +149 -0
  19. {acadfetch-0.6.3 → acadfetch-0.7.0}/tests/test_progress.py +30 -2
  20. {acadfetch-0.6.3 → acadfetch-0.7.0}/tests/test_remote.py +26 -0
  21. {acadfetch-0.6.3 → acadfetch-0.7.0}/LICENSE +0 -0
  22. {acadfetch-0.6.3 → acadfetch-0.7.0}/MANIFEST.in +0 -0
  23. {acadfetch-0.6.3 → acadfetch-0.7.0}/acadfetch/__init__.py +0 -0
  24. {acadfetch-0.6.3 → acadfetch-0.7.0}/acadfetch/__main__.py +0 -0
  25. {acadfetch-0.6.3 → acadfetch-0.7.0}/acadfetch/atomic.py +0 -0
  26. {acadfetch-0.6.3 → acadfetch-0.7.0}/acadfetch/auth.py +0 -0
  27. {acadfetch-0.6.3 → acadfetch-0.7.0}/acadfetch/client.py +0 -0
  28. {acadfetch-0.6.3 → acadfetch-0.7.0}/acadfetch/debug.py +0 -0
  29. {acadfetch-0.6.3 → acadfetch-0.7.0}/acadfetch/errors.py +0 -0
  30. {acadfetch-0.6.3 → acadfetch-0.7.0}/acadfetch/naming.py +0 -0
  31. {acadfetch-0.6.3 → acadfetch-0.7.0}/acadfetch/py.typed +0 -0
  32. {acadfetch-0.6.3 → acadfetch-0.7.0}/acadfetch/ratelimit.py +0 -0
  33. {acadfetch-0.6.3 → acadfetch-0.7.0}/acadfetch/sources/__init__.py +0 -0
  34. {acadfetch-0.6.3 → acadfetch-0.7.0}/acadfetch/sources/arxiv.py +0 -0
  35. {acadfetch-0.6.3 → acadfetch-0.7.0}/acadfetch/sources/base.py +0 -0
  36. {acadfetch-0.6.3 → acadfetch-0.7.0}/acadfetch/sources/direct.py +0 -0
  37. {acadfetch-0.6.3 → acadfetch-0.7.0}/acadfetch/sources/doi.py +0 -0
  38. {acadfetch-0.6.3 → acadfetch-0.7.0}/acadfetch/sources/registry.py +0 -0
  39. {acadfetch-0.6.3 → acadfetch-0.7.0}/acadfetch/sources/removed.py +0 -0
  40. {acadfetch-0.6.3 → acadfetch-0.7.0}/acadfetch/sources/zenodo.py +0 -0
  41. {acadfetch-0.6.3 → acadfetch-0.7.0}/acadfetch/units.py +0 -0
  42. {acadfetch-0.6.3 → acadfetch-0.7.0}/acadfetch/urls.py +0 -0
  43. {acadfetch-0.6.3 → acadfetch-0.7.0}/acadfetch.egg-info/dependency_links.txt +0 -0
  44. {acadfetch-0.6.3 → acadfetch-0.7.0}/acadfetch.egg-info/entry_points.txt +0 -0
  45. {acadfetch-0.6.3 → acadfetch-0.7.0}/acadfetch.egg-info/top_level.txt +0 -0
  46. {acadfetch-0.6.3 → acadfetch-0.7.0}/setup.cfg +0 -0
  47. {acadfetch-0.6.3 → acadfetch-0.7.0}/tests/conftest.py +0 -0
  48. {acadfetch-0.6.3 → acadfetch-0.7.0}/tests/fixtures/README.md +0 -0
  49. {acadfetch-0.6.3 → acadfetch-0.7.0}/tests/fixtures/ca.pem +0 -0
  50. {acadfetch-0.6.3 → acadfetch-0.7.0}/tests/fixtures/figshare_403_block.html +0 -0
  51. {acadfetch-0.6.3 → acadfetch-0.7.0}/tests/fixtures/figshare_403_forbidden.json +0 -0
  52. {acadfetch-0.6.3 → acadfetch-0.7.0}/tests/fixtures/figshare_article.json +0 -0
  53. {acadfetch-0.6.3 → acadfetch-0.7.0}/tests/fixtures/figshare_file.pdf +0 -0
  54. {acadfetch-0.6.3 → acadfetch-0.7.0}/tests/test_atomic.py +0 -0
  55. {acadfetch-0.6.3 → acadfetch-0.7.0}/tests/test_auth.py +0 -0
  56. {acadfetch-0.6.3 → acadfetch-0.7.0}/tests/test_checksum.py +0 -0
  57. {acadfetch-0.6.3 → acadfetch-0.7.0}/tests/test_citation.py +0 -0
  58. {acadfetch-0.6.3 → acadfetch-0.7.0}/tests/test_client.py +0 -0
  59. {acadfetch-0.6.3 → acadfetch-0.7.0}/tests/test_control_names.py +0 -0
  60. {acadfetch-0.6.3 → acadfetch-0.7.0}/tests/test_debug.py +0 -0
  61. {acadfetch-0.6.3 → acadfetch-0.7.0}/tests/test_disk.py +0 -0
  62. {acadfetch-0.6.3 → acadfetch-0.7.0}/tests/test_hash_once.py +0 -0
  63. {acadfetch-0.6.3 → acadfetch-0.7.0}/tests/test_identity.py +0 -0
  64. {acadfetch-0.6.3 → acadfetch-0.7.0}/tests/test_isolation.py +0 -0
  65. {acadfetch-0.6.3 → acadfetch-0.7.0}/tests/test_lock.py +0 -0
  66. {acadfetch-0.6.3 → acadfetch-0.7.0}/tests/test_manifest.py +0 -0
  67. {acadfetch-0.6.3 → acadfetch-0.7.0}/tests/test_naming.py +0 -0
  68. {acadfetch-0.6.3 → acadfetch-0.7.0}/tests/test_naming_race.py +0 -0
  69. {acadfetch-0.6.3 → acadfetch-0.7.0}/tests/test_network.py +0 -0
  70. {acadfetch-0.6.3 → acadfetch-0.7.0}/tests/test_packaging.py +0 -0
  71. {acadfetch-0.6.3 → acadfetch-0.7.0}/tests/test_ratelimit.py +0 -0
  72. {acadfetch-0.6.3 → acadfetch-0.7.0}/tests/test_redirect.py +0 -0
  73. {acadfetch-0.6.3 → acadfetch-0.7.0}/tests/test_removed.py +0 -0
  74. {acadfetch-0.6.3 → acadfetch-0.7.0}/tests/test_sources.py +0 -0
  75. {acadfetch-0.6.3 → acadfetch-0.7.0}/tests/test_urls.py +0 -0
  76. {acadfetch-0.6.3 → acadfetch-0.7.0}/tests/test_urls_glob.py +0 -0
  77. {acadfetch-0.6.3 → acadfetch-0.7.0}/tests/test_verify.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: acadfetch
3
- Version: 0.6.3
3
+ Version: 0.7.0
4
4
  Summary: Resumable, rate-limited, checksum-verified bulk downloads from academic data repositories
5
5
  Author: young-ski
6
6
  License-Expression: MIT
@@ -104,7 +104,7 @@ python -m pip install -e . pytest mypy && python -m pytest -q
104
104
 
105
105
  ## Usage
106
106
 
107
- These work as typed:
107
+ One target per run:
108
108
 
109
109
  ```bash
110
110
  acadfetch fetch 10.5281/zenodo.23612969 # a DOI
@@ -112,6 +112,14 @@ acadfetch fetch arxiv:1706.03762 # an arXiv id
112
112
  acadfetch fetch https://ndownloader.figshare.com/files/70600233/report.pdf # any URL
113
113
  ```
114
114
 
115
+ The first two are answered by Zenodo and arXiv and work from any network those
116
+ are reachable on. The third is a direct URL, so the host decides: figshare's CDN
117
+ answers 403 with a WAF block page from some networks — the same behaviour that
118
+ made its *adapter* not worth keeping in 0.2.0. acadfetch says so and keeps
119
+ nothing, rather than saving the block page as a PDF. If you hit that, the file
120
+ itself is fine; see the `403` section of
121
+ [docs/troubleshooting.md](docs/troubleshooting.md).
122
+
115
123
  figshare was removed in 0.2.0: pass the article's download URL instead, and
116
124
  [`--checksum`](docs/migration-0.2.md) keeps the verification the API used to
117
125
  give you.
@@ -123,10 +131,13 @@ acadfetch fetch 10.5281/zenodo.23612969 -n # dry run: which dir, which files
123
131
  acadfetch fetch 10.5281/zenodo.23612969 --only raw.tar.gz
124
132
  acadfetch fetch 10.5281/zenodo.23612969 --max-size 2GB # refuse anything bigger
125
133
  acadfetch fetch 10.5281/zenodo.23612969 -j 1 --rate 0.5 # be a good citizen
134
+ acadfetch fetch 10.5281/zenodo.23612969 --retries 2 --retry-403 0 # give up sooner
135
+ acadfetch fetch 10.5281/zenodo.23612969 --manifest-out build/m.json # a second copy
126
136
  acadfetch manifest 10.5281/zenodo.23612969 -o m.json # metadata only, no bytes
127
137
  acadfetch fetch zenodo:123456 --header "Authorization: Bearer $TOKEN" # restricted
128
138
 
129
139
  acadfetch verify ./data/manifest.json # re-hash what you downloaded
140
+ acadfetch --version # what is installed
130
141
  acadfetch sources # adapters, with an example each
131
142
  ```
132
143
 
@@ -164,16 +175,18 @@ block page is retried; see [docs/troubleshooting.md](docs/troubleshooting.md).
164
175
  Progress is one aggregate line for the whole run
165
176
  (`3 file(s) 12.4/44.1 MiB 1.8 MiB/s ETA 0:17`), redrawn in place on a TTY.
166
177
  When stdout is a pipe it becomes a new line at most every 5 s (one extra when a
167
- file starts or stops), so a CI log stays greppable without becoming a wall. `-q`
168
- turns it off.
178
+ file starts or stops), so a CI log stays greppable without becoming a wall.
179
+ `-q/--quiet` turns it off.
169
180
 
170
- Output defaults to `acadfetch-out/<source>-<id>` (a URL adds its host and a short
181
+ `-o/--out` defaults to `acadfetch-out/<source>-<id>` (a URL adds its host and a short
171
182
  hash), and `-n` prints it before
172
183
  you spend bandwidth. Files land in `<name>.part` and are renamed only after the
173
184
  checksum matches, so an interrupted or failed run never leaves something that
174
185
  *looks* finished -- and never throws away the bytes it already paid for: rerun and
175
- it resumes. `acadfetch-out/` is worth adding to a project's `.gitignore`; this
176
- repository already has it.
186
+ it resumes. The one case where a run must start over is a server that ignores
187
+ `Range`; the bytes already on disk are then moved aside and kept until a whole
188
+ new copy has arrived, so a restart that dies early costs nothing. `acadfetch-out/`
189
+ is worth adding to a project's `.gitignore`; this repository already has it.
177
190
 
178
191
  The default directory names the target, not only its filename — `data.csv` from
179
192
  two hosts is two datasets, and they do not share a directory. A file already
@@ -302,7 +315,7 @@ so the trace can be pasted into an issue.
302
315
 
303
316
  ## Politeness
304
317
 
305
- Defaults: 3 parallel transfers, 2 requests/second, 4 retries with jittered
318
+ Defaults: 3 parallel transfers (`-j/--concurrency`), 2 requests/second, 4 retries with jittered
306
319
  exponential backoff, `Retry-After` honoured up to 120 s. A block page is retried twice
307
320
  (`--retry-403 N`); Zenodo and arXiv are donation-funded —
308
321
  `--rate 1 -j 1` costs you minutes and costs them less.
@@ -324,7 +337,8 @@ honest way to test this code path.
324
337
  ## Getting involved
325
338
 
326
339
  A [30-second tour](docs/demo.md) shows the whole loop, and
327
- [docs/integrations.md](docs/integrations.md) has Make, CI, aria2 and Snakemake
340
+ [docs/integrations.md](docs/integrations.md) has Make, CI, aria2, Snakemake and a
341
+ pooch downloader
328
342
  recipes.
329
343
 
330
344
  A nightly job downloads three fixed public records and verifies them, so a
@@ -73,7 +73,7 @@ python -m pip install -e . pytest mypy && python -m pytest -q
73
73
 
74
74
  ## Usage
75
75
 
76
- These work as typed:
76
+ One target per run:
77
77
 
78
78
  ```bash
79
79
  acadfetch fetch 10.5281/zenodo.23612969 # a DOI
@@ -81,6 +81,14 @@ acadfetch fetch arxiv:1706.03762 # an arXiv id
81
81
  acadfetch fetch https://ndownloader.figshare.com/files/70600233/report.pdf # any URL
82
82
  ```
83
83
 
84
+ The first two are answered by Zenodo and arXiv and work from any network those
85
+ are reachable on. The third is a direct URL, so the host decides: figshare's CDN
86
+ answers 403 with a WAF block page from some networks — the same behaviour that
87
+ made its *adapter* not worth keeping in 0.2.0. acadfetch says so and keeps
88
+ nothing, rather than saving the block page as a PDF. If you hit that, the file
89
+ itself is fine; see the `403` section of
90
+ [docs/troubleshooting.md](docs/troubleshooting.md).
91
+
84
92
  figshare was removed in 0.2.0: pass the article's download URL instead, and
85
93
  [`--checksum`](docs/migration-0.2.md) keeps the verification the API used to
86
94
  give you.
@@ -92,10 +100,13 @@ acadfetch fetch 10.5281/zenodo.23612969 -n # dry run: which dir, which files
92
100
  acadfetch fetch 10.5281/zenodo.23612969 --only raw.tar.gz
93
101
  acadfetch fetch 10.5281/zenodo.23612969 --max-size 2GB # refuse anything bigger
94
102
  acadfetch fetch 10.5281/zenodo.23612969 -j 1 --rate 0.5 # be a good citizen
103
+ acadfetch fetch 10.5281/zenodo.23612969 --retries 2 --retry-403 0 # give up sooner
104
+ acadfetch fetch 10.5281/zenodo.23612969 --manifest-out build/m.json # a second copy
95
105
  acadfetch manifest 10.5281/zenodo.23612969 -o m.json # metadata only, no bytes
96
106
  acadfetch fetch zenodo:123456 --header "Authorization: Bearer $TOKEN" # restricted
97
107
 
98
108
  acadfetch verify ./data/manifest.json # re-hash what you downloaded
109
+ acadfetch --version # what is installed
99
110
  acadfetch sources # adapters, with an example each
100
111
  ```
101
112
 
@@ -133,16 +144,18 @@ block page is retried; see [docs/troubleshooting.md](docs/troubleshooting.md).
133
144
  Progress is one aggregate line for the whole run
134
145
  (`3 file(s) 12.4/44.1 MiB 1.8 MiB/s ETA 0:17`), redrawn in place on a TTY.
135
146
  When stdout is a pipe it becomes a new line at most every 5 s (one extra when a
136
- file starts or stops), so a CI log stays greppable without becoming a wall. `-q`
137
- turns it off.
147
+ file starts or stops), so a CI log stays greppable without becoming a wall.
148
+ `-q/--quiet` turns it off.
138
149
 
139
- Output defaults to `acadfetch-out/<source>-<id>` (a URL adds its host and a short
150
+ `-o/--out` defaults to `acadfetch-out/<source>-<id>` (a URL adds its host and a short
140
151
  hash), and `-n` prints it before
141
152
  you spend bandwidth. Files land in `<name>.part` and are renamed only after the
142
153
  checksum matches, so an interrupted or failed run never leaves something that
143
154
  *looks* finished -- and never throws away the bytes it already paid for: rerun and
144
- it resumes. `acadfetch-out/` is worth adding to a project's `.gitignore`; this
145
- repository already has it.
155
+ it resumes. The one case where a run must start over is a server that ignores
156
+ `Range`; the bytes already on disk are then moved aside and kept until a whole
157
+ new copy has arrived, so a restart that dies early costs nothing. `acadfetch-out/`
158
+ is worth adding to a project's `.gitignore`; this repository already has it.
146
159
 
147
160
  The default directory names the target, not only its filename — `data.csv` from
148
161
  two hosts is two datasets, and they do not share a directory. A file already
@@ -271,7 +284,7 @@ so the trace can be pasted into an issue.
271
284
 
272
285
  ## Politeness
273
286
 
274
- Defaults: 3 parallel transfers, 2 requests/second, 4 retries with jittered
287
+ Defaults: 3 parallel transfers (`-j/--concurrency`), 2 requests/second, 4 retries with jittered
275
288
  exponential backoff, `Retry-After` honoured up to 120 s. A block page is retried twice
276
289
  (`--retry-403 N`); Zenodo and arXiv are donation-funded —
277
290
  `--rate 1 -j 1` costs you minutes and costs them less.
@@ -293,7 +306,8 @@ honest way to test this code path.
293
306
  ## Getting involved
294
307
 
295
308
  A [30-second tour](docs/demo.md) shows the whole loop, and
296
- [docs/integrations.md](docs/integrations.md) has Make, CI, aria2 and Snakemake
309
+ [docs/integrations.md](docs/integrations.md) has Make, CI, aria2, Snakemake and a
310
+ pooch downloader
297
311
  recipes.
298
312
 
299
313
  A nightly job downloads three fixed public records and verifies them, so a
@@ -256,7 +256,7 @@ def _file_row(row, redact: bool = True) -> dict: # noqa: ANN001
256
256
  "http_status": row.http_status,
257
257
  "final_url": hide(row.final_url) if row.final_url else None,
258
258
  "verified": row.verified,
259
- "verified_reason": verify_reason(row.file.checksum, row.verified),
259
+ "verified_reason": verify_reason(row.file.checksum, row.verified, row.state),
260
260
  "resumed_from": row.resumed_from,
261
261
  "part": Path(row.part).name if row.part else None,
262
262
  "error": row.error,
@@ -427,7 +427,10 @@ def cmd_fetch(args: argparse.Namespace) -> int:
427
427
  + ("" if disk["ok"] else " NOT ENOUGH")
428
428
  )
429
429
  for file in chosen:
430
- print(f" would fetch {naming.sanitize(file.name)} ({human(file.size)})")
430
+ # "?" in a size column reads as a bug; arXiv genuinely publishes no
431
+ # size, so say that instead of printing a question mark.
432
+ size = human(file.size) if file.size is not None else "size not published"
433
+ print(f" would fetch {naming.sanitize(file.name)} ({size})")
431
434
  return EXIT_OK
432
435
 
433
436
  try:
@@ -519,6 +522,16 @@ def cmd_verify(args: argparse.Namespace) -> int:
519
522
  rows = manifest.verify_manifest(manifest_path, root, allow_escape=args.allow_escape, data=data)
520
523
  remote: dict[str, tuple[str, str]] = {}
521
524
  if args.against_remote:
525
+ if data.get("url_redacted"):
526
+ # A presigned URL is stripped on the way into the manifest, so the
527
+ # recorded URL cannot be resolved again. Say why instead of leaving
528
+ # a page of remote_unavailable to be puzzled over.
529
+ print(
530
+ "note: this manifest had credential query parameters stripped, so a "
531
+ "row whose URL cannot be resolved again reports remote_unavailable "
532
+ "(--record-full-url keeps the original)",
533
+ file=sys.stderr,
534
+ )
522
535
  published_now, reason = remote_check.published(_client(args), data)
523
536
  entries = [e for e in data.get("files") or [] if isinstance(e, dict)]
524
537
  # verify_manifest walks the same list in order; the rows do not carry
@@ -1,6 +1,6 @@
1
1
  """Package-wide constants and version."""
2
2
 
3
- __version__ = "0.6.3"
3
+ __version__ = "0.7.0"
4
4
 
5
5
  USER_AGENT = f"acadfetch/{__version__} (+https://github.com/young-ski/acadfetch)"
6
6
 
@@ -153,9 +153,29 @@ def _kept_bytes(part: Path) -> int:
153
153
  return 0
154
154
 
155
155
 
156
- def _drop(*paths: Path) -> None:
156
+ def _room_for(part: Path, need: int) -> bool:
157
+ """Whether keeping the old bytes while a restarting copy runs still fits."""
158
+ try:
159
+ return shutil.disk_usage(part.parent).free >= need
160
+ except OSError:
161
+ return False
162
+
163
+
164
+ def _drop(*paths: Path) -> bool:
165
+ """Remove files, and say whether they are all gone.
166
+
167
+ ``atomic.unlink`` swallows a sharing violation on purpose: losing a race with
168
+ a reader is usually harmless, and a cleanup step must not become a new way to
169
+ fail. Where the deletion is load-bearing -- a mismatched part that must not
170
+ be resumed onto, a part that is already over ``--max-size`` -- the caller has
171
+ to know it did not happen, or the next attempt silently resumes onto bytes it
172
+ was told to forget.
173
+ """
174
+ kept = False
157
175
  for path in paths:
158
176
  atomic.unlink(path)
177
+ kept = kept or path.exists()
178
+ return not kept
159
179
 
160
180
 
161
181
  def _content_length(headers, url: str) -> int | None:
@@ -340,6 +360,17 @@ def _transfer(client: Client, file: File, part: Path, meta_path: Path, opts: Opt
340
360
  else:
341
361
  mode, offset = "ab", have
342
362
 
363
+ # A 200 to a Range request means the server ignored it, so this attempt
364
+ # restarts from zero. The bytes already on disk are still the best copy
365
+ # of the file there is until a new one completes, so a restart is written
366
+ # beside them and swapped in only once it is whole: opening the part with
367
+ # "wb" threw away 6 MiB in order to save 1, which is the opposite of what
368
+ # "never throws away the bytes it already paid for" promises. Without
369
+ # room for both copies, the old behaviour returns.
370
+ sink, aside = part, None
371
+ if whole and have and _room_for(part, have):
372
+ aside = sink = part.with_name(part.name + ".new")
373
+
343
374
  # Hash while the bytes go by instead of reading the finished file again.
344
375
  # A resumed transfer has to be fed what is already on disk first.
345
376
  parsed = verify.parse(file.checksum)
@@ -352,58 +383,73 @@ def _transfer(client: Client, file: File, part: Path, meta_path: Path, opts: Opt
352
383
  for old in iter(lambda: existing.read(1 << 20), b""):
353
384
  for hasher in hashers.values():
354
385
  hasher.update(old)
355
- _write_meta(
356
- meta_path,
357
- {
358
- "url": file.url,
359
- "etag": resp.headers.get("ETag"),
360
- "last_modified": resp.headers.get("Last-Modified"),
361
- "size": file.size,
362
- },
363
- )
364
- if declared is not None and file.size is not None:
365
- expected = file.size - offset
366
- if declared != expected:
367
- raise HttpError(
368
- 0,
369
- file.url,
370
- f"Content-Length {declared} disagrees with the published size "
371
- f"({expected} bytes still expected)",
372
- )
373
- # Reads poll with a short socket timeout so an interrupt is noticed
374
- # within a second instead of after the whole 60 s socket timeout, and
375
- # so an idle connection fails on its own budget. Closing the response
376
- # from another thread additionally kicks a read that is already
377
- # blocked.
378
- with _LIVE_LOCK:
379
- _LIVE.add(resp)
380
386
  try:
381
- with part.open(mode) as fh:
382
- for chunk in _chunks(resp, client.timeout, file.url):
383
- fh.write(chunk)
384
- for hasher in hashers.values():
385
- hasher.update(chunk)
386
- written += len(chunk)
387
- if opts.progress:
388
- opts.progress.advance(len(chunk))
389
- if opts.max_bytes and offset + written > opts.max_bytes:
390
- raise HttpError(
391
- 0,
392
- file.url,
393
- f"refused locally: over --max-size ({human(opts.max_bytes)})",
394
- reason="max-size",
395
- )
396
- if opts.fsync:
397
- # The rename below is atomic, but only against crashes that
398
- # leave the page cache intact. This is what makes the bytes
399
- # survive a power cut before they get a name.
400
- fh.flush()
401
- os.fsync(fh.fileno())
402
- finally:
387
+ _write_meta(
388
+ meta_path,
389
+ {
390
+ "url": file.url,
391
+ "etag": resp.headers.get("ETag"),
392
+ "last_modified": resp.headers.get("Last-Modified"),
393
+ "size": file.size,
394
+ },
395
+ )
396
+ if declared is not None and file.size is not None:
397
+ expected = file.size - offset
398
+ if declared != expected:
399
+ raise HttpError(
400
+ 0,
401
+ file.url,
402
+ f"Content-Length {declared} disagrees with the published size "
403
+ f"({expected} bytes still expected)",
404
+ )
405
+ # Reads poll with a short socket timeout so an interrupt is noticed
406
+ # within a second instead of after the whole 60 s socket timeout, and
407
+ # so an idle connection fails on its own budget. Closing the response
408
+ # from another thread additionally kicks a read that is already
409
+ # blocked.
403
410
  with _LIVE_LOCK:
404
- _LIVE.discard(resp)
405
- if declared is not None and declared != written:
406
- raise HttpError(0, file.url, f"read {written} bytes, response declared {declared}")
411
+ _LIVE.add(resp)
412
+ try:
413
+ with sink.open(mode) as fh:
414
+ for chunk in _chunks(resp, client.timeout, file.url):
415
+ fh.write(chunk)
416
+ for hasher in hashers.values():
417
+ hasher.update(chunk)
418
+ written += len(chunk)
419
+ if opts.progress:
420
+ opts.progress.advance(len(chunk))
421
+ if opts.max_bytes and offset + written > opts.max_bytes:
422
+ raise HttpError(
423
+ 0,
424
+ file.url,
425
+ f"refused locally: over --max-size ({human(opts.max_bytes)})",
426
+ reason="max-size",
427
+ )
428
+ if opts.fsync:
429
+ # The rename below is atomic, but only against crashes that
430
+ # leave the page cache intact. This is what makes the bytes
431
+ # survive a power cut before they get a name.
432
+ fh.flush()
433
+ os.fsync(fh.fileno())
434
+ finally:
435
+ with _LIVE_LOCK:
436
+ _LIVE.discard(resp)
437
+ if declared is not None and declared != written:
438
+ raise HttpError(0, file.url, f"read {written} bytes, response declared {declared}")
439
+ if aside is not None:
440
+ # A whole body, agreeing with the declared length: only now does
441
+ # the restart replace what was already there.
442
+ atomic.replace(sink, part)
443
+ except BaseException:
444
+ if aside is not None:
445
+ # Keep the bytes that were there, and the sidecar that described
446
+ # them rather than this attempt's validator.
447
+ atomic.unlink(aside)
448
+ if meta:
449
+ _write_meta(meta_path, meta)
450
+ else:
451
+ atomic.unlink(meta_path)
452
+ raise
407
453
  return Transfer(
408
454
  written=written,
409
455
  final_url=resp.geturl() or file.url,
@@ -560,6 +606,8 @@ def _reserved(files) -> set[str]: # noqa: ANN001
560
606
  for file in files:
561
607
  name = naming.sanitize(file.name).lower()
562
608
  reserved.add(name + ".part")
609
+ # A restart beside a kept part (a server that ignored Range) writes here.
610
+ reserved.add(name + ".part.new")
563
611
  reserved.add(name + META_SUFFIX)
564
612
  return reserved
565
613
 
@@ -675,12 +723,19 @@ def _attempt(
675
723
  _drop(part, meta_path)
676
724
  if exc.reason == "max-size":
677
725
  # Do not leave an over-limit part for the next run to
678
- # resume straight back into the same wall.
679
- _drop(part, meta_path)
726
+ # resume straight back into the same wall. Saying so when the
727
+ # deletion fails is the difference between a rerun that works
728
+ # and one that fails the same way with no explanation.
729
+ kept = not _drop(part, meta_path)
680
730
  return Result(
681
731
  file=file,
682
732
  state="failed",
683
- error=str(exc),
733
+ error=str(exc)
734
+ + (
735
+ f" (the over-limit {part.name} could not be deleted; remove it by hand)"
736
+ if kept
737
+ else ""
738
+ ),
684
739
  http_status=exc.status,
685
740
  retrieved_at=_now(),
686
741
  )
@@ -695,7 +750,12 @@ def _attempt(
695
750
  return Result(file=file, state="failed", error="interrupted")
696
751
  if attempt < opts.retries:
697
752
  time.sleep(min(2**attempt, 10) * (0.7 + random.random() * 0.6))
698
- resumed_from = part.stat().st_size if part.exists() else 0
753
+ # ``_transfer`` re-reads the part size itself, so the offset this
754
+ # attempt resumes from needs no bookkeeping here. ``resumed_from``
755
+ # is deliberately *not* updated: it answers "how much was on disk
756
+ # when this run started", which a retry does not change. Rewriting
757
+ # it made a fresh download report ``(resumed at 1.6 MiB)``. See
758
+ # docs/json-report.md.
699
759
  else:
700
760
  assert last is not None # the loop only ends after a failure
701
761
  return Result(
@@ -712,7 +772,10 @@ def _attempt(
712
772
  return Result(
713
773
  file=file,
714
774
  state="failed",
715
- error=str(last),
775
+ # A terminal 403 breaks out of the loop above, so it lands here
776
+ # rather than on the retry-exhausted path below. Without the hint
777
+ # that made it the one error a user could not act on.
778
+ error=f"{last}{_hint(last)}",
716
779
  http_status=getattr(last, "status", None),
717
780
  retrieved_at=_now(),
718
781
  )
@@ -729,7 +792,19 @@ def _attempt(
729
792
 
730
793
  verdict = _verdict(file, info.digests if info else None)
731
794
  if verdict is False:
732
- _drop(part, meta_path)
795
+ if not _drop(part, meta_path):
796
+ # Re-downloading would splice onto the bytes just proven wrong.
797
+ return Result(
798
+ file=file,
799
+ state="failed",
800
+ error=(
801
+ f"checksum mismatch, and the bad {part.name} could not be deleted; "
802
+ f"close whatever is reading it and retry"
803
+ ),
804
+ verified=False,
805
+ bytes=_kept_bytes(part),
806
+ retrieved_at=_now(),
807
+ )
733
808
  try:
734
809
  info = _transfer(client, file, part, meta_path, opts)
735
810
  verdict = _verdict(file, info.digests if info else None)
@@ -119,10 +119,10 @@ def _entry( # noqa: ANN001
119
119
  "http_status": result.http_status if result else None,
120
120
  "retrieved_at": result.retrieved_at if result else None,
121
121
  # PR-2 adds final_url / etag / http_status / retrieved_at here.
122
- "verified_reason": (
123
- "transfer-failed"
124
- if result and result.state == "failed"
125
- else verify.reason(f.checksum, result.verified if result else None)
122
+ "verified_reason": verify.reason(
123
+ f.checksum,
124
+ result.verified if result else None,
125
+ result.state if result else None,
126
126
  ),
127
127
  }
128
128
  if state == "not-selected":
@@ -0,0 +1,157 @@
1
+ """A pooch downloader that borrows acadfetch's HTTP client.
2
+
3
+ ``pooch`` already caches, verifies and moves a file into place. What it does not
4
+ do is survive a link that stalls at 80%: its own downloader issues a flat GET,
5
+ and a drop means the whole transfer again. This adds the transport only — every
6
+ request goes through :class:`acadfetch.client.Client`, so it honours
7
+ ``Retry-After``, retries a 5xx or a WAF block page, keeps a ``.part`` and resumes
8
+ it with ``Range``/``If-Range`` inside the attempt, and drops a credential header
9
+ when a redirect leaves the origin.
10
+
11
+ .. code-block:: python
12
+
13
+ import pooch
14
+ from acadfetch.pooch import downloader
15
+
16
+ path = pooch.retrieve(url, known_hash="md5:…", downloader=downloader)
17
+
18
+ ``pooch`` still does the hashing and the move, so this cannot make verification
19
+ weaker — only the arrival more likely. It is a plain callable rather than a
20
+ subclass because a callable is the whole contract ``pooch`` asks for.
21
+
22
+ Two limits worth knowing, both from how ``pooch`` drives a downloader:
23
+
24
+ * it hands each attempt a **fresh temporary file**, so a partial is resumed
25
+ within one call and not across calls — ``pooch`` owns the retry loop between
26
+ them;
27
+ * ``pooch``'s ``progressbar=True`` is implemented inside its own downloader and
28
+ is not passed here, so a transfer through this one is silent. The CLI's
29
+ progress line is a separate thing.
30
+ """
31
+
32
+ from __future__ import annotations
33
+
34
+ from collections.abc import Mapping
35
+ from pathlib import Path
36
+ from typing import Any
37
+
38
+ from . import atomic
39
+ from .client import Client
40
+ from .consts import DEFAULT_RATE, DEFAULT_RETRIES, DEFAULT_TIMEOUT
41
+ from .downloader import CHUNK, Options, fetch_file
42
+ from .errors import AcFError, HttpError, TargetError
43
+ from .sources.base import File
44
+
45
+ __all__ = ["Downloader", "downloader"]
46
+
47
+
48
+ class Downloader:
49
+ """The callable ``pooch`` wants, configured like the CLI.
50
+
51
+ Every argument maps to a flag of the same name, so a pipeline that already
52
+ tuned the CLI does not have to learn a second vocabulary.
53
+ """
54
+
55
+ def __init__(
56
+ self,
57
+ *,
58
+ rate: float = DEFAULT_RATE,
59
+ retries: int = DEFAULT_RETRIES,
60
+ timeout: float = DEFAULT_TIMEOUT,
61
+ headers: Mapping[str, str] | None = None,
62
+ block_retries: int = 2,
63
+ allow_http: bool = False,
64
+ fsync: bool = True,
65
+ ) -> None:
66
+ self.rate = rate
67
+ self.retries = retries
68
+ self.timeout = timeout
69
+ self.headers = dict(headers or {})
70
+ self.block_retries = block_retries
71
+ self.allow_http = allow_http
72
+ self.fsync = fsync
73
+
74
+ def _client(self) -> Client:
75
+ return Client(
76
+ rate=self.rate,
77
+ retries=self.retries,
78
+ timeout=self.timeout,
79
+ headers=self.headers,
80
+ block_retries=self.block_retries,
81
+ allow_http=self.allow_http,
82
+ )
83
+
84
+ # -- the two things pooch asks for -------------------------------------
85
+ def __call__(
86
+ self,
87
+ url: str,
88
+ output_file: Any = None,
89
+ pooch: Any = None,
90
+ check_only: bool = False,
91
+ ) -> bool | None:
92
+ if check_only:
93
+ return self.available(url)
94
+ if output_file is None:
95
+ raise TargetError("pooch asked for a download but passed no output file")
96
+ if hasattr(output_file, "write"):
97
+ # A caller-supplied handle: pooch owns the file, so there is nothing
98
+ # to rename and nothing worth resuming onto.
99
+ self._stream(url, output_file)
100
+ return None
101
+
102
+ target = Path(output_file)
103
+ target.parent.mkdir(parents=True, exist_ok=True)
104
+ result = fetch_file(
105
+ self._client(),
106
+ File(url=url, name=target.name),
107
+ Options(
108
+ out=target.parent,
109
+ concurrency=1,
110
+ retries=self.retries,
111
+ fsync=self.fsync,
112
+ ),
113
+ set(),
114
+ )
115
+ if result.state == "failed":
116
+ # pooch does not catch this, so the sentence the CLI would print is
117
+ # the sentence the caller sees.
118
+ raise AcFError(result.error or f"could not fetch {url}")
119
+ produced = Path(result.path or target)
120
+ if produced != target:
121
+ # `naming` is allowed to change a name; pooch's own file must win.
122
+ atomic.replace(produced, target)
123
+ return None
124
+
125
+ def available(self, url: str) -> bool:
126
+ """Whether the URL answers at all — HEAD first, one ranged byte after.
127
+
128
+ ``Client.head`` treats a 403/405/501 as "no headers", which is
129
+ indistinguishable from a 200 that sent none, so an empty answer falls
130
+ through to a one-byte range request rather than being read as success.
131
+ """
132
+ client = self._client()
133
+ try:
134
+ if client.head(url):
135
+ return True
136
+ except HttpError:
137
+ return False
138
+ try:
139
+ with client.open(url, {"Range": "bytes=0-0"}, attempts=1):
140
+ return True
141
+ except HttpError:
142
+ return False
143
+
144
+ def _stream(self, url: str, handle: Any) -> None:
145
+ client = self._client()
146
+ with client.open(url, attempts=self.retries + 1) as resp:
147
+ reader = getattr(resp, "read1", None) or resp.read
148
+ while True:
149
+ chunk = reader(CHUNK)
150
+ if not chunk:
151
+ break
152
+ handle.write(chunk)
153
+ handle.flush()
154
+
155
+
156
+ #: The zero-configuration instance: ``pooch.retrieve(..., downloader=downloader)``.
157
+ downloader = Downloader()