pythonhere 0.3.0__py3-none-any.whl → 0.4.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,998 @@
1
+ """
2
+ A small cross-platform Hugging Face Hub downloader designed for PythonHere.
3
+
4
+ Uses requests to download public, private, or gated files with persistent resume,
5
+ progress callbacks, retries, integrity checks, and atomic completion. This module
6
+ implements only the Hub features needed for model downloads; it is not a general
7
+ replacement for huggingface_hub.
8
+
9
+ Typical use:
10
+
11
+ from pythonhere.ml_here import download_hf_model
12
+
13
+ path = download_hf_model(
14
+ "litert-community/some-model",
15
+ "model.tflite",
16
+ token=oauth_access_token,
17
+ progress=lambda p: print(p.percent, p.speed_mbps),
18
+ )
19
+
20
+ For a public model, omit `token`.
21
+ """
22
+
23
+ from __future__ import annotations
24
+
25
+ import hashlib
26
+ import json
27
+ import os
28
+ import random
29
+ import re
30
+ import time
31
+ from collections.abc import Callable
32
+ from dataclasses import asdict, dataclass, field
33
+ from http import HTTPStatus
34
+ from pathlib import Path
35
+ from typing import Any
36
+ from urllib.parse import quote
37
+
38
+ import requests
39
+
40
+ from .storage import model_path
41
+
42
+ __all__ = [
43
+ "HFClient",
44
+ "DownloadProgress",
45
+ "RemoteFile",
46
+ "HFError",
47
+ "HFAuthError",
48
+ "HFGatedError",
49
+ "HFNotFoundError",
50
+ "HFRateLimitError",
51
+ "HFIntegrityError",
52
+ "HFCancelled",
53
+ "download_hf_model",
54
+ ]
55
+
56
+
57
+ class HFError(RuntimeError):
58
+ """Base error for Hugging Face Hub operations."""
59
+
60
+
61
+ class HFAuthError(HFError):
62
+ """Authentication is missing, invalid, or insufficient."""
63
+
64
+
65
+ class HFGatedError(HFAuthError):
66
+ """The model is gated and this user does not currently have access."""
67
+
68
+
69
+ class HFNotFoundError(HFError):
70
+ """Repository, revision, or file was not found."""
71
+
72
+
73
+ class HFRateLimitError(HFError):
74
+ """Hugging Face rejected the request because of rate limiting."""
75
+
76
+
77
+ class HFIntegrityError(HFError):
78
+ """Downloaded data did not match expected size/hash."""
79
+
80
+
81
+ class HFCancelled(HFError):
82
+ """Download was cancelled by the caller."""
83
+
84
+
85
+ @dataclass(frozen=True)
86
+ class RemoteFile:
87
+ """Remote identity and metadata used to validate a download."""
88
+
89
+ repo_id: str
90
+ filename: str
91
+ revision: str
92
+ size: int | None
93
+ etag: str | None
94
+ commit: str | None
95
+ url: str
96
+
97
+
98
+ @dataclass(frozen=True)
99
+ class DownloadProgress:
100
+ """Byte counts and timing for a download progress notification."""
101
+
102
+ repo_id: str
103
+ filename: str
104
+ destination: str
105
+ downloaded: int
106
+ total: int | None
107
+ elapsed: float
108
+ speed_bps: float
109
+
110
+ @property
111
+ def fraction(self) -> float | None:
112
+ if not self.total:
113
+ return None
114
+ return min(1.0, self.downloaded / self.total)
115
+
116
+ @property
117
+ def percent(self) -> float | None:
118
+ value = self.fraction
119
+ return None if value is None else value * 100.0
120
+
121
+ @property
122
+ def speed_mbps(self) -> float:
123
+ return self.speed_bps / (1024 * 1024)
124
+
125
+ @property
126
+ def remaining(self) -> int | None:
127
+ if self.total is None:
128
+ return None
129
+ return max(0, self.total - self.downloaded)
130
+
131
+ @property
132
+ def eta(self) -> float | None:
133
+ remaining = self.remaining
134
+ if remaining is None or self.speed_bps <= 0:
135
+ return None
136
+ return remaining / self.speed_bps
137
+
138
+
139
+ @dataclass
140
+ class _PartialMeta:
141
+ repo_id: str
142
+ filename: str
143
+ revision: str
144
+ etag: str | None
145
+ commit: str | None
146
+ expected_size: int | None
147
+
148
+
149
+ @dataclass
150
+ class _ProgressReporter:
151
+ remote: RemoteFile
152
+ destination: Path
153
+ callback: Callable[[DownloadProgress], None] | None
154
+ interval: float
155
+ started: float = field(init=False, default=0.0)
156
+ initial_size: int = field(init=False, default=0)
157
+ last_time: float = field(init=False, default=0.0)
158
+ last_size: int = field(init=False, default=0)
159
+
160
+ def start(self, size: int) -> None:
161
+ self.started = self.last_time = time.monotonic()
162
+ self.initial_size = self.last_size = size
163
+
164
+ def report(self, current: int) -> None:
165
+ now = time.monotonic()
166
+ if self.callback and (
167
+ now - self.last_time >= self.interval
168
+ or (self.remote.size is not None and current >= self.remote.size)
169
+ ):
170
+ self.callback(
171
+ DownloadProgress(
172
+ repo_id=self.remote.repo_id,
173
+ filename=self.remote.filename,
174
+ destination=str(self.destination),
175
+ downloaded=current,
176
+ total=self.remote.size,
177
+ elapsed=now - self.started,
178
+ speed_bps=(current - self.last_size)
179
+ / max(now - self.last_time, 1e-9),
180
+ )
181
+ )
182
+ self.last_time = now
183
+ self.last_size = current
184
+
185
+ def finish(self) -> None:
186
+ if self.callback:
187
+ end = time.monotonic()
188
+ size = self.destination.stat().st_size
189
+ self.callback(
190
+ DownloadProgress(
191
+ repo_id=self.remote.repo_id,
192
+ filename=self.remote.filename,
193
+ destination=str(self.destination),
194
+ downloaded=size,
195
+ total=self.remote.size or size,
196
+ elapsed=end - self.started,
197
+ speed_bps=(size - self.initial_size)
198
+ / max(end - self.started, 1e-9),
199
+ )
200
+ )
201
+
202
+
203
+ def _raise_download_error(response: requests.Response, filename: str) -> None:
204
+ if response.status_code == HTTPStatus.TOO_MANY_REQUESTS:
205
+ raise HFRateLimitError(
206
+ f"Hugging Face rate limit exceeded while downloading {filename!r}."
207
+ )
208
+ raise HFError(f"HTTP {response.status_code} while downloading {filename!r}.")
209
+
210
+
211
+ _BYTES_PER_KIB = 1024
212
+ _SECONDS_PER_MINUTE = 60
213
+ _MINUTES_PER_HOUR = 60
214
+
215
+ _RETRYABLE_STATUS = {408, 425, 429, 500, 502, 503, 504}
216
+ _SAFE_ETAG = re.compile(r'^(?:W/)?"?(.*?)"?$')
217
+
218
+
219
+ def _clean_etag(value: str | None) -> str | None:
220
+ if not value:
221
+ return None
222
+ value = value.strip()
223
+ m = _SAFE_ETAG.match(value)
224
+ return m.group(1) if m else value.strip('"')
225
+
226
+
227
+ def _int_header(
228
+ headers: requests.structures.CaseInsensitiveDict, name: str
229
+ ) -> int | None:
230
+ value = headers.get(name)
231
+ if value is None:
232
+ return None
233
+ try:
234
+ return int(value)
235
+ except (TypeError, ValueError):
236
+ return None
237
+
238
+
239
+ def _remote_size(response: requests.Response) -> int | None:
240
+ # HF exposes the actual object size through X-Linked-Size for LFS-backed
241
+ # files. On a final ranged/CDN response, Content-Range is the strongest
242
+ # source. Fall back to ordinary Content-Length.
243
+ linked = _int_header(response.headers, "X-Linked-Size")
244
+ if linked is not None:
245
+ return linked
246
+
247
+ content_range = response.headers.get("Content-Range")
248
+ if content_range and "/" in content_range:
249
+ tail = content_range.rsplit("/", 1)[-1]
250
+ if tail != "*":
251
+ try:
252
+ return int(tail)
253
+ except ValueError:
254
+ pass
255
+
256
+ # Only trust Content-Length when the body is not content-encoded.
257
+ encoding = response.headers.get("Content-Encoding", "identity").lower()
258
+ if encoding == "identity":
259
+ return _int_header(response.headers, "Content-Length")
260
+ return None
261
+
262
+
263
+ def _retry_after(response: requests.Response) -> float | None:
264
+ value = response.headers.get("Retry-After")
265
+ if not value:
266
+ return None
267
+ try:
268
+ return max(0.0, float(value))
269
+ except ValueError:
270
+ return None
271
+
272
+
273
+ def _atomic_json_write(path: Path, data: dict[str, Any]) -> None:
274
+ tmp = path.with_name(path.name + ".tmp")
275
+ tmp.write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding="utf-8")
276
+ os.replace(tmp, path)
277
+
278
+
279
+ class HFClient:
280
+ """
281
+ Lightweight Hugging Face Hub download client.
282
+
283
+ Parameters
284
+ ----------
285
+ token:
286
+ Hugging Face user token or OAuth access token. Use None for public files.
287
+ endpoint:
288
+ Hub base URL. Mainly useful for testing or custom Hub-compatible hosts.
289
+ user_agent:
290
+ Sent on all requests.
291
+ timeout:
292
+ `(connect_timeout, read_timeout)` passed to requests.
293
+ The read timeout is per socket read, not a total download timeout.
294
+ chunk_size:
295
+ Streaming chunk size.
296
+ max_retries:
297
+ Retries after transient network/server failures.
298
+ backoff:
299
+ Base exponential retry delay.
300
+ """
301
+
302
+ # Public connection options remain individually configurable.
303
+ def __init__( # noqa: PLR0913
304
+ self,
305
+ token: str | None = None,
306
+ *,
307
+ endpoint: str = "https://huggingface.co",
308
+ user_agent: str = "pythonhere-hf-downloader/1.0",
309
+ timeout: tuple[float, float] = (15.0, 60.0),
310
+ chunk_size: int = 1024 * 1024,
311
+ max_retries: int = 5,
312
+ backoff: float = 1.0,
313
+ session: requests.Session | None = None,
314
+ ):
315
+ # pylint: disable=too-many-arguments
316
+ self.token = token
317
+ self.endpoint = endpoint.rstrip("/")
318
+ self.user_agent = user_agent
319
+ self.timeout = timeout
320
+ self.chunk_size = chunk_size
321
+ self.max_retries = max_retries
322
+ self.backoff = backoff
323
+ self.session = session or requests.Session()
324
+
325
+ def resolve_url(self, repo_id: str, filename: str, revision: str = "main") -> str:
326
+ repo = "/".join(quote(part, safe="") for part in repo_id.split("/"))
327
+ rev = quote(revision, safe="")
328
+ file_path = "/".join(quote(part, safe="") for part in filename.split("/"))
329
+ return f"{self.endpoint}/{repo}/resolve/{rev}/{file_path}"
330
+
331
+ def raw_url(self, repo_id: str, filename: str, revision: str = "main") -> str:
332
+ repo = "/".join(quote(part, safe="") for part in repo_id.split("/"))
333
+ rev = quote(revision, safe="")
334
+ file_path = "/".join(quote(part, safe="") for part in filename.split("/"))
335
+ return f"{self.endpoint}/{repo}/raw/{rev}/{file_path}"
336
+
337
+ def model_url(self, repo_id: str) -> str:
338
+ return f"{self.endpoint}/{repo_id}"
339
+
340
+ def _headers(self, extra: dict[str, str] | None = None) -> dict[str, str]:
341
+ # identity matters for reliable byte ranges / sizes.
342
+ headers = {
343
+ "User-Agent": self.user_agent,
344
+ "Accept-Encoding": "identity",
345
+ }
346
+ if self.token:
347
+ headers["Authorization"] = f"Bearer {self.token}"
348
+ if extra:
349
+ headers.update(extra)
350
+ return headers
351
+
352
+ def model_info(
353
+ self, repo_id: str, *, revision: str | None = None
354
+ ) -> dict[str, Any]:
355
+ """
356
+ Return raw model metadata from the Hub REST API.
357
+
358
+ The response includes useful fields such as `sha`, `gated`, and `siblings`.
359
+ """
360
+ repo = "/".join(quote(part, safe="") for part in repo_id.split("/"))
361
+ if revision:
362
+ rev = quote(revision, safe="")
363
+ url = f"{self.endpoint}/api/models/{repo}/revision/{rev}"
364
+ else:
365
+ url = f"{self.endpoint}/api/models/{repo}"
366
+
367
+ response = self._request_with_retry("GET", url, headers=self._headers())
368
+ self._raise_hf_error(response, repo_id=repo_id)
369
+ return response.json()
370
+
371
+ def list_files(self, repo_id: str, *, revision: str | None = None) -> list[str]:
372
+ info = self.model_info(repo_id, revision=revision)
373
+ return [
374
+ item["rfilename"]
375
+ for item in info.get("siblings", [])
376
+ if isinstance(item, dict) and item.get("rfilename")
377
+ ]
378
+
379
+ def file_info(
380
+ self,
381
+ repo_id: str,
382
+ filename: str,
383
+ *,
384
+ revision: str = "main",
385
+ ) -> RemoteFile:
386
+ """
387
+ Resolve a Hub file and return size/ETag/commit metadata without
388
+ downloading the body.
389
+ """
390
+ url = self.resolve_url(repo_id, filename, revision)
391
+
392
+ response = self._request_with_retry(
393
+ "HEAD",
394
+ url,
395
+ headers=self._headers(),
396
+ allow_redirects=True,
397
+ )
398
+ self._raise_hf_error(
399
+ response,
400
+ repo_id=repo_id,
401
+ filename=filename,
402
+ )
403
+
404
+ etag = _clean_etag(
405
+ response.headers.get("X-Linked-Etag") or response.headers.get("ETag")
406
+ )
407
+ commit = response.headers.get("X-Repo-Commit")
408
+ size = _remote_size(response)
409
+
410
+ return RemoteFile(
411
+ repo_id=repo_id,
412
+ filename=filename,
413
+ revision=revision,
414
+ size=size,
415
+ etag=etag,
416
+ commit=commit,
417
+ # Intentionally expose the stable HF resolve URL, not an expiring
418
+ # signed CDN/Xet redirect target.
419
+ url=url,
420
+ )
421
+
422
+ def git_lfs_info(
423
+ self,
424
+ repo_id: str,
425
+ filename: str,
426
+ *,
427
+ revision: str = "main",
428
+ ) -> dict[str, Any] | None:
429
+ """Return SHA-256 and size when the repository stores a Git LFS pointer."""
430
+
431
+ response = self._request_with_retry(
432
+ "GET",
433
+ self.raw_url(repo_id, filename, revision),
434
+ headers=self._headers({"Range": "bytes=0-1024"}),
435
+ stream=True,
436
+ )
437
+ try:
438
+ self._raise_hf_error(response, repo_id=repo_id, filename=filename)
439
+ data = response.raw.read(1025)
440
+ finally:
441
+ response.close()
442
+
443
+ try:
444
+ text = data.decode("utf-8")
445
+ except UnicodeDecodeError:
446
+ return None
447
+ match = re.fullmatch(
448
+ r"version https://git-lfs\.github\.com/spec/v1\r?\n"
449
+ r"oid sha256:([0-9a-fA-F]{64})\r?\n"
450
+ r"size ([0-9]+)\r?\n?",
451
+ text,
452
+ )
453
+ if match:
454
+ return {"sha256": match.group(1).lower(), "size": int(match.group(2))}
455
+ if text.startswith("version https://git-lfs.github.com/spec/v1"):
456
+ raise HFIntegrityError(f"Malformed Git LFS pointer for {filename}")
457
+ return None
458
+
459
+ # Preserve the public download keyword options.
460
+ def download( # noqa: PLR0913
461
+ self,
462
+ repo_id: str,
463
+ filename: str,
464
+ destination: str | os.PathLike[str],
465
+ *,
466
+ revision: str = "main",
467
+ resume: bool = True,
468
+ overwrite: bool = False,
469
+ progress: Callable[[DownloadProgress], None] | None = None,
470
+ cancelled: Callable[[], bool] | None = None,
471
+ expected_sha256: str | None = None,
472
+ progress_interval: float = 0.15,
473
+ ) -> Path:
474
+ """
475
+ Download one file from a Hugging Face model repository.
476
+
477
+ The final destination is written atomically:
478
+ destination.part
479
+ destination.part.json
480
+ -> destination
481
+
482
+ Existing partial downloads are resumed when their remote metadata still
483
+ matches. If the revision/file changed, the stale partial is discarded.
484
+
485
+ Parameters
486
+ ----------
487
+ progress:
488
+ Callback receiving DownloadProgress. It runs on the thread calling
489
+ `download`; for Kivy, marshal UI changes onto the UI thread.
490
+ cancelled:
491
+ Callback returning True when cancellation is requested. The .part
492
+ file is preserved, so calling download() later resumes it.
493
+ expected_sha256:
494
+ Optional explicit SHA-256 integrity check. Usually unnecessary for
495
+ ordinary HF use, but useful when your app ships a trusted catalog.
496
+ """
497
+ # Public options account for eleven of the seventeen local names.
498
+ # pylint: disable=too-many-arguments,too-many-locals
499
+ dest = Path(destination)
500
+ dest.parent.mkdir(parents=True, exist_ok=True)
501
+ part = dest.with_name(dest.name + ".part")
502
+ meta_path = dest.with_name(dest.name + ".part.json")
503
+
504
+ remote = self.file_info(repo_id, filename, revision=revision)
505
+
506
+ if dest.exists() and not overwrite:
507
+ self._check_existing(dest, remote, expected_sha256)
508
+ return dest
509
+
510
+ # Files above regular HF HTTP's supported range require Xet.
511
+ if remote.size is not None and remote.size > 50 * 1024**3:
512
+ raise HFError(
513
+ f"{filename!r} is larger than 50 GiB. Hugging Face does not "
514
+ "serve files of this size through its regular HTTP download "
515
+ "path; an Xet-capable client is required."
516
+ )
517
+
518
+ expected_meta = self._prepare_partial(part, meta_path, remote, resume)
519
+
520
+ # If a previous invocation finished the bytes but died before rename,
521
+ # finish locally without a network GET.
522
+ if remote.size is not None and part.stat().st_size == remote.size:
523
+ self._verify_and_commit(
524
+ part,
525
+ dest,
526
+ meta_path,
527
+ expected_size=remote.size,
528
+ expected_sha256=expected_sha256,
529
+ )
530
+ return dest
531
+
532
+ reporter = _ProgressReporter(remote, dest, progress, progress_interval)
533
+ reporter.start(part.stat().st_size)
534
+ self._download_with_retry(
535
+ part, meta_path, expected_meta, reporter, cancelled=cancelled, resume=resume
536
+ )
537
+
538
+ self._verify_and_commit(
539
+ part,
540
+ dest,
541
+ meta_path,
542
+ expected_size=remote.size,
543
+ expected_sha256=expected_sha256,
544
+ )
545
+
546
+ reporter.finish()
547
+
548
+ return dest
549
+
550
+ @staticmethod
551
+ def _check_existing(dest: Path, remote: RemoteFile, expected_sha256: str | None):
552
+ actual_size = dest.stat().st_size
553
+ if remote.size is not None and actual_size != remote.size:
554
+ raise HFIntegrityError(
555
+ f"Existing {dest.name} has {actual_size} bytes; expected "
556
+ f"{remote.size}. The file was left unchanged."
557
+ )
558
+ if expected_sha256 is not None:
559
+ actual_sha256 = HFClient._sha256(dest)
560
+ if actual_sha256.lower() != expected_sha256.lower():
561
+ raise HFIntegrityError(
562
+ f"Existing {dest.name} has SHA-256 {actual_sha256}; "
563
+ f"expected {expected_sha256}. The file was left unchanged."
564
+ )
565
+
566
+ def _prepare_partial(self, part, meta_path, remote, resume):
567
+ expected_meta = _PartialMeta(
568
+ repo_id=remote.repo_id,
569
+ filename=remote.filename,
570
+ revision=remote.revision,
571
+ etag=remote.etag,
572
+ commit=remote.commit,
573
+ expected_size=remote.size,
574
+ )
575
+
576
+ if not resume:
577
+ self._remove_partial(part, meta_path)
578
+ else:
579
+ self._validate_partial(part, meta_path, expected_meta)
580
+
581
+ if not part.exists():
582
+ part.touch()
583
+ _atomic_json_write(meta_path, asdict(expected_meta))
584
+
585
+ return expected_meta
586
+
587
+ def _download_with_retry( # noqa: PLR0913
588
+ self, part, meta_path, expected_meta, reporter, *, cancelled, resume
589
+ ):
590
+ # Retry state includes partial provenance and caller cancellation policy.
591
+ # pylint: disable=too-many-arguments
592
+ remote = reporter.remote
593
+ filename = remote.filename
594
+ attempt = 0
595
+ while True:
596
+ if cancelled and cancelled():
597
+ raise HFCancelled(f"Download cancelled: {filename}")
598
+
599
+ current = part.stat().st_size
600
+ if remote.size is not None and current > remote.size:
601
+ self._remove_partial(part, meta_path)
602
+ part.touch()
603
+ _atomic_json_write(meta_path, asdict(expected_meta))
604
+ current = 0
605
+
606
+ range_headers: dict[str, str] = {}
607
+ if resume and current > 0:
608
+ range_headers["Range"] = f"bytes={current}-"
609
+
610
+ try:
611
+ response = self.session.get(
612
+ remote.url,
613
+ headers=self._headers(range_headers),
614
+ stream=True,
615
+ allow_redirects=True,
616
+ timeout=self.timeout,
617
+ )
618
+
619
+ # A stale/exact-complete range can produce 416.
620
+ if response.status_code == HTTPStatus.REQUESTED_RANGE_NOT_SATISFIABLE:
621
+ if remote.size is not None and current == remote.size:
622
+ response.close()
623
+ break
624
+ response.close()
625
+ # Do not append to something the server rejects.
626
+ self._remove_partial(part, meta_path)
627
+ part.touch()
628
+ _atomic_json_write(meta_path, asdict(expected_meta))
629
+ attempt += 1
630
+ if attempt > self.max_retries:
631
+ raise HFIntegrityError(
632
+ f"Server rejected resume range for {filename!r}."
633
+ )
634
+ continue
635
+
636
+ if response.status_code in _RETRYABLE_STATUS:
637
+ delay = _retry_after(response)
638
+ response.close()
639
+ if attempt >= self.max_retries:
640
+ _raise_download_error(response, filename)
641
+ self._sleep_backoff(attempt, delay)
642
+ attempt += 1
643
+ continue
644
+
645
+ self._raise_hf_error(
646
+ response,
647
+ repo_id=remote.repo_id,
648
+ filename=filename,
649
+ )
650
+
651
+ self._stream_response(response, part, reporter, cancelled)
652
+
653
+ # A cleanly closed response should be complete. Verify below.
654
+ break
655
+
656
+ except (
657
+ requests.ConnectionError,
658
+ requests.Timeout,
659
+ requests.exceptions.ChunkedEncodingError,
660
+ ) as exc:
661
+ if attempt >= self.max_retries:
662
+ raise HFError(
663
+ f"Network error while downloading {filename!r} after "
664
+ f"{self.max_retries + 1} attempts: {exc}"
665
+ ) from exc
666
+ self._sleep_backoff(attempt)
667
+ attempt += 1
668
+ # Loop reopens .part and resumes from its current byte count.
669
+
670
+ def _stream_response(self, response, part, reporter, cancelled):
671
+ current = part.stat().st_size
672
+ # Never append a full response when the server ignores Range.
673
+ if current > 0 and response.status_code == HTTPStatus.PARTIAL_CONTENT:
674
+ mode = "ab"
675
+ else:
676
+ mode = "wb"
677
+ current = 0
678
+
679
+ with response, part.open(mode) as fp:
680
+ for chunk in response.iter_content(chunk_size=self.chunk_size):
681
+ if cancelled and cancelled():
682
+ fp.flush()
683
+ try:
684
+ os.fsync(fp.fileno())
685
+ except OSError:
686
+ pass
687
+ raise HFCancelled(f"Download cancelled: {reporter.remote.filename}")
688
+ if not chunk:
689
+ continue
690
+ fp.write(chunk)
691
+ current += len(chunk)
692
+ reporter.report(current)
693
+
694
+ def _request_with_retry(
695
+ self, method: str, url: str, **kwargs: Any
696
+ ) -> requests.Response:
697
+ last_exc: BaseException | None = None
698
+
699
+ for attempt in range(self.max_retries + 1):
700
+ try:
701
+ response = self.session.request(
702
+ method,
703
+ url,
704
+ timeout=self.timeout,
705
+ **kwargs,
706
+ )
707
+ except (requests.ConnectionError, requests.Timeout) as exc:
708
+ last_exc = exc
709
+ if attempt >= self.max_retries:
710
+ break
711
+ self._sleep_backoff(attempt)
712
+ continue
713
+
714
+ if response.status_code not in _RETRYABLE_STATUS:
715
+ return response
716
+
717
+ if attempt >= self.max_retries:
718
+ return response
719
+
720
+ delay = _retry_after(response)
721
+ response.close()
722
+ self._sleep_backoff(attempt, delay)
723
+
724
+ raise HFError(f"Network request failed: {last_exc}") from last_exc
725
+
726
+ def _sleep_backoff(self, attempt: int, explicit: float | None = None) -> None:
727
+ if explicit is not None:
728
+ delay = explicit
729
+ else:
730
+ delay = self.backoff * (2**attempt)
731
+ delay *= random.uniform(0.8, 1.2)
732
+ time.sleep(min(delay, 30.0))
733
+
734
+ def _validate_partial(
735
+ self,
736
+ part: Path,
737
+ meta_path: Path,
738
+ expected: _PartialMeta,
739
+ ) -> None:
740
+ if not part.exists():
741
+ if meta_path.exists():
742
+ meta_path.unlink()
743
+ return
744
+
745
+ if not meta_path.exists():
746
+ # Unknown provenance: unsafe to resume.
747
+ part.unlink()
748
+ return
749
+
750
+ try:
751
+ saved_raw = json.loads(meta_path.read_text(encoding="utf-8"))
752
+ saved = _PartialMeta(**saved_raw)
753
+ except Exception:
754
+ self._remove_partial(part, meta_path)
755
+ return
756
+
757
+ same_identity = (
758
+ saved.repo_id == expected.repo_id
759
+ and saved.filename == expected.filename
760
+ and saved.revision == expected.revision
761
+ )
762
+
763
+ # Prefer commit/etag as version identity. If HF didn't expose either,
764
+ # size still gives a weak but useful guard.
765
+ version_matches = True
766
+ if expected.commit and saved.commit:
767
+ version_matches = expected.commit == saved.commit
768
+ elif expected.etag and saved.etag:
769
+ version_matches = expected.etag == saved.etag
770
+ elif expected.expected_size is not None and saved.expected_size is not None:
771
+ version_matches = expected.expected_size == saved.expected_size
772
+
773
+ if not same_identity or not version_matches:
774
+ self._remove_partial(part, meta_path)
775
+
776
+ @staticmethod
777
+ def _remove_partial(part: Path, meta_path: Path) -> None:
778
+ for path in (part, meta_path):
779
+ try:
780
+ path.unlink()
781
+ except FileNotFoundError:
782
+ pass
783
+
784
+ @staticmethod
785
+ def _sha256(path: Path) -> str:
786
+ digest = hashlib.sha256()
787
+ with path.open("rb") as fp:
788
+ while True:
789
+ chunk = fp.read(1024 * 1024)
790
+ if not chunk:
791
+ break
792
+ digest.update(chunk)
793
+ return digest.hexdigest()
794
+
795
+ def _verify_and_commit(
796
+ self,
797
+ part: Path,
798
+ dest: Path,
799
+ meta_path: Path,
800
+ *,
801
+ expected_size: int | None,
802
+ expected_sha256: str | None,
803
+ ) -> None:
804
+ # Paths and integrity expectations are explicit at the commit boundary.
805
+ # pylint: disable=too-many-arguments
806
+ actual_size = part.stat().st_size
807
+
808
+ if expected_size is not None and actual_size != expected_size:
809
+ raise HFIntegrityError(
810
+ f"Incomplete download: expected {expected_size} bytes, "
811
+ f"got {actual_size} bytes. Partial file was kept for resume."
812
+ )
813
+
814
+ if expected_sha256 is not None:
815
+ actual = self._sha256(part)
816
+ if actual.lower() != expected_sha256.lower():
817
+ raise HFIntegrityError(
818
+ f"SHA-256 mismatch for {dest.name}: expected "
819
+ f"{expected_sha256}, got {actual}. Partial file was kept."
820
+ )
821
+
822
+ os.replace(part, dest)
823
+ try:
824
+ meta_path.unlink()
825
+ except FileNotFoundError:
826
+ pass
827
+
828
+ def _raise_hf_error(
829
+ self,
830
+ response: requests.Response,
831
+ *,
832
+ repo_id: str | None = None,
833
+ filename: str | None = None,
834
+ ) -> None:
835
+ status = response.status_code
836
+ if HTTPStatus.OK <= status < HTTPStatus.BAD_REQUEST:
837
+ return
838
+
839
+ request_id = response.headers.get("X-Request-Id") or response.headers.get(
840
+ "X-Amzn-Trace-Id"
841
+ )
842
+
843
+ body = ""
844
+ try:
845
+ body = response.text[:4096]
846
+ except Exception:
847
+ pass
848
+
849
+ lower = body.lower()
850
+ subject = repo_id or "Hugging Face resource"
851
+ if filename:
852
+ subject = f"{subject}/{filename}"
853
+
854
+ suffix = f" [request id: {request_id}]" if request_id else ""
855
+
856
+ if status == HTTPStatus.TOO_MANY_REQUESTS:
857
+ raise HFRateLimitError(
858
+ f"Hugging Face rate limit exceeded for {subject}.{suffix}"
859
+ )
860
+
861
+ # HF gated-resource responses are commonly 401/403 with an explanatory
862
+ # body. Detect this before the generic auth case.
863
+ if status in (401, 403) and (
864
+ "gated" in lower
865
+ or "access to model" in lower
866
+ or "access to this model" in lower
867
+ or "request access" in lower
868
+ ):
869
+ model_page = self.model_url(repo_id) if repo_id else self.endpoint
870
+ raise HFGatedError(
871
+ f"Access to {subject} is gated. The signed-in Hugging Face user "
872
+ f"must be granted/accept access first: {model_page}{suffix}"
873
+ )
874
+
875
+ if status in (401, 403):
876
+ if self.token:
877
+ raise HFAuthError(
878
+ f"Hugging Face denied access to {subject}. The token may "
879
+ f"be invalid or lack permission.{suffix}"
880
+ )
881
+ raise HFAuthError(
882
+ f"{subject} requires Hugging Face authentication.{suffix}"
883
+ )
884
+
885
+ if status == HTTPStatus.NOT_FOUND:
886
+ raise HFNotFoundError(
887
+ f"Hugging Face repository, revision, or file not found: "
888
+ f"{subject}.{suffix}"
889
+ )
890
+
891
+ message = f"Hugging Face returned HTTP {status} for {subject}"
892
+ if body:
893
+ # Keep server text useful but bounded.
894
+ compact = " ".join(body.split())
895
+ message += f": {compact[:500]}"
896
+ raise HFError(message + suffix)
897
+
898
+
899
+ # Keep the convenience wrapper compatible with existing callers.
900
+ def download_hf_model( # noqa: PLR0913
901
+ repo_id: str,
902
+ filename: str,
903
+ *,
904
+ revision: str = "main",
905
+ token: str | None = None,
906
+ destination: str | os.PathLike[str] | None = None,
907
+ progress: Callable[[DownloadProgress], None] | None = None,
908
+ cancelled: Callable[[], bool] | None = None,
909
+ overwrite: bool = False,
910
+ expected_sha256: str | None = None,
911
+ ) -> str:
912
+ """Download one Hugging Face model file and return its local path as a string."""
913
+
914
+ # pylint: disable=too-many-arguments
915
+ client = HFClient(token=token)
916
+ remote = client.file_info(repo_id, filename, revision=revision)
917
+ lfs_info = client.git_lfs_info(repo_id, filename, revision=revision)
918
+ if lfs_info is not None:
919
+ if remote.size is not None and lfs_info["size"] != remote.size:
920
+ raise HFIntegrityError(
921
+ f"Conflicting sizes for {filename}: download metadata reports "
922
+ f"{remote.size}, Git LFS reports {lfs_info['size']}"
923
+ )
924
+ if expected_sha256 is None:
925
+ expected_sha256 = lfs_info["sha256"]
926
+
927
+ if destination is None:
928
+ destination = model_path(repo_id, filename)
929
+ else:
930
+ destination = Path(destination)
931
+
932
+ path = client.download(
933
+ repo_id,
934
+ filename,
935
+ destination,
936
+ revision=revision,
937
+ overwrite=overwrite,
938
+ progress=progress,
939
+ cancelled=cancelled,
940
+ expected_sha256=expected_sha256,
941
+ )
942
+ return str(path)
943
+
944
+
945
+ def format_bytes(value: int | None) -> str:
946
+ if value is None:
947
+ return "?"
948
+ n = float(value)
949
+ for unit in ("B", "KiB", "MiB", "GiB", "TiB"):
950
+ if n < _BYTES_PER_KIB or unit == "TiB":
951
+ return f"{n:.1f} {unit}" if unit != "B" else f"{int(n)} B"
952
+ n /= _BYTES_PER_KIB
953
+ return f"{n:.1f} TiB"
954
+
955
+
956
+ def format_eta(seconds: float | None) -> str:
957
+ if seconds is None:
958
+ return "?"
959
+ seconds = max(0, int(seconds))
960
+ if seconds < _SECONDS_PER_MINUTE:
961
+ return f"{seconds}s"
962
+ minutes, sec = divmod(seconds, _SECONDS_PER_MINUTE)
963
+ if minutes < _MINUTES_PER_HOUR:
964
+ return f"{minutes}m {sec:02d}s"
965
+ hours, minutes = divmod(minutes, _MINUTES_PER_HOUR)
966
+ return f"{hours}h {minutes:02d}m"
967
+
968
+
969
+ if __name__ == "__main__":
970
+ import argparse
971
+
972
+ parser = argparse.ArgumentParser(description="Download one file from Hugging Face.")
973
+ parser.add_argument("repo_id")
974
+ parser.add_argument("filename")
975
+ parser.add_argument("destination")
976
+ parser.add_argument("--revision", default="main")
977
+ parser.add_argument("--token", default=os.environ.get("HF_TOKEN"))
978
+ args = parser.parse_args()
979
+
980
+ cli_client = HFClient(token=args.token)
981
+
982
+ def show(p: DownloadProgress) -> None:
983
+ pct = f"{p.percent:5.1f}%" if p.percent is not None else " ? "
984
+ print(
985
+ f"\r{pct} {format_bytes(p.downloaded)} / {format_bytes(p.total)} "
986
+ f"{p.speed_mbps:.1f} MiB/s ETA {format_eta(p.eta)}",
987
+ end="",
988
+ flush=True,
989
+ )
990
+
991
+ result = cli_client.download(
992
+ args.repo_id,
993
+ args.filename,
994
+ args.destination,
995
+ revision=args.revision,
996
+ progress=show,
997
+ )
998
+ print(f"\nSaved to {result}")