huggingface-hub 0.31.0rc0__py3-none-any.whl → 1.1.3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (150) hide show
  1. huggingface_hub/__init__.py +145 -46
  2. huggingface_hub/_commit_api.py +168 -119
  3. huggingface_hub/_commit_scheduler.py +15 -15
  4. huggingface_hub/_inference_endpoints.py +15 -12
  5. huggingface_hub/_jobs_api.py +301 -0
  6. huggingface_hub/_local_folder.py +18 -3
  7. huggingface_hub/_login.py +31 -63
  8. huggingface_hub/_oauth.py +460 -0
  9. huggingface_hub/_snapshot_download.py +239 -80
  10. huggingface_hub/_space_api.py +5 -5
  11. huggingface_hub/_tensorboard_logger.py +15 -19
  12. huggingface_hub/_upload_large_folder.py +172 -76
  13. huggingface_hub/_webhooks_payload.py +3 -3
  14. huggingface_hub/_webhooks_server.py +13 -25
  15. huggingface_hub/{commands → cli}/__init__.py +1 -15
  16. huggingface_hub/cli/_cli_utils.py +173 -0
  17. huggingface_hub/cli/auth.py +147 -0
  18. huggingface_hub/cli/cache.py +841 -0
  19. huggingface_hub/cli/download.py +189 -0
  20. huggingface_hub/cli/hf.py +60 -0
  21. huggingface_hub/cli/inference_endpoints.py +377 -0
  22. huggingface_hub/cli/jobs.py +772 -0
  23. huggingface_hub/cli/lfs.py +175 -0
  24. huggingface_hub/cli/repo.py +315 -0
  25. huggingface_hub/cli/repo_files.py +94 -0
  26. huggingface_hub/{commands/env.py → cli/system.py} +10 -13
  27. huggingface_hub/cli/upload.py +294 -0
  28. huggingface_hub/cli/upload_large_folder.py +117 -0
  29. huggingface_hub/community.py +20 -12
  30. huggingface_hub/constants.py +38 -53
  31. huggingface_hub/dataclasses.py +609 -0
  32. huggingface_hub/errors.py +80 -30
  33. huggingface_hub/fastai_utils.py +30 -41
  34. huggingface_hub/file_download.py +435 -351
  35. huggingface_hub/hf_api.py +2050 -1124
  36. huggingface_hub/hf_file_system.py +269 -152
  37. huggingface_hub/hub_mixin.py +43 -63
  38. huggingface_hub/inference/_client.py +347 -434
  39. huggingface_hub/inference/_common.py +133 -121
  40. huggingface_hub/inference/_generated/_async_client.py +397 -541
  41. huggingface_hub/inference/_generated/types/__init__.py +5 -1
  42. huggingface_hub/inference/_generated/types/automatic_speech_recognition.py +3 -3
  43. huggingface_hub/inference/_generated/types/base.py +10 -7
  44. huggingface_hub/inference/_generated/types/chat_completion.py +59 -23
  45. huggingface_hub/inference/_generated/types/depth_estimation.py +2 -2
  46. huggingface_hub/inference/_generated/types/document_question_answering.py +2 -2
  47. huggingface_hub/inference/_generated/types/feature_extraction.py +2 -2
  48. huggingface_hub/inference/_generated/types/fill_mask.py +2 -2
  49. huggingface_hub/inference/_generated/types/image_to_image.py +6 -2
  50. huggingface_hub/inference/_generated/types/image_to_video.py +60 -0
  51. huggingface_hub/inference/_generated/types/sentence_similarity.py +3 -3
  52. huggingface_hub/inference/_generated/types/summarization.py +2 -2
  53. huggingface_hub/inference/_generated/types/table_question_answering.py +5 -5
  54. huggingface_hub/inference/_generated/types/text2text_generation.py +2 -2
  55. huggingface_hub/inference/_generated/types/text_generation.py +10 -10
  56. huggingface_hub/inference/_generated/types/text_to_video.py +2 -2
  57. huggingface_hub/inference/_generated/types/token_classification.py +2 -2
  58. huggingface_hub/inference/_generated/types/translation.py +2 -2
  59. huggingface_hub/inference/_generated/types/zero_shot_classification.py +2 -2
  60. huggingface_hub/inference/_generated/types/zero_shot_image_classification.py +2 -2
  61. huggingface_hub/inference/_generated/types/zero_shot_object_detection.py +1 -3
  62. huggingface_hub/inference/_mcp/__init__.py +0 -0
  63. huggingface_hub/inference/_mcp/_cli_hacks.py +88 -0
  64. huggingface_hub/inference/_mcp/agent.py +100 -0
  65. huggingface_hub/inference/_mcp/cli.py +247 -0
  66. huggingface_hub/inference/_mcp/constants.py +81 -0
  67. huggingface_hub/inference/_mcp/mcp_client.py +395 -0
  68. huggingface_hub/inference/_mcp/types.py +45 -0
  69. huggingface_hub/inference/_mcp/utils.py +128 -0
  70. huggingface_hub/inference/_providers/__init__.py +82 -7
  71. huggingface_hub/inference/_providers/_common.py +129 -27
  72. huggingface_hub/inference/_providers/black_forest_labs.py +6 -6
  73. huggingface_hub/inference/_providers/cerebras.py +1 -1
  74. huggingface_hub/inference/_providers/clarifai.py +13 -0
  75. huggingface_hub/inference/_providers/cohere.py +20 -3
  76. huggingface_hub/inference/_providers/fal_ai.py +183 -56
  77. huggingface_hub/inference/_providers/featherless_ai.py +38 -0
  78. huggingface_hub/inference/_providers/fireworks_ai.py +18 -0
  79. huggingface_hub/inference/_providers/groq.py +9 -0
  80. huggingface_hub/inference/_providers/hf_inference.py +69 -30
  81. huggingface_hub/inference/_providers/hyperbolic.py +4 -4
  82. huggingface_hub/inference/_providers/nebius.py +33 -5
  83. huggingface_hub/inference/_providers/novita.py +5 -5
  84. huggingface_hub/inference/_providers/nscale.py +44 -0
  85. huggingface_hub/inference/_providers/openai.py +3 -1
  86. huggingface_hub/inference/_providers/publicai.py +6 -0
  87. huggingface_hub/inference/_providers/replicate.py +31 -13
  88. huggingface_hub/inference/_providers/sambanova.py +18 -4
  89. huggingface_hub/inference/_providers/scaleway.py +28 -0
  90. huggingface_hub/inference/_providers/together.py +20 -5
  91. huggingface_hub/inference/_providers/wavespeed.py +138 -0
  92. huggingface_hub/inference/_providers/zai_org.py +17 -0
  93. huggingface_hub/lfs.py +33 -100
  94. huggingface_hub/repocard.py +34 -38
  95. huggingface_hub/repocard_data.py +57 -57
  96. huggingface_hub/serialization/__init__.py +0 -1
  97. huggingface_hub/serialization/_base.py +12 -15
  98. huggingface_hub/serialization/_dduf.py +8 -8
  99. huggingface_hub/serialization/_torch.py +69 -69
  100. huggingface_hub/utils/__init__.py +19 -8
  101. huggingface_hub/utils/_auth.py +7 -7
  102. huggingface_hub/utils/_cache_manager.py +92 -147
  103. huggingface_hub/utils/_chunk_utils.py +2 -3
  104. huggingface_hub/utils/_deprecation.py +1 -1
  105. huggingface_hub/utils/_dotenv.py +55 -0
  106. huggingface_hub/utils/_experimental.py +7 -5
  107. huggingface_hub/utils/_fixes.py +0 -10
  108. huggingface_hub/utils/_git_credential.py +5 -5
  109. huggingface_hub/utils/_headers.py +8 -30
  110. huggingface_hub/utils/_http.py +398 -239
  111. huggingface_hub/utils/_pagination.py +4 -4
  112. huggingface_hub/utils/_parsing.py +98 -0
  113. huggingface_hub/utils/_paths.py +5 -5
  114. huggingface_hub/utils/_runtime.py +61 -24
  115. huggingface_hub/utils/_safetensors.py +21 -21
  116. huggingface_hub/utils/_subprocess.py +9 -9
  117. huggingface_hub/utils/_telemetry.py +4 -4
  118. huggingface_hub/{commands/_cli_utils.py → utils/_terminal.py} +4 -4
  119. huggingface_hub/utils/_typing.py +25 -5
  120. huggingface_hub/utils/_validators.py +55 -74
  121. huggingface_hub/utils/_verification.py +167 -0
  122. huggingface_hub/utils/_xet.py +64 -17
  123. huggingface_hub/utils/_xet_progress_reporting.py +162 -0
  124. huggingface_hub/utils/insecure_hashlib.py +3 -5
  125. huggingface_hub/utils/logging.py +8 -11
  126. huggingface_hub/utils/tqdm.py +5 -4
  127. {huggingface_hub-0.31.0rc0.dist-info → huggingface_hub-1.1.3.dist-info}/METADATA +94 -85
  128. huggingface_hub-1.1.3.dist-info/RECORD +155 -0
  129. {huggingface_hub-0.31.0rc0.dist-info → huggingface_hub-1.1.3.dist-info}/WHEEL +1 -1
  130. huggingface_hub-1.1.3.dist-info/entry_points.txt +6 -0
  131. huggingface_hub/commands/delete_cache.py +0 -474
  132. huggingface_hub/commands/download.py +0 -200
  133. huggingface_hub/commands/huggingface_cli.py +0 -61
  134. huggingface_hub/commands/lfs.py +0 -200
  135. huggingface_hub/commands/repo_files.py +0 -128
  136. huggingface_hub/commands/scan_cache.py +0 -181
  137. huggingface_hub/commands/tag.py +0 -159
  138. huggingface_hub/commands/upload.py +0 -314
  139. huggingface_hub/commands/upload_large_folder.py +0 -129
  140. huggingface_hub/commands/user.py +0 -304
  141. huggingface_hub/commands/version.py +0 -37
  142. huggingface_hub/inference_api.py +0 -217
  143. huggingface_hub/keras_mixin.py +0 -500
  144. huggingface_hub/repository.py +0 -1477
  145. huggingface_hub/serialization/_tensorflow.py +0 -95
  146. huggingface_hub/utils/_hf_folder.py +0 -68
  147. huggingface_hub-0.31.0rc0.dist-info/RECORD +0 -135
  148. huggingface_hub-0.31.0rc0.dist-info/entry_points.txt +0 -6
  149. {huggingface_hub-0.31.0rc0.dist-info → huggingface_hub-1.1.3.dist-info/licenses}/LICENSE +0 -0
  150. {huggingface_hub-0.31.0rc0.dist-info → huggingface_hub-1.1.3.dist-info}/top_level.txt +0 -0
@@ -4,7 +4,6 @@ Type definitions and utilities for the `create_commit` API
4
4
 
5
5
  import base64
6
6
  import io
7
- import math
8
7
  import os
9
8
  import warnings
10
9
  from collections import defaultdict
@@ -12,7 +11,7 @@ from contextlib import contextmanager
12
11
  from dataclasses import dataclass, field
13
12
  from itertools import groupby
14
13
  from pathlib import Path, PurePosixPath
15
- from typing import TYPE_CHECKING, Any, BinaryIO, Dict, Iterable, Iterator, List, Literal, Optional, Tuple, Union
14
+ from typing import TYPE_CHECKING, Any, BinaryIO, Iterable, Iterator, Literal, Optional, Union
16
15
 
17
16
  from tqdm.contrib.concurrent import thread_map
18
17
 
@@ -23,6 +22,7 @@ from .lfs import UploadInfo, lfs_upload, post_lfs_batch_info
23
22
  from .utils import (
24
23
  FORBIDDEN_FOLDERS,
25
24
  XetTokenType,
25
+ are_progress_bars_disabled,
26
26
  chunk_iterable,
27
27
  fetch_xet_connection_info_from_repo_info,
28
28
  get_session,
@@ -33,7 +33,7 @@ from .utils import (
33
33
  validate_hf_hub_args,
34
34
  )
35
35
  from .utils import tqdm as hf_tqdm
36
- from .utils.tqdm import _get_progress_bar_context
36
+ from .utils._runtime import is_xet_available
37
37
 
38
38
 
39
39
  if TYPE_CHECKING:
@@ -236,7 +236,7 @@ class CommitOperationAdd:
236
236
  config.json: 100%|█████████████████████████| 8.19k/8.19k [00:02<00:00, 3.72kB/s]
237
237
 
238
238
  >>> with operation.as_file(with_tqdm=True) as file:
239
- ... requests.put(..., data=file)
239
+ ... httpx.put(..., data=file)
240
240
  config.json: 100%|█████████████████████████| 8.19k/8.19k [00:02<00:00, 3.72kB/s]
241
241
  ```
242
242
  """
@@ -307,7 +307,7 @@ def _validate_path_in_repo(path_in_repo: str) -> str:
307
307
  CommitOperation = Union[CommitOperationAdd, CommitOperationCopy, CommitOperationDelete]
308
308
 
309
309
 
310
- def _warn_on_overwriting_operations(operations: List[CommitOperation]) -> None:
310
+ def _warn_on_overwriting_operations(operations: list[CommitOperation]) -> None:
311
311
  """
312
312
  Warn user when a list of operations is expected to overwrite itself in a single
313
313
  commit.
@@ -322,7 +322,7 @@ def _warn_on_overwriting_operations(operations: List[CommitOperation]) -> None:
322
322
  delete before upload) but can happen if a user deletes an entire folder and then
323
323
  add new files to it.
324
324
  """
325
- nb_additions_per_path: Dict[str, int] = defaultdict(int)
325
+ nb_additions_per_path: dict[str, int] = defaultdict(int)
326
326
  for operation in operations:
327
327
  path_in_repo = operation.path_in_repo
328
328
  if isinstance(operation, CommitOperationAdd):
@@ -354,15 +354,95 @@ def _warn_on_overwriting_operations(operations: List[CommitOperation]) -> None:
354
354
 
355
355
 
356
356
  @validate_hf_hub_args
357
- def _upload_lfs_files(
357
+ def _upload_files(
358
358
  *,
359
- additions: List[CommitOperationAdd],
359
+ additions: list[CommitOperationAdd],
360
360
  repo_type: str,
361
361
  repo_id: str,
362
- headers: Dict[str, str],
362
+ headers: dict[str, str],
363
363
  endpoint: Optional[str] = None,
364
364
  num_threads: int = 5,
365
365
  revision: Optional[str] = None,
366
+ create_pr: Optional[bool] = None,
367
+ ):
368
+ """
369
+ Negotiates per-file transfer (LFS vs Xet) and uploads in batches.
370
+ """
371
+ xet_additions: list[CommitOperationAdd] = []
372
+ lfs_actions: list[dict[str, Any]] = []
373
+ lfs_oid2addop: dict[str, CommitOperationAdd] = {}
374
+
375
+ for chunk in chunk_iterable(additions, chunk_size=UPLOAD_BATCH_MAX_NUM_FILES):
376
+ chunk_list = [op for op in chunk]
377
+
378
+ transfers: list[str] = ["basic", "multipart"]
379
+ has_buffered_io_data = any(isinstance(op.path_or_fileobj, io.BufferedIOBase) for op in chunk_list)
380
+ if is_xet_available():
381
+ if not has_buffered_io_data:
382
+ transfers.append("xet")
383
+ else:
384
+ logger.warning(
385
+ "Uploading files as a binary IO buffer is not supported by Xet Storage. "
386
+ "Falling back to HTTP upload."
387
+ )
388
+
389
+ actions_chunk, errors_chunk, chosen_transfer = post_lfs_batch_info(
390
+ upload_infos=[op.upload_info for op in chunk_list],
391
+ repo_id=repo_id,
392
+ repo_type=repo_type,
393
+ revision=revision,
394
+ endpoint=endpoint,
395
+ headers=headers,
396
+ token=None, # already passed in 'headers'
397
+ transfers=transfers,
398
+ )
399
+ if errors_chunk:
400
+ message = "\n".join(
401
+ [
402
+ f"Encountered error for file with OID {err.get('oid')}: `{err.get('error', {}).get('message')}"
403
+ for err in errors_chunk
404
+ ]
405
+ )
406
+ raise ValueError(f"LFS batch API returned errors:\n{message}")
407
+
408
+ # If server returns a transfer we didn't offer (e.g "xet" while uploading from BytesIO),
409
+ # fall back to LFS for this chunk.
410
+ if chosen_transfer == "xet" and ("xet" in transfers):
411
+ xet_additions.extend(chunk_list)
412
+ else:
413
+ lfs_actions.extend(actions_chunk)
414
+ for op in chunk_list:
415
+ lfs_oid2addop[op.upload_info.sha256.hex()] = op
416
+
417
+ if len(lfs_actions) > 0:
418
+ _upload_lfs_files(
419
+ actions=lfs_actions,
420
+ oid2addop=lfs_oid2addop,
421
+ headers=headers,
422
+ endpoint=endpoint,
423
+ num_threads=num_threads,
424
+ )
425
+
426
+ if len(xet_additions) > 0:
427
+ _upload_xet_files(
428
+ additions=xet_additions,
429
+ repo_type=repo_type,
430
+ repo_id=repo_id,
431
+ headers=headers,
432
+ endpoint=endpoint,
433
+ revision=revision,
434
+ create_pr=create_pr,
435
+ )
436
+
437
+
438
+ @validate_hf_hub_args
439
+ def _upload_lfs_files(
440
+ *,
441
+ actions: list[dict[str, Any]],
442
+ oid2addop: dict[str, CommitOperationAdd],
443
+ headers: dict[str, str],
444
+ endpoint: Optional[str] = None,
445
+ num_threads: int = 5,
366
446
  ):
367
447
  """
368
448
  Uploads the content of `additions` to the Hub using the large file storage protocol.
@@ -371,14 +451,26 @@ def _upload_lfs_files(
371
451
  - LFS Batch API: https://github.com/git-lfs/git-lfs/blob/main/docs/api/batch.md
372
452
 
373
453
  Args:
374
- additions (`List` of `CommitOperationAdd`):
375
- The files to be uploaded
376
- repo_type (`str`):
454
+ actions (`list[dict[str, Any]]`):
455
+ LFS batch actions returned by the server.
456
+ oid2addop (`dict[str, CommitOperationAdd]`):
457
+ A dictionary mapping the OID of the file to the corresponding `CommitOperationAdd` object.
458
+ headers (`dict[str, str]`):
459
+ Headers to use for the request, including authorization headers and user agent.
460
+ endpoint (`str`, *optional*):
461
+ The endpoint to use for the request. Defaults to `constants.ENDPOINT`.
462
+ num_threads (`int`, *optional*):
463
+ The number of concurrent threads to use when uploading. Defaults to 5.
464
+
465
+ Raises:
466
+ [`EnvironmentError`](https://docs.python.org/3/library/exceptions.html#EnvironmentError)
467
+ If an upload failed for any reason
468
+ [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError)
377
469
  Type of the repo to upload to: `"model"`, `"dataset"` or `"space"`.
378
470
  repo_id (`str`):
379
471
  A namespace (user or an organization) and a repo name separated
380
472
  by a `/`.
381
- headers (`Dict[str, str]`):
473
+ headers (`dict[str, str]`):
382
474
  Headers to use for the request, including authorization headers and user agent.
383
475
  num_threads (`int`, *optional*):
384
476
  The number of concurrent threads to use when uploading. Defaults to 5.
@@ -390,53 +482,20 @@ def _upload_lfs_files(
390
482
  If an upload failed for any reason
391
483
  [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError)
392
484
  If the server returns malformed responses
393
- [`HTTPError`](https://requests.readthedocs.io/en/latest/api/#requests.HTTPError)
485
+ [`HfHubHTTPError`]
394
486
  If the LFS batch endpoint returned an HTTP error.
395
487
  """
396
- # Step 1: retrieve upload instructions from the LFS batch endpoint.
397
- # Upload instructions are retrieved by chunk of 256 files to avoid reaching
398
- # the payload limit.
399
- batch_actions: List[Dict] = []
400
- for chunk in chunk_iterable(additions, chunk_size=UPLOAD_BATCH_MAX_NUM_FILES):
401
- batch_actions_chunk, batch_errors_chunk = post_lfs_batch_info(
402
- upload_infos=[op.upload_info for op in chunk],
403
- repo_id=repo_id,
404
- repo_type=repo_type,
405
- revision=revision,
406
- endpoint=endpoint,
407
- headers=headers,
408
- token=None, # already passed in 'headers'
409
- )
410
-
411
- # If at least 1 error, we do not retrieve information for other chunks
412
- if batch_errors_chunk:
413
- message = "\n".join(
414
- [
415
- f"Encountered error for file with OID {err.get('oid')}: `{err.get('error', {}).get('message')}"
416
- for err in batch_errors_chunk
417
- ]
418
- )
419
- raise ValueError(f"LFS batch endpoint returned errors:\n{message}")
420
-
421
- batch_actions += batch_actions_chunk
422
- oid2addop = {add_op.upload_info.sha256.hex(): add_op for add_op in additions}
423
-
424
- # Step 2: ignore files that have already been uploaded
488
+ # Filter out files already present upstream
425
489
  filtered_actions = []
426
- for action in batch_actions:
490
+ for action in actions:
427
491
  if action.get("actions") is None:
428
492
  logger.debug(
429
- f"Content of file {oid2addop[action['oid']].path_in_repo} is already"
430
- " present upstream - skipping upload."
493
+ f"Content of file {oid2addop[action['oid']].path_in_repo} is already present upstream - skipping upload."
431
494
  )
432
495
  else:
433
496
  filtered_actions.append(action)
434
497
 
435
- if len(filtered_actions) == 0:
436
- logger.debug("No LFS files to upload.")
437
- return
438
-
439
- # Step 3: upload files concurrently according to these instructions
498
+ # Upload according to server-provided actions
440
499
  def _wrapped_lfs_upload(batch_action) -> None:
441
500
  try:
442
501
  operation = oid2addop[batch_action["oid"]]
@@ -444,11 +503,7 @@ def _upload_lfs_files(
444
503
  except Exception as exc:
445
504
  raise RuntimeError(f"Error while uploading '{operation.path_in_repo}' to the Hub.") from exc
446
505
 
447
- if constants.HF_HUB_ENABLE_HF_TRANSFER:
448
- logger.debug(f"Uploading {len(filtered_actions)} LFS files to the Hub using `hf_transfer`.")
449
- for action in hf_tqdm(filtered_actions, name="huggingface_hub.lfs_upload"):
450
- _wrapped_lfs_upload(action)
451
- elif len(filtered_actions) == 1:
506
+ if len(filtered_actions) == 1:
452
507
  logger.debug("Uploading 1 LFS file to the Hub")
453
508
  _wrapped_lfs_upload(filtered_actions[0])
454
509
  else:
@@ -467,10 +522,10 @@ def _upload_lfs_files(
467
522
  @validate_hf_hub_args
468
523
  def _upload_xet_files(
469
524
  *,
470
- additions: List[CommitOperationAdd],
525
+ additions: list[CommitOperationAdd],
471
526
  repo_type: str,
472
527
  repo_id: str,
473
- headers: Dict[str, str],
528
+ headers: dict[str, str],
474
529
  endpoint: Optional[str] = None,
475
530
  revision: Optional[str] = None,
476
531
  create_pr: Optional[bool] = None,
@@ -480,14 +535,14 @@ def _upload_xet_files(
480
535
  This chunks the files and deduplicates the chunks before uploading them to xetcas storage.
481
536
 
482
537
  Args:
483
- additions (`List` of `CommitOperationAdd`):
538
+ additions (`` of `CommitOperationAdd`):
484
539
  The files to be uploaded.
485
540
  repo_type (`str`):
486
541
  Type of the repo to upload to: `"model"`, `"dataset"` or `"space"`.
487
542
  repo_id (`str`):
488
543
  A namespace (user or an organization) and a repo name separated
489
544
  by a `/`.
490
- headers (`Dict[str, str]`):
545
+ headers (`dict[str, str]`):
491
546
  Headers to use for the request, including authorization headers and user agent.
492
547
  endpoint: (`str`, *optional*):
493
548
  The endpoint to use for the xetcas service. Defaults to `constants.ENDPOINT`.
@@ -501,7 +556,7 @@ def _upload_xet_files(
501
556
  If an upload failed for any reason.
502
557
  [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError)
503
558
  If the server returns malformed responses or if the user is unauthorized to upload to xet storage.
504
- [`HTTPError`](https://requests.readthedocs.io/en/latest/api/#requests.HTTPError)
559
+ [`HfHubHTTPError`]
505
560
  If the LFS batch endpoint returned an HTTP error.
506
561
 
507
562
  **How it works:**
@@ -529,9 +584,12 @@ def _upload_xet_files(
529
584
  """
530
585
  if len(additions) == 0:
531
586
  return
587
+
532
588
  # at this point, we know that hf_xet is installed
533
589
  from hf_xet import upload_bytes, upload_files
534
590
 
591
+ from .utils._xet_progress_reporting import XetProgressReporter
592
+
535
593
  try:
536
594
  xet_connection_info = fetch_xet_connection_info_from_repo_info(
537
595
  token_type=XetTokenType.WRITE,
@@ -553,7 +611,7 @@ def _upload_xet_files(
553
611
  xet_endpoint = xet_connection_info.endpoint
554
612
  access_token_info = (xet_connection_info.access_token, xet_connection_info.expiration_unix_epoch)
555
613
 
556
- def token_refresher() -> Tuple[str, int]:
614
+ def token_refresher() -> tuple[str, int]:
557
615
  new_xet_connection = fetch_xet_connection_info_from_repo_info(
558
616
  token_type=XetTokenType.WRITE,
559
617
  repo_id=repo_id,
@@ -567,51 +625,42 @@ def _upload_xet_files(
567
625
  raise XetRefreshTokenError("Failed to refresh xet token")
568
626
  return new_xet_connection.access_token, new_xet_connection.expiration_unix_epoch
569
627
 
570
- num_chunks = math.ceil(len(additions) / UPLOAD_BATCH_MAX_NUM_FILES)
571
- num_chunks_num_digits = int(math.log10(num_chunks)) + 1
572
- for i, chunk in enumerate(chunk_iterable(additions, chunk_size=UPLOAD_BATCH_MAX_NUM_FILES)):
573
- _chunk = [op for op in chunk]
628
+ if not are_progress_bars_disabled():
629
+ progress = XetProgressReporter()
630
+ progress_callback = progress.update_progress
631
+ else:
632
+ progress, progress_callback = None, None
574
633
 
575
- bytes_ops = [op for op in _chunk if isinstance(op.path_or_fileobj, bytes)]
576
- paths_ops = [op for op in _chunk if isinstance(op.path_or_fileobj, (str, Path))]
577
- expected_size = sum(op.upload_info.size for op in bytes_ops + paths_ops)
634
+ try:
635
+ all_bytes_ops = [op for op in additions if isinstance(op.path_or_fileobj, bytes)]
636
+ all_paths_ops = [op for op in additions if isinstance(op.path_or_fileobj, (str, Path))]
637
+
638
+ if len(all_paths_ops) > 0:
639
+ all_paths = [str(op.path_or_fileobj) for op in all_paths_ops]
640
+ upload_files(
641
+ all_paths,
642
+ xet_endpoint,
643
+ access_token_info,
644
+ token_refresher,
645
+ progress_callback,
646
+ repo_type,
647
+ )
648
+
649
+ if len(all_bytes_ops) > 0:
650
+ all_bytes = [op.path_or_fileobj for op in all_bytes_ops]
651
+ upload_bytes(
652
+ all_bytes,
653
+ xet_endpoint,
654
+ access_token_info,
655
+ token_refresher,
656
+ progress_callback,
657
+ repo_type,
658
+ )
659
+
660
+ finally:
661
+ if progress is not None:
662
+ progress.close(False)
578
663
 
579
- if num_chunks > 1:
580
- description = f"Uploading Batch [{str(i + 1).zfill(num_chunks_num_digits)}/{num_chunks}]..."
581
- else:
582
- description = "Uploading..."
583
- progress_cm = _get_progress_bar_context(
584
- desc=description,
585
- total=expected_size,
586
- initial=0,
587
- unit="B",
588
- unit_scale=True,
589
- name="huggingface_hub.xet_put",
590
- log_level=logger.getEffectiveLevel(),
591
- )
592
- with progress_cm as progress:
593
-
594
- def update_progress(increment: int):
595
- progress.update(increment)
596
-
597
- if len(paths_ops) > 0:
598
- upload_files(
599
- [str(op.path_or_fileobj) for op in paths_ops],
600
- xet_endpoint,
601
- access_token_info,
602
- token_refresher,
603
- update_progress,
604
- repo_type,
605
- )
606
- if len(bytes_ops) > 0:
607
- upload_bytes(
608
- [op.path_or_fileobj for op in bytes_ops],
609
- xet_endpoint,
610
- access_token_info,
611
- token_refresher,
612
- update_progress,
613
- repo_type,
614
- )
615
664
  return
616
665
 
617
666
 
@@ -635,7 +684,7 @@ def _fetch_upload_modes(
635
684
  additions: Iterable[CommitOperationAdd],
636
685
  repo_type: str,
637
686
  repo_id: str,
638
- headers: Dict[str, str],
687
+ headers: dict[str, str],
639
688
  revision: str,
640
689
  endpoint: Optional[str] = None,
641
690
  create_pr: bool = False,
@@ -654,7 +703,7 @@ def _fetch_upload_modes(
654
703
  repo_id (`str`):
655
704
  A namespace (user or an organization) and a repo name separated
656
705
  by a `/`.
657
- headers (`Dict[str, str]`):
706
+ headers (`dict[str, str]`):
658
707
  Headers to use for the request, including authorization headers and user agent.
659
708
  revision (`str`):
660
709
  The git revision to upload the files to. Can be any valid git revision.
@@ -672,12 +721,12 @@ def _fetch_upload_modes(
672
721
  endpoint = endpoint if endpoint is not None else constants.ENDPOINT
673
722
 
674
723
  # Fetch upload mode (LFS or regular) chunk by chunk.
675
- upload_modes: Dict[str, UploadMode] = {}
676
- should_ignore_info: Dict[str, bool] = {}
677
- oid_info: Dict[str, Optional[str]] = {}
724
+ upload_modes: dict[str, UploadMode] = {}
725
+ should_ignore_info: dict[str, bool] = {}
726
+ oid_info: dict[str, Optional[str]] = {}
678
727
 
679
728
  for chunk in chunk_iterable(additions, 256):
680
- payload: Dict = {
729
+ payload: dict = {
681
730
  "files": [
682
731
  {
683
732
  "path": op.path_in_repo,
@@ -720,10 +769,10 @@ def _fetch_files_to_copy(
720
769
  copies: Iterable[CommitOperationCopy],
721
770
  repo_type: str,
722
771
  repo_id: str,
723
- headers: Dict[str, str],
772
+ headers: dict[str, str],
724
773
  revision: str,
725
774
  endpoint: Optional[str] = None,
726
- ) -> Dict[Tuple[str, Optional[str]], Union["RepoFile", bytes]]:
775
+ ) -> dict[tuple[str, Optional[str]], Union["RepoFile", bytes]]:
727
776
  """
728
777
  Fetch information about the files to copy.
729
778
 
@@ -739,12 +788,12 @@ def _fetch_files_to_copy(
739
788
  repo_id (`str`):
740
789
  A namespace (user or an organization) and a repo name separated
741
790
  by a `/`.
742
- headers (`Dict[str, str]`):
791
+ headers (`dict[str, str]`):
743
792
  Headers to use for the request, including authorization headers and user agent.
744
793
  revision (`str`):
745
794
  The git revision to upload the files to. Can be any valid git revision.
746
795
 
747
- Returns: `Dict[Tuple[str, Optional[str]], Union[RepoFile, bytes]]]`
796
+ Returns: `dict[tuple[str, Optional[str]], Union[RepoFile, bytes]]]`
748
797
  Key is the file path and revision of the file to copy.
749
798
  Value is the raw content as bytes (for regular files) or the file information as a RepoFile (for LFS files).
750
799
 
@@ -757,9 +806,9 @@ def _fetch_files_to_copy(
757
806
  from .hf_api import HfApi, RepoFolder
758
807
 
759
808
  hf_api = HfApi(endpoint=endpoint, headers=headers)
760
- files_to_copy: Dict[Tuple[str, Optional[str]], Union["RepoFile", bytes]] = {}
809
+ files_to_copy: dict[tuple[str, Optional[str]], Union["RepoFile", bytes]] = {}
761
810
  # Store (path, revision) -> oid mapping
762
- oid_info: Dict[Tuple[str, Optional[str]], Optional[str]] = {}
811
+ oid_info: dict[tuple[str, Optional[str]], Optional[str]] = {}
763
812
  # 1. Fetch OIDs for destination paths in batches.
764
813
  dest_paths = [op.path_in_repo for op in copies]
765
814
  for offset in range(0, len(dest_paths), FETCH_LFS_BATCH_SIZE):
@@ -819,11 +868,11 @@ def _fetch_files_to_copy(
819
868
 
820
869
  def _prepare_commit_payload(
821
870
  operations: Iterable[CommitOperation],
822
- files_to_copy: Dict[Tuple[str, Optional[str]], Union["RepoFile", bytes]],
871
+ files_to_copy: dict[tuple[str, Optional[str]], Union["RepoFile", bytes]],
823
872
  commit_message: str,
824
873
  commit_description: Optional[str] = None,
825
874
  parent_commit: Optional[str] = None,
826
- ) -> Iterable[Dict[str, Any]]:
875
+ ) -> Iterable[dict[str, Any]]:
827
876
  """
828
877
  Builds the payload to POST to the `/commit` API of the Hub.
829
878
 
@@ -7,7 +7,7 @@ from dataclasses import dataclass
7
7
  from io import SEEK_END, SEEK_SET, BytesIO
8
8
  from pathlib import Path
9
9
  from threading import Lock, Thread
10
- from typing import Dict, List, Optional, Union
10
+ from typing import Optional, Union
11
11
 
12
12
  from .hf_api import DEFAULT_IGNORE_PATTERNS, CommitInfo, CommitOperationAdd, HfApi
13
13
  from .utils import filter_repo_objects
@@ -53,9 +53,9 @@ class CommitScheduler:
53
53
  Whether to make the repo private. If `None` (default), the repo will be public unless the organization's default is private. This value is ignored if the repo already exists.
54
54
  token (`str`, *optional*):
55
55
  The token to use to commit to the repo. Defaults to the token saved on the machine.
56
- allow_patterns (`List[str]` or `str`, *optional*):
56
+ allow_patterns (`list[str]` or `str`, *optional*):
57
57
  If provided, only files matching at least one pattern are uploaded.
58
- ignore_patterns (`List[str]` or `str`, *optional*):
58
+ ignore_patterns (`list[str]` or `str`, *optional*):
59
59
  If provided, files matching any of the patterns are not uploaded.
60
60
  squash_history (`bool`, *optional*):
61
61
  Whether to squash the history of the repo after each commit. Defaults to `False`. Squashing commits is
@@ -108,8 +108,8 @@ class CommitScheduler:
108
108
  revision: Optional[str] = None,
109
109
  private: Optional[bool] = None,
110
110
  token: Optional[str] = None,
111
- allow_patterns: Optional[Union[List[str], str]] = None,
112
- ignore_patterns: Optional[Union[List[str], str]] = None,
111
+ allow_patterns: Optional[Union[list[str], str]] = None,
112
+ ignore_patterns: Optional[Union[list[str], str]] = None,
113
113
  squash_history: bool = False,
114
114
  hf_api: Optional["HfApi"] = None,
115
115
  ) -> None:
@@ -138,7 +138,7 @@ class CommitScheduler:
138
138
  self.token = token
139
139
 
140
140
  # Keep track of already uploaded files
141
- self.last_uploaded: Dict[Path, float] = {} # key is local path, value is timestamp
141
+ self.last_uploaded: dict[Path, float] = {} # key is local path, value is timestamp
142
142
 
143
143
  # Scheduler
144
144
  if not every > 0:
@@ -205,13 +205,10 @@ class CommitScheduler:
205
205
  """
206
206
  Push folder to the Hub and return the commit info.
207
207
 
208
- <Tip warning={true}>
209
-
210
- This method is not meant to be called directly. It is run in the background by the scheduler, respecting a
211
- queue mechanism to avoid concurrent commits. Making a direct call to the method might lead to concurrency
212
- issues.
213
-
214
- </Tip>
208
+ > [!WARNING]
209
+ > This method is not meant to be called directly. It is run in the background by the scheduler, respecting a
210
+ > queue mechanism to avoid concurrent commits. Making a direct call to the method might lead to concurrency
211
+ > issues.
215
212
 
216
213
  The default behavior of `push_to_hub` is to assume an append-only folder. It lists all files in the folder and
217
214
  uploads only changed files. If no changes are found, the method returns without committing anything. If you want
@@ -232,7 +229,7 @@ class CommitScheduler:
232
229
  prefix = f"{self.path_in_repo.strip('/')}/" if self.path_in_repo else ""
233
230
 
234
231
  # Filter with pattern + filter out unchanged files + retrieve current file size
235
- files_to_upload: List[_FileToUpload] = []
232
+ files_to_upload: list[_FileToUpload] = []
236
233
  for relpath in filter_repo_objects(
237
234
  relpath_to_abspath.keys(), allow_patterns=self.allow_patterns, ignore_patterns=self.ignore_patterns
238
235
  ):
@@ -315,10 +312,13 @@ class PartialFileIO(BytesIO):
315
312
  return self._size_limit
316
313
 
317
314
  def __getattribute__(self, name: str):
318
- if name.startswith("_") or name in ("read", "tell", "seek"): # only 3 public methods supported
315
+ if name.startswith("_") or name in ("read", "tell", "seek", "fileno"): # only 4 public methods supported
319
316
  return super().__getattribute__(name)
320
317
  raise NotImplementedError(f"PartialFileIO does not support '{name}'.")
321
318
 
319
+ def fileno(self):
320
+ raise AttributeError("PartialFileIO does not have a fileno.")
321
+
322
322
  def tell(self) -> int:
323
323
  """Return the current file position."""
324
324
  return self._file.tell()
@@ -2,7 +2,7 @@ import time
2
2
  from dataclasses import dataclass, field
3
3
  from datetime import datetime
4
4
  from enum import Enum
5
- from typing import TYPE_CHECKING, Dict, Optional, Union
5
+ from typing import TYPE_CHECKING, Optional, Union
6
6
 
7
7
  from huggingface_hub.errors import InferenceEndpointError, InferenceEndpointTimeoutError
8
8
 
@@ -62,7 +62,7 @@ class InferenceEndpoint:
62
62
  The timestamp of the last update of the Inference Endpoint.
63
63
  type ([`InferenceEndpointType`]):
64
64
  The type of the Inference Endpoint (public, protected, private).
65
- raw (`Dict`):
65
+ raw (`dict`):
66
66
  The raw dictionary data returned from the API.
67
67
  token (`str` or `bool`, *optional*):
68
68
  Authentication token for the Inference Endpoint, if set when requesting the API. Will default to the
@@ -100,6 +100,7 @@ class InferenceEndpoint:
100
100
  namespace: str
101
101
  repository: str = field(init=False)
102
102
  status: InferenceEndpointStatus = field(init=False)
103
+ health_route: str = field(init=False)
103
104
  url: Optional[str] = field(init=False)
104
105
 
105
106
  # Other fields
@@ -111,7 +112,7 @@ class InferenceEndpoint:
111
112
  type: InferenceEndpointType = field(repr=False, init=False)
112
113
 
113
114
  # Raw dict from the API
114
- raw: Dict = field(repr=False)
115
+ raw: dict = field(repr=False)
115
116
 
116
117
  # Internal fields
117
118
  _token: Union[str, bool, None] = field(repr=False, compare=False)
@@ -119,7 +120,7 @@ class InferenceEndpoint:
119
120
 
120
121
  @classmethod
121
122
  def from_raw(
122
- cls, raw: Dict, namespace: str, token: Union[str, bool, None] = None, api: Optional["HfApi"] = None
123
+ cls, raw: dict, namespace: str, token: Union[str, bool, None] = None, api: Optional["HfApi"] = None
123
124
  ) -> "InferenceEndpoint":
124
125
  """Initialize object from raw dictionary."""
125
126
  if api is None:
@@ -220,7 +221,8 @@ class InferenceEndpoint:
220
221
  )
221
222
  if self.status == InferenceEndpointStatus.RUNNING and self.url is not None:
222
223
  # Verify the endpoint is actually reachable
223
- response = get_session().get(self.url, headers=self._api._build_hf_headers(token=self._token))
224
+ _health_url = f"{self.url.rstrip('/')}/{self.health_route.lstrip('/')}"
225
+ response = get_session().get(_health_url, headers=self._api._build_hf_headers(token=self._token))
224
226
  if response.status_code == 200:
225
227
  logger.info("Inference Endpoint is ready to be used.")
226
228
  return self
@@ -258,8 +260,8 @@ class InferenceEndpoint:
258
260
  framework: Optional[str] = None,
259
261
  revision: Optional[str] = None,
260
262
  task: Optional[str] = None,
261
- custom_image: Optional[Dict] = None,
262
- secrets: Optional[Dict[str, str]] = None,
263
+ custom_image: Optional[dict] = None,
264
+ secrets: Optional[dict[str, str]] = None,
263
265
  ) -> "InferenceEndpoint":
264
266
  """Update the Inference Endpoint.
265
267
 
@@ -291,10 +293,10 @@ class InferenceEndpoint:
291
293
  The specific model revision to deploy on the Inference Endpoint (e.g. `"6c0e6080953db56375760c0471a8c5f2929baf11"`).
292
294
  task (`str`, *optional*):
293
295
  The task on which to deploy the model (e.g. `"text-classification"`).
294
- custom_image (`Dict`, *optional*):
296
+ custom_image (`dict`, *optional*):
295
297
  A custom Docker image to use for the Inference Endpoint. This is useful if you want to deploy an
296
298
  Inference Endpoint running on the `text-generation-inference` (TGI) framework (see examples).
297
- secrets (`Dict[str, str]`, *optional*):
299
+ secrets (`dict[str, str]`, *optional*):
298
300
  Secret values to inject in the container environment.
299
301
  Returns:
300
302
  [`InferenceEndpoint`]: the same Inference Endpoint, mutated in place with the latest data.
@@ -327,7 +329,7 @@ class InferenceEndpoint:
327
329
  """Pause the Inference Endpoint.
328
330
 
329
331
  A paused Inference Endpoint will not be charged. It can be resumed at any time using [`InferenceEndpoint.resume`].
330
- This is different than scaling the Inference Endpoint to zero with [`InferenceEndpoint.scale_to_zero`], which
332
+ This is different from scaling the Inference Endpoint to zero with [`InferenceEndpoint.scale_to_zero`], which
331
333
  would be automatically restarted when a request is made to it.
332
334
 
333
335
  This is an alias for [`HfApi.pause_inference_endpoint`]. The current object is mutated in place with the
@@ -365,8 +367,8 @@ class InferenceEndpoint:
365
367
  def scale_to_zero(self) -> "InferenceEndpoint":
366
368
  """Scale Inference Endpoint to zero.
367
369
 
368
- An Inference Endpoint scaled to zero will not be charged. It will be resume on the next request to it, with a
369
- cold start delay. This is different than pausing the Inference Endpoint with [`InferenceEndpoint.pause`], which
370
+ An Inference Endpoint scaled to zero will not be charged. It will be resumed on the next request to it, with a
371
+ cold start delay. This is different from pausing the Inference Endpoint with [`InferenceEndpoint.pause`], which
370
372
  would require a manual resume with [`InferenceEndpoint.resume`].
371
373
 
372
374
  This is an alias for [`HfApi.scale_to_zero_inference_endpoint`]. The current object is mutated in place with the
@@ -400,6 +402,7 @@ class InferenceEndpoint:
400
402
  self.repository = self.raw["model"]["repository"]
401
403
  self.status = self.raw["status"]["state"]
402
404
  self.url = self.raw["status"].get("url")
405
+ self.health_route = self.raw["healthRoute"]
403
406
 
404
407
  # Other fields
405
408
  self.framework = self.raw["model"]["framework"]