huggingface-hub 0.31.0rc0__py3-none-any.whl → 1.1.3__py3-none-any.whl

huggingface_hub/lfs.py

@@ -14,15 +14,12 @@
 # limitations under the License.
 """Git LFS related type definitions and utilities"""
 
-import inspect
 import io
 import re
-import warnings
 from dataclasses import dataclass
 from math import ceil
 from os.path import getsize
-from pathlib import Path
-from typing import TYPE_CHECKING, BinaryIO, Dict, Iterable, List, Optional, Tuple, TypedDict
+from typing import TYPE_CHECKING, BinaryIO, Iterable, Optional, TypedDict
 from urllib.parse import unquote
 
 from huggingface_hub import constants
@@ -34,12 +31,10 @@ from .utils import (
     hf_raise_for_status,
     http_backoff,
     logging,
-    tqdm,
     validate_hf_hub_args,
 )
 from .utils._lfs import SliceFileObj
 from .utils.sha import sha256, sha_fileobj
-from .utils.tqdm import is_tqdm_disabled
 
 
 if TYPE_CHECKING:
@@ -107,8 +102,9 @@ def post_lfs_batch_info(
     repo_id: str,
     revision: Optional[str] = None,
     endpoint: Optional[str] = None,
-    headers: Optional[Dict[str, str]] = None,
-) -> Tuple[List[dict], List[dict]]:
+    headers: Optional[dict[str, str]] = None,
+    transfers: Optional[list[str]] = None,
+) -> tuple[list[dict], list[dict], Optional[str]]:
     """
     Requests the LFS batch endpoint to retrieve upload instructions
 
@@ -127,16 +123,19 @@ def post_lfs_batch_info(
             The git revision to upload to.
         headers (`dict`, *optional*):
             Additional headers to include in the request
+        transfers (`list`, *optional*):
+            List of transfer methods to use. Defaults to ["basic", "multipart"].
 
     Returns:
-        `LfsBatchInfo`: 2-tuple:
+        `LfsBatchInfo`: 3-tuple:
             - First element is the list of upload instructions from the server
-            - Second element is an list of errors, if any
+            - Second element is a list of errors, if any
+            - Third element is the chosen transfer adapter if provided by the server (e.g. "basic", "multipart", "xet")
 
     Raises:
         [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError)
             If an argument is invalid or the server response is malformed.
-        [`HTTPError`](https://requests.readthedocs.io/en/latest/api/#requests.HTTPError)
+        [`HfHubHTTPError`]
             If the server returned an error.
     """
     endpoint = endpoint if endpoint is not None else constants.ENDPOINT
@@ -144,9 +143,9 @@ def post_lfs_batch_info(
     if repo_type in constants.REPO_TYPES_URL_PREFIXES:
         url_prefix = constants.REPO_TYPES_URL_PREFIXES[repo_type]
     batch_url = f"{endpoint}/{url_prefix}{repo_id}.git/info/lfs/objects/batch"
-    payload: Dict = {
+    payload: dict = {
         "operation": "upload",
-        "transfers": ["basic", "multipart"],
+        "transfers": transfers if transfers is not None else ["basic", "multipart"],
         "objects": [
             {
                 "oid": upload.sha256.hex(),
@@ -172,9 +171,13 @@ def post_lfs_batch_info(
     if not isinstance(objects, list):
         raise ValueError("Malformed response from server")
 
+    chosen_transfer = batch_info.get("transfer")
+    chosen_transfer = chosen_transfer if isinstance(chosen_transfer, str) else None
+
     return (
         [_validate_batch_actions(obj) for obj in objects if "error" not in obj],
         [_validate_batch_error(obj) for obj in objects if "error" in obj],
+        chosen_transfer,
     )
 
 
@@ -187,14 +190,14 @@ class CompletionPayloadT(TypedDict):
     """Payload that will be sent to the Hub when uploading multi-part."""
 
     oid: str
-    parts: List[PayloadPartT]
+    parts: list[PayloadPartT]
 
 
 def lfs_upload(
     operation: "CommitOperationAdd",
-    lfs_batch_action: Dict,
+    lfs_batch_action: dict,
     token: Optional[str] = None,
-    headers: Optional[Dict[str, str]] = None,
+    headers: Optional[dict[str, str]] = None,
     endpoint: Optional[str] = None,
 ) -> None:
     """
@@ -214,7 +217,7 @@ def lfs_upload(
     Raises:
         [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError)
             If `lfs_batch_action` is improperly formatted
-        [`HTTPError`](https://requests.readthedocs.io/en/latest/api/#requests.HTTPError)
+        [`HfHubHTTPError`]
             If the upload resulted in an error
     """
     # 0. If LFS file is already present, skip upload
@@ -308,42 +311,26 @@ def _upload_single_part(operation: "CommitOperationAdd", upload_url: str) -> Non
         fileobj:
             The file-like object holding the data to upload.
 
-    Returns: `requests.Response`
-
     Raises:
-     [`HTTPError`](https://requests.readthedocs.io/en/latest/api/#requests.HTTPError)
-        If the upload resulted in an error.
+        [`HfHubHTTPError`]
+            If the upload resulted in an error.
     """
     with operation.as_file(with_tqdm=True) as fileobj:
         # S3 might raise a transient 500 error -> let's retry if that happens
-        response = http_backoff("PUT", upload_url, data=fileobj, retry_on_status_codes=(500, 502, 503, 504))
+        response = http_backoff("PUT", upload_url, data=fileobj)
         hf_raise_for_status(response)
 
 
-def _upload_multi_part(operation: "CommitOperationAdd", header: Dict, chunk_size: int, upload_url: str) -> None:
+def _upload_multi_part(operation: "CommitOperationAdd", header: dict, chunk_size: int, upload_url: str) -> None:
     """
     Uploads file using HF multipart LFS transfer protocol.
     """
     # 1. Get upload URLs for each part
     sorted_parts_urls = _get_sorted_parts_urls(header=header, upload_info=operation.upload_info, chunk_size=chunk_size)
 
-    # 2. Upload parts (either with hf_transfer or in pure Python)
-    use_hf_transfer = constants.HF_HUB_ENABLE_HF_TRANSFER
-    if (
-        constants.HF_HUB_ENABLE_HF_TRANSFER
-        and not isinstance(operation.path_or_fileobj, str)
-        and not isinstance(operation.path_or_fileobj, Path)
-    ):
-        warnings.warn(
-            "hf_transfer is enabled but does not support uploading from bytes or BinaryIO, falling back to regular"
-            " upload"
-        )
-        use_hf_transfer = False
-
-    response_headers = (
-        _upload_parts_hf_transfer(operation=operation, sorted_parts_urls=sorted_parts_urls, chunk_size=chunk_size)
-        if use_hf_transfer
-        else _upload_parts_iteratively(operation=operation, sorted_parts_urls=sorted_parts_urls, chunk_size=chunk_size)
+    # 2. Upload parts (pure Python)
+    response_headers = _upload_parts_iteratively(
+        operation=operation, sorted_parts_urls=sorted_parts_urls, chunk_size=chunk_size
     )
 
     # 3. Send completion request
@@ -355,7 +342,7 @@ def _upload_multi_part(operation: "CommitOperationAdd", header: Dict, chunk_size
     hf_raise_for_status(completion_res)
 
 
-def _get_sorted_parts_urls(header: Dict, upload_info: UploadInfo, chunk_size: int) -> List[str]:
+def _get_sorted_parts_urls(header: dict, upload_info: UploadInfo, chunk_size: int) -> list[str]:
     sorted_part_upload_urls = [
         upload_url
         for _, upload_url in sorted(
@@ -373,8 +360,8 @@ def _get_sorted_parts_urls(header: Dict, upload_info: UploadInfo, chunk_size: in
     return sorted_part_upload_urls
 
 
-def _get_completion_payload(response_headers: List[Dict], oid: str) -> CompletionPayloadT:
-    parts: List[PayloadPartT] = []
+def _get_completion_payload(response_headers: list[dict], oid: str) -> CompletionPayloadT:
+    parts: list[PayloadPartT] = []
     for part_number, header in enumerate(response_headers):
         etag = header.get("etag")
         if etag is None or etag == "":
@@ -389,8 +376,8 @@ def _get_completion_payload(response_headers: List[Dict], oid: str) -> Completio
 
 
 def _upload_parts_iteratively(
-    operation: "CommitOperationAdd", sorted_parts_urls: List[str], chunk_size: int
-) -> List[Dict]:
+    operation: "CommitOperationAdd", sorted_parts_urls: list[str], chunk_size: int
+) -> list[dict]:
     headers = []
     with operation.as_file(with_tqdm=True) as fileobj:
         for part_idx, part_upload_url in enumerate(sorted_parts_urls):
@@ -400,61 +387,7 @@ def _upload_parts_iteratively(
                 read_limit=chunk_size,
             ) as fileobj_slice:
                 # S3 might raise a transient 500 error -> let's retry if that happens
-                part_upload_res = http_backoff(
-                    "PUT", part_upload_url, data=fileobj_slice, retry_on_status_codes=(500, 502, 503, 504)
-                )
+                part_upload_res = http_backoff("PUT", part_upload_url, data=fileobj_slice)
                 hf_raise_for_status(part_upload_res)
                 headers.append(part_upload_res.headers)
     return headers  # type: ignore
-
-
-def _upload_parts_hf_transfer(
-    operation: "CommitOperationAdd", sorted_parts_urls: List[str], chunk_size: int
-) -> List[Dict]:
-    # Upload file using an external Rust-based package. Upload is faster but support less features (no progress bars).
-    try:
-        from hf_transfer import multipart_upload
-    except ImportError:
-        raise ValueError(
-            "Fast uploading using 'hf_transfer' is enabled (HF_HUB_ENABLE_HF_TRANSFER=1) but 'hf_transfer' package is"
-            " not available in your environment. Try `pip install hf_transfer`."
-        )
-
-    supports_callback = "callback" in inspect.signature(multipart_upload).parameters
-    if not supports_callback:
-        warnings.warn(
-            "You are using an outdated version of `hf_transfer`. Consider upgrading to latest version to enable progress bars using `pip install -U hf_transfer`."
-        )
-
-    total = operation.upload_info.size
-    desc = operation.path_in_repo
-    if len(desc) > 40:
-        desc = f"(…){desc[-40:]}"
-
-    with tqdm(
-        unit="B",
-        unit_scale=True,
-        total=total,
-        initial=0,
-        desc=desc,
-        disable=is_tqdm_disabled(logger.getEffectiveLevel()),
-        name="huggingface_hub.lfs_upload",
-    ) as progress:
-        try:
-            output = multipart_upload(
-                file_path=operation.path_or_fileobj,
-                parts_urls=sorted_parts_urls,
-                chunk_size=chunk_size,
-                max_files=128,
-                parallel_failures=127,  # could be removed
-                max_retries=5,
-                **({"callback": progress.update} if supports_callback else {}),
-            )
-        except Exception as e:
-            raise RuntimeError(
-                "An error occurred while uploading using `hf_transfer`. Consider disabling HF_HUB_ENABLE_HF_TRANSFER for"
-                " better error handling."
-            ) from e
-        if not supports_callback:
-            progress.update(total)
-        return output

huggingface_hub/repocard.py

@@ -1,9 +1,8 @@
 import os
 import re
 from pathlib import Path
-from typing import Any, Dict, Literal, Optional, Type, Union
+from typing import Any, Literal, Optional, Union
 
-import requests
 import yaml
 
 from huggingface_hub.file_download import hf_hub_download
@@ -17,7 +16,7 @@ from huggingface_hub.repocard_data import (
     eval_results_to_model_index,
     model_index_to_eval_results,
 )
-from huggingface_hub.utils import get_session, is_jinja_available, yaml_dump
+from huggingface_hub.utils import HfHubHTTPError, get_session, hf_raise_for_status, is_jinja_available, yaml_dump
 
 from . import constants
 from .errors import EntryNotFoundError
@@ -65,13 +64,11 @@ class RepoCard:
             '\\n# My repo\\n'
 
             ```
-        <Tip>
-        Raises the following error:
-
-            - [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError)
-              when the content of the repo card metadata is not a dictionary.
-
-        </Tip>
+        > [!TIP]
+        > Raises the following error:
+        >
+        >     - [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError)
+        >       when the content of the repo card metadata is not a dictionary.
         """
 
         # Set the content of the RepoCard, as well as underlying .data and .text attributes.
@@ -149,7 +146,7 @@ class RepoCard:
             repo_id_or_path (`Union[str, Path]`):
                 The repo ID associated with a Hugging Face Hub repo or a local filepath.
             repo_type (`str`, *optional*):
-                The type of Hugging Face repo to push to. Defaults to None, which will use use "model". Other options
+                The type of Hugging Face repo to push to. Defaults to None, which will use "model". Other options
                 are "dataset" and "space". Not used when loading from a local filepath. If this is called from a child
                 class, the default value will be the child class's `repo_type`.
             token (`str`, *optional*):
@@ -199,15 +196,13 @@ class RepoCard:
                 The type of Hugging Face repo to push to. Options are "model", "dataset", and "space".
                 If this function is called from a child class, the default will be the child class's `repo_type`.
 
-        <Tip>
-        Raises the following errors:
-
-            - [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError)
-              if the card fails validation checks.
-            - [`HTTPError`](https://requests.readthedocs.io/en/latest/api/#requests.HTTPError)
-              if the request to the Hub API fails for any other reason.
-
-        </Tip>
+        > [!TIP]
+        > Raises the following errors:
+        >
+        >     - [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError)
+        >       if the card fails validation checks.
+        >     - [`HTTPError`](https://requests.readthedocs.io/en/latest/api/#requests.HTTPError)
+        >       if the request to the Hub API fails for any other reason.
         """
 
         # If repo type is provided, otherwise, use the repo type of the card.
@@ -220,11 +215,11 @@ class RepoCard:
         headers = {"Accept": "text/plain"}
 
         try:
-            r = get_session().post("https://huggingface.co/api/validate-yaml", body, headers=headers)
-            r.raise_for_status()
-        except requests.exceptions.HTTPError as exc:
-            if r.status_code == 400:
-                raise ValueError(r.text)
+            response = get_session().post("https://huggingface.co/api/validate-yaml", json=body, headers=headers)
+            hf_raise_for_status(response)
+        except HfHubHTTPError as exc:
+            if response.status_code == 400:
+                raise ValueError(response.text)
             else:
                 raise exc
 
@@ -263,7 +258,7 @@ class RepoCard:
                 If specified and `create_pr` is `False`, the commit will fail if `revision` does not point to `parent_commit`.
                 If specified and `create_pr` is `True`, the pull request will be created from `parent_commit`.
                 Specifying `parent_commit` ensures the repo has not changed before committing the changes, and can be
-                especially useful if the repo is updated / committed to concurrently.
+                especially useful if the repo is updated / committed too concurrently.
         Returns:
             `str`: URL of the commit which updated the card metadata.
         """
@@ -276,7 +271,7 @@ class RepoCard:
 
         with SoftTemporaryDirectory() as tmpdir:
             tmp_path = Path(tmpdir) / constants.REPOCARD_NAME
-            tmp_path.write_text(str(self))
+            tmp_path.write_text(str(self), encoding="utf-8")
             url = upload_file(
                 path_or_fileobj=str(tmp_path),
                 path_in_repo=constants.REPOCARD_NAME,
@@ -336,7 +331,7 @@ class RepoCard:
 
 
 class ModelCard(RepoCard):
-    card_data_class = ModelCardData
+    card_data_class = ModelCardData  # type: ignore[assignment]
     default_template_path = TEMPLATE_MODELCARD_PATH
     repo_type = "model"
 
@@ -417,7 +412,7 @@ class ModelCard(RepoCard):
 
 
 class DatasetCard(RepoCard):
-    card_data_class = DatasetCardData
+    card_data_class = DatasetCardData  # type: ignore[assignment]
     default_template_path = TEMPLATE_DATASETCARD_PATH
     repo_type = "dataset"
 
@@ -482,7 +477,7 @@ class DatasetCard(RepoCard):
 
 
 class SpaceCard(RepoCard):
-    card_data_class = SpaceCardData
+    card_data_class = SpaceCardData  # type: ignore[assignment]
     default_template_path = TEMPLATE_MODELCARD_PATH
     repo_type = "space"
 
@@ -508,7 +503,7 @@ def _detect_line_ending(content: str) -> Literal["\r", "\n", "\r\n", None]:  # n
         return "\n"
 
 
-def metadata_load(local_path: Union[str, Path]) -> Optional[Dict]:
+def metadata_load(local_path: Union[str, Path]) -> Optional[dict]:
     content = Path(local_path).read_text()
     match = REGEX_YAML_BLOCK.search(content)
     if match:
@@ -521,7 +516,7 @@ def metadata_load(local_path: Union[str, Path]) -> Optional[Dict]:
         return None
 
 
-def metadata_save(local_path: Union[str, Path], data: Dict) -> None:
+def metadata_save(local_path: Union[str, Path], data: dict) -> None:
     """
     Save the metadata dict in the upper YAML part Trying to preserve newlines as
     in the existing file. Docs about open() with newline="" parameter:
@@ -569,7 +564,7 @@ def metadata_eval_result(
     dataset_split: Optional[str] = None,
     dataset_revision: Optional[str] = None,
     metrics_verification_token: Optional[str] = None,
-) -> Dict:
+) -> dict:
     """
     Creates a metadata dict with the result from a model evaluated on a dataset.
 
@@ -684,7 +679,7 @@ def metadata_eval_result(
 @validate_hf_hub_args
 def metadata_update(
     repo_id: str,
-    metadata: Dict,
+    metadata: dict,
     *,
     repo_type: Optional[str] = None,
     overwrite: bool = False,
@@ -697,7 +692,7 @@ def metadata_update(
 ) -> str:
     """
     Updates the metadata in the README.md of a repository on the Hugging Face Hub.
-    If the README.md file doesn't exist yet, a new one is created with metadata and an
+    If the README.md file doesn't exist yet, a new one is created with metadata and
     the default ModelCard or DatasetCard template. For `space` repo, an error is thrown
     as a Space cannot exist without a `README.md` file.
 
@@ -730,7 +725,7 @@ def metadata_update(
             If specified and `create_pr` is `False`, the commit will fail if `revision` does not point to `parent_commit`.
             If specified and `create_pr` is `True`, the pull request will be created from `parent_commit`.
             Specifying `parent_commit` ensures the repo has not changed before committing the changes, and can be
-            especially useful if the repo is updated / committed to concurrently.
+            especially useful if the repo is updated / committed too concurrently.
     Returns:
         `str`: URL of the commit which updated the card metadata.
 
@@ -752,7 +747,7 @@ def metadata_update(
     commit_message = commit_message if commit_message is not None else "Update metadata with huggingface_hub"
 
     # Card class given repo_type
-    card_class: Type[RepoCard]
+    card_class: type[RepoCard]
     if repo_type is None or repo_type == "model":
         card_class = ModelCard
     elif repo_type == "dataset":
@@ -771,7 +766,8 @@ def metadata_update(
             raise ValueError("Cannot update metadata on a Space that doesn't contain a `README.md` file.")
 
         # Initialize a ModelCard or DatasetCard from default template and no data.
-        card = card_class.from_template(CardData())
+        # Cast to the concrete expected card type to satisfy type checkers.
+        card = card_class.from_template(CardData())  # type: ignore[return-value]
 
     for key, value in metadata.items():
         if key == "model-index":