huggingface-hub 0.29.0rc2__py3-none-any.whl → 1.1.3__py3-none-any.whl

huggingface_hub/utils/_pagination.py

@@ -14,17 +14,17 @@
 # limitations under the License.
 """Contains utilities to handle pagination on Huggingface Hub."""
 
-from typing import Dict, Iterable, Optional
+from typing import Iterable, Optional
 
-import requests
+import httpx
 
-from . import get_session, hf_raise_for_status, logging
+from . import get_session, hf_raise_for_status, http_backoff, logging
 
 
 logger = logging.get_logger(__name__)
 
 
-def paginate(path: str, params: Dict, headers: Dict) -> Iterable:
+def paginate(path: str, params: dict, headers: dict) -> Iterable:
     """Fetch a list of models/datasets/spaces and paginate through results.
 
     This is using the same "Link" header format as GitHub.
@@ -42,11 +42,11 @@ def paginate(path: str, params: Dict, headers: Dict) -> Iterable:
     next_page = _get_next_page(r)
     while next_page is not None:
         logger.debug(f"Pagination detected. Requesting next page: {next_page}")
-        r = session.get(next_page, headers=headers)
+        r = http_backoff("GET", next_page, max_retries=20, retry_on_status_codes=429, headers=headers)
         hf_raise_for_status(r)
         yield from r.json()
         next_page = _get_next_page(r)
 
 
-def _get_next_page(response: requests.Response) -> Optional[str]:
+def _get_next_page(response: httpx.Response) -> Optional[str]:
     return response.links.get("next", {}).get("url")

huggingface_hub/utils/_parsing.py

@@ -0,0 +1,98 @@
+# coding=utf-8
+# Copyright 2025-present, the HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Parsing helpers shared across modules."""
+
+import re
+import time
+from typing import Dict
+
+
+RE_NUMBER_WITH_UNIT = re.compile(r"(\d+)([a-z]+)", re.IGNORECASE)
+
+BYTE_UNITS: Dict[str, int] = {
+    "k": 1_000,
+    "m": 1_000_000,
+    "g": 1_000_000_000,
+    "t": 1_000_000_000_000,
+    "p": 1_000_000_000_000_000,
+}
+
+TIME_UNITS: Dict[str, int] = {
+    "s": 1,
+    "m": 60,
+    "h": 60 * 60,
+    "d": 24 * 60 * 60,
+    "w": 7 * 24 * 60 * 60,
+    "mo": 30 * 24 * 60 * 60,
+    "y": 365 * 24 * 60 * 60,
+}
+
+
+def parse_size(value: str) -> int:
+    """Parse a size expressed as a string with digits and unit (like `"10MB"`) to an integer (in bytes)."""
+    return _parse_with_unit(value, BYTE_UNITS)
+
+
+def parse_duration(value: str) -> int:
+    """Parse a duration expressed as a string with digits and unit (like `"10s"`) to an integer (in seconds)."""
+    return _parse_with_unit(value, TIME_UNITS)
+
+
+def _parse_with_unit(value: str, units: Dict[str, int]) -> int:
+    """Parse a numeric value with optional unit."""
+    stripped = value.strip()
+    if not stripped:
+        raise ValueError("Value cannot be empty.")
+    try:
+        return int(value)
+    except ValueError:
+        pass
+
+    match = RE_NUMBER_WITH_UNIT.fullmatch(stripped)
+    if not match:
+        raise ValueError(f"Invalid value '{value}'. Must match pattern '\\d+[a-z]+' or be a plain number.")
+
+    number = int(match.group(1))
+    unit = match.group(2).lower()
+
+    if unit not in units:
+        raise ValueError(f"Unknown unit '{unit}'. Must be one of {list(units.keys())}.")
+
+    return number * units[unit]
+
+
+def format_timesince(ts: float) -> str:
+    """Format timestamp in seconds into a human-readable string, relative to now.
+
+    Vaguely inspired by Django's `timesince` formatter.
+    """
+    _TIMESINCE_CHUNKS = (
+        # Label, divider, max value
+        ("second", 1, 60),
+        ("minute", 60, 60),
+        ("hour", 60 * 60, 24),
+        ("day", 60 * 60 * 24, 6),
+        ("week", 60 * 60 * 24 * 7, 6),
+        ("month", 60 * 60 * 24 * 30, 11),
+        ("year", 60 * 60 * 24 * 365, None),
+    )
+    delta = time.time() - ts
+    if delta < 20:
+        return "a few seconds ago"
+    for label, divider, max_value in _TIMESINCE_CHUNKS:  # noqa: B007
+        value = round(delta / divider)
+        if max_value is not None and value <= max_value:
+            break
+    return f"{value} {label}{'s' if value > 1 else ''} ago"

huggingface_hub/utils/_paths.py

@@ -16,7 +16,7 @@
 
 from fnmatch import fnmatch
 from pathlib import Path
-from typing import Callable, Generator, Iterable, List, Optional, TypeVar, Union
+from typing import Callable, Generator, Iterable, Optional, TypeVar, Union
 
 
 T = TypeVar("T")
@@ -39,8 +39,8 @@ FORBIDDEN_FOLDERS = [".git", ".cache"]
 def filter_repo_objects(
     items: Iterable[T],
     *,
-    allow_patterns: Optional[Union[List[str], str]] = None,
-    ignore_patterns: Optional[Union[List[str], str]] = None,
+    allow_patterns: Optional[Union[list[str], str]] = None,
+    ignore_patterns: Optional[Union[list[str], str]] = None,
     key: Optional[Callable[[T], str]] = None,
 ) -> Generator[T, None, None]:
     """Filter repo objects based on an allowlist and a denylist.
@@ -55,10 +55,10 @@ def filter_repo_objects(
     Args:
         items (`Iterable`):
             List of items to filter.
-        allow_patterns (`str` or `List[str]`, *optional*):
+        allow_patterns (`str` or `list[str]`, *optional*):
             Patterns constituting the allowlist. If provided, item paths must match at
             least one pattern from the allowlist.
-        ignore_patterns (`str` or `List[str]`, *optional*):
+        ignore_patterns (`str` or `list[str]`, *optional*):
             Patterns constituting the denylist. If provided, item paths must not match
             any patterns from the denylist.
         key (`Callable[[T], str]`, *optional*):

huggingface_hub/utils/_runtime.py

@@ -19,7 +19,8 @@ import os
 import platform
 import sys
 import warnings
-from typing import Any, Dict
+from pathlib import Path
+from typing import Any, Literal
 
 from .. import __version__, constants
 
@@ -35,8 +36,9 @@ _CANDIDATES = {
     "fastcore": {"fastcore"},
     "gradio": {"gradio"},
     "graphviz": {"graphviz"},
-    "hf_transfer": {"hf_transfer"},
+    "hf_xet": {"hf_xet"},
     "jinja": {"Jinja2"},
+    "httpx": {"httpx"},
     "keras": {"keras"},
     "numpy": {"numpy"},
     "pillow": {"Pillow"},
@@ -142,13 +144,26 @@ def get_graphviz_version() -> str:
     return _get_version("graphviz")
 
 
-# hf_transfer
-def is_hf_transfer_available() -> bool:
-    return is_package_available("hf_transfer")
+# httpx
+def is_httpx_available() -> bool:
+    return is_package_available("httpx")
 
 
-def get_hf_transfer_version() -> str:
-    return _get_version("hf_transfer")
+def get_httpx_version() -> str:
+    return _get_version("httpx")
+
+
+# xet
+def is_xet_available() -> bool:
+    # since hf_xet is automatically used if available, allow explicit disabling via environment variable
+    if constants.HF_HUB_DISABLE_XET:
+        return False
+
+    return is_package_available("hf_xet")
+
+
+def get_xet_version() -> str:
+    return _get_version("hf_xet")
 
 
 # keras
@@ -298,7 +313,50 @@ def is_colab_enterprise() -> bool:
     return os.environ.get("VERTEX_PRODUCT") == "COLAB_ENTERPRISE"
 
 
-def dump_environment_info() -> Dict[str, Any]:
+# Check how huggingface_hub has been installed
+
+
+def installation_method() -> Literal["brew", "hf_installer", "unknown"]:
+    """Return the installation method of the current environment.
+
+    - "hf_installer" if installed via the official installer script
+    - "brew" if installed via Homebrew
+    - "unknown" otherwise
+    """
+    if _is_brew_installation():
+        return "brew"
+    elif _is_hf_installer_installation():
+        return "hf_installer"
+    else:
+        return "unknown"
+
+
+def _is_brew_installation() -> bool:
+    """Check if running from a Homebrew installation.
+
+    Note: AI-generated by Claude.
+    """
+    exe_path = Path(sys.executable).resolve()
+    exe_str = str(exe_path)
+
+    # Check common Homebrew paths
+    # /opt/homebrew (Apple Silicon), /usr/local (Intel)
+    return "/Cellar/" in exe_str or "/opt/homebrew/" in exe_str or exe_str.startswith("/usr/local/Cellar/")
+
+
+def _is_hf_installer_installation() -> bool:
+    """Return `True` if the current environment was set up via the official hf installer script.
+
+    i.e. using one of
+        curl -LsSf https://hf.co/cli/install.sh | bash
+        powershell -ExecutionPolicy ByPass -c "irm https://hf.co/cli/install.ps1 | iex"
+    """
+    venv = sys.prefix  # points to venv root if active
+    marker = Path(venv) / ".hf_installer_marker"
+    return marker.exists()
+
+
+def dump_environment_info() -> dict[str, Any]:
     """Dump information about the machine to help debugging issues.
 
     Similar helper exist in:
@@ -312,7 +370,7 @@ def dump_environment_info() -> Dict[str, Any]:
     token = get_token()
 
     # Generic machine info
-    info: Dict[str, Any] = {
+    info: dict[str, Any] = {
         "huggingface_hub version": get_hf_hub_version(),
         "Platform": platform.platform(),
         "Python version": get_python_version(),
@@ -342,21 +400,14 @@ def dump_environment_info() -> Dict[str, Any]:
     except Exception:
         pass
 
+    # How huggingface_hub has been installed?
+    info["Installation method"] = installation_method()
+
     # Installed dependencies
-    info["FastAI"] = get_fastai_version()
-    info["Tensorflow"] = get_tf_version()
-    info["Torch"] = get_torch_version()
-    info["Jinja2"] = get_jinja_version()
-    info["Graphviz"] = get_graphviz_version()
-    info["keras"] = get_keras_version()
-    info["Pydot"] = get_pydot_version()
-    info["Pillow"] = get_pillow_version()
-    info["hf_transfer"] = get_hf_transfer_version()
+    info["httpx"] = get_httpx_version()
+    info["hf_xet"] = get_xet_version()
     info["gradio"] = get_gradio_version()
     info["tensorboard"] = get_tensorboard_version()
-    info["numpy"] = get_numpy_version()
-    info["pydantic"] = get_pydantic_version()
-    info["aiohttp"] = get_aiohttp_version()
 
     # Environment variables
     info["ENDPOINT"] = constants.ENDPOINT
@@ -370,9 +421,10 @@ def dump_environment_info() -> Dict[str, Any]:
     info["HF_HUB_DISABLE_SYMLINKS_WARNING"] = constants.HF_HUB_DISABLE_SYMLINKS_WARNING
     info["HF_HUB_DISABLE_EXPERIMENTAL_WARNING"] = constants.HF_HUB_DISABLE_EXPERIMENTAL_WARNING
     info["HF_HUB_DISABLE_IMPLICIT_TOKEN"] = constants.HF_HUB_DISABLE_IMPLICIT_TOKEN
-    info["HF_HUB_ENABLE_HF_TRANSFER"] = constants.HF_HUB_ENABLE_HF_TRANSFER
+    info["HF_HUB_DISABLE_XET"] = constants.HF_HUB_DISABLE_XET
     info["HF_HUB_ETAG_TIMEOUT"] = constants.HF_HUB_ETAG_TIMEOUT
     info["HF_HUB_DOWNLOAD_TIMEOUT"] = constants.HF_HUB_DOWNLOAD_TIMEOUT
+    info["HF_XET_HIGH_PERFORMANCE"] = constants.HF_XET_HIGH_PERFORMANCE
 
     print("\nCopy-and-paste the text below in your GitHub issue.\n")
     print("\n".join([f"- {prop}: {val}" for prop, val in info.items()]) + "\n")

huggingface_hub/utils/_safetensors.py

@@ -2,7 +2,7 @@ import functools
 import operator
 from collections import defaultdict
 from dataclasses import dataclass, field
-from typing import Dict, List, Literal, Optional, Tuple
+from typing import Literal, Optional
 
 
 FILENAME_T = str
@@ -19,17 +19,17 @@ class TensorInfo:
     Attributes:
         dtype (`str`):
             The data type of the tensor ("F64", "F32", "F16", "BF16", "I64", "I32", "I16", "I8", "U8", "BOOL").
-        shape (`List[int]`):
+        shape (`list[int]`):
             The shape of the tensor.
-        data_offsets (`Tuple[int, int]`):
+        data_offsets (`tuple[int, int]`):
             The offsets of the data in the file as a tuple `[BEGIN, END]`.
         parameter_count (`int`):
             The number of parameters in the tensor.
     """
 
     dtype: DTYPE_T
-    shape: List[int]
-    data_offsets: Tuple[int, int]
+    shape: list[int]
+    data_offsets: tuple[int, int]
     parameter_count: int = field(init=False)
 
     def __post_init__(self) -> None:
@@ -49,22 +49,22 @@ class SafetensorsFileMetadata:
     For more details regarding the safetensors format, check out https://huggingface.co/docs/safetensors/index#format.
 
     Attributes:
-        metadata (`Dict`):
+        metadata (`dict`):
             The metadata contained in the file.
-        tensors (`Dict[str, TensorInfo]`):
+        tensors (`dict[str, TensorInfo]`):
             A map of all tensors. Keys are tensor names and values are information about the corresponding tensor, as a
             [`TensorInfo`] object.
-        parameter_count (`Dict[str, int]`):
+        parameter_count (`dict[str, int]`):
             A map of the number of parameters per data type. Keys are data types and values are the number of parameters
             of that data type.
     """
 
-    metadata: Dict[str, str]
-    tensors: Dict[TENSOR_NAME_T, TensorInfo]
-    parameter_count: Dict[DTYPE_T, int] = field(init=False)
+    metadata: dict[str, str]
+    tensors: dict[TENSOR_NAME_T, TensorInfo]
+    parameter_count: dict[DTYPE_T, int] = field(init=False)
 
     def __post_init__(self) -> None:
-        parameter_count: Dict[DTYPE_T, int] = defaultdict(int)
+        parameter_count: dict[DTYPE_T, int] = defaultdict(int)
         for tensor in self.tensors.values():
             parameter_count[tensor.dtype] += tensor.parameter_count
         self.parameter_count = dict(parameter_count)
@@ -82,29 +82,29 @@ class SafetensorsRepoMetadata:
     For more details regarding the safetensors format, check out https://huggingface.co/docs/safetensors/index#format.
 
     Attributes:
-        metadata (`Dict`, *optional*):
+        metadata (`dict`, *optional*):
             The metadata contained in the 'model.safetensors.index.json' file, if it exists. Only populated for sharded
             models.
         sharded (`bool`):
             Whether the repo contains a sharded model or not.
-        weight_map (`Dict[str, str]`):
+        weight_map (`dict[str, str]`):
             A map of all weights. Keys are tensor names and values are filenames of the files containing the tensors.
-        files_metadata (`Dict[str, SafetensorsFileMetadata]`):
+        files_metadata (`dict[str, SafetensorsFileMetadata]`):
             A map of all files metadata. Keys are filenames and values are the metadata of the corresponding file, as
             a [`SafetensorsFileMetadata`] object.
-        parameter_count (`Dict[str, int]`):
+        parameter_count (`dict[str, int]`):
             A map of the number of parameters per data type. Keys are data types and values are the number of parameters
             of that data type.
     """
 
-    metadata: Optional[Dict]
+    metadata: Optional[dict]
     sharded: bool
-    weight_map: Dict[TENSOR_NAME_T, FILENAME_T]  # tensor name -> filename
-    files_metadata: Dict[FILENAME_T, SafetensorsFileMetadata]  # filename -> metadata
-    parameter_count: Dict[DTYPE_T, int] = field(init=False)
+    weight_map: dict[TENSOR_NAME_T, FILENAME_T]  # tensor name -> filename
+    files_metadata: dict[FILENAME_T, SafetensorsFileMetadata]  # filename -> metadata
+    parameter_count: dict[DTYPE_T, int] = field(init=False)
 
     def __post_init__(self) -> None:
-        parameter_count: Dict[DTYPE_T, int] = defaultdict(int)
+        parameter_count: dict[DTYPE_T, int] = defaultdict(int)
         for file_metadata in self.files_metadata.values():
             for dtype, nb_parameters_ in file_metadata.parameter_count.items():
                 parameter_count[dtype] += nb_parameters_

huggingface_hub/utils/_subprocess.py

@@ -20,7 +20,7 @@ import sys
 from contextlib import contextmanager
 from io import StringIO
 from pathlib import Path
-from typing import IO, Generator, List, Optional, Tuple, Union
+from typing import IO, Generator, Optional, Union
 
 from .logging import get_logger
 
@@ -44,12 +44,14 @@ def capture_output() -> Generator[StringIO, None, None]:
     output = StringIO()
     previous_output = sys.stdout
     sys.stdout = output
-    yield output
-    sys.stdout = previous_output
+    try:
+        yield output
+    finally:
+        sys.stdout = previous_output
 
 
 def run_subprocess(
-    command: Union[str, List[str]],
+    command: Union[str, list[str]],
     folder: Optional[Union[str, Path]] = None,
     check=True,
     **kwargs,
@@ -60,7 +62,7 @@ def run_subprocess(
     be captured.
 
     Args:
-        command (`str` or `List[str]`):
+        command (`str` or `list[str]`):
             The command to execute as a string or list of strings.
         folder (`str`, *optional*):
             The folder in which to run the command. Defaults to current working
@@ -68,7 +70,7 @@ def run_subprocess(
         check (`bool`, *optional*, defaults to `True`):
             Setting `check` to `True` will raise a `subprocess.CalledProcessError`
             when the subprocess has a non-zero exit code.
-        kwargs (`Dict[str]`):
+        kwargs (`dict[str]`):
             Keyword arguments to be passed to the `subprocess.run` underlying command.
 
     Returns:
@@ -94,23 +96,23 @@ def run_subprocess(
 
 @contextmanager
 def run_interactive_subprocess(
-    command: Union[str, List[str]],
+    command: Union[str, list[str]],
     folder: Optional[Union[str, Path]] = None,
     **kwargs,
-) -> Generator[Tuple[IO[str], IO[str]], None, None]:
+) -> Generator[tuple[IO[str], IO[str]], None, None]:
     """Run a subprocess in an interactive mode in a context manager.
 
     Args:
-        command (`str` or `List[str]`):
+        command (`str` or `list[str]`):
             The command to execute as a string or list of strings.
         folder (`str`, *optional*):
             The folder in which to run the command. Defaults to current working
             directory (from `os.getcwd()`).
-        kwargs (`Dict[str]`):
+        kwargs (`dict[str]`):
             Keyword arguments to be passed to the `subprocess.run` underlying command.
 
     Returns:
-        `Tuple[IO[str], IO[str]]`: A tuple with `stdin` and `stdout` to interact
+        `tuple[IO[str], IO[str]]`: A tuple with `stdin` and `stdout` to interact
         with the process (input and output are utf-8 encoded).
 
     Example:

huggingface_hub/utils/_telemetry.py

@@ -1,6 +1,6 @@
 from queue import Queue
 from threading import Lock, Thread
-from typing import Dict, Optional, Union
+from typing import Optional, Union
 from urllib.parse import quote
 
 from .. import constants, logging
@@ -22,10 +22,10 @@ def send_telemetry(
     *,
     library_name: Optional[str] = None,
     library_version: Optional[str] = None,
-    user_agent: Union[Dict, str, None] = None,
+    user_agent: Union[dict, str, None] = None,
 ) -> None:
     """
-    Sends telemetry that helps tracking usage of different HF libraries.
+    Sends telemetry that helps track usage of different HF libraries.
 
     This usage data helps us debug issues and prioritize new features. However, we understand that not everyone wants
     to share additional information, and we respect your privacy. You can disable telemetry collection by setting the
@@ -98,7 +98,7 @@ def _send_telemetry_in_thread(
     *,
     library_name: Optional[str] = None,
     library_version: Optional[str] = None,
-    user_agent: Union[Dict, str, None] = None,
+    user_agent: Union[dict, str, None] = None,
 ) -> None:
     """Contains the actual data sending data to the Hub.
 

huggingface_hub/{commands/_cli_utils.py → utils/_terminal.py}

@@ -1,4 +1,4 @@
-# Copyright 2022 The HuggingFace Team. All rights reserved.
+# Copyright 2025 The HuggingFace Team. All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -11,10 +11,10 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-"""Contains a utility for good-looking prints."""
+"""Contains utilities to print stuff to the terminal (styling, helpers)."""
 
 import os
-from typing import List, Union
+from typing import Union
 
 
 class ANSI:
@@ -52,7 +52,7 @@ class ANSI:
         return f"{code}{s}{cls._reset}"
 
 
-def tabulate(rows: List[List[Union[str, int]]], headers: List[str]) -> str:
+def tabulate(rows: list[list[Union[str, int]]], headers: list[str]) -> str:
     """
     Inspired by:
 

huggingface_hub/utils/_typing.py

@@ -15,10 +15,10 @@
 """Handle typing imports based on system compatibility."""
 
 import sys
-from typing import Any, Callable, List, Literal, Type, TypeVar, Union, get_args, get_origin
+from typing import Any, Callable, Literal, Optional, Type, TypeVar, Union, get_args, get_origin
 
 
-UNION_TYPES: List[Any] = [Union]
+UNION_TYPES: list[Any] = [Union]
 if sys.version_info >= (3, 10):
     from types import UnionType
 
@@ -33,7 +33,7 @@ CallableT = TypeVar("CallableT", bound=Callable)
 _JSON_SERIALIZABLE_TYPES = (int, float, str, bool, type(None))
 
 
-def is_jsonable(obj: Any) -> bool:
+def is_jsonable(obj: Any, _visited: Optional[set[int]] = None) -> bool:
     """Check if an object is JSON serializable.
 
     This is a weak check, as it does not check for the actual JSON serialization, but only for the types of the object.
@@ -43,19 +43,39 @@ def is_jsonable(obj: Any) -> bool:
     - it is an instance of int, float, str, bool, or NoneType
     - it is a list or tuple and all its items are json serializable
     - it is a dict and all its keys are strings and all its values are json serializable
+
+    Uses a visited set to avoid infinite recursion on circular references. If object has already been visited, it is
+    considered not json serializable.
     """
+    # Initialize visited set to track object ids and detect circular references
+    if _visited is None:
+        _visited = set()
+
+    # Detect circular reference
+    obj_id = id(obj)
+    if obj_id in _visited:
+        return False
+
+    # Add current object to visited before recursive checks
+    _visited.add(obj_id)
     try:
         if isinstance(obj, _JSON_SERIALIZABLE_TYPES):
             return True
         if isinstance(obj, (list, tuple)):
-            return all(is_jsonable(item) for item in obj)
+            return all(is_jsonable(item, _visited) for item in obj)
         if isinstance(obj, dict):
-            return all(isinstance(key, _JSON_SERIALIZABLE_TYPES) and is_jsonable(value) for key, value in obj.items())
+            return all(
+                isinstance(key, _JSON_SERIALIZABLE_TYPES) and is_jsonable(value, _visited)
+                for key, value in obj.items()
+            )
         if hasattr(obj, "__json__"):
             return True
         return False
     except RecursionError:
         return False
+    finally:
+        # Remove the object id from visited to avoid side‑effects for other branches
+        _visited.discard(obj_id)
 
 
 def is_simple_optional_type(type_: Type) -> bool: