huggingface-hub 0.34.4__py3-none-any.whl → 1.0.0rc0__py3-none-any.whl

huggingface_hub/utils/_pagination.py

@@ -14,9 +14,9 @@
 # limitations under the License.
 """Contains utilities to handle pagination on Huggingface Hub."""
 
-from typing import Dict, Iterable, Optional
+from typing import Iterable, Optional
 
-import requests
+import httpx
 
 from . import get_session, hf_raise_for_status, http_backoff, logging
 
@@ -24,7 +24,7 @@ from . import get_session, hf_raise_for_status, http_backoff, logging
 logger = logging.get_logger(__name__)
 
 
-def paginate(path: str, params: Dict, headers: Dict) -> Iterable:
+def paginate(path: str, params: dict, headers: dict) -> Iterable:
     """Fetch a list of models/datasets/spaces and paginate through results.
 
     This is using the same "Link" header format as GitHub.
@@ -48,5 +48,5 @@ def paginate(path: str, params: Dict, headers: Dict) -> Iterable:
         next_page = _get_next_page(r)
 
 
-def _get_next_page(response: requests.Response) -> Optional[str]:
+def _get_next_page(response: httpx.Response) -> Optional[str]:
     return response.links.get("next", {}).get("url")

huggingface_hub/utils/_paths.py

@@ -16,7 +16,7 @@
 
 from fnmatch import fnmatch
 from pathlib import Path
-from typing import Callable, Generator, Iterable, List, Optional, TypeVar, Union
+from typing import Callable, Generator, Iterable, Optional, TypeVar, Union
 
 
 T = TypeVar("T")
@@ -39,8 +39,8 @@ FORBIDDEN_FOLDERS = [".git", ".cache"]
 def filter_repo_objects(
     items: Iterable[T],
     *,
-    allow_patterns: Optional[Union[List[str], str]] = None,
-    ignore_patterns: Optional[Union[List[str], str]] = None,
+    allow_patterns: Optional[Union[list[str], str]] = None,
+    ignore_patterns: Optional[Union[list[str], str]] = None,
     key: Optional[Callable[[T], str]] = None,
 ) -> Generator[T, None, None]:
     """Filter repo objects based on an allowlist and a denylist.
@@ -55,10 +55,10 @@ def filter_repo_objects(
     Args:
         items (`Iterable`):
             List of items to filter.
-        allow_patterns (`str` or `List[str]`, *optional*):
+        allow_patterns (`str` or `list[str]`, *optional*):
             Patterns constituting the allowlist. If provided, item paths must match at
             least one pattern from the allowlist.
-        ignore_patterns (`str` or `List[str]`, *optional*):
+        ignore_patterns (`str` or `list[str]`, *optional*):
             Patterns constituting the denylist. If provided, item paths must not match
             any patterns from the denylist.
         key (`Callable[[T], str]`, *optional*):

huggingface_hub/utils/_runtime.py

@@ -19,7 +19,7 @@ import os
 import platform
 import sys
 import warnings
-from typing import Any, Dict
+from typing import Any
 
 from .. import __version__, constants
 
@@ -38,6 +38,7 @@ _CANDIDATES = {
     "hf_transfer": {"hf_transfer"},
     "hf_xet": {"hf_xet"},
     "jinja": {"Jinja2"},
+    "httpx": {"httpx"},
     "keras": {"keras"},
     "numpy": {"numpy"},
     "pillow": {"Pillow"},
@@ -152,6 +153,15 @@ def get_hf_transfer_version() -> str:
     return _get_version("hf_transfer")
 
 
+# httpx
+def is_httpx_available() -> bool:
+    return is_package_available("httpx")
+
+
+def get_httpx_version() -> str:
+    return _get_version("httpx")
+
+
 # xet
 def is_xet_available() -> bool:
     # since hf_xet is automatically used if available, allow explicit disabling via environment variable
@@ -312,7 +322,7 @@ def is_colab_enterprise() -> bool:
     return os.environ.get("VERTEX_PRODUCT") == "COLAB_ENTERPRISE"
 
 
-def dump_environment_info() -> Dict[str, Any]:
+def dump_environment_info() -> dict[str, Any]:
     """Dump information about the machine to help debugging issues.
 
     Similar helper exist in:
@@ -326,7 +336,7 @@ def dump_environment_info() -> Dict[str, Any]:
     token = get_token()
 
     # Generic machine info
-    info: Dict[str, Any] = {
+    info: dict[str, Any] = {
         "huggingface_hub version": get_hf_hub_version(),
         "Platform": platform.platform(),
         "Python version": get_python_version(),
@@ -357,21 +367,13 @@ def dump_environment_info() -> Dict[str, Any]:
         pass
 
     # Installed dependencies
-    info["FastAI"] = get_fastai_version()
-    info["Tensorflow"] = get_tf_version()
     info["Torch"] = get_torch_version()
-    info["Jinja2"] = get_jinja_version()
-    info["Graphviz"] = get_graphviz_version()
-    info["keras"] = get_keras_version()
-    info["Pydot"] = get_pydot_version()
-    info["Pillow"] = get_pillow_version()
+    info["httpx"] = get_httpx_version()
     info["hf_transfer"] = get_hf_transfer_version()
+    info["hf_xet"] = get_xet_version()
     info["gradio"] = get_gradio_version()
     info["tensorboard"] = get_tensorboard_version()
-    info["numpy"] = get_numpy_version()
     info["pydantic"] = get_pydantic_version()
-    info["aiohttp"] = get_aiohttp_version()
-    info["hf_xet"] = get_xet_version()
 
     # Environment variables
     info["ENDPOINT"] = constants.ENDPOINT

huggingface_hub/utils/_safetensors.py

@@ -2,7 +2,7 @@ import functools
 import operator
 from collections import defaultdict
 from dataclasses import dataclass, field
-from typing import Dict, List, Literal, Optional, Tuple
+from typing import Literal, Optional
 
 
 FILENAME_T = str
@@ -19,17 +19,17 @@ class TensorInfo:
     Attributes:
         dtype (`str`):
             The data type of the tensor ("F64", "F32", "F16", "BF16", "I64", "I32", "I16", "I8", "U8", "BOOL").
-        shape (`List[int]`):
+        shape (`list[int]`):
             The shape of the tensor.
-        data_offsets (`Tuple[int, int]`):
+        data_offsets (`tuple[int, int]`):
             The offsets of the data in the file as a tuple `[BEGIN, END]`.
         parameter_count (`int`):
             The number of parameters in the tensor.
     """
 
     dtype: DTYPE_T
-    shape: List[int]
-    data_offsets: Tuple[int, int]
+    shape: list[int]
+    data_offsets: tuple[int, int]
     parameter_count: int = field(init=False)
 
     def __post_init__(self) -> None:
@@ -49,22 +49,22 @@ class SafetensorsFileMetadata:
     For more details regarding the safetensors format, check out https://huggingface.co/docs/safetensors/index#format.
 
     Attributes:
-        metadata (`Dict`):
+        metadata (`dict`):
             The metadata contained in the file.
-        tensors (`Dict[str, TensorInfo]`):
+        tensors (`dict[str, TensorInfo]`):
             A map of all tensors. Keys are tensor names and values are information about the corresponding tensor, as a
             [`TensorInfo`] object.
-        parameter_count (`Dict[str, int]`):
+        parameter_count (`dict[str, int]`):
             A map of the number of parameters per data type. Keys are data types and values are the number of parameters
             of that data type.
     """
 
-    metadata: Dict[str, str]
-    tensors: Dict[TENSOR_NAME_T, TensorInfo]
-    parameter_count: Dict[DTYPE_T, int] = field(init=False)
+    metadata: dict[str, str]
+    tensors: dict[TENSOR_NAME_T, TensorInfo]
+    parameter_count: dict[DTYPE_T, int] = field(init=False)
 
     def __post_init__(self) -> None:
-        parameter_count: Dict[DTYPE_T, int] = defaultdict(int)
+        parameter_count: dict[DTYPE_T, int] = defaultdict(int)
         for tensor in self.tensors.values():
             parameter_count[tensor.dtype] += tensor.parameter_count
         self.parameter_count = dict(parameter_count)
@@ -82,29 +82,29 @@ class SafetensorsRepoMetadata:
     For more details regarding the safetensors format, check out https://huggingface.co/docs/safetensors/index#format.
 
     Attributes:
-        metadata (`Dict`, *optional*):
+        metadata (`dict`, *optional*):
             The metadata contained in the 'model.safetensors.index.json' file, if it exists. Only populated for sharded
             models.
         sharded (`bool`):
             Whether the repo contains a sharded model or not.
-        weight_map (`Dict[str, str]`):
+        weight_map (`dict[str, str]`):
             A map of all weights. Keys are tensor names and values are filenames of the files containing the tensors.
-        files_metadata (`Dict[str, SafetensorsFileMetadata]`):
+        files_metadata (`dict[str, SafetensorsFileMetadata]`):
             A map of all files metadata. Keys are filenames and values are the metadata of the corresponding file, as
             a [`SafetensorsFileMetadata`] object.
-        parameter_count (`Dict[str, int]`):
+        parameter_count (`dict[str, int]`):
             A map of the number of parameters per data type. Keys are data types and values are the number of parameters
             of that data type.
     """
 
-    metadata: Optional[Dict]
+    metadata: Optional[dict]
     sharded: bool
-    weight_map: Dict[TENSOR_NAME_T, FILENAME_T]  # tensor name -> filename
-    files_metadata: Dict[FILENAME_T, SafetensorsFileMetadata]  # filename -> metadata
-    parameter_count: Dict[DTYPE_T, int] = field(init=False)
+    weight_map: dict[TENSOR_NAME_T, FILENAME_T]  # tensor name -> filename
+    files_metadata: dict[FILENAME_T, SafetensorsFileMetadata]  # filename -> metadata
+    parameter_count: dict[DTYPE_T, int] = field(init=False)
 
     def __post_init__(self) -> None:
-        parameter_count: Dict[DTYPE_T, int] = defaultdict(int)
+        parameter_count: dict[DTYPE_T, int] = defaultdict(int)
         for file_metadata in self.files_metadata.values():
             for dtype, nb_parameters_ in file_metadata.parameter_count.items():
                 parameter_count[dtype] += nb_parameters_

huggingface_hub/utils/_subprocess.py

@@ -20,7 +20,7 @@ import sys
 from contextlib import contextmanager
 from io import StringIO
 from pathlib import Path
-from typing import IO, Generator, List, Optional, Tuple, Union
+from typing import IO, Generator, Optional, Union
 
 from .logging import get_logger
 
@@ -51,7 +51,7 @@ def capture_output() -> Generator[StringIO, None, None]:
 
 
 def run_subprocess(
-    command: Union[str, List[str]],
+    command: Union[str, list[str]],
     folder: Optional[Union[str, Path]] = None,
     check=True,
     **kwargs,
@@ -62,7 +62,7 @@ def run_subprocess(
     be captured.
 
     Args:
-        command (`str` or `List[str]`):
+        command (`str` or `list[str]`):
             The command to execute as a string or list of strings.
         folder (`str`, *optional*):
             The folder in which to run the command. Defaults to current working
@@ -70,7 +70,7 @@ def run_subprocess(
         check (`bool`, *optional*, defaults to `True`):
             Setting `check` to `True` will raise a `subprocess.CalledProcessError`
             when the subprocess has a non-zero exit code.
-        kwargs (`Dict[str]`):
+        kwargs (`dict[str]`):
             Keyword arguments to be passed to the `subprocess.run` underlying command.
 
     Returns:
@@ -96,23 +96,23 @@ def run_subprocess(
 
 @contextmanager
 def run_interactive_subprocess(
-    command: Union[str, List[str]],
+    command: Union[str, list[str]],
     folder: Optional[Union[str, Path]] = None,
     **kwargs,
-) -> Generator[Tuple[IO[str], IO[str]], None, None]:
+) -> Generator[tuple[IO[str], IO[str]], None, None]:
     """Run a subprocess in an interactive mode in a context manager.
 
     Args:
-        command (`str` or `List[str]`):
+        command (`str` or `list[str]`):
             The command to execute as a string or list of strings.
         folder (`str`, *optional*):
             The folder in which to run the command. Defaults to current working
             directory (from `os.getcwd()`).
-        kwargs (`Dict[str]`):
+        kwargs (`dict[str]`):
             Keyword arguments to be passed to the `subprocess.run` underlying command.
 
     Returns:
-        `Tuple[IO[str], IO[str]]`: A tuple with `stdin` and `stdout` to interact
+        `tuple[IO[str], IO[str]]`: A tuple with `stdin` and `stdout` to interact
         with the process (input and output are utf-8 encoded).
 
     Example:

huggingface_hub/utils/_telemetry.py

@@ -1,6 +1,6 @@
 from queue import Queue
 from threading import Lock, Thread
-from typing import Dict, Optional, Union
+from typing import Optional, Union
 from urllib.parse import quote
 
 from .. import constants, logging
@@ -22,7 +22,7 @@ def send_telemetry(
     *,
     library_name: Optional[str] = None,
     library_version: Optional[str] = None,
-    user_agent: Union[Dict, str, None] = None,
+    user_agent: Union[dict, str, None] = None,
 ) -> None:
     """
     Sends telemetry that helps tracking usage of different HF libraries.
@@ -98,7 +98,7 @@ def _send_telemetry_in_thread(
     *,
     library_name: Optional[str] = None,
     library_version: Optional[str] = None,
-    user_agent: Union[Dict, str, None] = None,
+    user_agent: Union[dict, str, None] = None,
 ) -> None:
     """Contains the actual data sending data to the Hub.
 

huggingface_hub/utils/_typing.py

@@ -15,10 +15,10 @@
 """Handle typing imports based on system compatibility."""
 
 import sys
-from typing import Any, Callable, List, Literal, Type, TypeVar, Union, get_args, get_origin
+from typing import Any, Callable, Literal, Optional, Type, TypeVar, Union, get_args, get_origin
 
 
-UNION_TYPES: List[Any] = [Union]
+UNION_TYPES: list[Any] = [Union]
 if sys.version_info >= (3, 10):
     from types import UnionType
 
@@ -33,7 +33,7 @@ CallableT = TypeVar("CallableT", bound=Callable)
 _JSON_SERIALIZABLE_TYPES = (int, float, str, bool, type(None))
 
 
-def is_jsonable(obj: Any) -> bool:
+def is_jsonable(obj: Any, _visited: Optional[set[int]] = None) -> bool:
     """Check if an object is JSON serializable.
 
     This is a weak check, as it does not check for the actual JSON serialization, but only for the types of the object.
@@ -43,19 +43,39 @@ def is_jsonable(obj: Any) -> bool:
     - it is an instance of int, float, str, bool, or NoneType
     - it is a list or tuple and all its items are json serializable
     - it is a dict and all its keys are strings and all its values are json serializable
+
+    Uses a visited set to avoid infinite recursion on circular references. If object has already been visited, it is
+    considered not json serializable.
     """
+    # Initialize visited set to track object ids and detect circular references
+    if _visited is None:
+        _visited = set()
+
+    # Detect circular reference
+    obj_id = id(obj)
+    if obj_id in _visited:
+        return False
+
+    # Add current object to visited before recursive checks
+    _visited.add(obj_id)
     try:
         if isinstance(obj, _JSON_SERIALIZABLE_TYPES):
             return True
         if isinstance(obj, (list, tuple)):
-            return all(is_jsonable(item) for item in obj)
+            return all(is_jsonable(item, _visited) for item in obj)
         if isinstance(obj, dict):
-            return all(isinstance(key, _JSON_SERIALIZABLE_TYPES) and is_jsonable(value) for key, value in obj.items())
+            return all(
+                isinstance(key, _JSON_SERIALIZABLE_TYPES) and is_jsonable(value, _visited)
+                for key, value in obj.items()
+            )
         if hasattr(obj, "__json__"):
             return True
         return False
     except RecursionError:
         return False
+    finally:
+        # Remove the object id from visited to avoid side‑effects for other branches
+        _visited.discard(obj_id)
 
 
 def is_simple_optional_type(type_: Type) -> bool:

huggingface_hub/utils/_validators.py

@@ -19,7 +19,7 @@ import re
 import warnings
 from functools import wraps
 from itertools import chain
-from typing import Any, Dict
+from typing import Any
 
 from huggingface_hub.errors import HFValidationError
 
@@ -48,9 +48,7 @@ def validate_hf_hub_args(fn: CallableT) -> CallableT:
     Validators:
         - [`~utils.validate_repo_id`]: `repo_id` must be `"repo_name"`
           or `"namespace/repo_name"`. Namespace is a username or an organization.
-        - [`~utils.smoothly_deprecate_use_auth_token`]: Use `token` instead of
-          `use_auth_token` (only if `use_auth_token` is not expected by the decorated
-          function - in practice, always the case in `huggingface_hub`).
+        - [`~utils.smoothly_deprecate_legacy_arguments`]: Ignore `proxies` when downloading files (should be set globally).
 
     Example:
     ```py
@@ -68,20 +66,6 @@ def validate_hf_hub_args(fn: CallableT) -> CallableT:
 
     >>> my_cool_method(repo_id="other..repo..id")
     huggingface_hub.utils._validators.HFValidationError: Cannot have -- or .. in repo_id: 'other..repo..id'.
-
-    >>> @validate_hf_hub_args
-    ... def my_cool_auth_method(token: str):
-    ...     print(token)
-
-    >>> my_cool_auth_method(token="a token")
-    "a token"
-
-    >>> my_cool_auth_method(use_auth_token="a use_auth_token")
-    "a use_auth_token"
-
-    >>> my_cool_auth_method(token="a token", use_auth_token="a use_auth_token")
-    UserWarning: Both `token` and `use_auth_token` are passed (...)
-    "a token"
     ```
 
     Raises:
@@ -91,13 +75,8 @@ def validate_hf_hub_args(fn: CallableT) -> CallableT:
     # TODO: add an argument to opt-out validation for specific argument?
     signature = inspect.signature(fn)
 
-    # Should the validator switch `use_auth_token` values to `token`? In practice, always
-    # True in `huggingface_hub`. Might not be the case in a downstream library.
-    check_use_auth_token = "use_auth_token" not in signature.parameters and "token" in signature.parameters
-
     @wraps(fn)
     def _inner_fn(*args, **kwargs):
-        has_token = False
         for arg_name, arg_value in chain(
             zip(signature.parameters, args),  # Args values
             kwargs.items(),  # Kwargs values
@@ -105,11 +84,7 @@ def validate_hf_hub_args(fn: CallableT) -> CallableT:
             if arg_name in ["repo_id", "from_id", "to_id"]:
                 validate_repo_id(arg_value)
 
-            elif arg_name == "token" and arg_value is not None:
-                has_token = True
-
-        if check_use_auth_token:
-            kwargs = smoothly_deprecate_use_auth_token(fn_name=fn.__name__, has_token=has_token, kwargs=kwargs)
+        kwargs = smoothly_deprecate_legacy_arguments(fn_name=fn.__name__, kwargs=kwargs)
 
         return fn(*args, **kwargs)
 
@@ -170,57 +145,63 @@ def validate_repo_id(repo_id: str) -> None:
         raise HFValidationError(f"Repo_id cannot end by '.git': '{repo_id}'.")
 
 
-def smoothly_deprecate_use_auth_token(fn_name: str, has_token: bool, kwargs: Dict[str, Any]) -> Dict[str, Any]:
-    """Smoothly deprecate `use_auth_token` in the `huggingface_hub` codebase.
-
-    The long-term goal is to remove any mention of `use_auth_token` in the codebase in
-    favor of a unique and less verbose `token` argument. This will be done a few steps:
+def smoothly_deprecate_legacy_arguments(fn_name: str, kwargs: dict[str, Any]) -> dict[str, Any]:
+    """Smoothly deprecate legacy arguments in the `huggingface_hub` codebase.
 
-    0. Step 0: methods that require a read-access to the Hub use the `use_auth_token`
-       argument (`str`, `bool` or `None`). Methods requiring write-access have a `token`
-       argument (`str`, `None`). This implicit rule exists to be able to not send the
-       token when not necessary (`use_auth_token=False`) even if logged in.
+    This function ignores some deprecated arguments from the kwargs and warns the user they are ignored.
+    The goal is to avoid breaking existing code while guiding the user to the new way of doing things.
 
-    1. Step 1: we want to harmonize everything and use `token` everywhere (supporting
-       `token=False` for read-only methods). In order not to break existing code, if
-       `use_auth_token` is passed to a function, the `use_auth_token` value is passed
-       as `token` instead, without any warning.
-       a. Corner case: if both `use_auth_token` and `token` values are passed, a warning
-          is thrown and the `use_auth_token` value is ignored.
+    List of deprecated arguments:
+        - `proxies`:
+            To set up proxies, user must either use the HTTP_PROXY environment variable or configure the `httpx.Client`
+            manually using the [`set_client_factory`] function.
 
-    2. Step 2: Once it is release, we should push downstream libraries to switch from
-       `use_auth_token` to `token` as much as possible, but without throwing a warning
-       (e.g. manually create issues on the corresponding repos).
+            In huggingface_hub 0.x, `proxies` was a dictionary directly passed to `requests.request`.
+            In huggingface_hub 1.x, we migrated to `httpx` which does not support `proxies` the same way.
+            In particular, it is not possible to configure proxies on a per-request basis. The solution is to configure
+            it globally using the [`set_client_factory`] function or using the HTTP_PROXY environment variable.
 
-    3. Step 3: After a transitional period (6 months e.g. until April 2023?), we update
-       `huggingface_hub` to throw a warning on `use_auth_token`. Hopefully, very few
-       users will be impacted as it would have already been fixed.
-       In addition, unit tests in `huggingface_hub` must be adapted to expect warnings
-       to be thrown (but still use `use_auth_token` as before).
+            More more details, see:
+            - https://www.python-httpx.org/advanced/proxies/
+            - https://www.python-httpx.org/compatibility/#proxy-keys.
 
-    4. Step 4: After a normal deprecation cycle (3 releases ?), remove this validator.
-       `use_auth_token` will definitely not be supported.
-       In addition, we update unit tests in `huggingface_hub` to use `token` everywhere.
-
-    This has been discussed in:
-    - https://github.com/huggingface/huggingface_hub/issues/1094.
-    - https://github.com/huggingface/huggingface_hub/pull/928
-    - (related) https://github.com/huggingface/huggingface_hub/pull/1064
+        - `resume_download`: deprecated without replacement. `huggingface_hub` always resumes downloads whenever possible.
+        - `force_filename`: deprecated without replacement. Filename is always the same as on the Hub.
+        - `local_dir_use_symlinks`: deprecated without replacement. Downloading to a local directory does not use symlinks anymore.
     """
     new_kwargs = kwargs.copy()  # do not mutate input !
 
-    use_auth_token = new_kwargs.pop("use_auth_token", None)  # remove from kwargs
-    if use_auth_token is not None:
-        if has_token:
-            warnings.warn(
-                "Both `token` and `use_auth_token` are passed to"
-                f" `{fn_name}` with non-None values. `token` is now the"
-                " preferred argument to pass a User Access Token."
-                " `use_auth_token` value will be ignored."
-            )
-        else:
-            # `token` argument is not passed and a non-None value is passed in
-            # `use_auth_token` => use `use_auth_token` value as `token` kwarg.
-            new_kwargs["token"] = use_auth_token
+    # proxies
+    proxies = new_kwargs.pop("proxies", None)  # remove from kwargs
+    if proxies is not None:
+        warnings.warn(
+            f"The `proxies` argument is ignored in `{fn_name}`. To set up proxies, use the HTTP_PROXY / HTTPS_PROXY"
+            " environment variables or configure the `httpx.Client` manually using `huggingface_hub.set_client_factory`."
+            " See https://www.python-httpx.org/advanced/proxies/ for more details."
+        )
+
+    # resume_download
+    resume_download = new_kwargs.pop("resume_download", None)  # remove from kwargs
+    if resume_download is not None:
+        warnings.warn(
+            f"The `resume_download` argument is deprecated and ignored in `{fn_name}`. Downloads always resume"
+            " whenever possible."
+        )
+
+    # force_filename
+    force_filename = new_kwargs.pop("force_filename", None)  # remove from kwargs
+    if force_filename is not None:
+        warnings.warn(
+            f"The `force_filename` argument is deprecated and ignored in `{fn_name}`. Filename is always the same "
+            "as on the Hub."
+        )
+
+    # local_dir_use_symlinks
+    local_dir_use_symlinks = new_kwargs.pop("local_dir_use_symlinks", None)  # remove from kwargs
+    if local_dir_use_symlinks is not None:
+        warnings.warn(
+            f"The `local_dir_use_symlinks` argument is deprecated and ignored in `{fn_name}`. Downloading to a local"
+            " directory does not use symlinks anymore."
+        )
 
     return new_kwargs