huggingface-hub 0.28.0rc5__py3-none-any.whl → 0.29.0rc0__py3-none-any.whl

huggingface_hub/__init__.py

@@ -46,7 +46,7 @@ import sys
 from typing import TYPE_CHECKING
 
 
-__version__ = "0.28.0.rc5"
+__version__ = "0.29.0.rc0"
 
 # Alphabetical order of definitions is ensured in tests
 # WARNING: any comment added in this dictionary definition will be lost when
@@ -392,7 +392,6 @@ _SUBMOD_ATTRS = {
         "TextToImageInput",
         "TextToImageOutput",
         "TextToImageParameters",
-        "TextToImageTargetSize",
         "TextToSpeechEarlyStoppingEnum",
         "TextToSpeechGenerationParameters",
         "TextToSpeechInput",
@@ -702,7 +701,6 @@ __all__ = [
     "TextToImageInput",
     "TextToImageOutput",
     "TextToImageParameters",
-    "TextToImageTargetSize",
     "TextToSpeechEarlyStoppingEnum",
     "TextToSpeechGenerationParameters",
     "TextToSpeechInput",
@@ -1334,7 +1332,6 @@ if TYPE_CHECKING:  # pragma: no cover
         TextToImageInput,  # noqa: F401
         TextToImageOutput,  # noqa: F401
         TextToImageParameters,  # noqa: F401
-        TextToImageTargetSize,  # noqa: F401
         TextToSpeechEarlyStoppingEnum,  # noqa: F401
         TextToSpeechGenerationParameters,  # noqa: F401
         TextToSpeechInput,  # noqa: F401

huggingface_hub/constants.py

@@ -2,7 +2,6 @@ import os
 import re
 import typing
 from typing import Literal, Optional, Tuple
-from urllib.parse import urljoin
 
 
 # Possible values for env variables
@@ -64,11 +63,13 @@ _staging_mode = _is_true(os.environ.get("HUGGINGFACE_CO_STAGING"))
 
 _HF_DEFAULT_ENDPOINT = "https://huggingface.co"
 _HF_DEFAULT_STAGING_ENDPOINT = "https://hub-ci.huggingface.co"
-ENDPOINT = os.getenv("HF_ENDPOINT", "").rstrip("/") or (
-    _HF_DEFAULT_STAGING_ENDPOINT if _staging_mode else _HF_DEFAULT_ENDPOINT
-)
+ENDPOINT = os.getenv("HF_ENDPOINT", _HF_DEFAULT_ENDPOINT).rstrip("/")
+HUGGINGFACE_CO_URL_TEMPLATE = ENDPOINT + "/{repo_id}/resolve/{revision}/{filename}"
+
+if _staging_mode:
+    ENDPOINT = _HF_DEFAULT_STAGING_ENDPOINT
+    HUGGINGFACE_CO_URL_TEMPLATE = _HF_DEFAULT_STAGING_ENDPOINT + "/{repo_id}/resolve/{revision}/{filename}"
 
-HUGGINGFACE_CO_URL_TEMPLATE = urljoin(ENDPOINT, "/{repo_id}/resolve/{revision}/{filename}")
 HUGGINGFACE_HEADER_X_REPO_COMMIT = "X-Repo-Commit"
 HUGGINGFACE_HEADER_X_LINKED_ETAG = "X-Linked-Etag"
 HUGGINGFACE_HEADER_X_LINKED_SIZE = "X-Linked-Size"
@@ -79,7 +80,7 @@ INFERENCE_ENDPOINT = os.environ.get("HF_INFERENCE_ENDPOINT", "https://api-infere
 INFERENCE_ENDPOINTS_ENDPOINT = "https://api.endpoints.huggingface.cloud/v2"
 
 # Proxy for third-party providers
-INFERENCE_PROXY_TEMPLATE = urljoin(ENDPOINT, "/api/inference-proxy/{provider}")
+INFERENCE_PROXY_TEMPLATE = "https://router.huggingface.co/{provider}"
 
 REPO_ID_SEPARATOR = "--"
 # ^ this substring is not allowed in repo_ids on hf.co
@@ -133,6 +134,10 @@ HF_ASSETS_CACHE = os.getenv("HF_ASSETS_CACHE", HUGGINGFACE_ASSETS_CACHE)
 
 HF_HUB_OFFLINE = _is_true(os.environ.get("HF_HUB_OFFLINE") or os.environ.get("TRANSFORMERS_OFFLINE"))
 
+# If set, log level will be set to DEBUG and all requests made to the Hub will be logged
+# as curl commands for reproducibility.
+HF_DEBUG = _is_true(os.environ.get("HF_DEBUG"))
+
 # Opt-out from telemetry requests
 HF_HUB_DISABLE_TELEMETRY = (
     _is_true(os.environ.get("HF_HUB_DISABLE_TELEMETRY"))  # HF-specific env variable
@@ -140,18 +145,15 @@ HF_HUB_DISABLE_TELEMETRY = (
     or _is_true(os.environ.get("DO_NOT_TRACK"))  # https://consoledonottrack.com/
 )
 
-# In the past, token was stored in a hardcoded location
-# `_OLD_HF_TOKEN_PATH` is deprecated and will be removed "at some point".
-# See https://github.com/huggingface/huggingface_hub/issues/1232
-_OLD_HF_TOKEN_PATH = os.path.expanduser("~/.huggingface/token")
 HF_TOKEN_PATH = os.environ.get("HF_TOKEN_PATH", os.path.join(HF_HOME, "token"))
 HF_STORED_TOKENS_PATH = os.path.join(os.path.dirname(HF_TOKEN_PATH), "stored_tokens")
 
 if _staging_mode:
     # In staging mode, we use a different cache to ensure we don't mix up production and staging data or tokens
+    # In practice in `huggingface_hub` tests, we monkeypatch these values with temporary directories. The following
+    # lines are only used in third-party libraries tests (e.g. `transformers`, `diffusers`, etc.).
     _staging_home = os.path.join(os.path.expanduser("~"), ".cache", "huggingface_staging")
     HUGGINGFACE_HUB_CACHE = os.path.join(_staging_home, "hub")
-    _OLD_HF_TOKEN_PATH = os.path.join(_staging_home, "_old_token")
     HF_TOKEN_PATH = os.path.join(_staging_home, "token")
 
 # Here, `True` will disable progress bars globally without possibility of enabling it
@@ -192,6 +194,9 @@ HF_HUB_ETAG_TIMEOUT: int = _as_int(os.environ.get("HF_HUB_ETAG_TIMEOUT")) or DEF
 # Used to override the get request timeout on a system level
 HF_HUB_DOWNLOAD_TIMEOUT: int = _as_int(os.environ.get("HF_HUB_DOWNLOAD_TIMEOUT")) or DEFAULT_DOWNLOAD_TIMEOUT
 
+# Allows to add information about the requester in the user-agent (eg. partner name)
+HF_HUB_USER_AGENT_ORIGIN: Optional[str] = os.environ.get("HF_HUB_USER_AGENT_ORIGIN")
+
 # List frameworks that are handled by the InferenceAPI service. Useful to scan endpoints and check which models are
 # deployed and running. Since 95% of the models are using the top 4 frameworks listed below, we scan only those by
 # default. We still keep the full list of supported frameworks in case we want to scan all of them.

huggingface_hub/file_download.py

@@ -29,6 +29,7 @@ from .errors import (
     EntryNotFoundError,
     FileMetadataError,
     GatedRepoError,
+    HfHubHTTPError,
     LocalEntryNotFoundError,
     RepositoryNotFoundError,
     RevisionNotFoundError,
@@ -59,6 +60,7 @@ from .utils import (
     tqdm,
     validate_hf_hub_args,
 )
+from .utils._http import _adjust_range_header
 from .utils._runtime import _PY_VERSION  # noqa: F401 # for backward compatibility
 from .utils._typing import HTTP_METHOD_T
 from .utils.sha import sha_fileobj
@@ -308,8 +310,8 @@ def http_get(
     temp_file: BinaryIO,
     *,
     proxies: Optional[Dict] = None,
-    resume_size: float = 0,
-    headers: Optional[Dict[str, str]] = None,
+    resume_size: int = 0,
+    headers: Optional[Dict[str, Any]] = None,
     expected_size: Optional[int] = None,
     displayed_filename: Optional[str] = None,
     _nb_retries: int = 5,
@@ -329,7 +331,7 @@ def http_get(
             The file-like object where to save the file.
         proxies (`dict`, *optional*):
             Dictionary mapping protocol to the URL of the proxy passed to `requests.request`.
-        resume_size (`float`, *optional*):
+        resume_size (`int`, *optional*):
             The number of bytes already downloaded. If set to 0 (default), the whole file is download. If set to a
             positive number, the download will resume at the given position.
         headers (`dict`, *optional*):
@@ -364,7 +366,7 @@ def http_get(
     initial_headers = headers
     headers = copy.deepcopy(headers) or {}
     if resume_size > 0:
-        headers["Range"] = "bytes=%d-" % (resume_size,)
+        headers["Range"] = _adjust_range_header(headers.get("Range"), resume_size)
 
     r = _request_wrapper(
         method="GET", url=url, stream=True, proxies=proxies, headers=headers, timeout=constants.HF_HUB_DOWNLOAD_TIMEOUT
@@ -1461,7 +1463,6 @@ def _get_metadata_or_catch_error(
 
 def _raise_on_head_call_error(head_call_error: Exception, force_download: bool, local_files_only: bool) -> NoReturn:
     """Raise an appropriate error when the HEAD call failed and we cannot locate a local file."""
-
     # No head call => we cannot force download.
     if force_download:
         if local_files_only:
@@ -1477,8 +1478,11 @@ def _raise_on_head_call_error(head_call_error: Exception, force_download: bool,
             "Cannot find the requested files in the disk cache and outgoing traffic has been disabled. To enable"
             " hf.co look-ups and downloads online, set 'local_files_only' to False."
         )
-    elif isinstance(head_call_error, RepositoryNotFoundError) or isinstance(head_call_error, GatedRepoError):
+    elif isinstance(head_call_error, (RepositoryNotFoundError, GatedRepoError)) or (
+        isinstance(head_call_error, HfHubHTTPError) and head_call_error.response.status_code == 401
+    ):
         # Repo not found or gated => let's raise the actual error
+        # Unauthorized => likely a token issue => let's raise the actual error
         raise head_call_error
     else:
         # Otherwise: most likely a connection issue or Hub downtime => let's warn the user

huggingface_hub/hf_api.py

@@ -115,6 +115,7 @@ from .utils import (
     filter_repo_objects,
     fix_hf_endpoint_in_url,
     get_session,
+    get_token,
     hf_raise_for_status,
     logging,
     paginate,
@@ -122,6 +123,7 @@ from .utils import (
     validate_hf_hub_args,
 )
 from .utils import tqdm as hf_tqdm
+from .utils._auth import _get_token_from_environment, _get_token_from_file, _get_token_from_google_colab
 from .utils._deprecation import _deprecate_method
 from .utils._typing import CallableT
 from .utils.endpoint_helpers import _is_emission_within_threshold
@@ -143,6 +145,7 @@ ExpandModelProperty_T = Literal[
     "gated",
     "gguf",
     "inference",
+    "inferenceProviderMapping",
     "lastModified",
     "library_name",
     "likes",
@@ -150,6 +153,7 @@ ExpandModelProperty_T = Literal[
     "model-index",
     "pipeline_tag",
     "private",
+    "resourceGroup",
     "safetensors",
     "sha",
     "siblings",
@@ -157,9 +161,8 @@ ExpandModelProperty_T = Literal[
     "tags",
     "transformersInfo",
     "trendingScore",
-    "widgetData",
     "usedStorage",
-    "resourceGroup",
+    "widgetData",
 ]
 
 ExpandDatasetProperty_T = Literal[
@@ -167,8 +170,8 @@ ExpandDatasetProperty_T = Literal[
     "cardData",
     "citation",
     "createdAt",
-    "disabled",
     "description",
+    "disabled",
     "downloads",
     "downloadsAllTime",
     "gated",
@@ -176,12 +179,12 @@ ExpandDatasetProperty_T = Literal[
     "likes",
     "paperswithcode_id",
     "private",
-    "siblings",
+    "resourceGroup",
     "sha",
-    "trendingScore",
+    "siblings",
     "tags",
+    "trendingScore",
     "usedStorage",
-    "resourceGroup",
 ]
 
 ExpandSpaceProperty_T = Literal[
@@ -194,15 +197,15 @@ ExpandSpaceProperty_T = Literal[
     "likes",
     "models",
     "private",
+    "resourceGroup",
     "runtime",
     "sdk",
-    "siblings",
     "sha",
+    "siblings",
     "subdomain",
     "tags",
     "trendingScore",
     "usedStorage",
-    "resourceGroup",
 ]
 
 USERNAME_PLACEHOLDER = "hf_user"
@@ -695,6 +698,19 @@ class RepoFolder:
         self.last_commit = last_commit
 
 
+@dataclass
+class InferenceProviderMapping:
+    status: Literal["live", "staging"]
+    provider_id: str
+    task: str
+
+    def __init__(self, **kwargs):
+        self.status = kwargs.pop("status")
+        self.provider_id = kwargs.pop("providerId")
+        self.task = kwargs.pop("task")
+        self.__dict__.update(**kwargs)
+
+
 @dataclass
 class ModelInfo:
     """
@@ -737,6 +753,8 @@ class ModelInfo:
             Status of the model on the inference API.
             Warm models are available for immediate use. Cold models will be loaded on first inference call.
             Frozen models are not available in Inference API.
+        inference_provider_mapping (`Dict`, *optional*):
+            Model's inference provider mapping.
         likes (`int`):
             Number of likes of the model.
         library_name (`str`, *optional*):
@@ -782,6 +800,7 @@ class ModelInfo:
     gated: Optional[Literal["auto", "manual", False]]
     gguf: Optional[Dict]
     inference: Optional[Literal["warm", "cold", "frozen"]]
+    inference_provider_mapping: Optional[Dict[str, InferenceProviderMapping]]
     likes: Optional[int]
     library_name: Optional[str]
     tags: Optional[List[str]]
@@ -814,7 +833,15 @@ class ModelInfo:
         self.likes = kwargs.pop("likes", None)
         self.library_name = kwargs.pop("library_name", None)
         self.gguf = kwargs.pop("gguf", None)
+
         self.inference = kwargs.pop("inference", None)
+        self.inference_provider_mapping = kwargs.pop("inferenceProviderMapping", None)
+        if self.inference_provider_mapping:
+            self.inference_provider_mapping = {
+                provider: InferenceProviderMapping(**value)
+                for provider, value in self.inference_provider_mapping.items()
+            }
+
         self.tags = kwargs.pop("tags", None)
         self.pipeline_tag = kwargs.pop("pipeline_tag", None)
         self.mask_token = kwargs.pop("mask_token", None)
@@ -1627,24 +1654,27 @@ class HfApi:
                 https://huggingface.co/docs/huggingface_hub/quick-start#authentication).
                 To disable authentication, pass `False`.
         """
+        # Get the effective token using the helper function get_token
+        effective_token = token or self.token or get_token() or True
         r = get_session().get(
             f"{self.endpoint}/api/whoami-v2",
-            headers=self._build_hf_headers(
-                # If `token` is provided and not `None`, it will be used by default.
-                # Otherwise, the token must be retrieved from cache or env variable.
-                token=(token or self.token or True),
-            ),
+            headers=self._build_hf_headers(token=effective_token),
         )
         try:
             hf_raise_for_status(r)
         except HTTPError as e:
-            raise HTTPError(
-                "Invalid user token. If you didn't pass a user token, make sure you "
-                "are properly logged in by executing `huggingface-cli login`, and "
-                "if you did pass a user token, double-check it's correct.",
-                request=e.request,
-                response=e.response,
-            ) from e
+            error_message = "Invalid user token."
+            # Check which token is the effective one and generate the error message accordingly
+            if effective_token == _get_token_from_google_colab():
+                error_message += " The token from Google Colab vault is invalid. Please update it from the UI."
+            elif effective_token == _get_token_from_environment():
+                error_message += (
+                    " The token from HF_TOKEN environment variable is invalid. "
+                    "Note that HF_TOKEN takes precedence over `huggingface-cli login`."
+                )
+            elif effective_token == _get_token_from_file():
+                error_message += " The token stored is invalid. Please run `huggingface-cli login` to update it."
+            raise HTTPError(error_message, request=e.request, response=e.response) from e
         return r.json()
 
     @_deprecate_method(
@@ -1788,7 +1818,7 @@ class HfApi:
             expand (`List[ExpandModelProperty_T]`, *optional*):
                 List properties to return in the response. When used, only the properties in the list will be returned.
                 This parameter cannot be used if `full`, `cardData` or `fetch_config` are passed.
-                Possible values are `"author"`, `"baseModels"`, `"cardData"`, `"childrenModelCount"`, `"config"`, `"createdAt"`, `"disabled"`, `"downloads"`, `"downloadsAllTime"`, `"gated"`, `"gguf"`, `"inference"`, `"lastModified"`, `"library_name"`, `"likes"`, `"mask_token"`, `"model-index"`, `"pipeline_tag"`, `"private"`, `"safetensors"`, `"sha"`, `"siblings"`, `"spaces"`, `"tags"`, `"transformersInfo"`, `"trendingScore"`, `"widgetData"`, `"usedStorage"` and `"resourceGroup"`.
+                Possible values are `"author"`, `"baseModels"`, `"cardData"`, `"childrenModelCount"`, `"config"`, `"createdAt"`, `"disabled"`, `"downloads"`, `"downloadsAllTime"`, `"gated"`, `"gguf"`, `"inference"`, `"inferenceProviderMapping"`, `"lastModified"`, `"library_name"`, `"likes"`, `"mask_token"`, `"model-index"`, `"pipeline_tag"`, `"private"`, `"safetensors"`, `"sha"`, `"siblings"`, `"spaces"`, `"tags"`, `"transformersInfo"`, `"trendingScore"`, `"widgetData"`, `"usedStorage"` and `"resourceGroup"`.
             full (`bool`, *optional*):
                 Whether to fetch all model data, including the `last_modified`,
                 the `sha`, the files and the `tags`. This is set to `True` by
@@ -2447,7 +2477,7 @@ class HfApi:
             expand (`List[ExpandModelProperty_T]`, *optional*):
                 List properties to return in the response. When used, only the properties in the list will be returned.
                 This parameter cannot be used if `securityStatus` or `files_metadata` are passed.
-                Possible values are `"author"`, `"baseModels"`, `"cardData"`, `"childrenModelCount"`, `"config"`, `"createdAt"`, `"disabled"`, `"downloads"`, `"downloadsAllTime"`, `"gated"`, `"gguf"`, `"inference"`, `"lastModified"`, `"library_name"`, `"likes"`, `"mask_token"`, `"model-index"`, `"pipeline_tag"`, `"private"`, `"safetensors"`, `"sha"`, `"siblings"`, `"spaces"`, `"tags"`, `"transformersInfo"`, `"trendingScore"`, `"widgetData"`, `"usedStorage"` and `"resourceGroup"`.
+                Possible values are `"author"`, `"baseModels"`, `"cardData"`, `"childrenModelCount"`, `"config"`, `"createdAt"`, `"disabled"`, `"downloads"`, `"downloadsAllTime"`, `"gated"`, `"gguf"`, `"inference"`, `"inferenceProviderMapping"`, `"lastModified"`, `"library_name"`, `"likes"`, `"mask_token"`, `"model-index"`, `"pipeline_tag"`, `"private"`, `"safetensors"`, `"sha"`, `"siblings"`, `"spaces"`, `"tags"`, `"transformersInfo"`, `"trendingScore"`, `"widgetData"`, `"usedStorage"` and `"resourceGroup"`.
             token (Union[bool, str, None], optional):
                 A valid user access token (string). Defaults to the locally saved
                 token, which is the recommended method for authentication (see
@@ -3548,7 +3578,7 @@ class HfApi:
             if not missing_ok:
                 raise
 
-    @_deprecate_method(version="0.29", message="Please use `update_repo_settings` instead.")
+    @_deprecate_method(version="0.32", message="Please use `update_repo_settings` instead.")
     @validate_hf_hub_args
     def update_repo_visibility(
         self,