huggingface-hub 0.20.3__py3-none-any.whl → 0.21.0__py3-none-any.whl

huggingface_hub/__init__.py

@@ -46,7 +46,7 @@ import sys
 from typing import TYPE_CHECKING
 
 
-__version__ = "0.20.3"
+__version__ = "0.21.0"
 
 # Alphabetical order of definitions is ensured in tests
 # WARNING: any comment added in this dictionary definition will be lost when
@@ -227,6 +227,7 @@ _SUBMOD_ATTRS = {
         "request_space_storage",
         "restart_space",
         "resume_inference_endpoint",
+        "revision_exists",
         "run_as_future",
         "scale_to_zero_inference_endpoint",
         "set_space_sleep_time",
@@ -245,6 +246,7 @@ _SUBMOD_ATTRS = {
         "HfFileSystem",
         "HfFileSystemFile",
         "HfFileSystemResolvedPath",
+        "HfFileSystemStreamFile",
     ],
     "hub_mixin": [
         "ModelHubMixin",
@@ -286,6 +288,13 @@ _SUBMOD_ATTRS = {
     "repository": [
         "Repository",
     ],
+    "serialization": [
+        "StateDictSplit",
+        "split_numpy_state_dict_into_shards",
+        "split_state_dict_into_shards_factory",
+        "split_tf_state_dict_into_shards",
+        "split_torch_state_dict_into_shards",
+    ],
     "utils": [
         "CacheNotFound",
         "CachedFileInfo",
@@ -574,6 +583,7 @@ if TYPE_CHECKING:  # pragma: no cover
         request_space_storage,  # noqa: F401
         restart_space,  # noqa: F401
         resume_inference_endpoint,  # noqa: F401
+        revision_exists,  # noqa: F401
         run_as_future,  # noqa: F401
         scale_to_zero_inference_endpoint,  # noqa: F401
         set_space_sleep_time,  # noqa: F401
@@ -592,6 +602,7 @@ if TYPE_CHECKING:  # pragma: no cover
         HfFileSystem,  # noqa: F401
         HfFileSystemFile,  # noqa: F401
         HfFileSystemResolvedPath,  # noqa: F401
+        HfFileSystemStreamFile,  # noqa: F401
     )
     from .hub_mixin import (
         ModelHubMixin,  # noqa: F401
@@ -627,6 +638,13 @@ if TYPE_CHECKING:  # pragma: no cover
         SpaceCardData,  # noqa: F401
     )
     from .repository import Repository  # noqa: F401
+    from .serialization import (
+        StateDictSplit,  # noqa: F401
+        split_numpy_state_dict_into_shards,  # noqa: F401
+        split_state_dict_into_shards_factory,  # noqa: F401
+        split_tf_state_dict_into_shards,  # noqa: F401
+        split_torch_state_dict_into_shards,  # noqa: F401
+    )
     from .utils import (
         CachedFileInfo,  # noqa: F401
         CachedRepoInfo,  # noqa: F401

huggingface_hub/_commit_api.py

@@ -17,6 +17,7 @@ from tqdm.contrib.concurrent import thread_map
 from huggingface_hub import get_session
 
 from .constants import ENDPOINT, HF_HUB_ENABLE_HF_TRANSFER
+from .file_download import hf_hub_url
 from .lfs import UploadInfo, lfs_upload, post_lfs_batch_info
 from .utils import (
     EntryNotFoundError,
@@ -521,16 +522,19 @@ def _fetch_upload_modes(
 
 
 @validate_hf_hub_args
-def _fetch_lfs_files_to_copy(
+def _fetch_files_to_copy(
     copies: Iterable[CommitOperationCopy],
     repo_type: str,
     repo_id: str,
     token: Optional[str],
     revision: str,
     endpoint: Optional[str] = None,
-) -> Dict[Tuple[str, Optional[str]], "RepoFile"]:
+) -> Dict[Tuple[str, Optional[str]], Union["RepoFile", bytes]]:
     """
-    Requests the Hub files information of the LFS files to be copied, including their sha256.
+    Fetch information about the files to copy.
+
+    For LFS files, we only need their metadata (file size and sha256) while for regular files
+    we need to download the raw content from the Hub.
 
     Args:
         copies (`Iterable` of :class:`CommitOperationCopy`):
@@ -546,8 +550,9 @@ def _fetch_lfs_files_to_copy(
         revision (`str`):
             The git revision to upload the files to. Can be any valid git revision.
 
-    Returns: `Dict[Tuple[str, Optional[str]], RepoFile]]`
-        Key is the file path and revision of the file to copy, value is the repo file.
+    Returns: `Dict[Tuple[str, Optional[str]], Union[RepoFile, bytes]]]`
+        Key is the file path and revision of the file to copy.
+        Value is the raw content as bytes (for regular files) or the file information as a RepoFile (for LFS files).
 
     Raises:
         [`~utils.HfHubHTTPError`]
@@ -558,7 +563,7 @@ def _fetch_lfs_files_to_copy(
     from .hf_api import HfApi, RepoFolder
 
     hf_api = HfApi(endpoint=endpoint, token=token)
-    files_to_copy = {}
+    files_to_copy: Dict[Tuple[str, Optional[str]], Union["RepoFile", bytes]] = {}
     for src_revision, operations in groupby(copies, key=lambda op: op.src_revision):
         operations = list(operations)  # type: ignore
         paths = [op.src_path_in_repo for op in operations]
@@ -572,9 +577,21 @@ def _fetch_lfs_files_to_copy(
             for src_repo_file in src_repo_files:
                 if isinstance(src_repo_file, RepoFolder):
                     raise NotImplementedError("Copying a folder is not implemented.")
-                if not src_repo_file.lfs:
-                    raise NotImplementedError("Copying a non-LFS file is not implemented")
-                files_to_copy[(src_repo_file.rfilename, src_revision)] = src_repo_file
+                if src_repo_file.lfs:
+                    files_to_copy[(src_repo_file.path, src_revision)] = src_repo_file
+                else:
+                    # TODO: (optimization) download regular files to copy concurrently
+                    headers = build_hf_headers(token=token)
+                    url = hf_hub_url(
+                        endpoint=endpoint,
+                        repo_type=repo_type,
+                        repo_id=repo_id,
+                        revision=src_revision or revision,
+                        filename=src_repo_file.path,
+                    )
+                    response = get_session().get(url, headers=headers)
+                    hf_raise_for_status(response)
+                    files_to_copy[(src_repo_file.path, src_revision)] = response.content
         for operation in operations:
             if (operation.src_path_in_repo, src_revision) not in files_to_copy:
                 raise EntryNotFoundError(
@@ -586,7 +603,7 @@ def _fetch_lfs_files_to_copy(
 
 def _prepare_commit_payload(
     operations: Iterable[CommitOperation],
-    files_to_copy: Dict[Tuple[str, Optional[str]], "RepoFile"],
+    files_to_copy: Dict[Tuple[str, Optional[str]], Union["RepoFile", bytes]],
     commit_message: str,
     commit_description: Optional[str] = None,
     parent_commit: Optional[str] = None,
@@ -649,16 +666,28 @@ def _prepare_commit_payload(
         # 2.d. Case copying a file or folder
         elif isinstance(operation, CommitOperationCopy):
             file_to_copy = files_to_copy[(operation.src_path_in_repo, operation.src_revision)]
-            if not file_to_copy.lfs:
-                raise NotImplementedError("Copying a non-LFS file is not implemented")
-            yield {
-                "key": "lfsFile",
-                "value": {
-                    "path": operation.path_in_repo,
-                    "algo": "sha256",
-                    "oid": file_to_copy.lfs["sha256"],
-                },
-            }
+            if isinstance(file_to_copy, bytes):
+                yield {
+                    "key": "file",
+                    "value": {
+                        "content": base64.b64encode(file_to_copy).decode(),
+                        "path": operation.path_in_repo,
+                        "encoding": "base64",
+                    },
+                }
+            elif file_to_copy.lfs:
+                yield {
+                    "key": "lfsFile",
+                    "value": {
+                        "path": operation.path_in_repo,
+                        "algo": "sha256",
+                        "oid": file_to_copy.lfs.sha256,
+                    },
+                }
+            else:
+                raise ValueError(
+                    "Malformed files_to_copy (should be raw file content as bytes or RepoFile objects with LFS info."
+                )
         # 2.e. Never expected to happen
         else:
             raise ValueError(

huggingface_hub/_inference_endpoints.py

@@ -192,6 +192,12 @@ class InferenceEndpoint:
 
         Returns:
             [`InferenceEndpoint`]: the same Inference Endpoint, mutated in place with the latest data.
+
+        Raises:
+            [`InferenceEndpointError`]
+                If the Inference Endpoint ended up in a failed state.
+            [`InferenceEndpointTimeoutError`]
+                If the Inference Endpoint is not deployed after `timeout` seconds.
         """
         if self.url is not None:  # Means the endpoint is deployed
             logger.info("Inference Endpoint is ready to be used.")
@@ -208,6 +214,10 @@ class InferenceEndpoint:
             if self.url is not None:  # Means the endpoint is deployed
                 logger.info("Inference Endpoint is ready to be used.")
                 return self
+            if self.status == InferenceEndpointStatus.FAILED:
+                raise InferenceEndpointError(
+                    f"Inference Endpoint {self.name} failed to deploy. Please check the logs for more information."
+                )
             if timeout is not None:
                 if time.time() - start > timeout:
                     raise InferenceEndpointTimeoutError("Timeout while waiting for Inference Endpoint to be deployed.")

huggingface_hub/_login.py

@@ -102,7 +102,7 @@ def login(
     if token is not None:
         if not add_to_git_credential:
             print(
-                "Token will not been saved to git credential helper. Pass"
+                "Token has not been saved to git credential helper. Pass"
                 " `add_to_git_credential=True` if you want to set the git"
                 " credential as well."
             )
@@ -186,7 +186,7 @@ def interpreter_login(new_session: bool = True, write_permission: bool = False)
     print("    To login, `huggingface_hub` requires a token generated from https://huggingface.co/settings/tokens .")
     if os.name == "nt":
         print("Token can be pasted using 'Right-Click'.")
-    token = getpass("Token: ")
+    token = getpass("Enter your token (input will not be visible): ")
     add_to_git_credential = _ask_for_confirmation_no_tui("Add token as git credential?")
 
     _login(token=token, add_to_git_credential=add_to_git_credential, write_permission=write_permission)

huggingface_hub/commands/download.py

@@ -64,7 +64,7 @@ class DownloadCommand(BaseHuggingfaceCLICommand):
             "--repo-type",
             choices=["model", "dataset", "space"],
             default="model",
-            help="Type of repo to download from (e.g. `dataset`).",
+            help="Type of repo to download from (defaults to 'model').",
         )
         download_parser.add_argument(
             "--revision",

huggingface_hub/file_download.py

@@ -414,18 +414,38 @@ def http_get(
     url: str,
     temp_file: BinaryIO,
     *,
-    proxies=None,
+    proxies: Optional[Dict] = None,
     resume_size: float = 0,
     headers: Optional[Dict[str, str]] = None,
     expected_size: Optional[int] = None,
+    displayed_filename: Optional[str] = None,
     _nb_retries: int = 5,
-):
+) -> None:
     """
     Download a remote file. Do not gobble up errors, and will return errors tailored to the Hugging Face Hub.
 
     If ConnectionError (SSLError) or ReadTimeout happen while streaming data from the server, it is most likely a
     transient error (network outage?). We log a warning message and try to resume the download a few times before
     giving up. The method gives up after 5 attempts if no new data has being received from the server.
+
+    Args:
+        url (`str`):
+            The URL of the file to download.
+        temp_file (`BinaryIO`):
+            The file-like object where to save the file.
+        proxies (`dict`, *optional*):
+            Dictionary mapping protocol to the URL of the proxy passed to `requests.request`.
+        resume_size (`float`, *optional*):
+            The number of bytes already downloaded. If set to 0 (default), the whole file is download. If set to a
+            positive number, the download will resume at the given position.
+        headers (`dict`, *optional*):
+            Dictionary of HTTP Headers to send with the request.
+        expected_size (`int`, *optional*):
+            The expected size of the file to download. If set, the download will raise an error if the size of the
+            received content is different from the expected one.
+        displayed_filename (`str`, *optional*):
+            The filename of the file that is being downloaded. Value is used only to display a nice progress bar. If
+            not set, the filename is guessed from the URL or the `Content-Disposition` header.
     """
     hf_transfer = None
     if HF_HUB_ENABLE_HF_TRANSFER:
@@ -458,21 +478,22 @@ def http_get(
     #       If the file is compressed, the number of bytes in the saved file will be higher than 'total'.
     total = resume_size + int(content_length) if content_length is not None else None
 
-    displayed_name = url
-    content_disposition = r.headers.get("Content-Disposition")
-    if content_disposition is not None:
-        match = HEADER_FILENAME_PATTERN.search(content_disposition)
-        if match is not None:
-            # Means file is on CDN
-            displayed_name = match.groupdict()["filename"]
+    if displayed_filename is None:
+        displayed_filename = url
+        content_disposition = r.headers.get("Content-Disposition")
+        if content_disposition is not None:
+            match = HEADER_FILENAME_PATTERN.search(content_disposition)
+            if match is not None:
+                # Means file is on CDN
+                displayed_filename = match.groupdict()["filename"]
 
     # Truncate filename if too long to display
-    if len(displayed_name) > 40:
-        displayed_name = f"(…){displayed_name[-40:]}"
+    if len(displayed_filename) > 40:
+        displayed_filename = f"(…){displayed_filename[-40:]}"
 
     consistency_error_message = (
         f"Consistency check failed: file should be of size {expected_size} but has size"
-        f" {{actual_size}} ({displayed_name}).\nWe are sorry for the inconvenience. Please retry download and"
+        f" {{actual_size}} ({displayed_filename}).\nWe are sorry for the inconvenience. Please retry download and"
         " pass `force_download=True, resume_download=False` as argument.\nIf the issue persists, please let us"
         " know by opening an issue on https://github.com/huggingface/huggingface_hub."
     )
@@ -483,8 +504,10 @@ def http_get(
         unit_scale=True,
         total=total,
         initial=resume_size,
-        desc=displayed_name,
-        disable=bool(logger.getEffectiveLevel() == logging.NOTSET),
+        desc=displayed_filename,
+        disable=True if (logger.getEffectiveLevel() == logging.NOTSET) else None,
+        # ^ set `disable=None` rather than `disable=False` by default to disable progress bar when no TTY attached
+        # see https://github.com/huggingface/huggingface_hub/pull/2000
     ) as progress:
         if hf_transfer and total is not None and total > 5 * DOWNLOAD_CHUNK_SIZE:
             supports_callback = "callback" in inspect.signature(hf_transfer.download).parameters
@@ -1279,11 +1302,14 @@ def hf_hub_download(
             # In case of a redirect, save an extra redirect on the request.get call,
             # and ensure we download the exact atomic version even if it changed
             # between the HEAD and the GET (unlikely, but hey).
-            # Useful for lfs blobs that are stored on a CDN.
+            #
+            # If url domain is different => we are downloading from a CDN => url is signed => don't send auth
+            # If url domain is the same => redirect due to repo rename AND downloading a regular file => keep auth
             if metadata.location != url:
                 url_to_download = metadata.location
-                # Remove authorization header when downloading a LFS blob
-                headers.pop("authorization", None)
+                if urlparse(url).netloc != urlparse(url_to_download).netloc:
+                    # Remove authorization header when downloading a LFS blob
+                    headers.pop("authorization", None)
         except (requests.exceptions.SSLError, requests.exceptions.ProxyError):
             # Actually raise for those subclasses of ConnectionError
             raise
@@ -1317,6 +1343,10 @@ def hf_hub_download(
             head_call_error = error
             pass
 
+    assert (
+        local_files_only or etag is not None or head_call_error is not None
+    ), "etag is empty due to uncovered problems"
+
     # etag can be None for several reasons:
     # 1. we passed local_files_only.
     # 2. we don't have a connection
@@ -1330,9 +1360,14 @@ def hf_hub_download(
     if etag is None:
         # In those cases, we cannot force download.
         if force_download:
-            raise ValueError(
-                "We have no connection or you passed local_files_only, so force_download is not an accepted option."
-            )
+            if local_files_only:
+                raise ValueError("Cannot pass 'force_download=True' and 'local_files_only=True' at the same time.")
+            elif isinstance(head_call_error, OfflineModeIsEnabled):
+                raise ValueError(
+                    "Cannot pass 'force_download=True' when offline mode is enabled."
+                ) from head_call_error
+            else:
+                raise ValueError("Force download failed due to the above error.") from head_call_error
 
         # Try to get "commit_hash" from "revision"
         commit_hash = None
@@ -1364,7 +1399,7 @@ def hf_hub_download(
                 " hf.co look-ups and downloads online, set 'local_files_only' to False."
             )
         elif isinstance(head_call_error, RepositoryNotFoundError) or isinstance(head_call_error, GatedRepoError):
-            # Repo not found => let's raise the actual error
+            # Repo not found or gated => let's raise the actual error
             raise head_call_error
         else:
             # Otherwise: most likely a connection issue or Hub downtime => let's warn the user
@@ -1461,6 +1496,7 @@ def hf_hub_download(
                 resume_size=resume_size,
                 headers=headers,
                 expected_size=expected_size,
+                displayed_filename=filename,
             )
 
         if local_dir is None: