huggingface-hub 0.13.4__py3-none-any.whl → 0.14.0.dev0__py3-none-any.whl

huggingface_hub/__init__.py

@@ -46,7 +46,7 @@ import sys
 from typing import TYPE_CHECKING
 
 
-__version__ = "0.13.4"
+__version__ = "0.14.0.dev0"
 
 # Alphabetical order of definitions is ensured in tests
 # WARNING: any comment added in this dictionary definition will be lost when
@@ -58,6 +58,10 @@ _SUBMOD_ATTRS = {
         "logout",
         "notebook_login",
     ],
+    "_multi_commits": [
+        "MultiCommitException",
+        "plan_multi_commits",
+    ],
     "_snapshot_download": [
         "snapshot_download",
     ],
@@ -66,6 +70,21 @@ _SUBMOD_ATTRS = {
         "SpaceRuntime",
         "SpaceStage",
     ],
+    "_webhooks_payload": [
+        "WebhookPayload",
+        "WebhookPayloadComment",
+        "WebhookPayloadDiscussion",
+        "WebhookPayloadDiscussionChanges",
+        "WebhookPayloadEvent",
+        "WebhookPayloadMovedTo",
+        "WebhookPayloadRepo",
+        "WebhookPayloadUrl",
+        "WebhookPayloadWebhook",
+    ],
+    "_webhooks_server": [
+        "WebhooksServer",
+        "webhook_endpoint",
+    ],
     "community": [
         "Discussion",
         "DiscussionComment",
@@ -119,6 +138,7 @@ _SUBMOD_ATTRS = {
         "comment_discussion",
         "create_branch",
         "create_commit",
+        "create_commits_on_pr",
         "create_discussion",
         "create_pull_request",
         "create_repo",
@@ -140,6 +160,7 @@ _SUBMOD_ATTRS = {
         "get_space_runtime",
         "like",
         "list_datasets",
+        "list_files_info",
         "list_liked_repos",
         "list_metrics",
         "list_models",
@@ -155,15 +176,19 @@ _SUBMOD_ATTRS = {
         "repo_type_and_id_from_hf_id",
         "request_space_hardware",
         "restart_space",
-        "set_access_token",
+        "set_space_sleep_time",
         "space_info",
         "unlike",
-        "unset_access_token",
         "update_repo_visibility",
         "upload_file",
         "upload_folder",
         "whoami",
     ],
+    "hf_file_system": [
+        "HfFileSystem",
+        "HfFileSystemFile",
+        "HfFileSystemResolvedPath",
+    ],
     "hub_mixin": [
         "ModelHubMixin",
         "PyTorchModelHubMixin",
@@ -207,7 +232,9 @@ _SUBMOD_ATTRS = {
         "HFCacheInfo",
         "HfFolder",
         "cached_assets_path",
+        "configure_http_backend",
         "dump_environment_info",
+        "get_session",
         "logging",
         "scan_cache_dir",
     ],
@@ -319,12 +346,31 @@ if TYPE_CHECKING:  # pragma: no cover
         logout,  # noqa: F401
         notebook_login,  # noqa: F401
     )
+    from ._multi_commits import (
+        MultiCommitException,  # noqa: F401
+        plan_multi_commits,  # noqa: F401
+    )
     from ._snapshot_download import snapshot_download  # noqa: F401
     from ._space_api import (
         SpaceHardware,  # noqa: F401
         SpaceRuntime,  # noqa: F401
         SpaceStage,  # noqa: F401
     )
+    from ._webhooks_payload import (
+        WebhookPayload,  # noqa: F401
+        WebhookPayloadComment,  # noqa: F401
+        WebhookPayloadDiscussion,  # noqa: F401
+        WebhookPayloadDiscussionChanges,  # noqa: F401
+        WebhookPayloadEvent,  # noqa: F401
+        WebhookPayloadMovedTo,  # noqa: F401
+        WebhookPayloadRepo,  # noqa: F401
+        WebhookPayloadUrl,  # noqa: F401
+        WebhookPayloadWebhook,  # noqa: F401
+    )
+    from ._webhooks_server import (
+        WebhooksServer,  # noqa: F401
+        webhook_endpoint,  # noqa: F401
+    )
     from .community import (
         Discussion,  # noqa: F401
         DiscussionComment,  # noqa: F401
@@ -378,6 +424,7 @@ if TYPE_CHECKING:  # pragma: no cover
         comment_discussion,  # noqa: F401
         create_branch,  # noqa: F401
         create_commit,  # noqa: F401
+        create_commits_on_pr,  # noqa: F401
         create_discussion,  # noqa: F401
         create_pull_request,  # noqa: F401
         create_repo,  # noqa: F401
@@ -399,6 +446,7 @@ if TYPE_CHECKING:  # pragma: no cover
         get_space_runtime,  # noqa: F401
         like,  # noqa: F401
         list_datasets,  # noqa: F401
+        list_files_info,  # noqa: F401
         list_liked_repos,  # noqa: F401
         list_metrics,  # noqa: F401
         list_models,  # noqa: F401
@@ -414,15 +462,19 @@ if TYPE_CHECKING:  # pragma: no cover
         repo_type_and_id_from_hf_id,  # noqa: F401
         request_space_hardware,  # noqa: F401
         restart_space,  # noqa: F401
-        set_access_token,  # noqa: F401
+        set_space_sleep_time,  # noqa: F401
         space_info,  # noqa: F401
         unlike,  # noqa: F401
-        unset_access_token,  # noqa: F401
         update_repo_visibility,  # noqa: F401
         upload_file,  # noqa: F401
         upload_folder,  # noqa: F401
         whoami,  # noqa: F401
     )
+    from .hf_file_system import (
+        HfFileSystem,  # noqa: F401
+        HfFileSystemFile,  # noqa: F401
+        HfFileSystemResolvedPath,  # noqa: F401
+    )
     from .hub_mixin import (
         ModelHubMixin,  # noqa: F401
         PyTorchModelHubMixin,  # noqa: F401
@@ -462,7 +514,9 @@ if TYPE_CHECKING:  # pragma: no cover
         HFCacheInfo,  # noqa: F401
         HfFolder,  # noqa: F401
         cached_assets_path,  # noqa: F401
+        configure_http_backend,  # noqa: F401
         dump_environment_info,  # noqa: F401
+        get_session,  # noqa: F401
         logging,  # noqa: F401
         scan_cache_dir,  # noqa: F401
     )

huggingface_hub/_commit_api.py

@@ -11,11 +11,12 @@ from dataclasses import dataclass, field
 from pathlib import Path, PurePosixPath
 from typing import Any, BinaryIO, Dict, Iterable, Iterator, List, Optional, Union
 
-import requests
 from tqdm.contrib.concurrent import thread_map
 
-from .constants import ENDPOINT
-from .lfs import UploadInfo, _validate_batch_actions, lfs_upload, post_lfs_batch_info
+from huggingface_hub import get_session
+
+from .constants import ENDPOINT, HF_HUB_ENABLE_HF_TRANSFER
+from .lfs import UploadInfo, lfs_upload, post_lfs_batch_info
 from .utils import (
     build_hf_headers,
     chunk_iterable,
@@ -25,7 +26,6 @@ from .utils import (
     validate_hf_hub_args,
 )
 from .utils import tqdm as hf_tqdm
-from .utils._deprecation import _deprecate_method
 from .utils._typing import Literal
 
 
@@ -130,14 +130,6 @@ class CommitOperationAdd:
         else:
             self.upload_info = UploadInfo.from_fileobj(self.path_or_fileobj)
 
-    @_deprecate_method(version="0.14", message="Operation is validated at initialization.")
-    def validate(self) -> None:
-        pass
-
-    @_deprecate_method(version="0.14", message="Use `upload_info` property instead.")
-    def _upload_info(self) -> UploadInfo:
-        return self.upload_info
-
     @contextmanager
     def as_file(self, with_tqdm: bool = False) -> Iterator[BinaryIO]:
         """
@@ -206,6 +198,11 @@ def _validate_path_in_repo(path_in_repo: str) -> str:
         raise ValueError(f"Invalid `path_in_repo` in CommitOperation: '{path_in_repo}'")
     if path_in_repo.startswith("./"):
         path_in_repo = path_in_repo[2:]
+    if any(part == ".git" for part in path_in_repo.split("/")):
+        raise ValueError(
+            "Invalid `path_in_repo` in CommitOperation: cannot update files under a '.git/' folder (path:"
+            f" '{path_in_repo}')."
+        )
     return path_in_repo
 
 
@@ -338,64 +335,28 @@ def upload_lfs_files(
         return
 
     # Step 3: upload files concurrently according to these instructions
-    def _inner_upload_lfs_object(batch_action):
+    def _wrapped_lfs_upload(batch_action) -> None:
         try:
             operation = oid2addop[batch_action["oid"]]
-            return _upload_lfs_object(operation=operation, lfs_batch_action=batch_action, token=token)
+            lfs_upload(operation=operation, lfs_batch_action=batch_action, token=token)
         except Exception as exc:
             raise RuntimeError(f"Error while uploading '{operation.path_in_repo}' to the Hub.") from exc
 
-    logger.debug(
-        f"Uploading {len(filtered_actions)} LFS files to the Hub using up to {num_threads} threads concurrently"
-    )
-    thread_map(
-        _inner_upload_lfs_object,
-        filtered_actions,
-        desc=f"Upload {len(filtered_actions)} LFS files",
-        max_workers=num_threads,
-        tqdm_class=hf_tqdm,
-    )
-
-
-def _upload_lfs_object(operation: CommitOperationAdd, lfs_batch_action: dict, token: Optional[str]):
-    """
-    Handles uploading a given object to the Hub with the LFS protocol.
-
-    Defers to [`~utils.lfs.lfs_upload`] for the actual upload logic.
-
-    Can be a No-op if the content of the file is already present on the hub
-    large file storage.
-
-    Args:
-        operation (`CommitOperationAdd`):
-            The add operation triggering this upload
-        lfs_batch_action (`dict`):
-            Upload instructions from the LFS batch endpoint for this object.
-            See [`~utils.lfs.post_lfs_batch_info`] for more details.
-        token (`str`, *optional*):
-            A [user access token](https://hf.co/settings/tokens) to authenticate requests against the Hub
-
-    Raises: `ValueError` if `lfs_batch_action` is improperly formatted
-    """
-    _validate_batch_actions(lfs_batch_action)
-    upload_info = operation.upload_info
-    actions = lfs_batch_action.get("actions")
-    if actions is None:
-        # The file was already uploaded
-        logger.debug(f"Content of file {operation.path_in_repo} is already present upstream - skipping upload")
-        return
-    upload_action = lfs_batch_action["actions"].get("upload")
-    verify_action = lfs_batch_action["actions"].get("verify")
-    with operation.as_file(with_tqdm=True) as fileobj:
-        logger.debug(f"Uploading {operation.path_in_repo} as LFS file...")
-        lfs_upload(
-            fileobj=fileobj,
-            upload_action=upload_action,
-            verify_action=verify_action,
-            upload_info=upload_info,
-            token=token,
+    if HF_HUB_ENABLE_HF_TRANSFER:
+        logger.debug(f"Uploading {len(filtered_actions)} LFS files to the Hub using `hf_transfer`.")
+        for action in filtered_actions:
+            _wrapped_lfs_upload(action)
+    else:
+        logger.debug(
+            f"Uploading {len(filtered_actions)} LFS files to the Hub using up to {num_threads} threads concurrently"
+        )
+        thread_map(
+            _wrapped_lfs_upload,
+            filtered_actions,
+            desc=f"Upload {len(filtered_actions)} LFS files",
+            max_workers=num_threads,
+            tqdm_class=hf_tqdm,
         )
-        logger.debug(f"{operation.path_in_repo}: Upload successful")
 
 
 def _validate_preupload_info(preupload_info: dict):
@@ -468,7 +429,7 @@ def fetch_upload_modes(
             ]
         }
 
-        resp = requests.post(
+        resp = get_session().post(
             f"{endpoint}/api/{repo_type}s/{repo_id}/preupload/{revision}",
             json=payload,
             headers=headers,
@@ -478,17 +439,11 @@ def fetch_upload_modes(
         preupload_info = _validate_preupload_info(resp.json())
         upload_modes.update(**{file["path"]: file["uploadMode"] for file in preupload_info["files"]})
 
-    # If a file is empty, it is most likely a mistake.
-    # => a warning message is triggered to warn the user.
-    #    => except if `.gitkeep` as it is a legit use case for an empty file.
-    #
     # Empty files cannot be uploaded as LFS (S3 would fail with a 501 Not Implemented)
     # => empty files are uploaded as "regular" to still allow users to commit them.
     for addition in additions:
         if addition.upload_info.size == 0:
             path = addition.path_in_repo
-            if not path.endswith(".gitkeep"):
-                warnings.warn(f"About to commit an empty file: '{path}'. Are you sure this is intended?")
             upload_modes[path] = "regular"
 
     return upload_modes

huggingface_hub/_login.py

@@ -15,13 +15,14 @@
 import os
 import subprocess
 from getpass import getpass
-from typing import List, Optional
+from typing import Optional
 
 from .commands._cli_utils import ANSI
 from .commands.delete_cache import _ask_for_confirmation_no_tui
 from .hf_api import HfApi
 from .utils import (
     HfFolder,
+    capture_output,
     is_google_colab,
     is_notebook,
     list_credential_helpers,
@@ -30,7 +31,6 @@ from .utils import (
     set_git_credential,
     unset_git_credential,
 )
-from .utils._deprecation import _deprecate_method
 
 
 logger = logging.get_logger(__name__)
@@ -127,7 +127,7 @@ def interpreter_login() -> None:
     _|_|_|_|  _|    _|  _|  _|_|  _|  _|_|    _|    _|  _|  _|  _|  _|_|      _|_|_|    _|_|_|_|  _|        _|_|_|
     _|    _|  _|    _|  _|    _|  _|    _|    _|    _|    _|_|  _|    _|      _|        _|    _|  _|        _|
     _|    _|    _|_|      _|_|_|    _|_|_|  _|_|_|  _|      _|    _|_|_|      _|        _|    _|    _|_|_|  _|_|_|_|
-    """)  # docstyle-ignore
+    """)
     if HfFolder.get_token() is not None:
         print(
             "    A token is already saved on your machine. Run `huggingface-cli"
@@ -181,7 +181,7 @@ def notebook_login() -> None:
     """
     try:
         import ipywidgets.widgets as widgets  # type: ignore
-        from IPython.display import clear_output, display  # type: ignore
+        from IPython.display import display  # type: ignore
     except ImportError:
         raise ImportError(
             "The `notebook_login` function can only be used in a notebook (Jupyter or"
@@ -212,8 +212,16 @@ def notebook_login() -> None:
         add_to_git_credential = git_checkbox_widget.value
         # Erase token and clear value to make sure it's not saved in the notebook.
         token_widget.value = ""
-        clear_output()
-        _login(token, add_to_git_credential=add_to_git_credential)
+        # Hide inputs
+        login_token_widget.children = [widgets.Label("Connecting...")]
+        try:
+            with capture_output() as captured:
+                _login(token, add_to_git_credential=add_to_git_credential)
+            message = captured.getvalue()
+        except Exception as error:
+            message = str(error)
+        # Print result (success message or error)
+        login_token_widget.children = [widgets.Label(line) for line in message.split("\n") if line.strip()]
 
     token_finish_button.on_click(login_token_event)
 
@@ -236,13 +244,13 @@ def _login(token: str, add_to_git_credential: bool) -> None:
             set_git_credential(token)
             print(
                 "Your token has been saved in your configured git credential helpers"
-                f" ({','.join(list_credential_helpers())})."
+                + f" ({','.join(list_credential_helpers())})."
             )
         else:
             print("Token has not been saved to git credential helper.")
 
     HfFolder.save_token(token)
-    print("Your token has been saved to", HfFolder.path_token)
+    print(f"Your token has been saved to {HfFolder.path_token}")
     print("Login successful")
 
 
@@ -293,8 +301,3 @@ def _set_store_as_git_credential_helper_globally() -> None:
         run_subprocess("git config --global credential.helper store")
     except subprocess.CalledProcessError as exc:
         raise EnvironmentError(exc.stderr)
-
-
-@_deprecate_method(version="0.14", message="Please use `list_credential_helpers` instead.")
-def _currently_setup_credential_helpers(directory: Optional[str] = None) -> List[str]:
-    return list_credential_helpers(directory)