huggingface-hub 1.0.0rc1__py3-none-any.whl → 1.0.0rc2__py3-none-any.whl

huggingface_hub/__init__.py

@@ -46,7 +46,7 @@ import sys
 from typing import TYPE_CHECKING
 
 
-__version__ = "1.0.0.rc1"
+__version__ = "1.0.0.rc2"
 
 # Alphabetical order of definitions is ensured in tests
 # WARNING: any comment added in this dictionary definition will be lost when

huggingface_hub/_commit_api.py

@@ -33,6 +33,7 @@ from .utils import (
     validate_hf_hub_args,
 )
 from .utils import tqdm as hf_tqdm
+from .utils._runtime import is_xet_available
 
 
 if TYPE_CHECKING:
@@ -353,7 +354,7 @@ def _warn_on_overwriting_operations(operations: list[CommitOperation]) -> None:
 
 
 @validate_hf_hub_args
-def _upload_lfs_files(
+def _upload_files(
     *,
     additions: list[CommitOperationAdd],
     repo_type: str,
@@ -362,6 +363,86 @@ def _upload_lfs_files(
     endpoint: Optional[str] = None,
     num_threads: int = 5,
     revision: Optional[str] = None,
+    create_pr: Optional[bool] = None,
+):
+    """
+    Negotiates per-file transfer (LFS vs Xet) and uploads in batches.
+    """
+    xet_additions: list[CommitOperationAdd] = []
+    lfs_actions: list[dict[str, Any]] = []
+    lfs_oid2addop: dict[str, CommitOperationAdd] = {}
+
+    for chunk in chunk_iterable(additions, chunk_size=UPLOAD_BATCH_MAX_NUM_FILES):
+        chunk_list = [op for op in chunk]
+
+        transfers: list[str] = ["basic", "multipart"]
+        has_buffered_io_data = any(isinstance(op.path_or_fileobj, io.BufferedIOBase) for op in chunk_list)
+        if is_xet_available():
+            if not has_buffered_io_data:
+                transfers.append("xet")
+            else:
+                logger.warning(
+                    "Uploading files as a binary IO buffer is not supported by Xet Storage. "
+                    "Falling back to HTTP upload."
+                )
+
+        actions_chunk, errors_chunk, chosen_transfer = post_lfs_batch_info(
+            upload_infos=[op.upload_info for op in chunk_list],
+            repo_id=repo_id,
+            repo_type=repo_type,
+            revision=revision,
+            endpoint=endpoint,
+            headers=headers,
+            token=None,  # already passed in 'headers'
+            transfers=transfers,
+        )
+        if errors_chunk:
+            message = "\n".join(
+                [
+                    f"Encountered error for file with OID {err.get('oid')}: `{err.get('error', {}).get('message')}"
+                    for err in errors_chunk
+                ]
+            )
+            raise ValueError(f"LFS batch API returned errors:\n{message}")
+
+        # If server returns a transfer we didn't offer (e.g "xet" while uploading from BytesIO),
+        # fall back to LFS for this chunk.
+        if chosen_transfer == "xet" and ("xet" in transfers):
+            xet_additions.extend(chunk_list)
+        else:
+            lfs_actions.extend(actions_chunk)
+            for op in chunk_list:
+                lfs_oid2addop[op.upload_info.sha256.hex()] = op
+
+    if len(lfs_actions) > 0:
+        _upload_lfs_files(
+            actions=lfs_actions,
+            oid2addop=lfs_oid2addop,
+            headers=headers,
+            endpoint=endpoint,
+            num_threads=num_threads,
+        )
+
+    if len(xet_additions) > 0:
+        _upload_xet_files(
+            additions=xet_additions,
+            repo_type=repo_type,
+            repo_id=repo_id,
+            headers=headers,
+            endpoint=endpoint,
+            revision=revision,
+            create_pr=create_pr,
+        )
+
+
+@validate_hf_hub_args
+def _upload_lfs_files(
+    *,
+    actions: list[dict[str, Any]],
+    oid2addop: dict[str, CommitOperationAdd],
+    headers: dict[str, str],
+    endpoint: Optional[str] = None,
+    num_threads: int = 5,
 ):
     """
     Uploads the content of `additions` to the Hub using the large file storage protocol.
@@ -370,9 +451,21 @@ def _upload_lfs_files(
         - LFS Batch API: https://github.com/git-lfs/git-lfs/blob/main/docs/api/batch.md
 
     Args:
-        additions (`List` of `CommitOperationAdd`):
-            The files to be uploaded
-        repo_type (`str`):
+        actions (`list[dict[str, Any]]`):
+            LFS batch actions returned by the server.
+        oid2addop (`dict[str, CommitOperationAdd]`):
+            A dictionary mapping the OID of the file to the corresponding `CommitOperationAdd` object.
+        headers (`dict[str, str]`):
+            Headers to use for the request, including authorization headers and user agent.
+        endpoint (`str`, *optional*):
+            The endpoint to use for the request. Defaults to `constants.ENDPOINT`.
+        num_threads (`int`, *optional*):
+            The number of concurrent threads to use when uploading. Defaults to 5.
+
+    Raises:
+        [`EnvironmentError`](https://docs.python.org/3/library/exceptions.html#EnvironmentError)
+            If an upload failed for any reason
+        [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError)
             Type of the repo to upload to: `"model"`, `"dataset"` or `"space"`.
         repo_id (`str`):
             A namespace (user or an organization) and a repo name separated
@@ -392,50 +485,17 @@ def _upload_lfs_files(
         [`HfHubHTTPError`]
             If the LFS batch endpoint returned an HTTP error.
     """
-    # Step 1: retrieve upload instructions from the LFS batch endpoint.
-    #         Upload instructions are retrieved by chunk of 256 files to avoid reaching
-    #         the payload limit.
-    batch_actions: list[dict] = []
-    for chunk in chunk_iterable(additions, chunk_size=UPLOAD_BATCH_MAX_NUM_FILES):
-        batch_actions_chunk, batch_errors_chunk = post_lfs_batch_info(
-            upload_infos=[op.upload_info for op in chunk],
-            repo_id=repo_id,
-            repo_type=repo_type,
-            revision=revision,
-            endpoint=endpoint,
-            headers=headers,
-            token=None,  # already passed in 'headers'
-        )
-
-        # If at least 1 error, we do not retrieve information for other chunks
-        if batch_errors_chunk:
-            message = "\n".join(
-                [
-                    f"Encountered error for file with OID {err.get('oid')}: `{err.get('error', {}).get('message')}"
-                    for err in batch_errors_chunk
-                ]
-            )
-            raise ValueError(f"LFS batch endpoint returned errors:\n{message}")
-
-        batch_actions += batch_actions_chunk
-    oid2addop = {add_op.upload_info.sha256.hex(): add_op for add_op in additions}
-
-    # Step 2: ignore files that have already been uploaded
+    # Filter out files already present upstream
     filtered_actions = []
-    for action in batch_actions:
+    for action in actions:
         if action.get("actions") is None:
             logger.debug(
-                f"Content of file {oid2addop[action['oid']].path_in_repo} is already"
-                " present upstream - skipping upload."
+                f"Content of file {oid2addop[action['oid']].path_in_repo} is already present upstream - skipping upload."
             )
         else:
             filtered_actions.append(action)
 
-    if len(filtered_actions) == 0:
-        logger.debug("No LFS files to upload.")
-        return
-
-    # Step 3: upload files concurrently according to these instructions
+    # Upload according to server-provided actions
     def _wrapped_lfs_upload(batch_action) -> None:
         try:
             operation = oid2addop[batch_action["oid"]]
@@ -479,7 +539,7 @@ def _upload_xet_files(
     This chunks the files and deduplicates the chunks before uploading them to xetcas storage.
 
     Args:
-        additions (`List` of `CommitOperationAdd`):
+        additions (`` of `CommitOperationAdd`):
             The files to be uploaded.
         repo_type (`str`):
             Type of the repo to upload to: `"model"`, `"dataset"` or `"space"`.
@@ -576,30 +636,30 @@ def _upload_xet_files(
         progress, progress_callback = None, None
 
     try:
-        for i, chunk in enumerate(chunk_iterable(additions, chunk_size=UPLOAD_BATCH_MAX_NUM_FILES)):
-            _chunk = [op for op in chunk]
-
-            bytes_ops = [op for op in _chunk if isinstance(op.path_or_fileobj, bytes)]
-            paths_ops = [op for op in _chunk if isinstance(op.path_or_fileobj, (str, Path))]
-
-            if len(paths_ops) > 0:
-                upload_files(
-                    [str(op.path_or_fileobj) for op in paths_ops],
-                    xet_endpoint,
-                    access_token_info,
-                    token_refresher,
-                    progress_callback,
-                    repo_type,
-                )
-            if len(bytes_ops) > 0:
-                upload_bytes(
-                    [op.path_or_fileobj for op in bytes_ops],
-                    xet_endpoint,
-                    access_token_info,
-                    token_refresher,
-                    progress_callback,
-                    repo_type,
-                )
+        all_bytes_ops = [op for op in additions if isinstance(op.path_or_fileobj, bytes)]
+        all_paths_ops = [op for op in additions if isinstance(op.path_or_fileobj, (str, Path))]
+
+        if len(all_paths_ops) > 0:
+            all_paths = [str(op.path_or_fileobj) for op in all_paths_ops]
+            upload_files(
+                all_paths,
+                xet_endpoint,
+                access_token_info,
+                token_refresher,
+                progress_callback,
+                repo_type,
+            )
+
+        if len(all_bytes_ops) > 0:
+            all_bytes = [op.path_or_fileobj for op in all_bytes_ops]
+            upload_bytes(
+                all_bytes,
+                xet_endpoint,
+                access_token_info,
+                token_refresher,
+                progress_callback,
+                repo_type,
+            )
 
     finally:
         if progress is not None:

huggingface_hub/_commit_scheduler.py

@@ -205,13 +205,10 @@ class CommitScheduler:
         """
         Push folder to the Hub and return the commit info.
 
-        <Tip warning={true}>
-
-        This method is not meant to be called directly. It is run in the background by the scheduler, respecting a
-        queue mechanism to avoid concurrent commits. Making a direct call to the method might lead to concurrency
-        issues.
-
-        </Tip>
+        > [!WARNING]
+        > This method is not meant to be called directly. It is run in the background by the scheduler, respecting a
+        > queue mechanism to avoid concurrent commits. Making a direct call to the method might lead to concurrency
+        > issues.
 
         The default behavior of `push_to_hub` is to assume an append-only folder. It lists all files in the folder and
         uploads only changed files. If no changes are found, the method returns without committing anything. If you want

huggingface_hub/_login.py

@@ -70,21 +70,15 @@ def login(
     To log in from outside of a script, one can also use `hf auth login` which is
     a cli command that wraps [`login`].
 
-    <Tip>
-
-    [`login`] is a drop-in replacement method for [`notebook_login`] as it wraps and
-    extends its capabilities.
-
-    </Tip>
-
-    <Tip>
-
-    When the token is not passed, [`login`] will automatically detect if the script runs
-    in a notebook or not. However, this detection might not be accurate due to the
-    variety of notebooks that exists nowadays. If that is the case, you can always force
-    the UI by using [`notebook_login`] or [`interpreter_login`].
-
-    </Tip>
+    > [!TIP]
+    > [`login`] is a drop-in replacement method for [`notebook_login`] as it wraps and
+    > extends its capabilities.
+
+    > [!TIP]
+    > When the token is not passed, [`login`] will automatically detect if the script runs
+    > in a notebook or not. However, this detection might not be accurate due to the
+    > variety of notebooks that exists nowadays. If that is the case, you can always force
+    > the UI by using [`notebook_login`] or [`interpreter_login`].
 
     Args:
         token (`str`, *optional*):

huggingface_hub/_tensorboard_logger.py

@@ -52,11 +52,8 @@ class HFSummaryWriter(_RuntimeSummaryWriter):
     issue), the main script will not be interrupted. Data is automatically pushed to the Hub every `commit_every`
     minutes (default to every 5 minutes).
 
-    <Tip warning={true}>
-
-    `HFSummaryWriter` is experimental. Its API is subject to change in the future without prior notice.
-
-    </Tip>
+    > [!WARNING]
+    > `HFSummaryWriter` is experimental. Its API is subject to change in the future without prior notice.
 
     Args:
         repo_id (`str`):

huggingface_hub/_webhooks_server.py

@@ -53,17 +53,11 @@ class WebhooksServer:
     Check out the [webhooks guide](../guides/webhooks_server) for a step-by-step tutorial on how to setup your
     WebhooksServer and deploy it on a Space.
 
-    <Tip warning={true}>
+    > [!WARNING]
+    > `WebhooksServer` is experimental. Its API is subject to change in the future.
 
-    `WebhooksServer` is experimental. Its API is subject to change in the future.
-
-    </Tip>
-
-    <Tip warning={true}>
-
-    You must have `gradio` installed to use `WebhooksServer` (`pip install --upgrade gradio`).
-
-    </Tip>
+    > [!WARNING]
+    > You must have `gradio` installed to use `WebhooksServer` (`pip install --upgrade gradio`).
 
     Args:
         ui (`gradio.Blocks`, optional):
@@ -240,17 +234,11 @@ def webhook_endpoint(path: Optional[str] = None) -> Callable:
     Check out the [webhooks guide](../guides/webhooks_server) for a step-by-step tutorial on how to setup your
     server and deploy it on a Space.
 
-    <Tip warning={true}>
-
-    `webhook_endpoint` is experimental. Its API is subject to change in the future.
-
-    </Tip>
-
-    <Tip warning={true}>
-
-    You must have `gradio` installed to use `webhook_endpoint` (`pip install --upgrade gradio`).
+    > [!WARNING]
+    > `webhook_endpoint` is experimental. Its API is subject to change in the future.
 
-    </Tip>
+    > [!WARNING]
+    > You must have `gradio` installed to use `webhook_endpoint` (`pip install --upgrade gradio`).
 
     Args:
         path (`str`, optional):

huggingface_hub/cli/repo.py

@@ -21,6 +21,7 @@ Usage:
     hf repo create my-cool-model --private
 """
 
+import enum
 from typing import Annotated, Optional
 
 import typer
@@ -44,8 +45,16 @@ from ._cli_utils import (
 logger = logging.get_logger(__name__)
 
 repo_cli = typer_factory(help="Manage repos on the Hub.")
-tag_app = typer_factory(help="Manage tags for a repo on the Hub.")
-repo_cli.add_typer(tag_app, name="tag")
+tag_cli = typer_factory(help="Manage tags for a repo on the Hub.")
+branch_cli = typer_factory(help="Manage branches for a repo on the Hub.")
+repo_cli.add_typer(tag_cli, name="tag")
+repo_cli.add_typer(branch_cli, name="branch")
+
+
+class GatedChoices(str, enum.Enum):
+    auto = "auto"
+    manual = "manual"
+    false = "false"
 
 
 @repo_cli.command("create", help="Create a new repo on the Hub.")
@@ -87,7 +96,130 @@ def repo_create(
     print(f"Your repo is now available at {ANSI.bold(repo_url)}")
 
 
-@tag_app.command("create", help="Create a tag for a repo.")
+@repo_cli.command("delete", help="Delete a repo from the Hub. this is an irreversible operation.")
+def repo_delete(
+    repo_id: RepoIdArg,
+    repo_type: RepoTypeOpt = RepoType.model,
+    token: TokenOpt = None,
+    missing_ok: Annotated[
+        bool,
+        typer.Option(
+            help="If set to True, do not raise an error if repo does not exist.",
+        ),
+    ] = False,
+) -> None:
+    api = get_hf_api(token=token)
+    api.delete_repo(
+        repo_id=repo_id,
+        repo_type=repo_type.value,
+        missing_ok=missing_ok,
+    )
+    print(f"Successfully deleted {ANSI.bold(repo_id)} on the Hub.")
+
+
+@repo_cli.command("move", help="Move a repository from a namespace to another namespace.")
+def repo_move(
+    from_id: RepoIdArg,
+    to_id: RepoIdArg,
+    token: TokenOpt = None,
+    repo_type: RepoTypeOpt = RepoType.model,
+) -> None:
+    api = get_hf_api(token=token)
+    api.move_repo(
+        from_id=from_id,
+        to_id=to_id,
+        repo_type=repo_type.value,
+    )
+    print(f"Successfully moved {ANSI.bold(from_id)} to {ANSI.bold(to_id)} on the Hub.")
+
+
+@repo_cli.command("settings", help="Update the settings of a repository.")
+def repo_settings(
+    repo_id: RepoIdArg,
+    gated: Annotated[
+        Optional[GatedChoices],
+        typer.Option(
+            help="The gated status for the repository.",
+        ),
+    ] = None,
+    private: Annotated[
+        Optional[bool],
+        typer.Option(
+            help="Whether the repository should be private.",
+        ),
+    ] = None,
+    xet_enabled: Annotated[
+        Optional[bool],
+        typer.Option(
+            help=" Whether the repository should be enabled for Xet Storage.",
+        ),
+    ] = None,
+    token: TokenOpt = None,
+    repo_type: RepoTypeOpt = RepoType.model,
+) -> None:
+    api = get_hf_api(token=token)
+    api.update_repo_settings(
+        repo_id=repo_id,
+        gated=(gated.value if gated else None),  # type: ignore [arg-type]
+        private=private,
+        xet_enabled=xet_enabled,
+        repo_type=repo_type.value,
+    )
+    print(f"Successfully updated the settings of {ANSI.bold(repo_id)} on the Hub.")
+
+
+@branch_cli.command("create", help="Create a new branch for a repo on the Hub.")
+def branch_create(
+    repo_id: RepoIdArg,
+    branch: Annotated[
+        str,
+        typer.Argument(
+            help="The name of the branch to create.",
+        ),
+    ],
+    revision: RevisionOpt = None,
+    token: TokenOpt = None,
+    repo_type: RepoTypeOpt = RepoType.model,
+    exist_ok: Annotated[
+        bool,
+        typer.Option(
+            help="If set to True, do not raise an error if branch already exists.",
+        ),
+    ] = False,
+) -> None:
+    api = get_hf_api(token=token)
+    api.create_branch(
+        repo_id=repo_id,
+        branch=branch,
+        revision=revision,
+        repo_type=repo_type.value,
+        exist_ok=exist_ok,
+    )
+    print(f"Successfully created {ANSI.bold(branch)} branch on {repo_type.value} {ANSI.bold(repo_id)}")
+
+
+@branch_cli.command("delete", help="Delete a branch from a repo on the Hub.")
+def branch_delete(
+    repo_id: RepoIdArg,
+    branch: Annotated[
+        str,
+        typer.Argument(
+            help="The name of the branch to delete.",
+        ),
+    ],
+    token: TokenOpt = None,
+    repo_type: RepoTypeOpt = RepoType.model,
+) -> None:
+    api = get_hf_api(token=token)
+    api.delete_branch(
+        repo_id=repo_id,
+        branch=branch,
+        repo_type=repo_type.value,
+    )
+    print(f"Successfully deleted {ANSI.bold(branch)} branch on {repo_type.value} {ANSI.bold(repo_id)}")
+
+
+@tag_cli.command("create", help="Create a tag for a repo.")
 def tag_create(
     repo_id: RepoIdArg,
     tag: Annotated[
@@ -127,7 +259,7 @@ def tag_create(
     print(f"Tag {ANSI.bold(tag)} created on {ANSI.bold(repo_id)}")
 
 
-@tag_app.command("list", help="List tags for a repo.")
+@tag_cli.command("list", help="List tags for a repo.")
 def tag_list(
     repo_id: RepoIdArg,
     token: TokenOpt = None,
@@ -152,7 +284,7 @@ def tag_list(
         print(t.name)
 
 
-@tag_app.command("delete", help="Delete a tag for a repo.")
+@tag_cli.command("delete", help="Delete a tag for a repo.")
 def tag_delete(
     repo_id: RepoIdArg,
     tag: Annotated[

huggingface_hub/dataclasses.py

@@ -1,18 +1,7 @@
 import inspect
 from dataclasses import _MISSING_TYPE, MISSING, Field, field, fields
 from functools import wraps
-from typing import (
-    Any,
-    Callable,
-    Literal,
-    Optional,
-    Type,
-    TypeVar,
-    Union,
-    get_args,
-    get_origin,
-    overload,
-)
+from typing import Any, Callable, ForwardRef, Literal, Optional, Type, TypeVar, Union, get_args, get_origin, overload
 
 from .errors import (
     StrictDataclassClassValidationError,
@@ -322,6 +311,8 @@ def type_validator(name: str, value: Any, expected_type: Any) -> None:
         validator(name, value, args)
     elif isinstance(expected_type, type):  # simple types
         _validate_simple_type(name, value, expected_type)
+    elif isinstance(expected_type, ForwardRef) or isinstance(expected_type, str):
+        return
     else:
         raise TypeError(f"Unsupported type for field '{name}': {expected_type}")
 

huggingface_hub/fastai_utils.py

@@ -34,13 +34,11 @@ def _check_fastai_fastcore_versions(
         fastcore_min_version (`str`, *optional*):
             The minimum fastcore version supported.
 
-    <Tip>
-    Raises the following error:
-
-        - [`ImportError`](https://docs.python.org/3/library/exceptions.html#ImportError)
-          if the fastai or fastcore libraries are not available or are of an invalid version.
-
-    </Tip>
+    > [!TIP]
+    > Raises the following error:
+    >
+    >     - [`ImportError`](https://docs.python.org/3/library/exceptions.html#ImportError)
+    >       if the fastai or fastcore libraries are not available or are of an invalid version.
     """
 
     if (get_fastcore_version() or get_fastai_version()) == "N/A":
@@ -88,15 +86,13 @@ def _check_fastai_fastcore_pyproject_versions(
         fastcore_min_version (`str`, *optional*):
             The minimum fastcore version supported.
 
-    <Tip>
-    Raises the following errors:
-
-        - [`ImportError`](https://docs.python.org/3/library/exceptions.html#ImportError)
-          if the `toml` module is not installed.
-        - [`ImportError`](https://docs.python.org/3/library/exceptions.html#ImportError)
-          if the `pyproject.toml` indicates a lower than minimum supported version of fastai or fastcore.
-
-    </Tip>
+    > [!TIP]
+    > Raises the following errors:
+    >
+    >     - [`ImportError`](https://docs.python.org/3/library/exceptions.html#ImportError)
+    >       if the `toml` module is not installed.
+    >     - [`ImportError`](https://docs.python.org/3/library/exceptions.html#ImportError)
+    >       if the `pyproject.toml` indicates a lower than minimum supported version of fastai or fastcore.
     """
 
     try:
@@ -253,14 +249,11 @@ def _save_pretrained_fastai(
         config (`dict`, *optional*):
             Configuration object. Will be uploaded as a .json file. Example: 'https://huggingface.co/espejelomar/fastai-pet-breeds-classification/blob/main/config.json'.
 
-    <Tip>
-
-    Raises the following error:
-
-        - [`RuntimeError`](https://docs.python.org/3/library/exceptions.html#RuntimeError)
-          if the config file provided is not a dictionary.
-
-    </Tip>
+    > [!TIP]
+    > Raises the following error:
+    >
+    >     - [`RuntimeError`](https://docs.python.org/3/library/exceptions.html#RuntimeError)
+    >       if the config file provided is not a dictionary.
     """
     _check_fastai_fastcore_versions()
 
@@ -394,14 +387,11 @@ def push_to_hub_fastai(
     Returns:
         The url of the commit of your model in the given repository.
 
-    <Tip>
-
-    Raises the following error:
-
-        - [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError)
-          if the user is not log on to the Hugging Face Hub.
-
-    </Tip>
+    > [!TIP]
+    > Raises the following error:
+    >
+    >     - [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError)
+    >       if the user is not log on to the Hugging Face Hub.
     """
     _check_fastai_fastcore_versions()
     api = HfApi(endpoint=api_endpoint)