huggingface-hub 0.35.3__py3-none-any.whl → 0.36.0__py3-none-any.whl

huggingface_hub/__init__.py

@@ -46,7 +46,7 @@ import sys
 from typing import TYPE_CHECKING
 
 
-__version__ = "0.35.3"
+__version__ = "0.36.0"
 
 # Alphabetical order of definitions is ensured in tests
 # WARNING: any comment added in this dictionary definition will be lost when
@@ -159,6 +159,7 @@ _SUBMOD_ATTRS = {
         "GitRefs",
         "HfApi",
         "ModelInfo",
+        "Organization",
         "RepoUrl",
         "SpaceInfo",
         "User",
@@ -212,6 +213,7 @@ _SUBMOD_ATTRS = {
         "get_full_repo_name",
         "get_inference_endpoint",
         "get_model_tags",
+        "get_organization_overview",
         "get_paths_info",
         "get_repo_discussions",
         "get_safetensors_metadata",
@@ -699,6 +701,7 @@ __all__ = [
     "ObjectDetectionInput",
     "ObjectDetectionOutputElement",
     "ObjectDetectionParameters",
+    "Organization",
     "PYTORCH_WEIGHTS_NAME",
     "Padding",
     "PyTorchModelHubMixin",
@@ -870,6 +873,7 @@ __all__ = [
     "get_hf_file_metadata",
     "get_inference_endpoint",
     "get_model_tags",
+    "get_organization_overview",
     "get_paths_info",
     "get_repo_discussions",
     "get_safetensors_metadata",
@@ -1179,6 +1183,7 @@ if TYPE_CHECKING:  # pragma: no cover
         GitRefs,  # noqa: F401
         HfApi,  # noqa: F401
         ModelInfo,  # noqa: F401
+        Organization,  # noqa: F401
         RepoUrl,  # noqa: F401
         SpaceInfo,  # noqa: F401
         User,  # noqa: F401
@@ -1232,6 +1237,7 @@ if TYPE_CHECKING:  # pragma: no cover
         get_full_repo_name,  # noqa: F401
         get_inference_endpoint,  # noqa: F401
         get_model_tags,  # noqa: F401
+        get_organization_overview,  # noqa: F401
         get_paths_info,  # noqa: F401
         get_repo_discussions,  # noqa: F401
         get_safetensors_metadata,  # noqa: F401

huggingface_hub/_commit_api.py

@@ -33,6 +33,7 @@ from .utils import (
     validate_hf_hub_args,
 )
 from .utils import tqdm as hf_tqdm
+from .utils._runtime import is_xet_available
 
 
 if TYPE_CHECKING:
@@ -353,7 +354,7 @@ def _warn_on_overwriting_operations(operations: List[CommitOperation]) -> None:
 
 
 @validate_hf_hub_args
-def _upload_lfs_files(
+def _upload_files(
     *,
     additions: List[CommitOperationAdd],
     repo_type: str,
@@ -362,6 +363,86 @@ def _upload_lfs_files(
     endpoint: Optional[str] = None,
     num_threads: int = 5,
     revision: Optional[str] = None,
+    create_pr: Optional[bool] = None,
+):
+    """
+    Negotiates per-file transfer (LFS vs Xet) and uploads in batches.
+    """
+    xet_additions: List[CommitOperationAdd] = []
+    lfs_actions: List[Dict] = []
+    lfs_oid2addop: Dict[str, CommitOperationAdd] = {}
+
+    for chunk in chunk_iterable(additions, chunk_size=UPLOAD_BATCH_MAX_NUM_FILES):
+        chunk_list = [op for op in chunk]
+
+        transfers: List[str] = ["basic", "multipart"]
+        has_buffered_io_data = any(isinstance(op.path_or_fileobj, io.BufferedIOBase) for op in chunk_list)
+        if is_xet_available():
+            if not has_buffered_io_data:
+                transfers.append("xet")
+            else:
+                logger.warning(
+                    "Uploading files as a binary IO buffer is not supported by Xet Storage. "
+                    "Falling back to HTTP upload."
+                )
+
+        actions_chunk, errors_chunk, chosen_transfer = post_lfs_batch_info(
+            upload_infos=[op.upload_info for op in chunk_list],
+            repo_id=repo_id,
+            repo_type=repo_type,
+            revision=revision,
+            endpoint=endpoint,
+            headers=headers,
+            token=None,  # already passed in 'headers'
+            transfers=transfers,
+        )
+        if errors_chunk:
+            message = "\n".join(
+                [
+                    f"Encountered error for file with OID {err.get('oid')}: `{err.get('error', {}).get('message')}"
+                    for err in errors_chunk
+                ]
+            )
+            raise ValueError(f"LFS batch API returned errors:\n{message}")
+
+        # If server returns a transfer we didn't offer (e.g "xet" while uploading from BytesIO),
+        # fall back to LFS for this chunk.
+        if chosen_transfer == "xet" and ("xet" in transfers):
+            xet_additions.extend(chunk_list)
+        else:
+            lfs_actions.extend(actions_chunk)
+            for op in chunk_list:
+                lfs_oid2addop[op.upload_info.sha256.hex()] = op
+
+    if len(lfs_actions) > 0:
+        _upload_lfs_files(
+            actions=lfs_actions,
+            oid2addop=lfs_oid2addop,
+            headers=headers,
+            endpoint=endpoint,
+            num_threads=num_threads,
+        )
+
+    if len(xet_additions) > 0:
+        _upload_xet_files(
+            additions=xet_additions,
+            repo_type=repo_type,
+            repo_id=repo_id,
+            headers=headers,
+            endpoint=endpoint,
+            revision=revision,
+            create_pr=create_pr,
+        )
+
+
+@validate_hf_hub_args
+def _upload_lfs_files(
+    *,
+    actions: List[Dict],
+    oid2addop: Dict[str, CommitOperationAdd],
+    headers: Dict[str, str],
+    endpoint: Optional[str] = None,
+    num_threads: int = 5,
 ):
     """
     Uploads the content of `additions` to the Hub using the large file storage protocol.
@@ -370,9 +451,21 @@ def _upload_lfs_files(
         - LFS Batch API: https://github.com/git-lfs/git-lfs/blob/main/docs/api/batch.md
 
     Args:
-        additions (`List` of `CommitOperationAdd`):
-            The files to be uploaded
-        repo_type (`str`):
+        actions (`List[Dict]`):
+            LFS batch actions returned by the server.
+        oid2addop (`Dict[str, CommitOperationAdd]`):
+            A dictionary mapping the OID of the file to the corresponding `CommitOperationAdd` object.
+        headers (`Dict[str, str]`):
+            Headers to use for the request, including authorization headers and user agent.
+        endpoint (`str`, *optional*):
+            The endpoint to use for the request. Defaults to `constants.ENDPOINT`.
+        num_threads (`int`, *optional*):
+            The number of concurrent threads to use when uploading. Defaults to 5.
+
+    Raises:
+        [`EnvironmentError`](https://docs.python.org/3/library/exceptions.html#EnvironmentError)
+            If an upload failed for any reason
+        [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError)
             Type of the repo to upload to: `"model"`, `"dataset"` or `"space"`.
         repo_id (`str`):
             A namespace (user or an organization) and a repo name separated
@@ -392,50 +485,17 @@ def _upload_lfs_files(
         [`HTTPError`](https://requests.readthedocs.io/en/latest/api/#requests.HTTPError)
             If the LFS batch endpoint returned an HTTP error.
     """
-    # Step 1: retrieve upload instructions from the LFS batch endpoint.
-    #         Upload instructions are retrieved by chunk of 256 files to avoid reaching
-    #         the payload limit.
-    batch_actions: List[Dict] = []
-    for chunk in chunk_iterable(additions, chunk_size=UPLOAD_BATCH_MAX_NUM_FILES):
-        batch_actions_chunk, batch_errors_chunk = post_lfs_batch_info(
-            upload_infos=[op.upload_info for op in chunk],
-            repo_id=repo_id,
-            repo_type=repo_type,
-            revision=revision,
-            endpoint=endpoint,
-            headers=headers,
-            token=None,  # already passed in 'headers'
-        )
-
-        # If at least 1 error, we do not retrieve information for other chunks
-        if batch_errors_chunk:
-            message = "\n".join(
-                [
-                    f"Encountered error for file with OID {err.get('oid')}: `{err.get('error', {}).get('message')}"
-                    for err in batch_errors_chunk
-                ]
-            )
-            raise ValueError(f"LFS batch endpoint returned errors:\n{message}")
-
-        batch_actions += batch_actions_chunk
-    oid2addop = {add_op.upload_info.sha256.hex(): add_op for add_op in additions}
-
-    # Step 2: ignore files that have already been uploaded
+    # Filter out files already present upstream
     filtered_actions = []
-    for action in batch_actions:
+    for action in actions:
         if action.get("actions") is None:
             logger.debug(
-                f"Content of file {oid2addop[action['oid']].path_in_repo} is already"
-                " present upstream - skipping upload."
+                f"Content of file {oid2addop[action['oid']].path_in_repo} is already present upstream - skipping upload."
             )
         else:
             filtered_actions.append(action)
 
-    if len(filtered_actions) == 0:
-        logger.debug("No LFS files to upload.")
-        return
-
-    # Step 3: upload files concurrently according to these instructions
+    # Upload according to server-provided actions
     def _wrapped_lfs_upload(batch_action) -> None:
         try:
             operation = oid2addop[batch_action["oid"]]
@@ -576,30 +636,30 @@ def _upload_xet_files(
         progress, progress_callback = None, None
 
     try:
-        for i, chunk in enumerate(chunk_iterable(additions, chunk_size=UPLOAD_BATCH_MAX_NUM_FILES)):
-            _chunk = [op for op in chunk]
-
-            bytes_ops = [op for op in _chunk if isinstance(op.path_or_fileobj, bytes)]
-            paths_ops = [op for op in _chunk if isinstance(op.path_or_fileobj, (str, Path))]
-
-            if len(paths_ops) > 0:
-                upload_files(
-                    [str(op.path_or_fileobj) for op in paths_ops],
-                    xet_endpoint,
-                    access_token_info,
-                    token_refresher,
-                    progress_callback,
-                    repo_type,
-                )
-            if len(bytes_ops) > 0:
-                upload_bytes(
-                    [op.path_or_fileobj for op in bytes_ops],
-                    xet_endpoint,
-                    access_token_info,
-                    token_refresher,
-                    progress_callback,
-                    repo_type,
-                )
+        all_bytes_ops = [op for op in additions if isinstance(op.path_or_fileobj, bytes)]
+        all_paths_ops = [op for op in additions if isinstance(op.path_or_fileobj, (str, Path))]
+
+        if len(all_paths_ops) > 0:
+            all_paths = [str(op.path_or_fileobj) for op in all_paths_ops]
+            upload_files(
+                all_paths,
+                xet_endpoint,
+                access_token_info,
+                token_refresher,
+                progress_callback,
+                repo_type,
+            )
+
+        if len(all_bytes_ops) > 0:
+            all_bytes = [op.path_or_fileobj for op in all_bytes_ops]
+            upload_bytes(
+                all_bytes,
+                xet_endpoint,
+                access_token_info,
+                token_refresher,
+                progress_callback,
+                repo_type,
+            )
 
     finally:
         if progress is not None:

huggingface_hub/_commit_scheduler.py

@@ -205,13 +205,10 @@ class CommitScheduler:
         """
         Push folder to the Hub and return the commit info.
 
-        <Tip warning={true}>
-
-        This method is not meant to be called directly. It is run in the background by the scheduler, respecting a
-        queue mechanism to avoid concurrent commits. Making a direct call to the method might lead to concurrency
-        issues.
-
-        </Tip>
+        > [!WARNING]
+        > This method is not meant to be called directly. It is run in the background by the scheduler, respecting a
+        > queue mechanism to avoid concurrent commits. Making a direct call to the method might lead to concurrency
+        > issues.
 
         The default behavior of `push_to_hub` is to assume an append-only folder. It lists all files in the folder and
         uploads only changed files. If no changes are found, the method returns without committing anything. If you want

huggingface_hub/_jobs_api.py

@@ -30,7 +30,7 @@ class JobStage(str, Enum):
     ```py
     assert JobStage.COMPLETED == "COMPLETED"
     ```
-
+    Possible values are: `COMPLETED`, `CANCELED`, `ERROR`, `DELETED`, `RUNNING`.
     Taken from https://github.com/huggingface/moon-landing/blob/main/server/job_types/JobInfo.ts#L61 (private url).
     """
 

huggingface_hub/_login.py

@@ -78,21 +78,15 @@ def login(
     To log in from outside of a script, one can also use `hf auth login` which is
     a cli command that wraps [`login`].
 
-    <Tip>
-
-    [`login`] is a drop-in replacement method for [`notebook_login`] as it wraps and
-    extends its capabilities.
-
-    </Tip>
-
-    <Tip>
-
-    When the token is not passed, [`login`] will automatically detect if the script runs
-    in a notebook or not. However, this detection might not be accurate due to the
-    variety of notebooks that exists nowadays. If that is the case, you can always force
-    the UI by using [`notebook_login`] or [`interpreter_login`].
-
-    </Tip>
+    > [!TIP]
+    > [`login`] is a drop-in replacement method for [`notebook_login`] as it wraps and
+    > extends its capabilities.
+
+    > [!TIP]
+    > When the token is not passed, [`login`] will automatically detect if the script runs
+    > in a notebook or not. However, this detection might not be accurate due to the
+    > variety of notebooks that exists nowadays. If that is the case, you can always force
+    > the UI by using [`notebook_login`] or [`interpreter_login`].
 
     Args:
         token (`str`, *optional*):

huggingface_hub/_tensorboard_logger.py

@@ -52,11 +52,8 @@ class HFSummaryWriter(_RuntimeSummaryWriter):
     issue), the main script will not be interrupted. Data is automatically pushed to the Hub every `commit_every`
     minutes (default to every 5 minutes).
 
-    <Tip warning={true}>
-
-    `HFSummaryWriter` is experimental. Its API is subject to change in the future without prior notice.
-
-    </Tip>
+    > [!WARNING]
+    > `HFSummaryWriter` is experimental. Its API is subject to change in the future without prior notice.
 
     Args:
         repo_id (`str`):

huggingface_hub/_webhooks_server.py

@@ -32,7 +32,7 @@ if is_fastapi_available():
     from fastapi.responses import JSONResponse
 else:
     # Will fail at runtime if FastAPI is not available
-    FastAPI = Request = JSONResponse = None  # type: ignore [misc, assignment]
+    FastAPI = Request = JSONResponse = None  # type: ignore
 
 
 _global_app: Optional["WebhooksServer"] = None
@@ -53,17 +53,11 @@ class WebhooksServer:
     Check out the [webhooks guide](../guides/webhooks_server) for a step-by-step tutorial on how to setup your
     WebhooksServer and deploy it on a Space.
 
-    <Tip warning={true}>
+    > [!WARNING]
+    > `WebhooksServer` is experimental. Its API is subject to change in the future.
 
-    `WebhooksServer` is experimental. Its API is subject to change in the future.
-
-    </Tip>
-
-    <Tip warning={true}>
-
-    You must have `gradio` installed to use `WebhooksServer` (`pip install --upgrade gradio`).
-
-    </Tip>
+    > [!WARNING]
+    > You must have `gradio` installed to use `WebhooksServer` (`pip install --upgrade gradio`).
 
     Args:
         ui (`gradio.Blocks`, optional):
@@ -240,17 +234,11 @@ def webhook_endpoint(path: Optional[str] = None) -> Callable:
     Check out the [webhooks guide](../guides/webhooks_server) for a step-by-step tutorial on how to setup your
     server and deploy it on a Space.
 
-    <Tip warning={true}>
-
-    `webhook_endpoint` is experimental. Its API is subject to change in the future.
-
-    </Tip>
-
-    <Tip warning={true}>
-
-    You must have `gradio` installed to use `webhook_endpoint` (`pip install --upgrade gradio`).
+    > [!WARNING]
+    > `webhook_endpoint` is experimental. Its API is subject to change in the future.
 
-    </Tip>
+    > [!WARNING]
+    > You must have `gradio` installed to use `webhook_endpoint` (`pip install --upgrade gradio`).
 
     Args:
         path (`str`, optional):

huggingface_hub/cli/download.py

@@ -155,7 +155,7 @@ class DownloadCommand(BaseHuggingfaceCLICommand):
                 force_download=self.force_download,
                 token=self.token,
                 local_dir=self.local_dir,
-                library_name="hf",
+                library_name="huggingface-cli",
             )
 
         # Otherwise: use `snapshot_download` to ensure all files comes from same revision
@@ -176,6 +176,6 @@ class DownloadCommand(BaseHuggingfaceCLICommand):
             cache_dir=self.cache_dir,
             token=self.token,
             local_dir=self.local_dir,
-            library_name="hf",
+            library_name="huggingface-cli",
             max_workers=self.max_workers,
         )

huggingface_hub/cli/repo_files.py

@@ -51,7 +51,7 @@ class DeleteFilesSubCommand:
         self.repo_id: str = args.repo_id
         self.repo_type: Optional[str] = args.repo_type
         self.revision: Optional[str] = args.revision
-        self.api: HfApi = HfApi(token=args.token, library_name="hf")
+        self.api: HfApi = HfApi(token=args.token, library_name="huggingface-cli")
         self.patterns: List[str] = args.patterns
         self.commit_message: Optional[str] = args.commit_message
         self.commit_description: Optional[str] = args.commit_description

huggingface_hub/cli/upload.py

@@ -151,7 +151,7 @@ class UploadCommand(BaseHuggingfaceCLICommand):
         self.commit_message: Optional[str] = args.commit_message
         self.commit_description: Optional[str] = args.commit_description
         self.create_pr: bool = args.create_pr
-        self.api: HfApi = HfApi(token=args.token, library_name="hf")
+        self.api: HfApi = HfApi(token=args.token, library_name="huggingface-cli")
         self.quiet: bool = args.quiet  # disable warnings and progress bars
 
         # Check `--every` is valid

huggingface_hub/cli/upload_large_folder.py

@@ -79,7 +79,7 @@ class UploadLargeFolderCommand(BaseHuggingfaceCLICommand):
         self.include: Optional[List[str]] = args.include
         self.exclude: Optional[List[str]] = args.exclude
 
-        self.api: HfApi = HfApi(token=args.token, library_name="hf")
+        self.api: HfApi = HfApi(token=args.token, library_name="huggingface-cli")
 
         self.num_workers: Optional[int] = args.num_workers
         self.no_report: bool = args.no_report

huggingface_hub/community.py

@@ -7,7 +7,7 @@ for more information on Pull Requests, Discussions, and the community tab.
 
 from dataclasses import dataclass
 from datetime import datetime
-from typing import List, Literal, Optional, Union
+from typing import List, Literal, Optional, TypedDict, Union
 
 from . import constants
 from .utils import parse_datetime
@@ -143,6 +143,14 @@ class DiscussionWithDetails(Discussion):
     diff: Optional[str]
 
 
+class DiscussionEventArgs(TypedDict):
+    id: str
+    type: str
+    created_at: datetime
+    author: str
+    _event: dict
+
+
 @dataclass
 class DiscussionEvent:
     """
@@ -319,13 +327,13 @@ def deserialize_event(event: dict) -> DiscussionEvent:
     event_type: str = event["type"]
     created_at = parse_datetime(event["createdAt"])
 
-    common_args = dict(
-        id=event_id,
-        type=event_type,
-        created_at=created_at,
-        author=event.get("author", {}).get("name", "deleted"),
-        _event=event,
-    )
+    common_args: DiscussionEventArgs = {
+        "id": event_id,
+        "type": event_type,
+        "created_at": created_at,
+        "author": event.get("author", {}).get("name", "deleted"),
+        "_event": event,
+    }
 
     if event_type == "comment":
         return DiscussionComment(

huggingface_hub/fastai_utils.py

@@ -35,13 +35,11 @@ def _check_fastai_fastcore_versions(
         fastcore_min_version (`str`, *optional*):
             The minimum fastcore version supported.
 
-    <Tip>
-    Raises the following error:
-
-        - [`ImportError`](https://docs.python.org/3/library/exceptions.html#ImportError)
-          if the fastai or fastcore libraries are not available or are of an invalid version.
-
-    </Tip>
+    > [!TIP]
+    > Raises the following error:
+    >
+    >     - [`ImportError`](https://docs.python.org/3/library/exceptions.html#ImportError)
+    >       if the fastai or fastcore libraries are not available or are of an invalid version.
     """
 
     if (get_fastcore_version() or get_fastai_version()) == "N/A":
@@ -89,15 +87,13 @@ def _check_fastai_fastcore_pyproject_versions(
         fastcore_min_version (`str`, *optional*):
             The minimum fastcore version supported.
 
-    <Tip>
-    Raises the following errors:
-
-        - [`ImportError`](https://docs.python.org/3/library/exceptions.html#ImportError)
-          if the `toml` module is not installed.
-        - [`ImportError`](https://docs.python.org/3/library/exceptions.html#ImportError)
-          if the `pyproject.toml` indicates a lower than minimum supported version of fastai or fastcore.
-
-    </Tip>
+    > [!TIP]
+    > Raises the following errors:
+    >
+    >     - [`ImportError`](https://docs.python.org/3/library/exceptions.html#ImportError)
+    >       if the `toml` module is not installed.
+    >     - [`ImportError`](https://docs.python.org/3/library/exceptions.html#ImportError)
+    >       if the `pyproject.toml` indicates a lower than minimum supported version of fastai or fastcore.
     """
 
     try:
@@ -254,14 +250,11 @@ def _save_pretrained_fastai(
         config (`dict`, *optional*):
             Configuration object. Will be uploaded as a .json file. Example: 'https://huggingface.co/espejelomar/fastai-pet-breeds-classification/blob/main/config.json'.
 
-    <Tip>
-
-    Raises the following error:
-
-        - [`RuntimeError`](https://docs.python.org/3/library/exceptions.html#RuntimeError)
-          if the config file provided is not a dictionary.
-
-    </Tip>
+    > [!TIP]
+    > Raises the following error:
+    >
+    >     - [`RuntimeError`](https://docs.python.org/3/library/exceptions.html#RuntimeError)
+    >       if the config file provided is not a dictionary.
     """
     _check_fastai_fastcore_versions()
 
@@ -395,14 +388,11 @@ def push_to_hub_fastai(
     Returns:
         The url of the commit of your model in the given repository.
 
-    <Tip>
-
-    Raises the following error:
-
-        - [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError)
-          if the user is not log on to the Hugging Face Hub.
-
-    </Tip>
+    > [!TIP]
+    > Raises the following error:
+    >
+    >     - [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError)
+    >       if the user is not log on to the Hugging Face Hub.
     """
     _check_fastai_fastcore_versions()
     api = HfApi(endpoint=api_endpoint)

huggingface_hub/file_download.py

@@ -214,26 +214,23 @@ def hf_hub_url(
     'https://huggingface.co/julien-c/EsperBERTo-small/resolve/main/pytorch_model.bin'
     ```
 
-    <Tip>
-
-    Notes:
-
-        Cloudfront is replicated over the globe so downloads are way faster for
-        the end user (and it also lowers our bandwidth costs).
-
-        Cloudfront aggressively caches files by default (default TTL is 24
-        hours), however this is not an issue here because we implement a
-        git-based versioning system on huggingface.co, which means that we store
-        the files on S3/Cloudfront in a content-addressable way (i.e., the file
-        name is its hash). Using content-addressable filenames means cache can't
-        ever be stale.
-
-        In terms of client-side caching from this library, we base our caching
-        on the objects' entity tag (`ETag`), which is an identifier of a
-        specific version of a resource [1]_. An object's ETag is: its git-sha1
-        if stored in git, or its sha256 if stored in git-lfs.
-
-    </Tip>
+    > [!TIP]
+    > Notes:
+    >
+    >     Cloudfront is replicated over the globe so downloads are way faster for
+    >     the end user (and it also lowers our bandwidth costs).
+    >
+    >     Cloudfront aggressively caches files by default (default TTL is 24
+    >     hours), however this is not an issue here because we implement a
+    >     git-based versioning system on huggingface.co, which means that we store
+    >     the files on S3/Cloudfront in a content-addressable way (i.e., the file
+    >     name is its hash). Using content-addressable filenames means cache can't
+    >     ever be stale.
+    >
+    >     In terms of client-side caching from this library, we base our caching
+    >     on the objects' entity tag (`ETag`), which is an identifier of a
+    >     specific version of a resource [1]_. An object's ETag is: its git-sha1
+    >     if stored in git, or its sha256 if stored in git-lfs.
 
     References: