huggingface-hub 1.0.0rc0__py3-none-any.whl → 1.0.0rc2__py3-none-any.whl

huggingface_hub/cli/upload_large_folder.py

@@ -15,118 +15,103 @@
 """Contains command to upload a large folder with the CLI."""
 
 import os
-from argparse import Namespace, _SubParsersAction
-from typing import Optional
+from typing import Annotated, Optional
+
+import typer
 
 from huggingface_hub import logging
-from huggingface_hub.commands import BaseHuggingfaceCLICommand
-from huggingface_hub.hf_api import HfApi
 from huggingface_hub.utils import disable_progress_bars
 
-from ._cli_utils import ANSI
+from ._cli_utils import ANSI, PrivateOpt, RepoIdArg, RepoType, RepoTypeOpt, RevisionOpt, TokenOpt, get_hf_api
 
 
 logger = logging.get_logger(__name__)
 
 
-class UploadLargeFolderCommand(BaseHuggingfaceCLICommand):
-    @staticmethod
-    def register_subcommand(parser: _SubParsersAction):
-        subparser = parser.add_parser(
-            "upload-large-folder",
-            help="Upload a large folder to the Hub. Recommended for resumable uploads.",
-        )
-        subparser.add_argument(
-            "repo_id", type=str, help="The ID of the repo to upload to (e.g. `username/repo-name`)."
-        )
-        subparser.add_argument("local_path", type=str, help="Local path to the file or folder to upload.")
-        subparser.add_argument(
-            "--repo-type",
-            choices=["model", "dataset", "space"],
-            help="Type of the repo to upload to (e.g. `dataset`).",
-        )
-        subparser.add_argument(
-            "--revision",
-            type=str,
-            help=("An optional Git revision to push to. It can be a branch name or a PR reference."),
-        )
-        subparser.add_argument(
-            "--private",
-            action="store_true",
-            help=(
-                "Whether to create a private repo if repo doesn't exist on the Hub. Ignored if the repo already exists."
-            ),
-        )
-        subparser.add_argument("--include", nargs="*", type=str, help="Glob patterns to match files to upload.")
-        subparser.add_argument("--exclude", nargs="*", type=str, help="Glob patterns to exclude from files to upload.")
-        subparser.add_argument(
-            "--token", type=str, help="A User Access Token generated from https://huggingface.co/settings/tokens"
-        )
-        subparser.add_argument(
-            "--num-workers", type=int, help="Number of workers to use to hash, upload and commit files."
-        )
-        subparser.add_argument("--no-report", action="store_true", help="Whether to disable regular status report.")
-        subparser.add_argument("--no-bars", action="store_true", help="Whether to disable progress bars.")
-        subparser.set_defaults(func=UploadLargeFolderCommand)
-
-    def __init__(self, args: Namespace) -> None:
-        self.repo_id: str = args.repo_id
-        self.local_path: str = args.local_path
-        self.repo_type: str = args.repo_type
-        self.revision: Optional[str] = args.revision
-        self.private: bool = args.private
-
-        self.include: Optional[list[str]] = args.include
-        self.exclude: Optional[list[str]] = args.exclude
-
-        self.api: HfApi = HfApi(token=args.token, library_name="hf")
-
-        self.num_workers: Optional[int] = args.num_workers
-        self.no_report: bool = args.no_report
-        self.no_bars: bool = args.no_bars
-
-        if not os.path.isdir(self.local_path):
-            raise ValueError("Large upload is only supported for folders.")
-
-    def run(self) -> None:
-        logging.set_verbosity_info()
-
-        print(
-            ANSI.yellow(
-                "You are about to upload a large folder to the Hub using `hf upload-large-folder`. "
-                "This is a new feature so feedback is very welcome!\n"
-                "\n"
-                "A few things to keep in mind:\n"
-                "  - Repository limits still apply: https://huggingface.co/docs/hub/repositories-recommendations\n"
-                "  - Do not start several processes in parallel.\n"
-                "  - You can interrupt and resume the process at any time. "
-                "The script will pick up where it left off except for partially uploaded files that would have to be entirely reuploaded.\n"
-                "  - Do not upload the same folder to several repositories. If you need to do so, you must delete the `./.cache/huggingface/` folder first.\n"
-                "\n"
-                f"Some temporary metadata will be stored under `{self.local_path}/.cache/huggingface`.\n"
-                "  - You must not modify those files manually.\n"
-                "  - You must not delete the `./.cache/huggingface/` folder while a process is running.\n"
-                "  - You can delete the `./.cache/huggingface/` folder to reinitialize the upload state when process is not running. Files will have to be hashed and preuploaded again, except for already committed files.\n"
-                "\n"
-                "If the process output is too verbose, you can disable the progress bars with `--no-bars`. "
-                "You can also entirely disable the status report with `--no-report`.\n"
-                "\n"
-                "For more details, run `hf upload-large-folder --help` or check the documentation at "
-                "https://huggingface.co/docs/huggingface_hub/guides/upload#upload-a-large-folder."
-            )
-        )
-
-        if self.no_bars:
-            disable_progress_bars()
-
-        self.api.upload_large_folder(
-            repo_id=self.repo_id,
-            folder_path=self.local_path,
-            repo_type=self.repo_type,
-            revision=self.revision,
-            private=self.private,
-            allow_patterns=self.include,
-            ignore_patterns=self.exclude,
-            num_workers=self.num_workers,
-            print_report=not self.no_report,
+def upload_large_folder(
+    repo_id: RepoIdArg,
+    local_path: Annotated[
+        str,
+        typer.Argument(
+            help="Local path to the folder to upload.",
+        ),
+    ],
+    repo_type: RepoTypeOpt = RepoType.model,
+    revision: RevisionOpt = None,
+    private: PrivateOpt = False,
+    include: Annotated[
+        Optional[list[str]],
+        typer.Option(
+            help="Glob patterns to match files to upload.",
+        ),
+    ] = None,
+    exclude: Annotated[
+        Optional[list[str]],
+        typer.Option(
+            help="Glob patterns to exclude from files to upload.",
+        ),
+    ] = None,
+    token: TokenOpt = None,
+    num_workers: Annotated[
+        Optional[int],
+        typer.Option(
+            help="Number of workers to use to hash, upload and commit files.",
+        ),
+    ] = None,
+    no_report: Annotated[
+        bool,
+        typer.Option(
+            help="Whether to disable regular status report.",
+        ),
+    ] = False,
+    no_bars: Annotated[
+        bool,
+        typer.Option(
+            help="Whether to disable progress bars.",
+        ),
+    ] = False,
+) -> None:
+    """Upload a large folder to the Hub. Recommended for resumable uploads."""
+    if not os.path.isdir(local_path):
+        raise typer.BadParameter("Large upload is only supported for folders.", param_hint="local_path")
+
+    print(
+        ANSI.yellow(
+            "You are about to upload a large folder to the Hub using `hf upload-large-folder`. "
+            "This is a new feature so feedback is very welcome!\n"
+            "\n"
+            "A few things to keep in mind:\n"
+            "  - Repository limits still apply: https://huggingface.co/docs/hub/repositories-recommendations\n"
+            "  - Do not start several processes in parallel.\n"
+            "  - You can interrupt and resume the process at any time. "
+            "The script will pick up where it left off except for partially uploaded files that would have to be entirely reuploaded.\n"
+            "  - Do not upload the same folder to several repositories. If you need to do so, you must delete the `./.cache/huggingface/` folder first.\n"
+            "\n"
+            f"Some temporary metadata will be stored under `{local_path}/.cache/huggingface`.\n"
+            "  - You must not modify those files manually.\n"
+            "  - You must not delete the `./.cache/huggingface/` folder while a process is running.\n"
+            "  - You can delete the `./.cache/huggingface/` folder to reinitialize the upload state when process is not running. Files will have to be hashed and preuploaded again, except for already committed files.\n"
+            "\n"
+            "If the process output is too verbose, you can disable the progress bars with `--no-bars`. "
+            "You can also entirely disable the status report with `--no-report`.\n"
+            "\n"
+            "For more details, run `hf upload-large-folder --help` or check the documentation at "
+            "https://huggingface.co/docs/huggingface_hub/guides/upload#upload-a-large-folder."
         )
+    )
+
+    if no_bars:
+        disable_progress_bars()
+
+    api = get_hf_api(token=token)
+    api.upload_large_folder(
+        repo_id=repo_id,
+        folder_path=local_path,
+        repo_type=repo_type.value,
+        revision=revision,
+        private=private,
+        allow_patterns=include,
+        ignore_patterns=exclude,
+        num_workers=num_workers,
+        print_report=not no_report,
+    )

huggingface_hub/dataclasses.py

@@ -1,18 +1,7 @@
 import inspect
 from dataclasses import _MISSING_TYPE, MISSING, Field, field, fields
 from functools import wraps
-from typing import (
-    Any,
-    Callable,
-    Literal,
-    Optional,
-    Type,
-    TypeVar,
-    Union,
-    get_args,
-    get_origin,
-    overload,
-)
+from typing import Any, Callable, ForwardRef, Literal, Optional, Type, TypeVar, Union, get_args, get_origin, overload
 
 from .errors import (
     StrictDataclassClassValidationError,
@@ -322,6 +311,8 @@ def type_validator(name: str, value: Any, expected_type: Any) -> None:
         validator(name, value, args)
     elif isinstance(expected_type, type):  # simple types
         _validate_simple_type(name, value, expected_type)
+    elif isinstance(expected_type, ForwardRef) or isinstance(expected_type, str):
+        return
     else:
         raise TypeError(f"Unsupported type for field '{name}': {expected_type}")
 

huggingface_hub/errors.py

@@ -37,7 +37,7 @@ class OfflineModeIsEnabled(ConnectionError):
     """Raised when a request is made but `HF_HUB_OFFLINE=1` is set as environment variable."""
 
 
-class HfHubHTTPError(HTTPError):
+class HfHubHTTPError(HTTPError, OSError):
     """
     HTTPError to inherit from for any custom HTTP Error raised in HF Hub.
 

huggingface_hub/fastai_utils.py

@@ -34,13 +34,11 @@ def _check_fastai_fastcore_versions(
         fastcore_min_version (`str`, *optional*):
             The minimum fastcore version supported.
 
-    <Tip>
-    Raises the following error:
-
-        - [`ImportError`](https://docs.python.org/3/library/exceptions.html#ImportError)
-          if the fastai or fastcore libraries are not available or are of an invalid version.
-
-    </Tip>
+    > [!TIP]
+    > Raises the following error:
+    >
+    >     - [`ImportError`](https://docs.python.org/3/library/exceptions.html#ImportError)
+    >       if the fastai or fastcore libraries are not available or are of an invalid version.
     """
 
     if (get_fastcore_version() or get_fastai_version()) == "N/A":
@@ -88,15 +86,13 @@ def _check_fastai_fastcore_pyproject_versions(
         fastcore_min_version (`str`, *optional*):
             The minimum fastcore version supported.
 
-    <Tip>
-    Raises the following errors:
-
-        - [`ImportError`](https://docs.python.org/3/library/exceptions.html#ImportError)
-          if the `toml` module is not installed.
-        - [`ImportError`](https://docs.python.org/3/library/exceptions.html#ImportError)
-          if the `pyproject.toml` indicates a lower than minimum supported version of fastai or fastcore.
-
-    </Tip>
+    > [!TIP]
+    > Raises the following errors:
+    >
+    >     - [`ImportError`](https://docs.python.org/3/library/exceptions.html#ImportError)
+    >       if the `toml` module is not installed.
+    >     - [`ImportError`](https://docs.python.org/3/library/exceptions.html#ImportError)
+    >       if the `pyproject.toml` indicates a lower than minimum supported version of fastai or fastcore.
     """
 
     try:
@@ -253,14 +249,11 @@ def _save_pretrained_fastai(
         config (`dict`, *optional*):
             Configuration object. Will be uploaded as a .json file. Example: 'https://huggingface.co/espejelomar/fastai-pet-breeds-classification/blob/main/config.json'.
 
-    <Tip>
-
-    Raises the following error:
-
-        - [`RuntimeError`](https://docs.python.org/3/library/exceptions.html#RuntimeError)
-          if the config file provided is not a dictionary.
-
-    </Tip>
+    > [!TIP]
+    > Raises the following error:
+    >
+    >     - [`RuntimeError`](https://docs.python.org/3/library/exceptions.html#RuntimeError)
+    >       if the config file provided is not a dictionary.
     """
     _check_fastai_fastcore_versions()
 
@@ -394,14 +387,11 @@ def push_to_hub_fastai(
     Returns:
         The url of the commit of your model in the given repository.
 
-    <Tip>
-
-    Raises the following error:
-
-        - [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError)
-          if the user is not log on to the Hugging Face Hub.
-
-    </Tip>
+    > [!TIP]
+    > Raises the following error:
+    >
+    >     - [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError)
+    >       if the user is not log on to the Hugging Face Hub.
     """
     _check_fastai_fastcore_versions()
     api = HfApi(endpoint=api_endpoint)

huggingface_hub/file_download.py

@@ -191,26 +191,23 @@ def hf_hub_url(
     'https://huggingface.co/julien-c/EsperBERTo-small/resolve/main/pytorch_model.bin'
     ```
 
-    <Tip>
-
-    Notes:
-
-        Cloudfront is replicated over the globe so downloads are way faster for
-        the end user (and it also lowers our bandwidth costs).
-
-        Cloudfront aggressively caches files by default (default TTL is 24
-        hours), however this is not an issue here because we implement a
-        git-based versioning system on huggingface.co, which means that we store
-        the files on S3/Cloudfront in a content-addressable way (i.e., the file
-        name is its hash). Using content-addressable filenames means cache can't
-        ever be stale.
-
-        In terms of client-side caching from this library, we base our caching
-        on the objects' entity tag (`ETag`), which is an identifier of a
-        specific version of a resource [1]_. An object's ETag is: its git-sha1
-        if stored in git, or its sha256 if stored in git-lfs.
-
-    </Tip>
+    > [!TIP]
+    > Notes:
+    >
+    >     Cloudfront is replicated over the globe so downloads are way faster for
+    >     the end user (and it also lowers our bandwidth costs).
+    >
+    >     Cloudfront aggressively caches files by default (default TTL is 24
+    >     hours), however this is not an issue here because we implement a
+    >     git-based versioning system on huggingface.co, which means that we store
+    >     the files on S3/Cloudfront in a content-addressable way (i.e., the file
+    >     name is its hash). Using content-addressable filenames means cache can't
+    >     ever be stale.
+    >
+    >     In terms of client-side caching from this library, we base our caching
+    >     on the objects' entity tag (`ETag`), which is an identifier of a
+    >     specific version of a resource [1]_. An object's ETag is: its git-sha1
+    >     if stored in git, or its sha256 if stored in git-lfs.
 
     References:
 
@@ -243,7 +240,7 @@ def _httpx_follow_relative_redirects(method: HTTP_METHOD_T, url: str, **httpx_kw
 
     This is useful to follow a redirection to a renamed repository without following redirection to a CDN.
 
-    A backoff mechanism retries the HTTP call on 429, 503 and 504 errors.
+    A backoff mechanism retries the HTTP call on 5xx errors and network errors.
 
     Args:
         method (`str`):