huggingface-hub 0.36.0__py3-none-any.whl → 1.0.0__py3-none-any.whl

huggingface_hub/_space_api.py

@@ -15,7 +15,7 @@
 from dataclasses import dataclass
 from datetime import datetime
 from enum import Enum
-from typing import Dict, Optional
+from typing import Optional
 
 from huggingface_hub.utils import parse_datetime
 
@@ -128,9 +128,9 @@ class SpaceRuntime:
     requested_hardware: Optional[SpaceHardware]
     sleep_time: Optional[int]
     storage: Optional[SpaceStorage]
-    raw: Dict
+    raw: dict
 
-    def __init__(self, data: Dict) -> None:
+    def __init__(self, data: dict) -> None:
         self.stage = data["stage"]
         self.hardware = data.get("hardware", {}).get("current")
         self.requested_hardware = data.get("hardware", {}).get("requested")
@@ -160,7 +160,7 @@ class SpaceVariable:
     description: Optional[str]
     updated_at: Optional[datetime]
 
-    def __init__(self, key: str, values: Dict) -> None:
+    def __init__(self, key: str, values: dict) -> None:
         self.key = key
         self.value = values["value"]
         self.description = values.get("description")

huggingface_hub/_tensorboard_logger.py

@@ -14,7 +14,7 @@
 """Contains a logger to push training logs to the Hub, using Tensorboard."""
 
 from pathlib import Path
-from typing import List, Optional, Union
+from typing import Optional, Union
 
 from ._commit_scheduler import CommitScheduler
 from .errors import EntryNotFoundError
@@ -74,10 +74,10 @@ class HFSummaryWriter(_RuntimeSummaryWriter):
             Whether to make the repo private. If `None` (default), the repo will be public unless the organization's default is private. This value is ignored if the repo already exists.
         path_in_repo (`str`, *optional*):
             The path to the folder in the repo where the logs will be pushed. Defaults to "tensorboard/".
-        repo_allow_patterns (`List[str]` or `str`, *optional*):
+        repo_allow_patterns (`list[str]` or `str`, *optional*):
             A list of patterns to include in the upload. Defaults to `"*.tfevents.*"`. Check out the
             [upload guide](https://huggingface.co/docs/huggingface_hub/guides/upload#upload-a-folder) for more details.
-        repo_ignore_patterns (`List[str]` or `str`, *optional*):
+        repo_ignore_patterns (`list[str]` or `str`, *optional*):
             A list of patterns to exclude in the upload. Check out the
             [upload guide](https://huggingface.co/docs/huggingface_hub/guides/upload#upload-a-folder) for more details.
         token (`str`, *optional*):
@@ -134,8 +134,8 @@ class HFSummaryWriter(_RuntimeSummaryWriter):
         repo_revision: Optional[str] = None,
         repo_private: Optional[bool] = None,
         path_in_repo: Optional[str] = "tensorboard",
-        repo_allow_patterns: Optional[Union[List[str], str]] = "*.tfevents.*",
-        repo_ignore_patterns: Optional[Union[List[str], str]] = None,
+        repo_allow_patterns: Optional[Union[list[str], str]] = "*.tfevents.*",
+        repo_ignore_patterns: Optional[Union[list[str], str]] = None,
         token: Optional[str] = None,
         **kwargs,
     ):

huggingface_hub/_upload_large_folder.py

@@ -24,15 +24,13 @@ import traceback
 from datetime import datetime
 from pathlib import Path
 from threading import Lock
-from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union
+from typing import TYPE_CHECKING, Any, Optional, Union
 from urllib.parse import quote
 
-from . import constants
 from ._commit_api import CommitOperationAdd, UploadInfo, _fetch_upload_modes
 from ._local_folder import LocalUploadFileMetadata, LocalUploadFilePaths, get_local_upload_paths, read_upload_metadata
 from .constants import DEFAULT_REVISION, REPO_TYPES
-from .utils import DEFAULT_IGNORE_PATTERNS, filter_repo_objects, tqdm
-from .utils._cache_manager import _format_size
+from .utils import DEFAULT_IGNORE_PATTERNS, _format_size, filter_repo_objects, tqdm
 from .utils._runtime import is_xet_available
 from .utils.sha import sha_fileobj
 
@@ -44,7 +42,7 @@ logger = logging.getLogger(__name__)
 
 WAITING_TIME_IF_NO_TASKS = 10  # seconds
 MAX_NB_FILES_FETCH_UPLOAD_MODE = 100
-COMMIT_SIZE_SCALE: List[int] = [20, 50, 75, 100, 125, 200, 250, 400, 600, 1000]
+COMMIT_SIZE_SCALE: list[int] = [20, 50, 75, 100, 125, 200, 250, 400, 600, 1000]
 
 UPLOAD_BATCH_SIZE_XET = 256  # Max 256 files per upload batch for XET-enabled repos
 UPLOAD_BATCH_SIZE_LFS = 1  # Otherwise, batches of 1 for regular LFS upload
@@ -56,7 +54,7 @@ MAX_FILE_SIZE_GB = 50  # Hard limit for individual file size
 RECOMMENDED_FILE_SIZE_GB = 20  # Recommended maximum for individual file size
 
 
-def _validate_upload_limits(paths_list: List[LocalUploadFilePaths]) -> None:
+def _validate_upload_limits(paths_list: list[LocalUploadFilePaths]) -> None:
     """
     Validate upload against repository limits and warn about potential issues.
 
@@ -85,7 +83,7 @@ def _validate_upload_limits(paths_list: List[LocalUploadFilePaths]) -> None:
     # Track immediate children (files and subdirs) for each folder
     from collections import defaultdict
 
-    entries_per_folder: Dict[str, Any] = defaultdict(lambda: {"files": 0, "subdirs": set()})
+    entries_per_folder: dict[str, Any] = defaultdict(lambda: {"files": 0, "subdirs": set()})
 
     for paths in paths_list:
         path = Path(paths.path_in_repo)
@@ -160,8 +158,8 @@ def upload_large_folder_internal(
     repo_type: str,  # Repo type is required!
     revision: Optional[str] = None,
     private: Optional[bool] = None,
-    allow_patterns: Optional[Union[List[str], str]] = None,
-    ignore_patterns: Optional[Union[List[str], str]] = None,
+    allow_patterns: Optional[Union[list[str], str]] = None,
+    ignore_patterns: Optional[Union[list[str], str]] = None,
     num_workers: Optional[int] = None,
     print_report: bool = True,
     print_report_every: int = 60,
@@ -200,16 +198,7 @@ def upload_large_folder_internal(
     logger.info(f"Repo created: {repo_url}")
     repo_id = repo_url.repo_id
     # 2.1 Check if xet is enabled to set batch file upload size
-    is_xet_enabled = (
-        is_xet_available()
-        and api.repo_info(
-            repo_id=repo_id,
-            repo_type=repo_type,
-            revision=revision,
-            expand="xetEnabled",
-        ).xet_enabled
-    )
-    upload_batch_size = UPLOAD_BATCH_SIZE_XET if is_xet_enabled else UPLOAD_BATCH_SIZE_LFS
+    upload_batch_size = UPLOAD_BATCH_SIZE_XET if is_xet_available() else UPLOAD_BATCH_SIZE_LFS
 
     # 3. List files to upload
     filtered_paths_list = filter_repo_objects(
@@ -284,13 +273,13 @@ class WorkerJob(enum.Enum):
     WAIT = enum.auto()  # if no tasks are available but we don't want to exit
 
 
-JOB_ITEM_T = Tuple[LocalUploadFilePaths, LocalUploadFileMetadata]
+JOB_ITEM_T = tuple[LocalUploadFilePaths, LocalUploadFileMetadata]
 
 
 class LargeUploadStatus:
     """Contains information, queues and tasks for a large upload process."""
 
-    def __init__(self, items: List[JOB_ITEM_T], upload_batch_size: int = 1):
+    def __init__(self, items: list[JOB_ITEM_T], upload_batch_size: int = 1):
         self.items = items
         self.queue_sha256: "queue.Queue[JOB_ITEM_T]" = queue.Queue()
         self.queue_get_upload_mode: "queue.Queue[JOB_ITEM_T]" = queue.Queue()
@@ -423,7 +412,7 @@ def _worker_job(
     Read `upload_large_folder` docstring for more information on how tasks are prioritized.
     """
     while True:
-        next_job: Optional[Tuple[WorkerJob, List[JOB_ITEM_T]]] = None
+        next_job: Optional[tuple[WorkerJob, list[JOB_ITEM_T]]] = None
 
         # Determine next task
         next_job = _determine_next_job(status)
@@ -516,7 +505,7 @@ def _worker_job(
                 status.nb_workers_waiting -= 1
 
 
-def _determine_next_job(status: LargeUploadStatus) -> Optional[Tuple[WorkerJob, List[JOB_ITEM_T]]]:
+def _determine_next_job(status: LargeUploadStatus) -> Optional[tuple[WorkerJob, list[JOB_ITEM_T]]]:
     with status.lock:
         # 1. Commit if more than 5 minutes since last commit attempt (and at least 1 file)
         if (
@@ -560,10 +549,7 @@ def _determine_next_job(status: LargeUploadStatus) -> Optional[Tuple[WorkerJob,
             return (WorkerJob.GET_UPLOAD_MODE, _get_n(status.queue_get_upload_mode, MAX_NB_FILES_FETCH_UPLOAD_MODE))
 
         # 7. Preupload LFS file if at least `status.upload_batch_size` files
-        #    Skip if hf_transfer is enabled and there is already a worker preuploading LFS
-        elif status.queue_preupload_lfs.qsize() >= status.upload_batch_size and (
-            status.nb_workers_preupload_lfs == 0 or not constants.HF_HUB_ENABLE_HF_TRANSFER
-        ):
+        elif status.queue_preupload_lfs.qsize() >= status.upload_batch_size:
             status.nb_workers_preupload_lfs += 1
             logger.debug("Job: preupload LFS")
             return (WorkerJob.PREUPLOAD_LFS, _get_n(status.queue_preupload_lfs, status.upload_batch_size))
@@ -639,7 +625,7 @@ def _compute_sha256(item: JOB_ITEM_T) -> None:
     metadata.save(paths)
 
 
-def _get_upload_mode(items: List[JOB_ITEM_T], api: "HfApi", repo_id: str, repo_type: str, revision: str) -> None:
+def _get_upload_mode(items: list[JOB_ITEM_T], api: "HfApi", repo_id: str, repo_type: str, revision: str) -> None:
     """Get upload mode for each file and update metadata.
 
     Also receive info if the file should be ignored.
@@ -661,7 +647,7 @@ def _get_upload_mode(items: List[JOB_ITEM_T], api: "HfApi", repo_id: str, repo_t
         metadata.save(paths)
 
 
-def _preupload_lfs(items: List[JOB_ITEM_T], api: "HfApi", repo_id: str, repo_type: str, revision: str) -> None:
+def _preupload_lfs(items: list[JOB_ITEM_T], api: "HfApi", repo_id: str, repo_type: str, revision: str) -> None:
     """Preupload LFS files and update metadata."""
     additions = [_build_hacky_operation(item) for item in items]
     api.preupload_lfs_files(
@@ -676,7 +662,7 @@ def _preupload_lfs(items: List[JOB_ITEM_T], api: "HfApi", repo_id: str, repo_typ
         metadata.save(paths)
 
 
-def _commit(items: List[JOB_ITEM_T], api: "HfApi", repo_id: str, repo_type: str, revision: str) -> None:
+def _commit(items: list[JOB_ITEM_T], api: "HfApi", repo_id: str, repo_type: str, revision: str) -> None:
     """Commit files to the repo."""
     additions = [_build_hacky_operation(item) for item in items]
     api.create_commit(
@@ -721,11 +707,11 @@ def _build_hacky_operation(item: JOB_ITEM_T) -> HackyCommitOperationAdd:
 ####################
 
 
-def _get_one(queue: "queue.Queue[JOB_ITEM_T]") -> List[JOB_ITEM_T]:
+def _get_one(queue: "queue.Queue[JOB_ITEM_T]") -> list[JOB_ITEM_T]:
     return [queue.get()]
 
 
-def _get_n(queue: "queue.Queue[JOB_ITEM_T]", n: int) -> List[JOB_ITEM_T]:
+def _get_n(queue: "queue.Queue[JOB_ITEM_T]", n: int) -> list[JOB_ITEM_T]:
     return [queue.get() for _ in range(min(queue.qsize(), n))]
 
 

huggingface_hub/_webhooks_payload.py

@@ -14,7 +14,7 @@
 # limitations under the License.
 """Contains data structures to parse the webhooks payload."""
 
-from typing import List, Literal, Optional
+from typing import Literal, Optional
 
 from .utils import is_pydantic_available
 
@@ -116,7 +116,7 @@ class WebhookPayloadRepo(ObjectId):
     name: str
     private: bool
     subdomain: Optional[str] = None
-    tags: Optional[List[str]] = None
+    tags: Optional[list[str]] = None
     type: Literal["dataset", "model", "space"]
     url: WebhookPayloadUrl
 
@@ -134,4 +134,4 @@ class WebhookPayload(BaseModel):
     comment: Optional[WebhookPayloadComment] = None
     webhook: WebhookPayloadWebhook
     movedTo: Optional[WebhookPayloadMovedTo] = None
-    updatedRefs: Optional[List[WebhookPayloadUpdatedRef]] = None
+    updatedRefs: Optional[list[WebhookPayloadUpdatedRef]] = None

huggingface_hub/_webhooks_server.py

@@ -18,7 +18,7 @@ import atexit
 import inspect
 import os
 from functools import wraps
-from typing import TYPE_CHECKING, Any, Callable, Dict, Optional
+from typing import TYPE_CHECKING, Any, Callable, Optional
 
 from .utils import experimental, is_fastapi_available, is_gradio_available
 
@@ -109,7 +109,7 @@ class WebhooksServer:
         self._ui = ui
 
         self.webhook_secret = webhook_secret or os.getenv("WEBHOOK_SECRET")
-        self.registered_webhooks: Dict[str, Callable] = {}
+        self.registered_webhooks: dict[str, Callable] = {}
         _warn_on_empty_secret(self.webhook_secret)
 
     def add_webhook(self, path: Optional[str] = None) -> Callable:

huggingface_hub/cli/__init__.py

@@ -11,17 +11,3 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
-from abc import ABC, abstractmethod
-from argparse import _SubParsersAction
-
-
-class BaseHuggingfaceCLICommand(ABC):
-    @staticmethod
-    @abstractmethod
-    def register_subcommand(parser: _SubParsersAction):
-        raise NotImplementedError()
-
-    @abstractmethod
-    def run(self):
-        raise NotImplementedError()

huggingface_hub/cli/_cli_utils.py

@@ -11,59 +11,163 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-"""Contains a utility for good-looking prints."""
+"""Contains CLI utilities (styling, helpers)."""
 
+import importlib.metadata
 import os
-from typing import List, Union
+import time
+from enum import Enum
+from pathlib import Path
+from typing import TYPE_CHECKING, Annotated, Optional
 
+import click
+import typer
 
-class ANSI:
+from huggingface_hub import __version__, constants
+from huggingface_hub.utils import ANSI, get_session, hf_raise_for_status, installation_method, logging
+
+
+logger = logging.get_logger()
+
+
+if TYPE_CHECKING:
+    from huggingface_hub.hf_api import HfApi
+
+
+def get_hf_api(token: Optional[str] = None) -> "HfApi":
+    # Import here to avoid circular import
+    from huggingface_hub.hf_api import HfApi
+
+    return HfApi(token=token, library_name="huggingface-cli", library_version=__version__)
+
+
+#### TYPER UTILS
+
+
+class AlphabeticalMixedGroup(typer.core.TyperGroup):
     """
-    Helper for en.wikipedia.org/wiki/ANSI_escape_code
+    Typer Group that lists commands and sub-apps mixed and alphabetically.
     """
 
-    _bold = "\u001b[1m"
-    _gray = "\u001b[90m"
-    _red = "\u001b[31m"
-    _reset = "\u001b[0m"
-    _yellow = "\u001b[33m"
+    def list_commands(self, ctx: click.Context) -> list[str]:  # type: ignore[name-defined]
+        # click.Group stores both commands and sub-groups in `self.commands`
+        return sorted(self.commands.keys())
+
+
+def typer_factory(help: str) -> typer.Typer:
+    return typer.Typer(
+        help=help,
+        add_completion=True,
+        no_args_is_help=True,
+        cls=AlphabeticalMixedGroup,
+        # Disable rich completely for consistent experience
+        rich_markup_mode=None,
+        rich_help_panel=None,
+        pretty_exceptions_enable=False,
+    )
+
 
-    @classmethod
-    def bold(cls, s: str) -> str:
-        return cls._format(s, cls._bold)
+class RepoType(str, Enum):
+    model = "model"
+    dataset = "dataset"
+    space = "space"
 
-    @classmethod
-    def gray(cls, s: str) -> str:
-        return cls._format(s, cls._gray)
 
-    @classmethod
-    def red(cls, s: str) -> str:
-        return cls._format(s, cls._bold + cls._red)
+RepoIdArg = Annotated[
+    str,
+    typer.Argument(
+        help="The ID of the repo (e.g. `username/repo-name`).",
+    ),
+]
 
-    @classmethod
-    def yellow(cls, s: str) -> str:
-        return cls._format(s, cls._yellow)
 
-    @classmethod
-    def _format(cls, s: str, code: str) -> str:
-        if os.environ.get("NO_COLOR"):
-            # See https://no-color.org/
-            return s
-        return f"{code}{s}{cls._reset}"
+RepoTypeOpt = Annotated[
+    RepoType,
+    typer.Option(
+        help="The type of repository (model, dataset, or space).",
+    ),
+]
 
+TokenOpt = Annotated[
+    Optional[str],
+    typer.Option(
+        help="A User Access Token generated from https://huggingface.co/settings/tokens.",
+    ),
+]
 
-def tabulate(rows: List[List[Union[str, int]]], headers: List[str]) -> str:
+PrivateOpt = Annotated[
+    bool,
+    typer.Option(
+        help="Whether to create a private repo if repo doesn't exist on the Hub. Ignored if the repo already exists.",
+    ),
+]
+
+RevisionOpt = Annotated[
+    Optional[str],
+    typer.Option(
+        help="Git revision id which can be a branch name, a tag, or a commit hash.",
+    ),
+]
+
+
+### PyPI VERSION CHECKER
+
+
+def check_cli_update() -> None:
     """
-    Inspired by:
+    Check whether a newer version of `huggingface_hub` is available on PyPI.
+
+    If a newer version is found, notify the user and suggest updating.
+    If current version is a pre-release (e.g. `1.0.0.rc1`), or a dev version (e.g. `1.0.0.dev1`), no check is performed.
 
-    - stackoverflow.com/a/8356620/593036
-    - stackoverflow.com/questions/9535954/printing-lists-as-tabular-data
+    This function is called at the entry point of the CLI. It only performs the check once every 24 hours, and any error
+    during the check is caught and logged, to avoid breaking the CLI.
     """
-    col_widths = [max(len(str(x)) for x in col) for col in zip(*rows, headers)]
-    row_format = ("{{:{}}} " * len(headers)).format(*col_widths)
-    lines = []
-    lines.append(row_format.format(*headers))
-    lines.append(row_format.format(*["-" * w for w in col_widths]))
-    for row in rows:
-        lines.append(row_format.format(*row))
-    return "\n".join(lines)
+    try:
+        _check_cli_update()
+    except Exception:
+        # We don't want the CLI to fail on version checks, no matter the reason.
+        logger.debug("Error while checking for CLI update.", exc_info=True)
+
+
+def _check_cli_update() -> None:
+    current_version = importlib.metadata.version("huggingface_hub")
+
+    # Skip if current version is a pre-release or dev version
+    if any(tag in current_version for tag in ["rc", "dev"]):
+        return
+
+    # Skip if already checked in the last 24 hours
+    if os.path.exists(constants.CHECK_FOR_UPDATE_DONE_PATH):
+        mtime = os.path.getmtime(constants.CHECK_FOR_UPDATE_DONE_PATH)
+        if (time.time() - mtime) < 24 * 3600:
+            return
+
+    # Touch the file to mark that we did the check now
+    Path(constants.CHECK_FOR_UPDATE_DONE_PATH).touch()
+
+    # Check latest version from PyPI
+    response = get_session().get("https://pypi.org/pypi/huggingface_hub/json", timeout=2)
+    hf_raise_for_status(response)
+    data = response.json()
+    latest_version = data["info"]["version"]
+
+    # If latest version is different from current, notify user
+    if current_version != latest_version:
+        method = installation_method()
+        if method == "brew":
+            update_command = "brew upgrade huggingface-cli"
+        elif method == "hf_installer" and os.name == "nt":
+            update_command = 'powershell -NoProfile -Command "iwr -useb https://hf.co/cli/install.ps1 | iex"'
+        elif method == "hf_installer":
+            update_command = "curl -LsSf https://hf.co/cli/install.sh | sh -"
+        else:  # unknown => likely pip
+            update_command = "pip install -U huggingface_hub"
+
+        click.echo(
+            ANSI.yellow(
+                f"A new version of huggingface_hub ({latest_version}) is available! "
+                f"You are using version {current_version}.\n"
+                f"To update, run: {ANSI.bold(update_command)}\n",
+            )
+        )