huggingface-hub 0.31.0rc0__py3-none-any.whl → 1.1.3__py3-none-any.whl

huggingface_hub/cli/lfs.py

@@ -0,0 +1,175 @@
+"""
+Implementation of a custom transfer agent for the transfer type "multipart" for
+git-lfs.
+
+Inspired by:
+github.com/cbartz/git-lfs-swift-transfer-agent/blob/master/git_lfs_swift_transfer.py
+
+Spec is: github.com/git-lfs/git-lfs/blob/master/docs/custom-transfers.md
+
+
+To launch debugger while developing:
+
+``` [lfs "customtransfer.multipart"]
+path = /path/to/huggingface_hub/.env/bin/python args = -m debugpy --listen 5678
+--wait-for-client
+/path/to/huggingface_hub/src/huggingface_hub/commands/huggingface_cli.py
+lfs-multipart-upload ```"""
+
+import json
+import os
+import subprocess
+import sys
+from typing import Annotated, Optional
+
+import typer
+
+from huggingface_hub.lfs import LFS_MULTIPART_UPLOAD_COMMAND
+
+from ..utils import get_session, hf_raise_for_status, logging
+from ..utils._lfs import SliceFileObj
+
+
+logger = logging.get_logger(__name__)
+
+
+def lfs_enable_largefiles(
+    path: Annotated[
+        str,
+        typer.Argument(
+            help="Local path to repository you want to configure.",
+        ),
+    ],
+) -> None:
+    """
+    Configure a local git repository to use the multipart transfer agent for large files.
+
+    This command sets up git-lfs to use the custom multipart transfer agent
+    which enables efficient uploading of large files in chunks.
+    """
+    local_path = os.path.abspath(path)
+    if not os.path.isdir(local_path):
+        print("This does not look like a valid git repo.")
+        raise typer.Exit(code=1)
+    subprocess.run(
+        "git config lfs.customtransfer.multipart.path hf".split(),
+        check=True,
+        cwd=local_path,
+    )
+    subprocess.run(
+        f"git config lfs.customtransfer.multipart.args {LFS_MULTIPART_UPLOAD_COMMAND}".split(),
+        check=True,
+        cwd=local_path,
+    )
+    print("Local repo set up for largefiles")
+
+
+def write_msg(msg: dict):
+    """Write out the message in Line delimited JSON."""
+    msg_str = json.dumps(msg) + "\n"
+    sys.stdout.write(msg_str)
+    sys.stdout.flush()
+
+
+def read_msg() -> Optional[dict]:
+    """Read Line delimited JSON from stdin."""
+    msg = json.loads(sys.stdin.readline().strip())
+
+    if "terminate" in (msg.get("type"), msg.get("event")):
+        # terminate message received
+        return None
+
+    if msg.get("event") not in ("download", "upload"):
+        logger.critical("Received unexpected message")
+        sys.exit(1)
+
+    return msg
+
+
+def lfs_multipart_upload() -> None:
+    """Internal git-lfs custom transfer agent for multipart uploads.
+
+    This function implements the custom transfer protocol for git-lfs multipart uploads.
+    Handles chunked uploads of large files to Hugging Face Hub.
+    """
+    # Immediately after invoking a custom transfer process, git-lfs
+    # sends initiation data to the process over stdin.
+    # This tells the process useful information about the configuration.
+    init_msg = json.loads(sys.stdin.readline().strip())
+    if not (init_msg.get("event") == "init" and init_msg.get("operation") == "upload"):
+        write_msg({"error": {"code": 32, "message": "Wrong lfs init operation"}})
+        sys.exit(1)
+
+    # The transfer process should use the information it needs from the
+    # initiation structure, and also perform any one-off setup tasks it
+    # needs to do. It should then respond on stdout with a simple empty
+    # confirmation structure, as follows:
+    write_msg({})
+
+    # After the initiation exchange, git-lfs will send any number of
+    # transfer requests to the stdin of the transfer process, in a serial sequence.
+    while True:
+        msg = read_msg()
+        if msg is None:
+            # When all transfers have been processed, git-lfs will send
+            # a terminate event to the stdin of the transfer process.
+            # On receiving this message the transfer process should
+            # clean up and terminate. No response is expected.
+            sys.exit(0)
+
+        oid = msg["oid"]
+        filepath = msg["path"]
+        completion_url = msg["action"]["href"]
+        header = msg["action"]["header"]
+        chunk_size = int(header.pop("chunk_size"))
+        presigned_urls: list[str] = list(header.values())
+
+        # Send a "started" progress event to allow other workers to start.
+        # Otherwise they're delayed until first "progress" event is reported,
+        # i.e. after the first 5GB by default (!)
+        write_msg(
+            {
+                "event": "progress",
+                "oid": oid,
+                "bytesSoFar": 1,
+                "bytesSinceLast": 0,
+            }
+        )
+
+        parts = []
+        with open(filepath, "rb") as file:
+            for i, presigned_url in enumerate(presigned_urls):
+                with SliceFileObj(
+                    file,
+                    seek_from=i * chunk_size,
+                    read_limit=chunk_size,
+                ) as data:
+                    r = get_session().put(presigned_url, data=data)
+                    hf_raise_for_status(r)
+                    parts.append(
+                        {
+                            "etag": r.headers.get("etag"),
+                            "partNumber": i + 1,
+                        }
+                    )
+                    # In order to support progress reporting while data is uploading / downloading,
+                    # the transfer process should post messages to stdout
+                    write_msg(
+                        {
+                            "event": "progress",
+                            "oid": oid,
+                            "bytesSoFar": (i + 1) * chunk_size,
+                            "bytesSinceLast": chunk_size,
+                        }
+                    )
+
+        r = get_session().post(
+            completion_url,
+            json={
+                "oid": oid,
+                "parts": parts,
+            },
+        )
+        hf_raise_for_status(r)
+
+        write_msg({"event": "complete", "oid": oid})

huggingface_hub/cli/repo.py

@@ -0,0 +1,315 @@
+# Copyright 2025 The HuggingFace Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Contains commands to interact with repositories on the Hugging Face Hub.
+
+Usage:
+    # create a new dataset repo on the Hub
+    hf repo create my-cool-dataset --repo-type=dataset
+
+    # create a private model repo on the Hub
+    hf repo create my-cool-model --private
+"""
+
+import enum
+from typing import Annotated, Optional
+
+import typer
+
+from huggingface_hub.errors import HfHubHTTPError, RepositoryNotFoundError, RevisionNotFoundError
+from huggingface_hub.utils import ANSI, logging
+
+from ._cli_utils import (
+    PrivateOpt,
+    RepoIdArg,
+    RepoType,
+    RepoTypeOpt,
+    RevisionOpt,
+    TokenOpt,
+    get_hf_api,
+    typer_factory,
+)
+
+
+logger = logging.get_logger(__name__)
+
+repo_cli = typer_factory(help="Manage repos on the Hub.")
+tag_cli = typer_factory(help="Manage tags for a repo on the Hub.")
+branch_cli = typer_factory(help="Manage branches for a repo on the Hub.")
+repo_cli.add_typer(tag_cli, name="tag")
+repo_cli.add_typer(branch_cli, name="branch")
+
+
+class GatedChoices(str, enum.Enum):
+    auto = "auto"
+    manual = "manual"
+    false = "false"
+
+
+@repo_cli.command("create", help="Create a new repo on the Hub.")
+def repo_create(
+    repo_id: RepoIdArg,
+    repo_type: RepoTypeOpt = RepoType.model,
+    space_sdk: Annotated[
+        Optional[str],
+        typer.Option(
+            help="Hugging Face Spaces SDK type. Required when --type is set to 'space'.",
+        ),
+    ] = None,
+    private: PrivateOpt = False,
+    token: TokenOpt = None,
+    exist_ok: Annotated[
+        bool,
+        typer.Option(
+            help="Do not raise an error if repo already exists.",
+        ),
+    ] = False,
+    resource_group_id: Annotated[
+        Optional[str],
+        typer.Option(
+            help="Resource group in which to create the repo. Resource groups is only available for Enterprise Hub organizations.",
+        ),
+    ] = None,
+) -> None:
+    api = get_hf_api(token=token)
+    repo_url = api.create_repo(
+        repo_id=repo_id,
+        repo_type=repo_type.value,
+        private=private,
+        token=token,
+        exist_ok=exist_ok,
+        resource_group_id=resource_group_id,
+        space_sdk=space_sdk,
+    )
+    print(f"Successfully created {ANSI.bold(repo_url.repo_id)} on the Hub.")
+    print(f"Your repo is now available at {ANSI.bold(repo_url)}")
+
+
+@repo_cli.command("delete", help="Delete a repo from the Hub. this is an irreversible operation.")
+def repo_delete(
+    repo_id: RepoIdArg,
+    repo_type: RepoTypeOpt = RepoType.model,
+    token: TokenOpt = None,
+    missing_ok: Annotated[
+        bool,
+        typer.Option(
+            help="If set to True, do not raise an error if repo does not exist.",
+        ),
+    ] = False,
+) -> None:
+    api = get_hf_api(token=token)
+    api.delete_repo(
+        repo_id=repo_id,
+        repo_type=repo_type.value,
+        missing_ok=missing_ok,
+    )
+    print(f"Successfully deleted {ANSI.bold(repo_id)} on the Hub.")
+
+
+@repo_cli.command("move", help="Move a repository from a namespace to another namespace.")
+def repo_move(
+    from_id: RepoIdArg,
+    to_id: RepoIdArg,
+    token: TokenOpt = None,
+    repo_type: RepoTypeOpt = RepoType.model,
+) -> None:
+    api = get_hf_api(token=token)
+    api.move_repo(
+        from_id=from_id,
+        to_id=to_id,
+        repo_type=repo_type.value,
+    )
+    print(f"Successfully moved {ANSI.bold(from_id)} to {ANSI.bold(to_id)} on the Hub.")
+
+
+@repo_cli.command("settings", help="Update the settings of a repository.")
+def repo_settings(
+    repo_id: RepoIdArg,
+    gated: Annotated[
+        Optional[GatedChoices],
+        typer.Option(
+            help="The gated status for the repository.",
+        ),
+    ] = None,
+    private: Annotated[
+        Optional[bool],
+        typer.Option(
+            help="Whether the repository should be private.",
+        ),
+    ] = None,
+    token: TokenOpt = None,
+    repo_type: RepoTypeOpt = RepoType.model,
+) -> None:
+    api = get_hf_api(token=token)
+    api.update_repo_settings(
+        repo_id=repo_id,
+        gated=(gated.value if gated else None),  # type: ignore [arg-type]
+        private=private,
+        repo_type=repo_type.value,
+    )
+    print(f"Successfully updated the settings of {ANSI.bold(repo_id)} on the Hub.")
+
+
+@branch_cli.command("create", help="Create a new branch for a repo on the Hub.")
+def branch_create(
+    repo_id: RepoIdArg,
+    branch: Annotated[
+        str,
+        typer.Argument(
+            help="The name of the branch to create.",
+        ),
+    ],
+    revision: RevisionOpt = None,
+    token: TokenOpt = None,
+    repo_type: RepoTypeOpt = RepoType.model,
+    exist_ok: Annotated[
+        bool,
+        typer.Option(
+            help="If set to True, do not raise an error if branch already exists.",
+        ),
+    ] = False,
+) -> None:
+    api = get_hf_api(token=token)
+    api.create_branch(
+        repo_id=repo_id,
+        branch=branch,
+        revision=revision,
+        repo_type=repo_type.value,
+        exist_ok=exist_ok,
+    )
+    print(f"Successfully created {ANSI.bold(branch)} branch on {repo_type.value} {ANSI.bold(repo_id)}")
+
+
+@branch_cli.command("delete", help="Delete a branch from a repo on the Hub.")
+def branch_delete(
+    repo_id: RepoIdArg,
+    branch: Annotated[
+        str,
+        typer.Argument(
+            help="The name of the branch to delete.",
+        ),
+    ],
+    token: TokenOpt = None,
+    repo_type: RepoTypeOpt = RepoType.model,
+) -> None:
+    api = get_hf_api(token=token)
+    api.delete_branch(
+        repo_id=repo_id,
+        branch=branch,
+        repo_type=repo_type.value,
+    )
+    print(f"Successfully deleted {ANSI.bold(branch)} branch on {repo_type.value} {ANSI.bold(repo_id)}")
+
+
+@tag_cli.command("create", help="Create a tag for a repo.")
+def tag_create(
+    repo_id: RepoIdArg,
+    tag: Annotated[
+        str,
+        typer.Argument(
+            help="The name of the tag to create.",
+        ),
+    ],
+    message: Annotated[
+        Optional[str],
+        typer.Option(
+            "-m",
+            "--message",
+            help="The description of the tag to create.",
+        ),
+    ] = None,
+    revision: RevisionOpt = None,
+    token: TokenOpt = None,
+    repo_type: RepoTypeOpt = RepoType.model,
+) -> None:
+    repo_type_str = repo_type.value
+    api = get_hf_api(token=token)
+    print(f"You are about to create tag {ANSI.bold(tag)} on {repo_type_str} {ANSI.bold(repo_id)}")
+    try:
+        api.create_tag(repo_id=repo_id, tag=tag, tag_message=message, revision=revision, repo_type=repo_type_str)
+    except RepositoryNotFoundError:
+        print(f"{repo_type_str.capitalize()} {ANSI.bold(repo_id)} not found.")
+        raise typer.Exit(code=1)
+    except RevisionNotFoundError:
+        print(f"Revision {ANSI.bold(str(revision))} not found.")
+        raise typer.Exit(code=1)
+    except HfHubHTTPError as e:
+        if e.response.status_code == 409:
+            print(f"Tag {ANSI.bold(tag)} already exists on {ANSI.bold(repo_id)}")
+            raise typer.Exit(code=1)
+        raise e
+    print(f"Tag {ANSI.bold(tag)} created on {ANSI.bold(repo_id)}")
+
+
+@tag_cli.command("list", help="List tags for a repo.")
+def tag_list(
+    repo_id: RepoIdArg,
+    token: TokenOpt = None,
+    repo_type: RepoTypeOpt = RepoType.model,
+) -> None:
+    repo_type_str = repo_type.value
+    api = get_hf_api(token=token)
+    try:
+        refs = api.list_repo_refs(repo_id=repo_id, repo_type=repo_type_str)
+    except RepositoryNotFoundError:
+        print(f"{repo_type_str.capitalize()} {ANSI.bold(repo_id)} not found.")
+        raise typer.Exit(code=1)
+    except HfHubHTTPError as e:
+        print(e)
+        print(ANSI.red(e.response.text))
+        raise typer.Exit(code=1)
+    if len(refs.tags) == 0:
+        print("No tags found")
+        raise typer.Exit(code=0)
+    print(f"Tags for {repo_type_str} {ANSI.bold(repo_id)}:")
+    for t in refs.tags:
+        print(t.name)
+
+
+@tag_cli.command("delete", help="Delete a tag for a repo.")
+def tag_delete(
+    repo_id: RepoIdArg,
+    tag: Annotated[
+        str,
+        typer.Argument(
+            help="The name of the tag to delete.",
+        ),
+    ],
+    yes: Annotated[
+        bool,
+        typer.Option(
+            "-y",
+            "--yes",
+            help="Answer Yes to prompt automatically",
+        ),
+    ] = False,
+    token: TokenOpt = None,
+    repo_type: RepoTypeOpt = RepoType.model,
+) -> None:
+    repo_type_str = repo_type.value
+    print(f"You are about to delete tag {ANSI.bold(tag)} on {repo_type_str} {ANSI.bold(repo_id)}")
+    if not yes:
+        choice = input("Proceed? [Y/n] ").lower()
+        if choice not in ("", "y", "yes"):
+            print("Abort")
+            raise typer.Exit()
+    api = get_hf_api(token=token)
+    try:
+        api.delete_tag(repo_id=repo_id, tag=tag, repo_type=repo_type_str)
+    except RepositoryNotFoundError:
+        print(f"{repo_type_str.capitalize()} {ANSI.bold(repo_id)} not found.")
+        raise typer.Exit(code=1)
+    except RevisionNotFoundError:
+        print(f"Tag {ANSI.bold(tag)} not found on {ANSI.bold(repo_id)}")
+        raise typer.Exit(code=1)
+    print(f"Tag {ANSI.bold(tag)} deleted on {ANSI.bold(repo_id)}")

huggingface_hub/cli/repo_files.py

@@ -0,0 +1,94 @@
+# coding=utf-8
+# Copyright 2023-present, the HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Contains command to update or delete files in a repository using the CLI.
+
+Usage:
+    # delete all
+    hf repo-files delete <repo_id> "*"
+
+    # delete single file
+    hf repo-files delete <repo_id> file.txt
+
+    # delete single folder
+    hf repo-files delete <repo_id> folder/
+
+    # delete multiple
+    hf repo-files delete <repo_id> file.txt folder/ file2.txt
+
+    # delete multiple patterns
+    hf repo-files delete <repo_id> file.txt "*.json" "folder/*.parquet"
+
+    # delete from different revision / repo-type
+    hf repo-files delete <repo_id> file.txt --revision=refs/pr/1 --repo-type=dataset
+"""
+
+from typing import Annotated, Optional
+
+import typer
+
+from huggingface_hub import logging
+
+from ._cli_utils import RepoIdArg, RepoType, RepoTypeOpt, RevisionOpt, TokenOpt, get_hf_api, typer_factory
+
+
+logger = logging.get_logger(__name__)
+
+
+repo_files_cli = typer_factory(help="Manage files in a repo on the Hub.")
+
+
+@repo_files_cli.command("delete")
+def repo_files_delete(
+    repo_id: RepoIdArg,
+    patterns: Annotated[
+        list[str],
+        typer.Argument(
+            help="Glob patterns to match files to delete.",
+        ),
+    ],
+    repo_type: RepoTypeOpt = RepoType.model,
+    revision: RevisionOpt = None,
+    commit_message: Annotated[
+        Optional[str],
+        typer.Option(
+            help="The summary / title / first line of the generated commit.",
+        ),
+    ] = None,
+    commit_description: Annotated[
+        Optional[str],
+        typer.Option(
+            help="The description of the generated commit.",
+        ),
+    ] = None,
+    create_pr: Annotated[
+        bool,
+        typer.Option(
+            help="Whether to create a new Pull Request for these changes.",
+        ),
+    ] = False,
+    token: TokenOpt = None,
+) -> None:
+    api = get_hf_api(token=token)
+    url = api.delete_files(
+        delete_patterns=patterns,
+        repo_id=repo_id,
+        repo_type=repo_type.value,
+        revision=revision,
+        commit_message=commit_message,
+        commit_description=commit_description,
+        create_pr=create_pr,
+    )
+    print(f"Files correctly deleted from repo. Commit: {url}.")
+    logging.set_verbosity_warning()

huggingface_hub/{commands/env.py → cli/system.py}

@@ -11,26 +11,23 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-"""Contains command to print information about the environment.
+"""Contains commands to print information about the environment and version.
 
 Usage:
-    huggingface-cli env
+    hf env
+    hf version
 """
 
-from argparse import _SubParsersAction
+from huggingface_hub import __version__
 
 from ..utils import dump_environment_info
-from . import BaseHuggingfaceCLICommand
 
 
-class EnvironmentCommand(BaseHuggingfaceCLICommand):
-    def __init__(self, args):
-        self.args = args
+def env() -> None:
+    """Print information about the environment."""
+    dump_environment_info()
 
-    @staticmethod
-    def register_subcommand(parser: _SubParsersAction):
-        env_parser = parser.add_parser("env", help="Print information about the environment.")
-        env_parser.set_defaults(func=EnvironmentCommand)
 
-    def run(self) -> None:
-        dump_environment_info()
+def version() -> None:
+    """Print CLI version."""
+    print(__version__)