huggingface-hub 0.33.4__py3-none-any.whl → 0.34.0rc0__py3-none-any.whl

huggingface_hub/cli/lfs.py

@@ -0,0 +1,198 @@
+"""
+Implementation of a custom transfer agent for the transfer type "multipart" for
+git-lfs.
+
+Inspired by:
+github.com/cbartz/git-lfs-swift-transfer-agent/blob/master/git_lfs_swift_transfer.py
+
+Spec is: github.com/git-lfs/git-lfs/blob/master/docs/custom-transfers.md
+
+
+To launch debugger while developing:
+
+``` [lfs "customtransfer.multipart"]
+path = /path/to/huggingface_hub/.env/bin/python args = -m debugpy --listen 5678
+--wait-for-client
+/path/to/huggingface_hub/src/huggingface_hub/commands/huggingface_cli.py
+lfs-multipart-upload ```"""
+
+import json
+import os
+import subprocess
+import sys
+from argparse import _SubParsersAction
+from typing import Dict, List, Optional
+
+from huggingface_hub.commands import BaseHuggingfaceCLICommand
+from huggingface_hub.lfs import LFS_MULTIPART_UPLOAD_COMMAND
+
+from ..utils import get_session, hf_raise_for_status, logging
+from ..utils._lfs import SliceFileObj
+
+
+logger = logging.get_logger(__name__)
+
+
+class LfsCommands(BaseHuggingfaceCLICommand):
+    """
+    Implementation of a custom transfer agent for the transfer type "multipart"
+    for git-lfs. This lets users upload large files >5GB 🔥. Spec for LFS custom
+    transfer agent is:
+    https://github.com/git-lfs/git-lfs/blob/master/docs/custom-transfers.md
+
+    This introduces two commands to the CLI:
+
+    1. $ hf lfs-enable-largefiles
+
+    This should be executed once for each model repo that contains a model file
+    >5GB. It's documented in the error message you get if you just try to git
+    push a 5GB file without having enabled it before.
+
+    2. $ hf lfs-multipart-upload
+
+    This command is called by lfs directly and is not meant to be called by the
+    user.
+    """
+
+    @staticmethod
+    def register_subcommand(parser: _SubParsersAction):
+        enable_parser = parser.add_parser("lfs-enable-largefiles", add_help=False)
+        enable_parser.add_argument("path", type=str, help="Local path to repository you want to configure.")
+        enable_parser.set_defaults(func=lambda args: LfsEnableCommand(args))
+
+        # Command will get called by git-lfs, do not call it directly.
+        upload_parser = parser.add_parser(LFS_MULTIPART_UPLOAD_COMMAND, add_help=False)
+        upload_parser.set_defaults(func=lambda args: LfsUploadCommand(args))
+
+
+class LfsEnableCommand:
+    def __init__(self, args):
+        self.args = args
+
+    def run(self):
+        local_path = os.path.abspath(self.args.path)
+        if not os.path.isdir(local_path):
+            print("This does not look like a valid git repo.")
+            exit(1)
+        subprocess.run(
+            "git config lfs.customtransfer.multipart.path hf".split(),
+            check=True,
+            cwd=local_path,
+        )
+        subprocess.run(
+            f"git config lfs.customtransfer.multipart.args {LFS_MULTIPART_UPLOAD_COMMAND}".split(),
+            check=True,
+            cwd=local_path,
+        )
+        print("Local repo set up for largefiles")
+
+
+def write_msg(msg: Dict):
+    """Write out the message in Line delimited JSON."""
+    msg_str = json.dumps(msg) + "\n"
+    sys.stdout.write(msg_str)
+    sys.stdout.flush()
+
+
+def read_msg() -> Optional[Dict]:
+    """Read Line delimited JSON from stdin."""
+    msg = json.loads(sys.stdin.readline().strip())
+
+    if "terminate" in (msg.get("type"), msg.get("event")):
+        # terminate message received
+        return None
+
+    if msg.get("event") not in ("download", "upload"):
+        logger.critical("Received unexpected message")
+        sys.exit(1)
+
+    return msg
+
+
+class LfsUploadCommand:
+    def __init__(self, args) -> None:
+        self.args = args
+
+    def run(self) -> None:
+        # Immediately after invoking a custom transfer process, git-lfs
+        # sends initiation data to the process over stdin.
+        # This tells the process useful information about the configuration.
+        init_msg = json.loads(sys.stdin.readline().strip())
+        if not (init_msg.get("event") == "init" and init_msg.get("operation") == "upload"):
+            write_msg({"error": {"code": 32, "message": "Wrong lfs init operation"}})
+            sys.exit(1)
+
+        # The transfer process should use the information it needs from the
+        # initiation structure, and also perform any one-off setup tasks it
+        # needs to do. It should then respond on stdout with a simple empty
+        # confirmation structure, as follows:
+        write_msg({})
+
+        # After the initiation exchange, git-lfs will send any number of
+        # transfer requests to the stdin of the transfer process, in a serial sequence.
+        while True:
+            msg = read_msg()
+            if msg is None:
+                # When all transfers have been processed, git-lfs will send
+                # a terminate event to the stdin of the transfer process.
+                # On receiving this message the transfer process should
+                # clean up and terminate. No response is expected.
+                sys.exit(0)
+
+            oid = msg["oid"]
+            filepath = msg["path"]
+            completion_url = msg["action"]["href"]
+            header = msg["action"]["header"]
+            chunk_size = int(header.pop("chunk_size"))
+            presigned_urls: List[str] = list(header.values())
+
+            # Send a "started" progress event to allow other workers to start.
+            # Otherwise they're delayed until first "progress" event is reported,
+            # i.e. after the first 5GB by default (!)
+            write_msg(
+                {
+                    "event": "progress",
+                    "oid": oid,
+                    "bytesSoFar": 1,
+                    "bytesSinceLast": 0,
+                }
+            )
+
+            parts = []
+            with open(filepath, "rb") as file:
+                for i, presigned_url in enumerate(presigned_urls):
+                    with SliceFileObj(
+                        file,
+                        seek_from=i * chunk_size,
+                        read_limit=chunk_size,
+                    ) as data:
+                        r = get_session().put(presigned_url, data=data)
+                        hf_raise_for_status(r)
+                        parts.append(
+                            {
+                                "etag": r.headers.get("etag"),
+                                "partNumber": i + 1,
+                            }
+                        )
+                        # In order to support progress reporting while data is uploading / downloading,
+                        # the transfer process should post messages to stdout
+                        write_msg(
+                            {
+                                "event": "progress",
+                                "oid": oid,
+                                "bytesSoFar": (i + 1) * chunk_size,
+                                "bytesSinceLast": chunk_size,
+                            }
+                        )
+                        # Not precise but that's ok.
+
+            r = get_session().post(
+                completion_url,
+                json={
+                    "oid": oid,
+                    "parts": parts,
+                },
+            )
+            hf_raise_for_status(r)
+
+            write_msg({"event": "complete", "oid": oid})

huggingface_hub/cli/repo.py

@@ -0,0 +1,243 @@
+# Copyright 2025 The HuggingFace Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Contains commands to interact with repositories on the Hugging Face Hub.
+
+Usage:
+    # create a new dataset repo on the Hub
+    hf repo create my-cool-dataset --repo-type=dataset
+
+    # create a private model repo on the Hub
+    hf repo create my-cool-model --private
+"""
+
+import argparse
+from argparse import _SubParsersAction
+from typing import Optional
+
+from requests.exceptions import HTTPError
+
+from huggingface_hub.commands import BaseHuggingfaceCLICommand
+from huggingface_hub.commands._cli_utils import ANSI
+from huggingface_hub.constants import REPO_TYPES, SPACES_SDK_TYPES
+from huggingface_hub.errors import HfHubHTTPError, RepositoryNotFoundError, RevisionNotFoundError
+from huggingface_hub.hf_api import HfApi
+from huggingface_hub.utils import logging
+
+
+logger = logging.get_logger(__name__)
+
+
+class RepoCommands(BaseHuggingfaceCLICommand):
+    @staticmethod
+    def register_subcommand(parser: _SubParsersAction):
+        repo_parser = parser.add_parser("repo", help="Manage repos on the Hub.")
+        repo_subparsers = repo_parser.add_subparsers(help="huggingface.co repos related commands")
+        # CREATE
+        repo_create_parser = repo_subparsers.add_parser("create", help="Create a new repo on huggingface.co")
+        repo_create_parser.add_argument(
+            "repo_id",
+            type=str,
+            help="The ID of the repo to create to (e.g. `username/repo-name`). The username is optional and will be set to your username if not provided.",
+        )
+        repo_create_parser.add_argument(
+            "--repo-type",
+            type=str,
+            help='Optional: set to "dataset" or "space" if creating a dataset or space, default is model.',
+        )
+        repo_create_parser.add_argument(
+            "--space_sdk",
+            type=str,
+            help='Optional: Hugging Face Spaces SDK type. Required when --type is set to "space".',
+            choices=SPACES_SDK_TYPES,
+        )
+        repo_create_parser.add_argument(
+            "--private",
+            action="store_true",
+            help="Whether to create a private repository. Defaults to public unless the organization's default is private.",
+        )
+        repo_create_parser.add_argument(
+            "--token",
+            type=str,
+            help="Hugging Face token. Will default to the locally saved token if not provided.",
+        )
+        repo_create_parser.add_argument(
+            "--exist-ok",
+            action="store_true",
+            help="Do not raise an error if repo already exists.",
+        )
+        repo_create_parser.add_argument(
+            "--resource-group-id",
+            type=str,
+            help="Resource group in which to create the repo. Resource groups is only available for Enterprise Hub organizations.",
+        )
+        repo_create_parser.set_defaults(func=lambda args: RepoCreateCommand(args))
+
+        # TAG SUBCOMMANDS
+        repo_tag_parser = repo_subparsers.add_parser("tag", help="Manage tags for a repo on the Hub.")
+        tag_subparsers = repo_tag_parser.add_subparsers(help="Tag actions", dest="tag_action", required=True)
+
+        # tag create
+        tag_create_parser = tag_subparsers.add_parser("create", help="Create a tag for a repo.")
+        tag_create_parser.add_argument(
+            "repo_id", type=str, help="The ID of the repo to tag (e.g. `username/repo-name`)."
+        )
+        tag_create_parser.add_argument("tag", type=str, help="The name of the tag to create.")
+        tag_create_parser.add_argument("-m", "--message", type=str, help="The description of the tag to create.")
+        tag_create_parser.add_argument("--revision", type=str, help="The git revision to tag.")
+        tag_create_parser.add_argument(
+            "--token", type=str, help="A User Access Token generated from https://huggingface.co/settings/tokens."
+        )
+        tag_create_parser.add_argument(
+            "--repo-type",
+            choices=["model", "dataset", "space"],
+            default="model",
+            help="Set the type of repository (model, dataset, or space).",
+        )
+        tag_create_parser.set_defaults(func=lambda args: RepoTagCreateCommand(args))
+
+        # tag list
+        tag_list_parser = tag_subparsers.add_parser("list", help="List tags for a repo.")
+        tag_list_parser.add_argument(
+            "repo_id", type=str, help="The ID of the repo to list tags for (e.g. `username/repo-name`)."
+        )
+        tag_list_parser.add_argument(
+            "--token", type=str, help="A User Access Token generated from https://huggingface.co/settings/tokens."
+        )
+        tag_list_parser.add_argument(
+            "--repo-type",
+            choices=["model", "dataset", "space"],
+            default="model",
+            help="Set the type of repository (model, dataset, or space).",
+        )
+        tag_list_parser.set_defaults(func=lambda args: RepoTagListCommand(args))
+
+        # tag delete
+        tag_delete_parser = tag_subparsers.add_parser("delete", help="Delete a tag from a repo.")
+        tag_delete_parser.add_argument(
+            "repo_id", type=str, help="The ID of the repo to delete the tag from (e.g. `username/repo-name`)."
+        )
+        tag_delete_parser.add_argument("tag", type=str, help="The name of the tag to delete.")
+        tag_delete_parser.add_argument("-y", "--yes", action="store_true", help="Answer Yes to prompts automatically.")
+        tag_delete_parser.add_argument(
+            "--token", type=str, help="A User Access Token generated from https://huggingface.co/settings/tokens."
+        )
+        tag_delete_parser.add_argument(
+            "--repo-type",
+            choices=["model", "dataset", "space"],
+            default="model",
+            help="Set the type of repository (model, dataset, or space).",
+        )
+        tag_delete_parser.set_defaults(func=lambda args: RepoTagDeleteCommand(args))
+
+
+class RepoCreateCommand:
+    def __init__(self, args: argparse.Namespace):
+        self.repo_id: str = args.repo_id
+        self.repo_type: Optional[str] = args.repo_type
+        self.space_sdk: Optional[str] = args.space_sdk
+        self.private: bool = args.private
+        self.token: Optional[str] = args.token
+        self.exist_ok: bool = args.exist_ok
+        self.resource_group_id: Optional[str] = args.resource_group_id
+        self._api = HfApi()
+
+    def run(self):
+        repo_url = self._api.create_repo(
+            repo_id=self.repo_id,
+            repo_type=self.repo_type,
+            private=self.private,
+            token=self.token,
+            exist_ok=self.exist_ok,
+            resource_group_id=self.resource_group_id,
+            space_sdk=self.space_sdk,
+        )
+        print(f"Successfully created {ANSI.bold(repo_url.repo_id)} on the Hub.")
+        print(f"Your repo is now available at {ANSI.bold(repo_url)}")
+
+
+class RepoTagCommand:
+    def __init__(self, args):
+        self.args = args
+        self.api = HfApi(token=getattr(args, "token", None))
+        self.repo_id = args.repo_id
+        self.repo_type = getattr(args, "repo_type", "model")
+        if self.repo_type not in REPO_TYPES:
+            print("Invalid repo --repo-type")
+            exit(1)
+
+
+class RepoTagCreateCommand(RepoTagCommand):
+    def run(self):
+        print(f"You are about to create tag {ANSI.bold(self.args.tag)} on {self.repo_type} {ANSI.bold(self.repo_id)}")
+        try:
+            self.api.create_tag(
+                repo_id=self.repo_id,
+                tag=self.args.tag,
+                tag_message=getattr(self.args, "message", None),
+                revision=getattr(self.args, "revision", None),
+                repo_type=self.repo_type,
+            )
+        except RepositoryNotFoundError:
+            print(f"{self.repo_type.capitalize()} {ANSI.bold(self.repo_id)} not found.")
+            exit(1)
+        except RevisionNotFoundError:
+            print(f"Revision {ANSI.bold(getattr(self.args, 'revision', None))} not found.")
+            exit(1)
+        except HfHubHTTPError as e:
+            if e.response.status_code == 409:
+                print(f"Tag {ANSI.bold(self.args.tag)} already exists on {ANSI.bold(self.repo_id)}")
+                exit(1)
+            raise e
+        print(f"Tag {ANSI.bold(self.args.tag)} created on {ANSI.bold(self.repo_id)}")
+
+
+class RepoTagListCommand(RepoTagCommand):
+    def run(self):
+        try:
+            refs = self.api.list_repo_refs(
+                repo_id=self.repo_id,
+                repo_type=self.repo_type,
+            )
+        except RepositoryNotFoundError:
+            print(f"{self.repo_type.capitalize()} {ANSI.bold(self.repo_id)} not found.")
+            exit(1)
+        except HTTPError as e:
+            print(e)
+            print(ANSI.red(e.response.text))
+            exit(1)
+        if len(refs.tags) == 0:
+            print("No tags found")
+            exit(0)
+        print(f"Tags for {self.repo_type} {ANSI.bold(self.repo_id)}:")
+        for tag in refs.tags:
+            print(tag.name)
+
+
+class RepoTagDeleteCommand(RepoTagCommand):
+    def run(self):
+        print(f"You are about to delete tag {ANSI.bold(self.args.tag)} on {self.repo_type} {ANSI.bold(self.repo_id)}")
+        if not getattr(self.args, "yes", False):
+            choice = input("Proceed? [Y/n] ").lower()
+            if choice not in ("", "y", "yes"):
+                print("Abort")
+                exit()
+        try:
+            self.api.delete_tag(repo_id=self.repo_id, tag=self.args.tag, repo_type=self.repo_type)
+        except RepositoryNotFoundError:
+            print(f"{self.repo_type.capitalize()} {ANSI.bold(self.repo_id)} not found.")
+            exit(1)
+        except RevisionNotFoundError:
+            print(f"Tag {ANSI.bold(self.args.tag)} not found on {ANSI.bold(self.repo_id)}")
+            exit(1)
+        print(f"Tag {ANSI.bold(self.args.tag)} deleted on {ANSI.bold(self.repo_id)}")

huggingface_hub/cli/repo_files.py

@@ -0,0 +1,128 @@
+# coding=utf-8
+# Copyright 2023-present, the HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Contains command to update or delete files in a repository using the CLI.
+
+Usage:
+    # delete all
+    hf repo-files delete <repo_id> "*"
+
+    # delete single file
+    hf repo-files delete <repo_id> file.txt
+
+    # delete single folder
+    hf repo-files delete <repo_id> folder/
+
+    # delete multiple
+    hf repo-files delete <repo_id> file.txt folder/ file2.txt
+
+    # delete multiple patterns
+    hf repo-files delete <repo_id> file.txt "*.json" "folder/*.parquet"
+
+    # delete from different revision / repo-type
+    hf repo-files delete <repo_id> file.txt --revision=refs/pr/1 --repo-type=dataset
+"""
+
+from argparse import _SubParsersAction
+from typing import List, Optional
+
+from huggingface_hub import logging
+from huggingface_hub.commands import BaseHuggingfaceCLICommand
+from huggingface_hub.hf_api import HfApi
+
+
+logger = logging.get_logger(__name__)
+
+
+class DeleteFilesSubCommand:
+    def __init__(self, args) -> None:
+        self.args = args
+        self.repo_id: str = args.repo_id
+        self.repo_type: Optional[str] = args.repo_type
+        self.revision: Optional[str] = args.revision
+        self.api: HfApi = HfApi(token=args.token, library_name="hf")
+        self.patterns: List[str] = args.patterns
+        self.commit_message: Optional[str] = args.commit_message
+        self.commit_description: Optional[str] = args.commit_description
+        self.create_pr: bool = args.create_pr
+        self.token: Optional[str] = args.token
+
+    def run(self) -> None:
+        logging.set_verbosity_info()
+        url = self.api.delete_files(
+            delete_patterns=self.patterns,
+            repo_id=self.repo_id,
+            repo_type=self.repo_type,
+            revision=self.revision,
+            commit_message=self.commit_message,
+            commit_description=self.commit_description,
+            create_pr=self.create_pr,
+        )
+        print(f"Files correctly deleted from repo. Commit: {url}.")
+        logging.set_verbosity_warning()
+
+
+class RepoFilesCommand(BaseHuggingfaceCLICommand):
+    @staticmethod
+    def register_subcommand(parser: _SubParsersAction):
+        repo_files_parser = parser.add_parser("repo-files", help="Manage files in a repo on the Hub.")
+        repo_files_subparsers = repo_files_parser.add_subparsers(
+            help="Action to execute against the files.",
+            required=True,
+        )
+        delete_subparser = repo_files_subparsers.add_parser(
+            "delete",
+            help="Delete files from a repo on the Hub",
+        )
+        delete_subparser.set_defaults(func=lambda args: DeleteFilesSubCommand(args))
+        delete_subparser.add_argument(
+            "repo_id", type=str, help="The ID of the repo to manage (e.g. `username/repo-name`)."
+        )
+        delete_subparser.add_argument(
+            "patterns",
+            nargs="+",
+            type=str,
+            help="Glob patterns to match files to delete.",
+        )
+        delete_subparser.add_argument(
+            "--repo-type",
+            choices=["model", "dataset", "space"],
+            default="model",
+            help="Type of the repo to upload to (e.g. `dataset`).",
+        )
+        delete_subparser.add_argument(
+            "--revision",
+            type=str,
+            help=(
+                "An optional Git revision to push to. It can be a branch name "
+                "or a PR reference. If revision does not"
+                " exist and `--create-pr` is not set, a branch will be automatically created."
+            ),
+        )
+        delete_subparser.add_argument(
+            "--commit-message", type=str, help="The summary / title / first line of the generated commit."
+        )
+        delete_subparser.add_argument(
+            "--commit-description", type=str, help="The description of the generated commit."
+        )
+        delete_subparser.add_argument(
+            "--create-pr", action="store_true", help="Whether to create a new Pull Request for these changes."
+        )
+        delete_subparser.add_argument(
+            "--token",
+            type=str,
+            help="A User Access Token generated from https://huggingface.co/settings/tokens",
+        )
+
+        repo_files_parser.set_defaults(func=RepoFilesCommand)

huggingface_hub/cli/system.py

@@ -0,0 +1,52 @@
+# Copyright 2022 The HuggingFace Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Contains commands to print information about the environment and version.
+
+Usage:
+    hf env
+    hf version
+"""
+
+from argparse import _SubParsersAction
+
+from huggingface_hub import __version__
+
+from ..utils import dump_environment_info
+from . import BaseHuggingfaceCLICommand
+
+
+class EnvironmentCommand(BaseHuggingfaceCLICommand):
+    def __init__(self, args):
+        self.args = args
+
+    @staticmethod
+    def register_subcommand(parser: _SubParsersAction):
+        env_parser = parser.add_parser("env", help="Print information about the environment.")
+        env_parser.set_defaults(func=EnvironmentCommand)
+
+    def run(self) -> None:
+        dump_environment_info()
+
+
+class VersionCommand(BaseHuggingfaceCLICommand):
+    def __init__(self, args):
+        self.args = args
+
+    @staticmethod
+    def register_subcommand(parser: _SubParsersAction):
+        version_parser = parser.add_parser("version", help="Print information about the hf version.")
+        version_parser.set_defaults(func=VersionCommand)
+
+    def run(self) -> None:
+        print(f"huggingface_hub version: {__version__}")