huggingface-hub 1.0.0rc2__py3-none-any.whl → 1.0.0rc3__py3-none-any.whl

huggingface_hub/commands/download.py

@@ -1,195 +0,0 @@
-# coding=utf-8
-# Copyright 2023-present, the HuggingFace Inc. team.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-"""Contains command to download files from the Hub with the CLI.
-
-Usage:
-    huggingface-cli download --help
-
-    # Download file
-    huggingface-cli download gpt2 config.json
-
-    # Download entire repo
-    huggingface-cli download fffiloni/zeroscope --repo-type=space --revision=refs/pr/78
-
-    # Download repo with filters
-    huggingface-cli download gpt2 --include="*.safetensors"
-
-    # Download with token
-    huggingface-cli download Wauplin/private-model --token=hf_***
-
-    # Download quietly (no progress bar, no warnings, only the returned path)
-    huggingface-cli download gpt2 config.json --quiet
-
-    # Download to local dir
-    huggingface-cli download gpt2 --local-dir=./models/gpt2
-"""
-
-import warnings
-from argparse import Namespace, _SubParsersAction
-from typing import Optional
-
-from huggingface_hub import logging
-from huggingface_hub._snapshot_download import snapshot_download
-from huggingface_hub.commands import BaseHuggingfaceCLICommand
-from huggingface_hub.file_download import hf_hub_download
-from huggingface_hub.utils import disable_progress_bars, enable_progress_bars
-
-from ._cli_utils import show_deprecation_warning
-
-
-logger = logging.get_logger(__name__)
-
-
-class DownloadCommand(BaseHuggingfaceCLICommand):
-    @staticmethod
-    def register_subcommand(parser: _SubParsersAction):
-        download_parser = parser.add_parser("download", help="Download files from the Hub")
-        download_parser.add_argument(
-            "repo_id", type=str, help="ID of the repo to download from (e.g. `username/repo-name`)."
-        )
-        download_parser.add_argument(
-            "filenames", type=str, nargs="*", help="Files to download (e.g. `config.json`, `data/metadata.jsonl`)."
-        )
-        download_parser.add_argument(
-            "--repo-type",
-            choices=["model", "dataset", "space"],
-            default="model",
-            help="Type of repo to download from (defaults to 'model').",
-        )
-        download_parser.add_argument(
-            "--revision",
-            type=str,
-            help="An optional Git revision id which can be a branch name, a tag, or a commit hash.",
-        )
-        download_parser.add_argument(
-            "--include", nargs="*", type=str, help="Glob patterns to match files to download."
-        )
-        download_parser.add_argument(
-            "--exclude", nargs="*", type=str, help="Glob patterns to exclude from files to download."
-        )
-        download_parser.add_argument(
-            "--cache-dir", type=str, help="Path to the directory where to save the downloaded files."
-        )
-        download_parser.add_argument(
-            "--local-dir",
-            type=str,
-            help=(
-                "If set, the downloaded file will be placed under this directory. Check out"
-                " https://huggingface.co/docs/huggingface_hub/guides/download#download-files-to-local-folder for more"
-                " details."
-            ),
-        )
-        download_parser.add_argument(
-            "--local-dir-use-symlinks",
-            choices=["auto", "True", "False"],
-            help=("Deprecated and ignored. Downloading to a local directory does not use symlinks anymore."),
-        )
-        download_parser.add_argument(
-            "--force-download",
-            action="store_true",
-            help="If True, the files will be downloaded even if they are already cached.",
-        )
-        download_parser.add_argument(
-            "--resume-download",
-            action="store_true",
-            help="Deprecated and ignored. Downloading a file to local dir always attempts to resume previously interrupted downloads (unless hf-transfer is enabled).",
-        )
-        download_parser.add_argument(
-            "--token", type=str, help="A User Access Token generated from https://huggingface.co/settings/tokens"
-        )
-        download_parser.add_argument(
-            "--quiet",
-            action="store_true",
-            help="If True, progress bars are disabled and only the path to the download files is printed.",
-        )
-        download_parser.add_argument(
-            "--max-workers",
-            type=int,
-            default=8,
-            help="Maximum number of workers to use for downloading files. Default is 8.",
-        )
-        download_parser.set_defaults(func=DownloadCommand)
-
-    def __init__(self, args: Namespace) -> None:
-        self.token = args.token
-        self.repo_id: str = args.repo_id
-        self.filenames: list[str] = args.filenames
-        self.repo_type: str = args.repo_type
-        self.revision: Optional[str] = args.revision
-        self.include: Optional[list[str]] = args.include
-        self.exclude: Optional[list[str]] = args.exclude
-        self.cache_dir: Optional[str] = args.cache_dir
-        self.local_dir: Optional[str] = args.local_dir
-        self.force_download: bool = args.force_download
-        self.quiet: bool = args.quiet
-        self.max_workers: int = args.max_workers
-
-    def run(self) -> None:
-        show_deprecation_warning("huggingface-cli download", "hf download")
-
-        if self.quiet:
-            disable_progress_bars()
-            with warnings.catch_warnings():
-                warnings.simplefilter("ignore")
-                print(self._download())  # Print path to downloaded files
-            enable_progress_bars()
-        else:
-            logging.set_verbosity_info()
-            print(self._download())  # Print path to downloaded files
-            logging.set_verbosity_warning()
-
-    def _download(self) -> str:
-        # Warn user if patterns are ignored
-        if len(self.filenames) > 0:
-            if self.include is not None and len(self.include) > 0:
-                warnings.warn("Ignoring `--include` since filenames have being explicitly set.")
-            if self.exclude is not None and len(self.exclude) > 0:
-                warnings.warn("Ignoring `--exclude` since filenames have being explicitly set.")
-
-        # Single file to download: use `hf_hub_download`
-        if len(self.filenames) == 1:
-            return hf_hub_download(
-                repo_id=self.repo_id,
-                repo_type=self.repo_type,
-                revision=self.revision,
-                filename=self.filenames[0],
-                cache_dir=self.cache_dir,
-                force_download=self.force_download,
-                token=self.token,
-                local_dir=self.local_dir,
-                library_name="huggingface-cli",
-            )
-
-        # Otherwise: use `snapshot_download` to ensure all files comes from same revision
-        elif len(self.filenames) == 0:
-            allow_patterns = self.include
-            ignore_patterns = self.exclude
-        else:
-            allow_patterns = self.filenames
-            ignore_patterns = None
-
-        return snapshot_download(
-            repo_id=self.repo_id,
-            repo_type=self.repo_type,
-            revision=self.revision,
-            allow_patterns=allow_patterns,
-            ignore_patterns=ignore_patterns,
-            force_download=self.force_download,
-            cache_dir=self.cache_dir,
-            token=self.token,
-            local_dir=self.local_dir,
-            library_name="huggingface-cli",
-            max_workers=self.max_workers,
-        )

huggingface_hub/commands/env.py

@@ -1,39 +0,0 @@
-# Copyright 2022 The HuggingFace Team. All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-"""Contains command to print information about the environment.
-
-Usage:
-    huggingface-cli env
-"""
-
-from argparse import _SubParsersAction
-
-from ..utils import dump_environment_info
-from . import BaseHuggingfaceCLICommand
-from ._cli_utils import show_deprecation_warning
-
-
-class EnvironmentCommand(BaseHuggingfaceCLICommand):
-    def __init__(self, args):
-        self.args = args
-
-    @staticmethod
-    def register_subcommand(parser: _SubParsersAction):
-        env_parser = parser.add_parser("env", help="Print information about the environment.")
-        env_parser.set_defaults(func=EnvironmentCommand)
-
-    def run(self) -> None:
-        show_deprecation_warning("huggingface-cli env", "hf env")
-
-        dump_environment_info()

huggingface_hub/commands/huggingface_cli.py

@@ -1,65 +0,0 @@
-# Copyright 2020 The HuggingFace Team. All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from argparse import ArgumentParser
-
-from huggingface_hub.commands._cli_utils import show_deprecation_warning
-from huggingface_hub.commands.delete_cache import DeleteCacheCommand
-from huggingface_hub.commands.download import DownloadCommand
-from huggingface_hub.commands.env import EnvironmentCommand
-from huggingface_hub.commands.lfs import LfsCommands
-from huggingface_hub.commands.repo import RepoCommands
-from huggingface_hub.commands.repo_files import RepoFilesCommand
-from huggingface_hub.commands.scan_cache import ScanCacheCommand
-from huggingface_hub.commands.tag import TagCommands
-from huggingface_hub.commands.upload import UploadCommand
-from huggingface_hub.commands.upload_large_folder import UploadLargeFolderCommand
-from huggingface_hub.commands.user import UserCommands
-from huggingface_hub.commands.version import VersionCommand
-
-
-def main():
-    parser = ArgumentParser("huggingface-cli", usage="huggingface-cli <command> [<args>]")
-    commands_parser = parser.add_subparsers(help="huggingface-cli command helpers")
-
-    # Register commands
-    DownloadCommand.register_subcommand(commands_parser)
-    UploadCommand.register_subcommand(commands_parser)
-    RepoFilesCommand.register_subcommand(commands_parser)
-    EnvironmentCommand.register_subcommand(commands_parser)
-    UserCommands.register_subcommand(commands_parser)
-    RepoCommands.register_subcommand(commands_parser)
-    LfsCommands.register_subcommand(commands_parser)
-    ScanCacheCommand.register_subcommand(commands_parser)
-    DeleteCacheCommand.register_subcommand(commands_parser)
-    TagCommands.register_subcommand(commands_parser)
-    VersionCommand.register_subcommand(commands_parser)
-
-    # Experimental
-    UploadLargeFolderCommand.register_subcommand(commands_parser)
-
-    # Let's go
-    args = parser.parse_args()
-    if not hasattr(args, "func"):
-        show_deprecation_warning("huggingface-cli", "hf")
-        parser.print_help()
-        exit(1)
-
-    # Run
-    service = args.func(args)
-    service.run()
-
-
-if __name__ == "__main__":
-    main()

huggingface_hub/commands/lfs.py

@@ -1,200 +0,0 @@
-"""
-Implementation of a custom transfer agent for the transfer type "multipart" for
-git-lfs.
-
-Inspired by:
-github.com/cbartz/git-lfs-swift-transfer-agent/blob/master/git_lfs_swift_transfer.py
-
-Spec is: github.com/git-lfs/git-lfs/blob/master/docs/custom-transfers.md
-
-
-To launch debugger while developing:
-
-``` [lfs "customtransfer.multipart"]
-path = /path/to/huggingface_hub/.env/bin/python args = -m debugpy --listen 5678
---wait-for-client
-/path/to/huggingface_hub/src/huggingface_hub/commands/huggingface_cli.py
-lfs-multipart-upload ```"""
-
-import json
-import os
-import subprocess
-import sys
-from argparse import _SubParsersAction
-from typing import Optional
-
-from huggingface_hub.commands import BaseHuggingfaceCLICommand
-from huggingface_hub.lfs import LFS_MULTIPART_UPLOAD_COMMAND
-
-from ..utils import get_session, hf_raise_for_status, logging
-from ..utils._lfs import SliceFileObj
-
-
-logger = logging.get_logger(__name__)
-
-
-class LfsCommands(BaseHuggingfaceCLICommand):
-    """
-    Implementation of a custom transfer agent for the transfer type "multipart"
-    for git-lfs. This lets users upload large files >5GB 🔥. Spec for LFS custom
-    transfer agent is:
-    https://github.com/git-lfs/git-lfs/blob/master/docs/custom-transfers.md
-
-    This introduces two commands to the CLI:
-
-    1. $ huggingface-cli lfs-enable-largefiles
-
-    This should be executed once for each model repo that contains a model file
-    >5GB. It's documented in the error message you get if you just try to git
-    push a 5GB file without having enabled it before.
-
-    2. $ huggingface-cli lfs-multipart-upload
-
-    This command is called by lfs directly and is not meant to be called by the
-    user.
-    """
-
-    @staticmethod
-    def register_subcommand(parser: _SubParsersAction):
-        enable_parser = parser.add_parser(
-            "lfs-enable-largefiles", help="Configure your repository to enable upload of files > 5GB."
-        )
-        enable_parser.add_argument("path", type=str, help="Local path to repository you want to configure.")
-        enable_parser.set_defaults(func=lambda args: LfsEnableCommand(args))
-
-        # Command will get called by git-lfs, do not call it directly.
-        upload_parser = parser.add_parser(LFS_MULTIPART_UPLOAD_COMMAND, add_help=False)
-        upload_parser.set_defaults(func=lambda args: LfsUploadCommand(args))
-
-
-class LfsEnableCommand:
-    def __init__(self, args):
-        self.args = args
-
-    def run(self):
-        local_path = os.path.abspath(self.args.path)
-        if not os.path.isdir(local_path):
-            print("This does not look like a valid git repo.")
-            exit(1)
-        subprocess.run(
-            "git config lfs.customtransfer.multipart.path huggingface-cli".split(),
-            check=True,
-            cwd=local_path,
-        )
-        subprocess.run(
-            f"git config lfs.customtransfer.multipart.args {LFS_MULTIPART_UPLOAD_COMMAND}".split(),
-            check=True,
-            cwd=local_path,
-        )
-        print("Local repo set up for largefiles")
-
-
-def write_msg(msg: dict):
-    """Write out the message in Line delimited JSON."""
-    msg_str = json.dumps(msg) + "\n"
-    sys.stdout.write(msg_str)
-    sys.stdout.flush()
-
-
-def read_msg() -> Optional[dict]:
-    """Read Line delimited JSON from stdin."""
-    msg = json.loads(sys.stdin.readline().strip())
-
-    if "terminate" in (msg.get("type"), msg.get("event")):
-        # terminate message received
-        return None
-
-    if msg.get("event") not in ("download", "upload"):
-        logger.critical("Received unexpected message")
-        sys.exit(1)
-
-    return msg
-
-
-class LfsUploadCommand:
-    def __init__(self, args) -> None:
-        self.args = args
-
-    def run(self) -> None:
-        # Immediately after invoking a custom transfer process, git-lfs
-        # sends initiation data to the process over stdin.
-        # This tells the process useful information about the configuration.
-        init_msg = json.loads(sys.stdin.readline().strip())
-        if not (init_msg.get("event") == "init" and init_msg.get("operation") == "upload"):
-            write_msg({"error": {"code": 32, "message": "Wrong lfs init operation"}})
-            sys.exit(1)
-
-        # The transfer process should use the information it needs from the
-        # initiation structure, and also perform any one-off setup tasks it
-        # needs to do. It should then respond on stdout with a simple empty
-        # confirmation structure, as follows:
-        write_msg({})
-
-        # After the initiation exchange, git-lfs will send any number of
-        # transfer requests to the stdin of the transfer process, in a serial sequence.
-        while True:
-            msg = read_msg()
-            if msg is None:
-                # When all transfers have been processed, git-lfs will send
-                # a terminate event to the stdin of the transfer process.
-                # On receiving this message the transfer process should
-                # clean up and terminate. No response is expected.
-                sys.exit(0)
-
-            oid = msg["oid"]
-            filepath = msg["path"]
-            completion_url = msg["action"]["href"]
-            header = msg["action"]["header"]
-            chunk_size = int(header.pop("chunk_size"))
-            presigned_urls: list[str] = list(header.values())
-
-            # Send a "started" progress event to allow other workers to start.
-            # Otherwise they're delayed until first "progress" event is reported,
-            # i.e. after the first 5GB by default (!)
-            write_msg(
-                {
-                    "event": "progress",
-                    "oid": oid,
-                    "bytesSoFar": 1,
-                    "bytesSinceLast": 0,
-                }
-            )
-
-            parts = []
-            with open(filepath, "rb") as file:
-                for i, presigned_url in enumerate(presigned_urls):
-                    with SliceFileObj(
-                        file,
-                        seek_from=i * chunk_size,
-                        read_limit=chunk_size,
-                    ) as data:
-                        r = get_session().put(presigned_url, data=data)
-                        hf_raise_for_status(r)
-                        parts.append(
-                            {
-                                "etag": r.headers.get("etag"),
-                                "partNumber": i + 1,
-                            }
-                        )
-                        # In order to support progress reporting while data is uploading / downloading,
-                        # the transfer process should post messages to stdout
-                        write_msg(
-                            {
-                                "event": "progress",
-                                "oid": oid,
-                                "bytesSoFar": (i + 1) * chunk_size,
-                                "bytesSinceLast": chunk_size,
-                            }
-                        )
-                        # Not precise but that's ok.
-
-            r = get_session().post(
-                completion_url,
-                json={
-                    "oid": oid,
-                    "parts": parts,
-                },
-            )
-            hf_raise_for_status(r)
-
-            write_msg({"event": "complete", "oid": oid})

huggingface_hub/commands/repo.py

@@ -1,151 +0,0 @@
-# Copyright 2025 The HuggingFace Team. All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-"""Contains commands to interact with repositories on the Hugging Face Hub.
-
-Usage:
-    # create a new dataset repo on the Hub
-    huggingface-cli repo create my-cool-dataset --repo-type=dataset
-
-    # create a private model repo on the Hub
-    huggingface-cli repo create my-cool-model --private
-"""
-
-import argparse
-from argparse import _SubParsersAction
-from typing import Optional
-
-from huggingface_hub.commands import BaseHuggingfaceCLICommand
-from huggingface_hub.commands._cli_utils import ANSI
-from huggingface_hub.constants import SPACES_SDK_TYPES
-from huggingface_hub.hf_api import HfApi
-from huggingface_hub.utils import logging
-
-from ._cli_utils import show_deprecation_warning
-
-
-logger = logging.get_logger(__name__)
-
-
-class RepoCommands(BaseHuggingfaceCLICommand):
-    @staticmethod
-    def register_subcommand(parser: _SubParsersAction):
-        repo_parser = parser.add_parser("repo", help="{create} Commands to interact with your huggingface.co repos.")
-        repo_subparsers = repo_parser.add_subparsers(help="huggingface.co repos related commands")
-        repo_create_parser = repo_subparsers.add_parser("create", help="Create a new repo on huggingface.co")
-        repo_create_parser.add_argument(
-            "repo_id",
-            type=str,
-            help="The ID of the repo to create to (e.g. `username/repo-name`). The username is optional and will be set to your username if not provided.",
-        )
-        repo_create_parser.add_argument(
-            "--repo-type",
-            type=str,
-            help='Optional: set to "dataset" or "space" if creating a dataset or space, default is model.',
-        )
-        repo_create_parser.add_argument(
-            "--space_sdk",
-            type=str,
-            help='Optional: Hugging Face Spaces SDK type. Required when --type is set to "space".',
-            choices=SPACES_SDK_TYPES,
-        )
-        repo_create_parser.add_argument(
-            "--private",
-            action="store_true",
-            help="Whether to create a private repository. Defaults to public unless the organization's default is private.",
-        )
-        repo_create_parser.add_argument(
-            "--token",
-            type=str,
-            help="Hugging Face token. Will default to the locally saved token if not provided.",
-        )
-        repo_create_parser.add_argument(
-            "--exist-ok",
-            action="store_true",
-            help="Do not raise an error if repo already exists.",
-        )
-        repo_create_parser.add_argument(
-            "--resource-group-id",
-            type=str,
-            help="Resource group in which to create the repo. Resource groups is only available for Enterprise Hub organizations.",
-        )
-        repo_create_parser.add_argument(
-            "--type",
-            type=str,
-            help="[Deprecated]: use --repo-type instead.",
-        )
-        repo_create_parser.add_argument(
-            "-y",
-            "--yes",
-            action="store_true",
-            help="[Deprecated] no effect.",
-        )
-        repo_create_parser.add_argument(
-            "--organization", type=str, help="[Deprecated] Pass the organization namespace directly in the repo_id."
-        )
-        repo_create_parser.set_defaults(func=lambda args: RepoCreateCommand(args))
-
-
-class RepoCreateCommand:
-    def __init__(self, args: argparse.Namespace):
-        self.repo_id: str = args.repo_id
-        self.repo_type: Optional[str] = args.repo_type or args.type
-        self.space_sdk: Optional[str] = args.space_sdk
-        self.organization: Optional[str] = args.organization
-        self.yes: bool = args.yes
-        self.private: bool = args.private
-        self.token: Optional[str] = args.token
-        self.exist_ok: bool = args.exist_ok
-        self.resource_group_id: Optional[str] = args.resource_group_id
-
-        if args.type is not None:
-            print(
-                ANSI.yellow(
-                    "The --type argument is deprecated and will be removed in a future version. Use --repo-type instead."
-                )
-            )
-        if self.organization is not None:
-            print(
-                ANSI.yellow(
-                    "The --organization argument is deprecated and will be removed in a future version. Pass the organization namespace directly in the repo_id."
-                )
-            )
-        if self.yes:
-            print(
-                ANSI.yellow(
-                    "The --yes argument is deprecated and will be removed in a future version. It does not have any effect."
-                )
-            )
-
-        self._api = HfApi()
-
-    def run(self):
-        show_deprecation_warning("huggingface-cli repo", "hf repo")
-
-        if self.organization is not None:
-            if "/" in self.repo_id:
-                print(ANSI.red("You cannot pass both --organization and a repo_id with a namespace."))
-                exit(1)
-            self.repo_id = f"{self.organization}/{self.repo_id}"
-
-        repo_url = self._api.create_repo(
-            repo_id=self.repo_id,
-            repo_type=self.repo_type,
-            private=self.private,
-            token=self.token,
-            exist_ok=self.exist_ok,
-            resource_group_id=self.resource_group_id,
-            space_sdk=self.space_sdk,
-        )
-        print(f"Successfully created {ANSI.bold(repo_url.repo_id)} on the Hub.")
-        print(f"Your repo is now available at {ANSI.bold(repo_url)}")