huggingface-hub 0.31.0rc0__py3-none-any.whl → 1.1.3__py3-none-any.whl

huggingface_hub/commands/download.py

@@ -1,200 +0,0 @@
-# coding=utf-8
-# Copyright 2023-present, the HuggingFace Inc. team.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-"""Contains command to download files from the Hub with the CLI.
-
-Usage:
-    huggingface-cli download --help
-
-    # Download file
-    huggingface-cli download gpt2 config.json
-
-    # Download entire repo
-    huggingface-cli download fffiloni/zeroscope --repo-type=space --revision=refs/pr/78
-
-    # Download repo with filters
-    huggingface-cli download gpt2 --include="*.safetensors"
-
-    # Download with token
-    huggingface-cli download Wauplin/private-model --token=hf_***
-
-    # Download quietly (no progress bar, no warnings, only the returned path)
-    huggingface-cli download gpt2 config.json --quiet
-
-    # Download to local dir
-    huggingface-cli download gpt2 --local-dir=./models/gpt2
-"""
-
-import warnings
-from argparse import Namespace, _SubParsersAction
-from typing import List, Optional
-
-from huggingface_hub import logging
-from huggingface_hub._snapshot_download import snapshot_download
-from huggingface_hub.commands import BaseHuggingfaceCLICommand
-from huggingface_hub.file_download import hf_hub_download
-from huggingface_hub.utils import disable_progress_bars, enable_progress_bars
-
-
-logger = logging.get_logger(__name__)
-
-
-class DownloadCommand(BaseHuggingfaceCLICommand):
-    @staticmethod
-    def register_subcommand(parser: _SubParsersAction):
-        download_parser = parser.add_parser("download", help="Download files from the Hub")
-        download_parser.add_argument(
-            "repo_id", type=str, help="ID of the repo to download from (e.g. `username/repo-name`)."
-        )
-        download_parser.add_argument(
-            "filenames", type=str, nargs="*", help="Files to download (e.g. `config.json`, `data/metadata.jsonl`)."
-        )
-        download_parser.add_argument(
-            "--repo-type",
-            choices=["model", "dataset", "space"],
-            default="model",
-            help="Type of repo to download from (defaults to 'model').",
-        )
-        download_parser.add_argument(
-            "--revision",
-            type=str,
-            help="An optional Git revision id which can be a branch name, a tag, or a commit hash.",
-        )
-        download_parser.add_argument(
-            "--include", nargs="*", type=str, help="Glob patterns to match files to download."
-        )
-        download_parser.add_argument(
-            "--exclude", nargs="*", type=str, help="Glob patterns to exclude from files to download."
-        )
-        download_parser.add_argument(
-            "--cache-dir", type=str, help="Path to the directory where to save the downloaded files."
-        )
-        download_parser.add_argument(
-            "--local-dir",
-            type=str,
-            help=(
-                "If set, the downloaded file will be placed under this directory. Check out"
-                " https://huggingface.co/docs/huggingface_hub/guides/download#download-files-to-local-folder for more"
-                " details."
-            ),
-        )
-        download_parser.add_argument(
-            "--local-dir-use-symlinks",
-            choices=["auto", "True", "False"],
-            help=("Deprecated and ignored. Downloading to a local directory does not use symlinks anymore."),
-        )
-        download_parser.add_argument(
-            "--force-download",
-            action="store_true",
-            help="If True, the files will be downloaded even if they are already cached.",
-        )
-        download_parser.add_argument(
-            "--resume-download",
-            action="store_true",
-            help="Deprecated and ignored. Downloading a file to local dir always attempts to resume previously interrupted downloads (unless hf-transfer is enabled).",
-        )
-        download_parser.add_argument(
-            "--token", type=str, help="A User Access Token generated from https://huggingface.co/settings/tokens"
-        )
-        download_parser.add_argument(
-            "--quiet",
-            action="store_true",
-            help="If True, progress bars are disabled and only the path to the download files is printed.",
-        )
-        download_parser.add_argument(
-            "--max-workers",
-            type=int,
-            default=8,
-            help="Maximum number of workers to use for downloading files. Default is 8.",
-        )
-        download_parser.set_defaults(func=DownloadCommand)
-
-    def __init__(self, args: Namespace) -> None:
-        self.token = args.token
-        self.repo_id: str = args.repo_id
-        self.filenames: List[str] = args.filenames
-        self.repo_type: str = args.repo_type
-        self.revision: Optional[str] = args.revision
-        self.include: Optional[List[str]] = args.include
-        self.exclude: Optional[List[str]] = args.exclude
-        self.cache_dir: Optional[str] = args.cache_dir
-        self.local_dir: Optional[str] = args.local_dir
-        self.force_download: bool = args.force_download
-        self.resume_download: Optional[bool] = args.resume_download or None
-        self.quiet: bool = args.quiet
-        self.max_workers: int = args.max_workers
-
-        if args.local_dir_use_symlinks is not None:
-            warnings.warn(
-                "Ignoring --local-dir-use-symlinks. Downloading to a local directory does not use symlinks anymore.",
-                FutureWarning,
-            )
-
-    def run(self) -> None:
-        if self.quiet:
-            disable_progress_bars()
-            with warnings.catch_warnings():
-                warnings.simplefilter("ignore")
-                print(self._download())  # Print path to downloaded files
-            enable_progress_bars()
-        else:
-            logging.set_verbosity_info()
-            print(self._download())  # Print path to downloaded files
-            logging.set_verbosity_warning()
-
-    def _download(self) -> str:
-        # Warn user if patterns are ignored
-        if len(self.filenames) > 0:
-            if self.include is not None and len(self.include) > 0:
-                warnings.warn("Ignoring `--include` since filenames have being explicitly set.")
-            if self.exclude is not None and len(self.exclude) > 0:
-                warnings.warn("Ignoring `--exclude` since filenames have being explicitly set.")
-
-        # Single file to download: use `hf_hub_download`
-        if len(self.filenames) == 1:
-            return hf_hub_download(
-                repo_id=self.repo_id,
-                repo_type=self.repo_type,
-                revision=self.revision,
-                filename=self.filenames[0],
-                cache_dir=self.cache_dir,
-                resume_download=self.resume_download,
-                force_download=self.force_download,
-                token=self.token,
-                local_dir=self.local_dir,
-                library_name="huggingface-cli",
-            )
-
-        # Otherwise: use `snapshot_download` to ensure all files comes from same revision
-        elif len(self.filenames) == 0:
-            allow_patterns = self.include
-            ignore_patterns = self.exclude
-        else:
-            allow_patterns = self.filenames
-            ignore_patterns = None
-
-        return snapshot_download(
-            repo_id=self.repo_id,
-            repo_type=self.repo_type,
-            revision=self.revision,
-            allow_patterns=allow_patterns,
-            ignore_patterns=ignore_patterns,
-            resume_download=self.resume_download,
-            force_download=self.force_download,
-            cache_dir=self.cache_dir,
-            token=self.token,
-            local_dir=self.local_dir,
-            library_name="huggingface-cli",
-            max_workers=self.max_workers,
-        )

huggingface_hub/commands/huggingface_cli.py

@@ -1,61 +0,0 @@
-# Copyright 2020 The HuggingFace Team. All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from argparse import ArgumentParser
-
-from huggingface_hub.commands.delete_cache import DeleteCacheCommand
-from huggingface_hub.commands.download import DownloadCommand
-from huggingface_hub.commands.env import EnvironmentCommand
-from huggingface_hub.commands.lfs import LfsCommands
-from huggingface_hub.commands.repo_files import RepoFilesCommand
-from huggingface_hub.commands.scan_cache import ScanCacheCommand
-from huggingface_hub.commands.tag import TagCommands
-from huggingface_hub.commands.upload import UploadCommand
-from huggingface_hub.commands.upload_large_folder import UploadLargeFolderCommand
-from huggingface_hub.commands.user import UserCommands
-from huggingface_hub.commands.version import VersionCommand
-
-
-def main():
-    parser = ArgumentParser("huggingface-cli", usage="huggingface-cli <command> [<args>]")
-    commands_parser = parser.add_subparsers(help="huggingface-cli command helpers")
-
-    # Register commands
-    DownloadCommand.register_subcommand(commands_parser)
-    UploadCommand.register_subcommand(commands_parser)
-    RepoFilesCommand.register_subcommand(commands_parser)
-    EnvironmentCommand.register_subcommand(commands_parser)
-    UserCommands.register_subcommand(commands_parser)
-    LfsCommands.register_subcommand(commands_parser)
-    ScanCacheCommand.register_subcommand(commands_parser)
-    DeleteCacheCommand.register_subcommand(commands_parser)
-    TagCommands.register_subcommand(commands_parser)
-    VersionCommand.register_subcommand(commands_parser)
-
-    # Experimental
-    UploadLargeFolderCommand.register_subcommand(commands_parser)
-
-    # Let's go
-    args = parser.parse_args()
-    if not hasattr(args, "func"):
-        parser.print_help()
-        exit(1)
-
-    # Run
-    service = args.func(args)
-    service.run()
-
-
-if __name__ == "__main__":
-    main()

huggingface_hub/commands/lfs.py

@@ -1,200 +0,0 @@
-"""
-Implementation of a custom transfer agent for the transfer type "multipart" for
-git-lfs.
-
-Inspired by:
-github.com/cbartz/git-lfs-swift-transfer-agent/blob/master/git_lfs_swift_transfer.py
-
-Spec is: github.com/git-lfs/git-lfs/blob/master/docs/custom-transfers.md
-
-
-To launch debugger while developing:
-
-``` [lfs "customtransfer.multipart"]
-path = /path/to/huggingface_hub/.env/bin/python args = -m debugpy --listen 5678
---wait-for-client
-/path/to/huggingface_hub/src/huggingface_hub/commands/huggingface_cli.py
-lfs-multipart-upload ```"""
-
-import json
-import os
-import subprocess
-import sys
-from argparse import _SubParsersAction
-from typing import Dict, List, Optional
-
-from huggingface_hub.commands import BaseHuggingfaceCLICommand
-from huggingface_hub.lfs import LFS_MULTIPART_UPLOAD_COMMAND
-
-from ..utils import get_session, hf_raise_for_status, logging
-from ..utils._lfs import SliceFileObj
-
-
-logger = logging.get_logger(__name__)
-
-
-class LfsCommands(BaseHuggingfaceCLICommand):
-    """
-    Implementation of a custom transfer agent for the transfer type "multipart"
-    for git-lfs. This lets users upload large files >5GB 🔥. Spec for LFS custom
-    transfer agent is:
-    https://github.com/git-lfs/git-lfs/blob/master/docs/custom-transfers.md
-
-    This introduces two commands to the CLI:
-
-    1. $ huggingface-cli lfs-enable-largefiles
-
-    This should be executed once for each model repo that contains a model file
-    >5GB. It's documented in the error message you get if you just try to git
-    push a 5GB file without having enabled it before.
-
-    2. $ huggingface-cli lfs-multipart-upload
-
-    This command is called by lfs directly and is not meant to be called by the
-    user.
-    """
-
-    @staticmethod
-    def register_subcommand(parser: _SubParsersAction):
-        enable_parser = parser.add_parser(
-            "lfs-enable-largefiles", help="Configure your repository to enable upload of files > 5GB."
-        )
-        enable_parser.add_argument("path", type=str, help="Local path to repository you want to configure.")
-        enable_parser.set_defaults(func=lambda args: LfsEnableCommand(args))
-
-        # Command will get called by git-lfs, do not call it directly.
-        upload_parser = parser.add_parser(LFS_MULTIPART_UPLOAD_COMMAND, add_help=False)
-        upload_parser.set_defaults(func=lambda args: LfsUploadCommand(args))
-
-
-class LfsEnableCommand:
-    def __init__(self, args):
-        self.args = args
-
-    def run(self):
-        local_path = os.path.abspath(self.args.path)
-        if not os.path.isdir(local_path):
-            print("This does not look like a valid git repo.")
-            exit(1)
-        subprocess.run(
-            "git config lfs.customtransfer.multipart.path huggingface-cli".split(),
-            check=True,
-            cwd=local_path,
-        )
-        subprocess.run(
-            f"git config lfs.customtransfer.multipart.args {LFS_MULTIPART_UPLOAD_COMMAND}".split(),
-            check=True,
-            cwd=local_path,
-        )
-        print("Local repo set up for largefiles")
-
-
-def write_msg(msg: Dict):
-    """Write out the message in Line delimited JSON."""
-    msg_str = json.dumps(msg) + "\n"
-    sys.stdout.write(msg_str)
-    sys.stdout.flush()
-
-
-def read_msg() -> Optional[Dict]:
-    """Read Line delimited JSON from stdin."""
-    msg = json.loads(sys.stdin.readline().strip())
-
-    if "terminate" in (msg.get("type"), msg.get("event")):
-        # terminate message received
-        return None
-
-    if msg.get("event") not in ("download", "upload"):
-        logger.critical("Received unexpected message")
-        sys.exit(1)
-
-    return msg
-
-
-class LfsUploadCommand:
-    def __init__(self, args) -> None:
-        self.args = args
-
-    def run(self) -> None:
-        # Immediately after invoking a custom transfer process, git-lfs
-        # sends initiation data to the process over stdin.
-        # This tells the process useful information about the configuration.
-        init_msg = json.loads(sys.stdin.readline().strip())
-        if not (init_msg.get("event") == "init" and init_msg.get("operation") == "upload"):
-            write_msg({"error": {"code": 32, "message": "Wrong lfs init operation"}})
-            sys.exit(1)
-
-        # The transfer process should use the information it needs from the
-        # initiation structure, and also perform any one-off setup tasks it
-        # needs to do. It should then respond on stdout with a simple empty
-        # confirmation structure, as follows:
-        write_msg({})
-
-        # After the initiation exchange, git-lfs will send any number of
-        # transfer requests to the stdin of the transfer process, in a serial sequence.
-        while True:
-            msg = read_msg()
-            if msg is None:
-                # When all transfers have been processed, git-lfs will send
-                # a terminate event to the stdin of the transfer process.
-                # On receiving this message the transfer process should
-                # clean up and terminate. No response is expected.
-                sys.exit(0)
-
-            oid = msg["oid"]
-            filepath = msg["path"]
-            completion_url = msg["action"]["href"]
-            header = msg["action"]["header"]
-            chunk_size = int(header.pop("chunk_size"))
-            presigned_urls: List[str] = list(header.values())
-
-            # Send a "started" progress event to allow other workers to start.
-            # Otherwise they're delayed until first "progress" event is reported,
-            # i.e. after the first 5GB by default (!)
-            write_msg(
-                {
-                    "event": "progress",
-                    "oid": oid,
-                    "bytesSoFar": 1,
-                    "bytesSinceLast": 0,
-                }
-            )
-
-            parts = []
-            with open(filepath, "rb") as file:
-                for i, presigned_url in enumerate(presigned_urls):
-                    with SliceFileObj(
-                        file,
-                        seek_from=i * chunk_size,
-                        read_limit=chunk_size,
-                    ) as data:
-                        r = get_session().put(presigned_url, data=data)
-                        hf_raise_for_status(r)
-                        parts.append(
-                            {
-                                "etag": r.headers.get("etag"),
-                                "partNumber": i + 1,
-                            }
-                        )
-                        # In order to support progress reporting while data is uploading / downloading,
-                        # the transfer process should post messages to stdout
-                        write_msg(
-                            {
-                                "event": "progress",
-                                "oid": oid,
-                                "bytesSoFar": (i + 1) * chunk_size,
-                                "bytesSinceLast": chunk_size,
-                            }
-                        )
-                        # Not precise but that's ok.
-
-            r = get_session().post(
-                completion_url,
-                json={
-                    "oid": oid,
-                    "parts": parts,
-                },
-            )
-            hf_raise_for_status(r)
-
-            write_msg({"event": "complete", "oid": oid})

huggingface_hub/commands/repo_files.py

@@ -1,128 +0,0 @@
-# coding=utf-8
-# Copyright 2023-present, the HuggingFace Inc. team.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-"""Contains command to update or delete files in a repository using the CLI.
-
-Usage:
-    # delete all
-    huggingface-cli repo-files <repo_id> delete "*"
-
-    # delete single file
-    huggingface-cli repo-files <repo_id> delete file.txt
-
-    # delete single folder
-    huggingface-cli repo-files <repo_id> delete folder/
-
-    # delete multiple
-    huggingface-cli repo-files <repo_id> delete file.txt folder/ file2.txt
-
-    # delete multiple patterns
-    huggingface-cli repo-files <repo_id> delete file.txt "*.json" "folder/*.parquet"
-
-    # delete from different revision / repo-type
-    huggingface-cli repo-files <repo_id> delete file.txt --revision=refs/pr/1 --repo-type=dataset
-"""
-
-from argparse import _SubParsersAction
-from typing import List, Optional
-
-from huggingface_hub import logging
-from huggingface_hub.commands import BaseHuggingfaceCLICommand
-from huggingface_hub.hf_api import HfApi
-
-
-logger = logging.get_logger(__name__)
-
-
-class DeleteFilesSubCommand:
-    def __init__(self, args) -> None:
-        self.args = args
-        self.repo_id: str = args.repo_id
-        self.repo_type: Optional[str] = args.repo_type
-        self.revision: Optional[str] = args.revision
-        self.api: HfApi = HfApi(token=args.token, library_name="huggingface-cli")
-        self.patterns: List[str] = args.patterns
-        self.commit_message: Optional[str] = args.commit_message
-        self.commit_description: Optional[str] = args.commit_description
-        self.create_pr: bool = args.create_pr
-        self.token: Optional[str] = args.token
-
-    def run(self) -> None:
-        logging.set_verbosity_info()
-        url = self.api.delete_files(
-            delete_patterns=self.patterns,
-            repo_id=self.repo_id,
-            repo_type=self.repo_type,
-            revision=self.revision,
-            commit_message=self.commit_message,
-            commit_description=self.commit_description,
-            create_pr=self.create_pr,
-        )
-        print(f"Files correctly deleted from repo. Commit: {url}.")
-        logging.set_verbosity_warning()
-
-
-class RepoFilesCommand(BaseHuggingfaceCLICommand):
-    @staticmethod
-    def register_subcommand(parser: _SubParsersAction):
-        repo_files_parser = parser.add_parser("repo-files", help="Manage files in a repo on the Hub")
-        repo_files_parser.add_argument(
-            "repo_id", type=str, help="The ID of the repo to manage (e.g. `username/repo-name`)."
-        )
-        repo_files_subparsers = repo_files_parser.add_subparsers(
-            help="Action to execute against the files.",
-            required=True,
-        )
-        delete_subparser = repo_files_subparsers.add_parser(
-            "delete",
-            help="Delete files from a repo on the Hub",
-        )
-        delete_subparser.set_defaults(func=lambda args: DeleteFilesSubCommand(args))
-        delete_subparser.add_argument(
-            "patterns",
-            nargs="+",
-            type=str,
-            help="Glob patterns to match files to delete.",
-        )
-        delete_subparser.add_argument(
-            "--repo-type",
-            choices=["model", "dataset", "space"],
-            default="model",
-            help="Type of the repo to upload to (e.g. `dataset`).",
-        )
-        delete_subparser.add_argument(
-            "--revision",
-            type=str,
-            help=(
-                "An optional Git revision to push to. It can be a branch name "
-                "or a PR reference. If revision does not"
-                " exist and `--create-pr` is not set, a branch will be automatically created."
-            ),
-        )
-        delete_subparser.add_argument(
-            "--commit-message", type=str, help="The summary / title / first line of the generated commit."
-        )
-        delete_subparser.add_argument(
-            "--commit-description", type=str, help="The description of the generated commit."
-        )
-        delete_subparser.add_argument(
-            "--create-pr", action="store_true", help="Whether to create a new Pull Request for these changes."
-        )
-        repo_files_parser.add_argument(
-            "--token",
-            type=str,
-            help="A User Access Token generated from https://huggingface.co/settings/tokens",
-        )
-
-        repo_files_parser.set_defaults(func=RepoFilesCommand)