huggingface-hub 0.36.0rc0__py3-none-any.whl → 1.0.0__py3-none-any.whl

huggingface_hub/cli/download.py

@@ -37,145 +37,153 @@ Usage:
 """
 
 import warnings
-from argparse import Namespace, _SubParsersAction
-from typing import List, Optional
+from typing import Annotated, Optional, Union
+
+import typer
 
 from huggingface_hub import logging
 from huggingface_hub._snapshot_download import snapshot_download
-from huggingface_hub.commands import BaseHuggingfaceCLICommand
-from huggingface_hub.file_download import hf_hub_download
-from huggingface_hub.utils import disable_progress_bars, enable_progress_bars
+from huggingface_hub.file_download import DryRunFileInfo, hf_hub_download
+from huggingface_hub.utils import _format_size, disable_progress_bars, enable_progress_bars, tabulate
+
+from ._cli_utils import RepoIdArg, RepoTypeOpt, RevisionOpt, TokenOpt
 
 
 logger = logging.get_logger(__name__)
 
 
-class DownloadCommand(BaseHuggingfaceCLICommand):
-    @staticmethod
-    def register_subcommand(parser: _SubParsersAction):
-        download_parser = parser.add_parser("download", help="Download files from the Hub")
-        download_parser.add_argument(
-            "repo_id", type=str, help="ID of the repo to download from (e.g. `username/repo-name`)."
-        )
-        download_parser.add_argument(
-            "filenames", type=str, nargs="*", help="Files to download (e.g. `config.json`, `data/metadata.jsonl`)."
-        )
-        download_parser.add_argument(
-            "--repo-type",
-            choices=["model", "dataset", "space"],
-            default="model",
-            help="Type of repo to download from (defaults to 'model').",
-        )
-        download_parser.add_argument(
-            "--revision",
-            type=str,
-            help="An optional Git revision id which can be a branch name, a tag, or a commit hash.",
-        )
-        download_parser.add_argument(
-            "--include", nargs="*", type=str, help="Glob patterns to match files to download."
-        )
-        download_parser.add_argument(
-            "--exclude", nargs="*", type=str, help="Glob patterns to exclude from files to download."
-        )
-        download_parser.add_argument(
-            "--cache-dir", type=str, help="Path to the directory where to save the downloaded files."
-        )
-        download_parser.add_argument(
-            "--local-dir",
-            type=str,
-            help=(
-                "If set, the downloaded file will be placed under this directory. Check out"
-                " https://huggingface.co/docs/huggingface_hub/guides/download#download-files-to-local-folder for more"
-                " details."
-            ),
-        )
-        download_parser.add_argument(
-            "--force-download",
-            action="store_true",
+def download(
+    repo_id: RepoIdArg,
+    filenames: Annotated[
+        Optional[list[str]],
+        typer.Argument(
+            help="Files to download (e.g. `config.json`, `data/metadata.jsonl`).",
+        ),
+    ] = None,
+    repo_type: RepoTypeOpt = RepoTypeOpt.model,
+    revision: RevisionOpt = None,
+    include: Annotated[
+        Optional[list[str]],
+        typer.Option(
+            help="Glob patterns to include from files to download. eg: *.json",
+        ),
+    ] = None,
+    exclude: Annotated[
+        Optional[list[str]],
+        typer.Option(
+            help="Glob patterns to exclude from files to download.",
+        ),
+    ] = None,
+    cache_dir: Annotated[
+        Optional[str],
+        typer.Option(
+            help="Directory where to save files.",
+        ),
+    ] = None,
+    local_dir: Annotated[
+        Optional[str],
+        typer.Option(
+            help="If set, the downloaded file will be placed under this directory. Check out https://huggingface.co/docs/huggingface_hub/guides/download#download-files-to-local-folder for more details.",
+        ),
+    ] = None,
+    force_download: Annotated[
+        bool,
+        typer.Option(
             help="If True, the files will be downloaded even if they are already cached.",
-        )
-        download_parser.add_argument(
-            "--token", type=str, help="A User Access Token generated from https://huggingface.co/settings/tokens"
-        )
-        download_parser.add_argument(
-            "--quiet",
-            action="store_true",
+        ),
+    ] = False,
+    dry_run: Annotated[
+        bool,
+        typer.Option(
+            help="If True, perform a dry run without actually downloading the file.",
+        ),
+    ] = False,
+    token: TokenOpt = None,
+    quiet: Annotated[
+        bool,
+        typer.Option(
             help="If True, progress bars are disabled and only the path to the download files is printed.",
-        )
-        download_parser.add_argument(
-            "--max-workers",
-            type=int,
-            default=8,
+        ),
+    ] = False,
+    max_workers: Annotated[
+        int,
+        typer.Option(
             help="Maximum number of workers to use for downloading files. Default is 8.",
-        )
-        download_parser.set_defaults(func=DownloadCommand)
-
-    def __init__(self, args: Namespace) -> None:
-        self.token = args.token
-        self.repo_id: str = args.repo_id
-        self.filenames: List[str] = args.filenames
-        self.repo_type: str = args.repo_type
-        self.revision: Optional[str] = args.revision
-        self.include: Optional[List[str]] = args.include
-        self.exclude: Optional[List[str]] = args.exclude
-        self.cache_dir: Optional[str] = args.cache_dir
-        self.local_dir: Optional[str] = args.local_dir
-        self.force_download: bool = args.force_download
-        self.quiet: bool = args.quiet
-        self.max_workers: int = args.max_workers
-
-    def run(self) -> None:
-        if self.quiet:
-            disable_progress_bars()
-            with warnings.catch_warnings():
-                warnings.simplefilter("ignore")
-                print(self._download())  # Print path to downloaded files
-            enable_progress_bars()
-        else:
-            logging.set_verbosity_info()
-            print(self._download())  # Print path to downloaded files
-            logging.set_verbosity_warning()
+        ),
+    ] = 8,
+) -> None:
+    """Download files from the Hub."""
 
-    def _download(self) -> str:
+    def run_download() -> Union[str, DryRunFileInfo, list[DryRunFileInfo]]:
+        filenames_list = filenames if filenames is not None else []
         # Warn user if patterns are ignored
-        if len(self.filenames) > 0:
-            if self.include is not None and len(self.include) > 0:
+        if len(filenames_list) > 0:
+            if include is not None and len(include) > 0:
                 warnings.warn("Ignoring `--include` since filenames have being explicitly set.")
-            if self.exclude is not None and len(self.exclude) > 0:
+            if exclude is not None and len(exclude) > 0:
                 warnings.warn("Ignoring `--exclude` since filenames have being explicitly set.")
 
         # Single file to download: use `hf_hub_download`
-        if len(self.filenames) == 1:
+        if len(filenames_list) == 1:
             return hf_hub_download(
-                repo_id=self.repo_id,
-                repo_type=self.repo_type,
-                revision=self.revision,
-                filename=self.filenames[0],
-                cache_dir=self.cache_dir,
-                force_download=self.force_download,
-                token=self.token,
-                local_dir=self.local_dir,
+                repo_id=repo_id,
+                repo_type=repo_type.value,
+                revision=revision,
+                filename=filenames_list[0],
+                cache_dir=cache_dir,
+                force_download=force_download,
+                token=token,
+                local_dir=local_dir,
                 library_name="huggingface-cli",
+                dry_run=dry_run,
             )
 
         # Otherwise: use `snapshot_download` to ensure all files comes from same revision
-        elif len(self.filenames) == 0:
-            allow_patterns = self.include
-            ignore_patterns = self.exclude
+        if len(filenames_list) == 0:
+            allow_patterns = include
+            ignore_patterns = exclude
         else:
-            allow_patterns = self.filenames
+            allow_patterns = filenames_list
             ignore_patterns = None
 
         return snapshot_download(
-            repo_id=self.repo_id,
-            repo_type=self.repo_type,
-            revision=self.revision,
+            repo_id=repo_id,
+            repo_type=repo_type.value,
+            revision=revision,
             allow_patterns=allow_patterns,
             ignore_patterns=ignore_patterns,
-            force_download=self.force_download,
-            cache_dir=self.cache_dir,
-            token=self.token,
-            local_dir=self.local_dir,
+            force_download=force_download,
+            cache_dir=cache_dir,
+            token=token,
+            local_dir=local_dir,
             library_name="huggingface-cli",
-            max_workers=self.max_workers,
+            max_workers=max_workers,
+            dry_run=dry_run,
+        )
+
+    def _print_result(result: Union[str, DryRunFileInfo, list[DryRunFileInfo]]) -> None:
+        if isinstance(result, str):
+            print(result)
+            return
+
+        # Print dry run info
+        if isinstance(result, DryRunFileInfo):
+            result = [result]
+        print(
+            f"[dry-run] Will download {len([r for r in result if r.will_download])} files (out of {len(result)}) totalling {_format_size(sum(r.file_size for r in result if r.will_download))}."
         )
+        columns = ["File", "Bytes to download"]
+        items: list[list[Union[str, int]]] = []
+        for info in sorted(result, key=lambda x: x.filename):
+            items.append([info.filename, _format_size(info.file_size) if info.will_download else "-"])
+        print(tabulate(items, headers=columns))
+
+    if quiet:
+        disable_progress_bars()
+        with warnings.catch_warnings():
+            warnings.simplefilter("ignore")
+            _print_result(run_download())
+        enable_progress_bars()
+    else:
+        _print_result(run_download())
+        logging.set_verbosity_warning()

huggingface_hub/cli/hf.py

@@ -12,51 +12,48 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from argparse import ArgumentParser
 
-from huggingface_hub.cli.auth import AuthCommands
-from huggingface_hub.cli.cache import CacheCommand
-from huggingface_hub.cli.download import DownloadCommand
-from huggingface_hub.cli.jobs import JobsCommands
-from huggingface_hub.cli.lfs import LfsCommands
-from huggingface_hub.cli.repo import RepoCommands
-from huggingface_hub.cli.repo_files import RepoFilesCommand
-from huggingface_hub.cli.system import EnvironmentCommand, VersionCommand
-from huggingface_hub.cli.upload import UploadCommand
-from huggingface_hub.cli.upload_large_folder import UploadLargeFolderCommand
+from huggingface_hub.cli._cli_utils import check_cli_update, typer_factory
+from huggingface_hub.cli.auth import auth_cli
+from huggingface_hub.cli.cache import cache_cli
+from huggingface_hub.cli.download import download
+from huggingface_hub.cli.jobs import jobs_cli
+from huggingface_hub.cli.lfs import lfs_enable_largefiles, lfs_multipart_upload
+from huggingface_hub.cli.repo import repo_cli
+from huggingface_hub.cli.repo_files import repo_files_cli
+from huggingface_hub.cli.system import env, version
+
+# from huggingface_hub.cli.jobs import jobs_app
+from huggingface_hub.cli.upload import upload
+from huggingface_hub.cli.upload_large_folder import upload_large_folder
+from huggingface_hub.utils import logging
+
+
+app = typer_factory(help="Hugging Face Hub CLI")
+
+
+# top level single commands (defined in their respective files)
+app.command(help="Download files from the Hub.")(download)
+app.command(help="Upload a file or a folder to the Hub.")(upload)
+app.command(help="Upload a large folder to the Hub. Recommended for resumable uploads.")(upload_large_folder)
+app.command(name="env", help="Print information about the environment.")(env)
+app.command(help="Print information about the hf version.")(version)
+app.command(help="Configure your repository to enable upload of files > 5GB.", hidden=True)(lfs_enable_largefiles)
+app.command(help="Upload large files to the Hub.", hidden=True)(lfs_multipart_upload)
+
+
+# command groups
+app.add_typer(auth_cli, name="auth")
+app.add_typer(cache_cli, name="cache")
+app.add_typer(repo_cli, name="repo")
+app.add_typer(repo_files_cli, name="repo-files")
+app.add_typer(jobs_cli, name="jobs")
 
 
 def main():
-    parser = ArgumentParser("hf", usage="hf <command> [<args>]")
-    commands_parser = parser.add_subparsers(help="hf command helpers")
-
-    # Register commands
-    AuthCommands.register_subcommand(commands_parser)
-    CacheCommand.register_subcommand(commands_parser)
-    DownloadCommand.register_subcommand(commands_parser)
-    JobsCommands.register_subcommand(commands_parser)
-    RepoCommands.register_subcommand(commands_parser)
-    RepoFilesCommand.register_subcommand(commands_parser)
-    UploadCommand.register_subcommand(commands_parser)
-    UploadLargeFolderCommand.register_subcommand(commands_parser)
-
-    # System commands
-    EnvironmentCommand.register_subcommand(commands_parser)
-    VersionCommand.register_subcommand(commands_parser)
-
-    # LFS commands (hidden in --help)
-    LfsCommands.register_subcommand(commands_parser)
-
-    # Let's go
-    args = parser.parse_args()
-    if not hasattr(args, "func"):
-        parser.print_help()
-        exit(1)
-
-    # Run
-    service = args.func(args)
-    if service is not None:
-        service.run()
+    logging.set_verbosity_info()
+    check_cli_update()
+    app()
 
 
 if __name__ == "__main__":