huggingface-hub 0.33.4__py3-none-any.whl → 0.34.0rc0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of huggingface-hub might be problematic. Click here for more details.

Files changed (68) hide show
  1. huggingface_hub/__init__.py +47 -1
  2. huggingface_hub/_commit_api.py +21 -28
  3. huggingface_hub/_jobs_api.py +145 -0
  4. huggingface_hub/_local_folder.py +7 -1
  5. huggingface_hub/_login.py +5 -5
  6. huggingface_hub/_oauth.py +1 -1
  7. huggingface_hub/_snapshot_download.py +11 -6
  8. huggingface_hub/_upload_large_folder.py +46 -23
  9. huggingface_hub/cli/__init__.py +27 -0
  10. huggingface_hub/cli/_cli_utils.py +69 -0
  11. huggingface_hub/cli/auth.py +210 -0
  12. huggingface_hub/cli/cache.py +405 -0
  13. huggingface_hub/cli/download.py +181 -0
  14. huggingface_hub/cli/hf.py +66 -0
  15. huggingface_hub/cli/jobs.py +522 -0
  16. huggingface_hub/cli/lfs.py +198 -0
  17. huggingface_hub/cli/repo.py +243 -0
  18. huggingface_hub/cli/repo_files.py +128 -0
  19. huggingface_hub/cli/system.py +52 -0
  20. huggingface_hub/cli/upload.py +316 -0
  21. huggingface_hub/cli/upload_large_folder.py +132 -0
  22. huggingface_hub/commands/_cli_utils.py +5 -0
  23. huggingface_hub/commands/delete_cache.py +3 -1
  24. huggingface_hub/commands/download.py +4 -0
  25. huggingface_hub/commands/env.py +3 -0
  26. huggingface_hub/commands/huggingface_cli.py +2 -0
  27. huggingface_hub/commands/repo.py +4 -0
  28. huggingface_hub/commands/repo_files.py +4 -0
  29. huggingface_hub/commands/scan_cache.py +3 -1
  30. huggingface_hub/commands/tag.py +3 -1
  31. huggingface_hub/commands/upload.py +4 -0
  32. huggingface_hub/commands/upload_large_folder.py +3 -1
  33. huggingface_hub/commands/user.py +11 -1
  34. huggingface_hub/commands/version.py +3 -0
  35. huggingface_hub/constants.py +1 -0
  36. huggingface_hub/file_download.py +16 -5
  37. huggingface_hub/hf_api.py +519 -7
  38. huggingface_hub/hf_file_system.py +8 -16
  39. huggingface_hub/hub_mixin.py +3 -3
  40. huggingface_hub/inference/_client.py +38 -39
  41. huggingface_hub/inference/_common.py +44 -14
  42. huggingface_hub/inference/_generated/_async_client.py +50 -51
  43. huggingface_hub/inference/_generated/types/__init__.py +1 -0
  44. huggingface_hub/inference/_generated/types/image_to_video.py +60 -0
  45. huggingface_hub/inference/_mcp/cli.py +36 -18
  46. huggingface_hub/inference/_mcp/constants.py +8 -0
  47. huggingface_hub/inference/_mcp/types.py +3 -0
  48. huggingface_hub/inference/_providers/__init__.py +4 -1
  49. huggingface_hub/inference/_providers/_common.py +3 -6
  50. huggingface_hub/inference/_providers/fal_ai.py +85 -42
  51. huggingface_hub/inference/_providers/hf_inference.py +17 -9
  52. huggingface_hub/inference/_providers/replicate.py +19 -1
  53. huggingface_hub/keras_mixin.py +2 -2
  54. huggingface_hub/repocard.py +1 -1
  55. huggingface_hub/repository.py +2 -2
  56. huggingface_hub/utils/_auth.py +1 -1
  57. huggingface_hub/utils/_cache_manager.py +2 -2
  58. huggingface_hub/utils/_dotenv.py +51 -0
  59. huggingface_hub/utils/_headers.py +1 -1
  60. huggingface_hub/utils/_runtime.py +1 -1
  61. huggingface_hub/utils/_xet.py +6 -2
  62. huggingface_hub/utils/_xet_progress_reporting.py +141 -0
  63. {huggingface_hub-0.33.4.dist-info → huggingface_hub-0.34.0rc0.dist-info}/METADATA +7 -8
  64. {huggingface_hub-0.33.4.dist-info → huggingface_hub-0.34.0rc0.dist-info}/RECORD +68 -51
  65. {huggingface_hub-0.33.4.dist-info → huggingface_hub-0.34.0rc0.dist-info}/entry_points.txt +1 -0
  66. {huggingface_hub-0.33.4.dist-info → huggingface_hub-0.34.0rc0.dist-info}/LICENSE +0 -0
  67. {huggingface_hub-0.33.4.dist-info → huggingface_hub-0.34.0rc0.dist-info}/WHEEL +0 -0
  68. {huggingface_hub-0.33.4.dist-info → huggingface_hub-0.34.0rc0.dist-info}/top_level.txt +0 -0
@@ -0,0 +1,405 @@
1
+ # coding=utf-8
2
+ # Copyright 2025-present, the HuggingFace Inc. team.
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ """Contains the 'hf cache' command group with 'scan' and 'delete' subcommands."""
16
+
17
+ import os
18
+ import time
19
+ from argparse import Namespace, _SubParsersAction
20
+ from functools import wraps
21
+ from tempfile import mkstemp
22
+ from typing import Any, Callable, Iterable, List, Literal, Optional, Union
23
+
24
+ from ..utils import (
25
+ CachedRepoInfo,
26
+ CachedRevisionInfo,
27
+ CacheNotFound,
28
+ HFCacheInfo,
29
+ scan_cache_dir,
30
+ )
31
+ from . import BaseHuggingfaceCLICommand
32
+ from ._cli_utils import ANSI, tabulate
33
+
34
+
35
+ # --- DELETE helpers (from delete_cache.py) ---
36
+ try:
37
+ from InquirerPy import inquirer
38
+ from InquirerPy.base.control import Choice
39
+ from InquirerPy.separator import Separator
40
+
41
+ _inquirer_py_available = True
42
+ except ImportError:
43
+ _inquirer_py_available = False
44
+
45
+ SortingOption_T = Literal["alphabetical", "lastUpdated", "lastUsed", "size"]
46
+ _CANCEL_DELETION_STR = "CANCEL_DELETION"
47
+
48
+
49
+ def require_inquirer_py(fn: Callable) -> Callable:
50
+ @wraps(fn)
51
+ def _inner(*args, **kwargs):
52
+ if not _inquirer_py_available:
53
+ raise ImportError(
54
+ "The 'cache delete' command requires extra dependencies for the TUI.\n"
55
+ "Please run 'pip install huggingface_hub[cli]' to install them.\n"
56
+ "Otherwise, disable TUI using the '--disable-tui' flag."
57
+ )
58
+ return fn(*args, **kwargs)
59
+
60
+ return _inner
61
+
62
+
63
+ class CacheCommand(BaseHuggingfaceCLICommand):
64
+ @staticmethod
65
+ def register_subcommand(parser: _SubParsersAction):
66
+ cache_parser = parser.add_parser("cache", help="Manage local cache directory.")
67
+ cache_subparsers = cache_parser.add_subparsers(dest="cache_command", help="Cache subcommands")
68
+ # Scan subcommand
69
+ scan_parser = cache_subparsers.add_parser("scan", help="Scan cache directory.")
70
+ scan_parser.add_argument(
71
+ "--dir",
72
+ type=str,
73
+ default=None,
74
+ help="cache directory to scan (optional). Default to the default HuggingFace cache.",
75
+ )
76
+ scan_parser.add_argument(
77
+ "-v",
78
+ "--verbose",
79
+ action="count",
80
+ default=0,
81
+ help="show a more verbose output",
82
+ )
83
+ scan_parser.set_defaults(func=CacheCommand, cache_command="scan")
84
+ # Delete subcommand
85
+ delete_parser = cache_subparsers.add_parser("delete", help="Delete revisions from the cache directory.")
86
+ delete_parser.add_argument(
87
+ "--dir",
88
+ type=str,
89
+ default=None,
90
+ help="cache directory (optional). Default to the default HuggingFace cache.",
91
+ )
92
+ delete_parser.add_argument(
93
+ "--disable-tui",
94
+ action="store_true",
95
+ help=(
96
+ "Disable Terminal User Interface (TUI) mode. Useful if your platform/terminal doesn't support the multiselect menu."
97
+ ),
98
+ )
99
+ delete_parser.add_argument(
100
+ "--sort",
101
+ nargs="?",
102
+ choices=["alphabetical", "lastUpdated", "lastUsed", "size"],
103
+ help=(
104
+ "Sort repositories by the specified criteria. Options: "
105
+ "'alphabetical' (A-Z), "
106
+ "'lastUpdated' (newest first), "
107
+ "'lastUsed' (most recent first), "
108
+ "'size' (largest first)."
109
+ ),
110
+ )
111
+ delete_parser.set_defaults(func=CacheCommand, cache_command="delete")
112
+
113
+ def __init__(self, args: Namespace) -> None:
114
+ self.args = args
115
+ self.verbosity: int = getattr(args, "verbose", 0)
116
+ self.cache_dir: Optional[str] = getattr(args, "dir", None)
117
+ self.disable_tui: bool = getattr(args, "disable_tui", False)
118
+ self.sort_by: Optional[SortingOption_T] = getattr(args, "sort", None)
119
+ self.cache_command: Optional[str] = getattr(args, "cache_command", None)
120
+
121
+ def run(self):
122
+ if self.cache_command == "scan":
123
+ self._run_scan()
124
+ elif self.cache_command == "delete":
125
+ self._run_delete()
126
+ else:
127
+ print("Please specify a cache subcommand (scan or delete). Use -h for help.")
128
+
129
+ def _run_scan(self):
130
+ try:
131
+ t0 = time.time()
132
+ hf_cache_info = scan_cache_dir(self.cache_dir)
133
+ t1 = time.time()
134
+ except CacheNotFound as exc:
135
+ cache_dir = exc.cache_dir
136
+ print(f"Cache directory not found: {cache_dir}")
137
+ return
138
+ print(get_table(hf_cache_info, verbosity=self.verbosity))
139
+ print(
140
+ f"\nDone in {round(t1 - t0, 1)}s. Scanned {len(hf_cache_info.repos)} repo(s)"
141
+ f" for a total of {ANSI.red(hf_cache_info.size_on_disk_str)}."
142
+ )
143
+ if len(hf_cache_info.warnings) > 0:
144
+ message = f"Got {len(hf_cache_info.warnings)} warning(s) while scanning."
145
+ if self.verbosity >= 3:
146
+ print(ANSI.gray(message))
147
+ for warning in hf_cache_info.warnings:
148
+ print(ANSI.gray(warning))
149
+ else:
150
+ print(ANSI.gray(message + " Use -vvv to print details."))
151
+
152
+ def _run_delete(self):
153
+ hf_cache_info = scan_cache_dir(self.cache_dir)
154
+ if self.disable_tui:
155
+ selected_hashes = _manual_review_no_tui(hf_cache_info, preselected=[], sort_by=self.sort_by)
156
+ else:
157
+ selected_hashes = _manual_review_tui(hf_cache_info, preselected=[], sort_by=self.sort_by)
158
+ if len(selected_hashes) > 0 and _CANCEL_DELETION_STR not in selected_hashes:
159
+ confirm_message = _get_expectations_str(hf_cache_info, selected_hashes) + " Confirm deletion ?"
160
+ if self.disable_tui:
161
+ confirmed = _ask_for_confirmation_no_tui(confirm_message)
162
+ else:
163
+ confirmed = _ask_for_confirmation_tui(confirm_message)
164
+ if confirmed:
165
+ strategy = hf_cache_info.delete_revisions(*selected_hashes)
166
+ print("Start deletion.")
167
+ strategy.execute()
168
+ print(
169
+ f"Done. Deleted {len(strategy.repos)} repo(s) and"
170
+ f" {len(strategy.snapshots)} revision(s) for a total of"
171
+ f" {strategy.expected_freed_size_str}."
172
+ )
173
+ return
174
+ print("Deletion is cancelled. Do nothing.")
175
+
176
+
177
+ def get_table(hf_cache_info: HFCacheInfo, *, verbosity: int = 0) -> str:
178
+ if verbosity == 0:
179
+ return tabulate(
180
+ rows=[
181
+ [
182
+ repo.repo_id,
183
+ repo.repo_type,
184
+ "{:>12}".format(repo.size_on_disk_str),
185
+ repo.nb_files,
186
+ repo.last_accessed_str,
187
+ repo.last_modified_str,
188
+ ", ".join(sorted(repo.refs)),
189
+ str(repo.repo_path),
190
+ ]
191
+ for repo in sorted(hf_cache_info.repos, key=lambda repo: repo.repo_path)
192
+ ],
193
+ headers=[
194
+ "REPO ID",
195
+ "REPO TYPE",
196
+ "SIZE ON DISK",
197
+ "NB FILES",
198
+ "LAST_ACCESSED",
199
+ "LAST_MODIFIED",
200
+ "REFS",
201
+ "LOCAL PATH",
202
+ ],
203
+ )
204
+ else:
205
+ return tabulate(
206
+ rows=[
207
+ [
208
+ repo.repo_id,
209
+ repo.repo_type,
210
+ revision.commit_hash,
211
+ "{:>12}".format(revision.size_on_disk_str),
212
+ revision.nb_files,
213
+ revision.last_modified_str,
214
+ ", ".join(sorted(revision.refs)),
215
+ str(revision.snapshot_path),
216
+ ]
217
+ for repo in sorted(hf_cache_info.repos, key=lambda repo: repo.repo_path)
218
+ for revision in sorted(repo.revisions, key=lambda revision: revision.commit_hash)
219
+ ],
220
+ headers=[
221
+ "REPO ID",
222
+ "REPO TYPE",
223
+ "REVISION",
224
+ "SIZE ON DISK",
225
+ "NB FILES",
226
+ "LAST_MODIFIED",
227
+ "REFS",
228
+ "LOCAL PATH",
229
+ ],
230
+ )
231
+
232
+
233
+ def _get_repo_sorting_key(repo: CachedRepoInfo, sort_by: Optional[SortingOption_T] = None):
234
+ if sort_by == "alphabetical":
235
+ return (repo.repo_type, repo.repo_id.lower())
236
+ elif sort_by == "lastUpdated":
237
+ return -max(rev.last_modified for rev in repo.revisions)
238
+ elif sort_by == "lastUsed":
239
+ return -repo.last_accessed
240
+ elif sort_by == "size":
241
+ return -repo.size_on_disk
242
+ else:
243
+ return (repo.repo_type, repo.repo_id)
244
+
245
+
246
+ @require_inquirer_py
247
+ def _manual_review_tui(
248
+ hf_cache_info: HFCacheInfo, preselected: List[str], sort_by: Optional[SortingOption_T] = None
249
+ ) -> List[str]:
250
+ choices = _get_tui_choices_from_scan(repos=hf_cache_info.repos, preselected=preselected, sort_by=sort_by)
251
+ checkbox = inquirer.checkbox(
252
+ message="Select revisions to delete:",
253
+ choices=choices,
254
+ cycle=False,
255
+ height=100,
256
+ instruction=_get_expectations_str(
257
+ hf_cache_info, selected_hashes=[c.value for c in choices if isinstance(c, Choice) and c.enabled]
258
+ ),
259
+ long_instruction="Press <space> to select, <enter> to validate and <ctrl+c> to quit without modification.",
260
+ transformer=lambda result: f"{len(result)} revision(s) selected.",
261
+ )
262
+
263
+ def _update_expectations(_):
264
+ checkbox._instruction = _get_expectations_str(
265
+ hf_cache_info,
266
+ selected_hashes=[choice["value"] for choice in checkbox.content_control.choices if choice["enabled"]],
267
+ )
268
+
269
+ checkbox.kb_func_lookup["toggle"].append({"func": _update_expectations})
270
+ try:
271
+ return checkbox.execute()
272
+ except KeyboardInterrupt:
273
+ return []
274
+
275
+
276
+ @require_inquirer_py
277
+ def _ask_for_confirmation_tui(message: str, default: bool = True) -> bool:
278
+ return inquirer.confirm(message, default=default).execute()
279
+
280
+
281
+ def _get_tui_choices_from_scan(
282
+ repos: Iterable[CachedRepoInfo], preselected: List[str], sort_by: Optional[SortingOption_T] = None
283
+ ) -> List:
284
+ choices: List[Union["Choice", "Separator"]] = []
285
+ choices.append(
286
+ Choice(
287
+ _CANCEL_DELETION_STR, name="None of the following (if selected, nothing will be deleted).", enabled=False
288
+ )
289
+ )
290
+ sorted_repos = sorted(repos, key=lambda repo: _get_repo_sorting_key(repo, sort_by))
291
+ for repo in sorted_repos:
292
+ choices.append(
293
+ Separator(
294
+ f"\n{repo.repo_type.capitalize()} {repo.repo_id} ({repo.size_on_disk_str}, used {repo.last_accessed_str})"
295
+ )
296
+ )
297
+ for revision in sorted(repo.revisions, key=_revision_sorting_order):
298
+ choices.append(
299
+ Choice(
300
+ revision.commit_hash,
301
+ name=(
302
+ f"{revision.commit_hash[:8]}: {', '.join(sorted(revision.refs)) or '(detached)'} # modified {revision.last_modified_str}"
303
+ ),
304
+ enabled=revision.commit_hash in preselected,
305
+ )
306
+ )
307
+ return choices
308
+
309
+
310
+ def _manual_review_no_tui(
311
+ hf_cache_info: HFCacheInfo, preselected: List[str], sort_by: Optional[SortingOption_T] = None
312
+ ) -> List[str]:
313
+ fd, tmp_path = mkstemp(suffix=".txt")
314
+ os.close(fd)
315
+ lines = []
316
+ sorted_repos = sorted(hf_cache_info.repos, key=lambda repo: _get_repo_sorting_key(repo, sort_by))
317
+ for repo in sorted_repos:
318
+ lines.append(
319
+ f"\n# {repo.repo_type.capitalize()} {repo.repo_id} ({repo.size_on_disk_str}, used {repo.last_accessed_str})"
320
+ )
321
+ for revision in sorted(repo.revisions, key=_revision_sorting_order):
322
+ lines.append(
323
+ f"{'' if revision.commit_hash in preselected else '#'} {revision.commit_hash} # Refs: {', '.join(sorted(revision.refs)) or '(detached)'} # modified {revision.last_modified_str}"
324
+ )
325
+ with open(tmp_path, "w") as f:
326
+ f.write(_MANUAL_REVIEW_NO_TUI_INSTRUCTIONS)
327
+ f.write("\n".join(lines))
328
+ instructions = f"""
329
+ TUI is disabled. In order to select which revisions you want to delete, please edit
330
+ the following file using the text editor of your choice. Instructions for manual
331
+ editing are located at the beginning of the file. Edit the file, save it and confirm
332
+ to continue.
333
+ File to edit: {ANSI.bold(tmp_path)}
334
+ """
335
+ print("\n".join(line.strip() for line in instructions.strip().split("\n")))
336
+ while True:
337
+ selected_hashes = _read_manual_review_tmp_file(tmp_path)
338
+ if _ask_for_confirmation_no_tui(
339
+ _get_expectations_str(hf_cache_info, selected_hashes) + " Continue ?", default=False
340
+ ):
341
+ break
342
+ os.remove(tmp_path)
343
+ return sorted(selected_hashes)
344
+
345
+
346
+ def _ask_for_confirmation_no_tui(message: str, default: bool = True) -> bool:
347
+ YES = ("y", "yes", "1")
348
+ NO = ("n", "no", "0")
349
+ DEFAULT = ""
350
+ ALL = YES + NO + (DEFAULT,)
351
+ full_message = message + (" (Y/n) " if default else " (y/N) ")
352
+ while True:
353
+ answer = input(full_message).lower()
354
+ if answer == DEFAULT:
355
+ return default
356
+ if answer in YES:
357
+ return True
358
+ if answer in NO:
359
+ return False
360
+ print(f"Invalid input. Must be one of {ALL}")
361
+
362
+
363
+ def _get_expectations_str(hf_cache_info: HFCacheInfo, selected_hashes: List[str]) -> str:
364
+ if _CANCEL_DELETION_STR in selected_hashes:
365
+ return "Nothing will be deleted."
366
+ strategy = hf_cache_info.delete_revisions(*selected_hashes)
367
+ return f"{len(selected_hashes)} revisions selected counting for {strategy.expected_freed_size_str}."
368
+
369
+
370
+ def _read_manual_review_tmp_file(tmp_path: str) -> List[str]:
371
+ with open(tmp_path) as f:
372
+ content = f.read()
373
+ lines = [line.strip() for line in content.split("\n")]
374
+ selected_lines = [line for line in lines if not line.startswith("#")]
375
+ selected_hashes = [line.split("#")[0].strip() for line in selected_lines]
376
+ return [hash for hash in selected_hashes if len(hash) > 0]
377
+
378
+
379
+ _MANUAL_REVIEW_NO_TUI_INSTRUCTIONS = f"""
380
+ # INSTRUCTIONS
381
+ # ------------
382
+ # This is a temporary file created by running `hf cache delete --disable-tui`. It contains a set of revisions that can be deleted from your local cache directory.
383
+ #
384
+ # Please manually review the revisions you want to delete:
385
+ # - Revision hashes can be commented out with '#'.
386
+ # - Only non-commented revisions in this file will be deleted.
387
+ # - Revision hashes that are removed from this file are ignored as well.
388
+ # - If `{_CANCEL_DELETION_STR}` line is uncommented, the all cache deletion is cancelled and no changes will be applied.
389
+ #
390
+ # Once you've manually reviewed this file, please confirm deletion in the terminal. This file will be automatically removed once done.
391
+ # ------------
392
+
393
+ # KILL SWITCH
394
+ # ------------
395
+ # Un-comment following line to completely cancel the deletion process
396
+ # {_CANCEL_DELETION_STR}
397
+ # ------------
398
+
399
+ # REVISIONS
400
+ # ------------
401
+ """.strip()
402
+
403
+
404
+ def _revision_sorting_order(revision: CachedRevisionInfo) -> Any:
405
+ return revision.last_modified
@@ -0,0 +1,181 @@
1
+ # coding=utf-8
2
+ # Copyright 202-present, the HuggingFace Inc. team.
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+ """Contains command to download files from the Hub with the CLI.
16
+
17
+ Usage:
18
+ hf download --help
19
+
20
+ # Download file
21
+ hf download gpt2 config.json
22
+
23
+ # Download entire repo
24
+ hf download fffiloni/zeroscope --repo-type=space --revision=refs/pr/78
25
+
26
+ # Download repo with filters
27
+ hf download gpt2 --include="*.safetensors"
28
+
29
+ # Download with token
30
+ hf download Wauplin/private-model --token=hf_***
31
+
32
+ # Download quietly (no progress bar, no warnings, only the returned path)
33
+ hf download gpt2 config.json --quiet
34
+
35
+ # Download to local dir
36
+ hf download gpt2 --local-dir=./models/gpt2
37
+ """
38
+
39
+ import warnings
40
+ from argparse import Namespace, _SubParsersAction
41
+ from typing import List, Optional
42
+
43
+ from huggingface_hub import logging
44
+ from huggingface_hub._snapshot_download import snapshot_download
45
+ from huggingface_hub.commands import BaseHuggingfaceCLICommand
46
+ from huggingface_hub.file_download import hf_hub_download
47
+ from huggingface_hub.utils import disable_progress_bars, enable_progress_bars
48
+
49
+
50
+ logger = logging.get_logger(__name__)
51
+
52
+
53
+ class DownloadCommand(BaseHuggingfaceCLICommand):
54
+ @staticmethod
55
+ def register_subcommand(parser: _SubParsersAction):
56
+ download_parser = parser.add_parser("download", help="Download files from the Hub")
57
+ download_parser.add_argument(
58
+ "repo_id", type=str, help="ID of the repo to download from (e.g. `username/repo-name`)."
59
+ )
60
+ download_parser.add_argument(
61
+ "filenames", type=str, nargs="*", help="Files to download (e.g. `config.json`, `data/metadata.jsonl`)."
62
+ )
63
+ download_parser.add_argument(
64
+ "--repo-type",
65
+ choices=["model", "dataset", "space"],
66
+ default="model",
67
+ help="Type of repo to download from (defaults to 'model').",
68
+ )
69
+ download_parser.add_argument(
70
+ "--revision",
71
+ type=str,
72
+ help="An optional Git revision id which can be a branch name, a tag, or a commit hash.",
73
+ )
74
+ download_parser.add_argument(
75
+ "--include", nargs="*", type=str, help="Glob patterns to match files to download."
76
+ )
77
+ download_parser.add_argument(
78
+ "--exclude", nargs="*", type=str, help="Glob patterns to exclude from files to download."
79
+ )
80
+ download_parser.add_argument(
81
+ "--cache-dir", type=str, help="Path to the directory where to save the downloaded files."
82
+ )
83
+ download_parser.add_argument(
84
+ "--local-dir",
85
+ type=str,
86
+ help=(
87
+ "If set, the downloaded file will be placed under this directory. Check out"
88
+ " https://huggingface.co/docs/huggingface_hub/guides/download#download-files-to-local-folder for more"
89
+ " details."
90
+ ),
91
+ )
92
+ download_parser.add_argument(
93
+ "--force-download",
94
+ action="store_true",
95
+ help="If True, the files will be downloaded even if they are already cached.",
96
+ )
97
+ download_parser.add_argument(
98
+ "--token", type=str, help="A User Access Token generated from https://huggingface.co/settings/tokens"
99
+ )
100
+ download_parser.add_argument(
101
+ "--quiet",
102
+ action="store_true",
103
+ help="If True, progress bars are disabled and only the path to the download files is printed.",
104
+ )
105
+ download_parser.add_argument(
106
+ "--max-workers",
107
+ type=int,
108
+ default=8,
109
+ help="Maximum number of workers to use for downloading files. Default is 8.",
110
+ )
111
+ download_parser.set_defaults(func=DownloadCommand)
112
+
113
+ def __init__(self, args: Namespace) -> None:
114
+ self.token = args.token
115
+ self.repo_id: str = args.repo_id
116
+ self.filenames: List[str] = args.filenames
117
+ self.repo_type: str = args.repo_type
118
+ self.revision: Optional[str] = args.revision
119
+ self.include: Optional[List[str]] = args.include
120
+ self.exclude: Optional[List[str]] = args.exclude
121
+ self.cache_dir: Optional[str] = args.cache_dir
122
+ self.local_dir: Optional[str] = args.local_dir
123
+ self.force_download: bool = args.force_download
124
+ self.quiet: bool = args.quiet
125
+ self.max_workers: int = args.max_workers
126
+
127
+ def run(self) -> None:
128
+ if self.quiet:
129
+ disable_progress_bars()
130
+ with warnings.catch_warnings():
131
+ warnings.simplefilter("ignore")
132
+ print(self._download()) # Print path to downloaded files
133
+ enable_progress_bars()
134
+ else:
135
+ logging.set_verbosity_info()
136
+ print(self._download()) # Print path to downloaded files
137
+ logging.set_verbosity_warning()
138
+
139
+ def _download(self) -> str:
140
+ # Warn user if patterns are ignored
141
+ if len(self.filenames) > 0:
142
+ if self.include is not None and len(self.include) > 0:
143
+ warnings.warn("Ignoring `--include` since filenames have being explicitly set.")
144
+ if self.exclude is not None and len(self.exclude) > 0:
145
+ warnings.warn("Ignoring `--exclude` since filenames have being explicitly set.")
146
+
147
+ # Single file to download: use `hf_hub_download`
148
+ if len(self.filenames) == 1:
149
+ return hf_hub_download(
150
+ repo_id=self.repo_id,
151
+ repo_type=self.repo_type,
152
+ revision=self.revision,
153
+ filename=self.filenames[0],
154
+ cache_dir=self.cache_dir,
155
+ force_download=self.force_download,
156
+ token=self.token,
157
+ local_dir=self.local_dir,
158
+ library_name="hf",
159
+ )
160
+
161
+ # Otherwise: use `snapshot_download` to ensure all files comes from same revision
162
+ elif len(self.filenames) == 0:
163
+ allow_patterns = self.include
164
+ ignore_patterns = self.exclude
165
+ else:
166
+ allow_patterns = self.filenames
167
+ ignore_patterns = None
168
+
169
+ return snapshot_download(
170
+ repo_id=self.repo_id,
171
+ repo_type=self.repo_type,
172
+ revision=self.revision,
173
+ allow_patterns=allow_patterns,
174
+ ignore_patterns=ignore_patterns,
175
+ force_download=self.force_download,
176
+ cache_dir=self.cache_dir,
177
+ token=self.token,
178
+ local_dir=self.local_dir,
179
+ library_name="hf",
180
+ max_workers=self.max_workers,
181
+ )
@@ -0,0 +1,66 @@
1
+ # Copyright 2020 The HuggingFace Team. All rights reserved.
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ from argparse import ArgumentParser
16
+
17
+ from huggingface_hub.cli.auth import AuthCommands
18
+ from huggingface_hub.cli.cache import CacheCommand
19
+ from huggingface_hub.cli.download import DownloadCommand
20
+ from huggingface_hub.cli.jobs import JobsCommands
21
+ from huggingface_hub.cli.lfs import LfsCommands
22
+ from huggingface_hub.cli.repo import RepoCommands
23
+ from huggingface_hub.cli.repo_files import RepoFilesCommand
24
+ from huggingface_hub.cli.system import EnvironmentCommand, VersionCommand
25
+ from huggingface_hub.cli.upload import UploadCommand
26
+ from huggingface_hub.cli.upload_large_folder import UploadLargeFolderCommand
27
+
28
+
29
+ def main():
30
+ parser = ArgumentParser("hf", usage="hf <command> [<args>]")
31
+ commands_parser = parser.add_subparsers(help="hf command helpers")
32
+
33
+ # Register commands
34
+ AuthCommands.register_subcommand(commands_parser)
35
+ CacheCommand.register_subcommand(commands_parser)
36
+ DownloadCommand.register_subcommand(commands_parser)
37
+ JobsCommands.register_subcommand(commands_parser)
38
+ RepoCommands.register_subcommand(commands_parser)
39
+ RepoFilesCommand.register_subcommand(commands_parser)
40
+ UploadCommand.register_subcommand(commands_parser)
41
+ UploadLargeFolderCommand.register_subcommand(commands_parser)
42
+
43
+ # System commands
44
+ EnvironmentCommand.register_subcommand(commands_parser)
45
+ VersionCommand.register_subcommand(commands_parser)
46
+
47
+ # LFS commands (hidden in --help)
48
+ LfsCommands.register_subcommand(commands_parser)
49
+
50
+ # Legacy commands
51
+
52
+ # Experimental
53
+
54
+ # Let's go
55
+ args = parser.parse_args()
56
+ if not hasattr(args, "func"):
57
+ parser.print_help()
58
+ exit(1)
59
+
60
+ # Run
61
+ service = args.func(args)
62
+ service.run()
63
+
64
+
65
+ if __name__ == "__main__":
66
+ main()