huggingface-hub 1.0.0rc1__py3-none-any.whl → 1.0.0rc3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of huggingface-hub might be problematic. Click here for more details.

Files changed (59) hide show
  1. huggingface_hub/__init__.py +4 -7
  2. huggingface_hub/_commit_api.py +126 -66
  3. huggingface_hub/_commit_scheduler.py +4 -7
  4. huggingface_hub/_login.py +10 -16
  5. huggingface_hub/_snapshot_download.py +119 -21
  6. huggingface_hub/_tensorboard_logger.py +2 -5
  7. huggingface_hub/_upload_large_folder.py +1 -2
  8. huggingface_hub/_webhooks_server.py +8 -20
  9. huggingface_hub/cli/_cli_utils.py +12 -6
  10. huggingface_hub/cli/download.py +32 -7
  11. huggingface_hub/cli/repo.py +137 -5
  12. huggingface_hub/dataclasses.py +122 -2
  13. huggingface_hub/errors.py +4 -0
  14. huggingface_hub/fastai_utils.py +22 -32
  15. huggingface_hub/file_download.py +234 -38
  16. huggingface_hub/hf_api.py +385 -424
  17. huggingface_hub/hf_file_system.py +55 -65
  18. huggingface_hub/inference/_client.py +27 -48
  19. huggingface_hub/inference/_generated/_async_client.py +27 -48
  20. huggingface_hub/inference/_generated/types/image_to_image.py +6 -2
  21. huggingface_hub/inference/_mcp/agent.py +2 -5
  22. huggingface_hub/inference/_mcp/mcp_client.py +6 -8
  23. huggingface_hub/inference/_providers/__init__.py +16 -0
  24. huggingface_hub/inference/_providers/_common.py +2 -0
  25. huggingface_hub/inference/_providers/fal_ai.py +2 -0
  26. huggingface_hub/inference/_providers/publicai.py +6 -0
  27. huggingface_hub/inference/_providers/scaleway.py +28 -0
  28. huggingface_hub/inference/_providers/zai_org.py +17 -0
  29. huggingface_hub/lfs.py +14 -8
  30. huggingface_hub/repocard.py +12 -16
  31. huggingface_hub/serialization/_base.py +3 -6
  32. huggingface_hub/serialization/_torch.py +16 -34
  33. huggingface_hub/utils/__init__.py +1 -2
  34. huggingface_hub/utils/_cache_manager.py +42 -72
  35. huggingface_hub/utils/_chunk_utils.py +2 -3
  36. huggingface_hub/utils/_http.py +37 -68
  37. huggingface_hub/utils/_validators.py +2 -2
  38. huggingface_hub/utils/logging.py +8 -11
  39. {huggingface_hub-1.0.0rc1.dist-info → huggingface_hub-1.0.0rc3.dist-info}/METADATA +2 -2
  40. {huggingface_hub-1.0.0rc1.dist-info → huggingface_hub-1.0.0rc3.dist-info}/RECORD +44 -56
  41. {huggingface_hub-1.0.0rc1.dist-info → huggingface_hub-1.0.0rc3.dist-info}/entry_points.txt +0 -1
  42. huggingface_hub/commands/__init__.py +0 -27
  43. huggingface_hub/commands/_cli_utils.py +0 -74
  44. huggingface_hub/commands/delete_cache.py +0 -476
  45. huggingface_hub/commands/download.py +0 -195
  46. huggingface_hub/commands/env.py +0 -39
  47. huggingface_hub/commands/huggingface_cli.py +0 -65
  48. huggingface_hub/commands/lfs.py +0 -200
  49. huggingface_hub/commands/repo.py +0 -151
  50. huggingface_hub/commands/repo_files.py +0 -132
  51. huggingface_hub/commands/scan_cache.py +0 -183
  52. huggingface_hub/commands/tag.py +0 -159
  53. huggingface_hub/commands/upload.py +0 -318
  54. huggingface_hub/commands/upload_large_folder.py +0 -131
  55. huggingface_hub/commands/user.py +0 -207
  56. huggingface_hub/commands/version.py +0 -40
  57. {huggingface_hub-1.0.0rc1.dist-info → huggingface_hub-1.0.0rc3.dist-info}/LICENSE +0 -0
  58. {huggingface_hub-1.0.0rc1.dist-info → huggingface_hub-1.0.0rc3.dist-info}/WHEEL +0 -0
  59. {huggingface_hub-1.0.0rc1.dist-info → huggingface_hub-1.0.0rc3.dist-info}/top_level.txt +0 -0
@@ -1,476 +0,0 @@
1
- # coding=utf-8
2
- # Copyright 2022-present, the HuggingFace Inc. team.
3
- #
4
- # Licensed under the Apache License, Version 2.0 (the "License");
5
- # you may not use this file except in compliance with the License.
6
- # You may obtain a copy of the License at
7
- #
8
- # http://www.apache.org/licenses/LICENSE-2.0
9
- #
10
- # Unless required by applicable law or agreed to in writing, software
11
- # distributed under the License is distributed on an "AS IS" BASIS,
12
- # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
- # See the License for the specific language governing permissions and
14
- # limitations under the License.
15
- """Contains command to delete some revisions from the HF cache directory.
16
-
17
- Usage:
18
- huggingface-cli delete-cache
19
- huggingface-cli delete-cache --disable-tui
20
- huggingface-cli delete-cache --dir ~/.cache/huggingface/hub
21
- huggingface-cli delete-cache --sort=size
22
-
23
- NOTE:
24
- This command is based on `InquirerPy` to build the multiselect menu in the terminal.
25
- This dependency has to be installed with `pip install "huggingface_hub[cli]"`. Since
26
- we want to avoid as much as possible cross-platform issues, I chose a library that
27
- is built on top of `python-prompt-toolkit` which seems to be a reference in terminal
28
- GUI (actively maintained on both Unix and Windows, 7.9k stars).
29
-
30
- For the moment, the TUI feature is in beta.
31
-
32
- See:
33
- - https://github.com/kazhala/InquirerPy
34
- - https://inquirerpy.readthedocs.io/en/latest/
35
- - https://github.com/prompt-toolkit/python-prompt-toolkit
36
-
37
- Other solutions could have been:
38
- - `simple_term_menu`: would be good as well for our use case but some issues suggest
39
- that Windows is less supported.
40
- See: https://github.com/IngoMeyer441/simple-term-menu
41
- - `PyInquirer`: very similar to `InquirerPy` but older and not maintained anymore.
42
- In particular, no support of Python3.10.
43
- See: https://github.com/CITGuru/PyInquirer
44
- - `pick` (or `pickpack`): easy to use and flexible but built on top of Python's
45
- standard library `curses` that is specific to Unix (not implemented on Windows).
46
- See https://github.com/wong2/pick and https://github.com/anafvana/pickpack.
47
- - `inquirer`: lot of traction (700 stars) but explicitly states "experimental
48
- support of Windows". Not built on top of `python-prompt-toolkit`.
49
- See https://github.com/magmax/python-inquirer
50
-
51
- TODO: add support for `huggingface-cli delete-cache aaaaaa bbbbbb cccccc (...)` ?
52
- TODO: add "--keep-last" arg to delete revisions that are not on `main` ref
53
- TODO: add "--filter" arg to filter repositories by name ?
54
- TODO: add "--limit" arg to limit to X repos ?
55
- TODO: add "-y" arg for immediate deletion ?
56
- See discussions in https://github.com/huggingface/huggingface_hub/issues/1025.
57
- """
58
-
59
- import os
60
- from argparse import Namespace, _SubParsersAction
61
- from functools import wraps
62
- from tempfile import mkstemp
63
- from typing import Any, Callable, Iterable, Literal, Optional, Union
64
-
65
- from ..utils import CachedRepoInfo, CachedRevisionInfo, HFCacheInfo, scan_cache_dir
66
- from . import BaseHuggingfaceCLICommand
67
- from ._cli_utils import ANSI, show_deprecation_warning
68
-
69
-
70
- try:
71
- from InquirerPy import inquirer
72
- from InquirerPy.base.control import Choice
73
- from InquirerPy.separator import Separator
74
-
75
- _inquirer_py_available = True
76
- except ImportError:
77
- _inquirer_py_available = False
78
-
79
- SortingOption_T = Literal["alphabetical", "lastUpdated", "lastUsed", "size"]
80
-
81
-
82
- def require_inquirer_py(fn: Callable) -> Callable:
83
- """Decorator to flag methods that require `InquirerPy`."""
84
-
85
- # TODO: refactor this + imports in a unified pattern across codebase
86
- @wraps(fn)
87
- def _inner(*args, **kwargs):
88
- if not _inquirer_py_available:
89
- raise ImportError(
90
- "The `delete-cache` command requires extra dependencies to work with"
91
- ' the TUI.\nPlease run `pip install "huggingface_hub[cli]"` to install'
92
- " them.\nOtherwise, disable TUI using the `--disable-tui` flag."
93
- )
94
-
95
- return fn(*args, **kwargs)
96
-
97
- return _inner
98
-
99
-
100
- # Possibility for the user to cancel deletion
101
- _CANCEL_DELETION_STR = "CANCEL_DELETION"
102
-
103
-
104
- class DeleteCacheCommand(BaseHuggingfaceCLICommand):
105
- @staticmethod
106
- def register_subcommand(parser: _SubParsersAction):
107
- delete_cache_parser = parser.add_parser("delete-cache", help="Delete revisions from the cache directory.")
108
-
109
- delete_cache_parser.add_argument(
110
- "--dir",
111
- type=str,
112
- default=None,
113
- help="cache directory (optional). Default to the default HuggingFace cache.",
114
- )
115
-
116
- delete_cache_parser.add_argument(
117
- "--disable-tui",
118
- action="store_true",
119
- help=(
120
- "Disable Terminal User Interface (TUI) mode. Useful if your"
121
- " platform/terminal doesn't support the multiselect menu."
122
- ),
123
- )
124
-
125
- delete_cache_parser.add_argument(
126
- "--sort",
127
- nargs="?",
128
- choices=["alphabetical", "lastUpdated", "lastUsed", "size"],
129
- help=(
130
- "Sort repositories by the specified criteria. Options: "
131
- "'alphabetical' (A-Z), "
132
- "'lastUpdated' (newest first), "
133
- "'lastUsed' (most recent first), "
134
- "'size' (largest first)."
135
- ),
136
- )
137
-
138
- delete_cache_parser.set_defaults(func=DeleteCacheCommand)
139
-
140
- def __init__(self, args: Namespace) -> None:
141
- self.cache_dir: Optional[str] = args.dir
142
- self.disable_tui: bool = args.disable_tui
143
- self.sort_by: Optional[SortingOption_T] = args.sort
144
-
145
- def run(self):
146
- """Run `delete-cache` command with or without TUI."""
147
- show_deprecation_warning("huggingface-cli delete-cache", "hf cache delete")
148
-
149
- # Scan cache directory
150
- hf_cache_info = scan_cache_dir(self.cache_dir)
151
-
152
- # Manual review from the user
153
- if self.disable_tui:
154
- selected_hashes = _manual_review_no_tui(hf_cache_info, preselected=[], sort_by=self.sort_by)
155
- else:
156
- selected_hashes = _manual_review_tui(hf_cache_info, preselected=[], sort_by=self.sort_by)
157
-
158
- # If deletion is not cancelled
159
- if len(selected_hashes) > 0 and _CANCEL_DELETION_STR not in selected_hashes:
160
- confirm_message = _get_expectations_str(hf_cache_info, selected_hashes) + " Confirm deletion ?"
161
-
162
- # Confirm deletion
163
- if self.disable_tui:
164
- confirmed = _ask_for_confirmation_no_tui(confirm_message)
165
- else:
166
- confirmed = _ask_for_confirmation_tui(confirm_message)
167
-
168
- # Deletion is confirmed
169
- if confirmed:
170
- strategy = hf_cache_info.delete_revisions(*selected_hashes)
171
- print("Start deletion.")
172
- strategy.execute()
173
- print(
174
- f"Done. Deleted {len(strategy.repos)} repo(s) and"
175
- f" {len(strategy.snapshots)} revision(s) for a total of"
176
- f" {strategy.expected_freed_size_str}."
177
- )
178
- return
179
-
180
- # Deletion is cancelled
181
- print("Deletion is cancelled. Do nothing.")
182
-
183
-
184
- def _get_repo_sorting_key(repo: CachedRepoInfo, sort_by: Optional[SortingOption_T] = None):
185
- if sort_by == "alphabetical":
186
- return (repo.repo_type, repo.repo_id.lower()) # by type then name
187
- elif sort_by == "lastUpdated":
188
- return -max(rev.last_modified for rev in repo.revisions) # newest first
189
- elif sort_by == "lastUsed":
190
- return -repo.last_accessed # most recently used first
191
- elif sort_by == "size":
192
- return -repo.size_on_disk # largest first
193
- else:
194
- return (repo.repo_type, repo.repo_id) # default stable order
195
-
196
-
197
- @require_inquirer_py
198
- def _manual_review_tui(
199
- hf_cache_info: HFCacheInfo,
200
- preselected: list[str],
201
- sort_by: Optional[SortingOption_T] = None,
202
- ) -> list[str]:
203
- """Ask the user for a manual review of the revisions to delete.
204
-
205
- Displays a multi-select menu in the terminal (TUI).
206
- """
207
- # Define multiselect list
208
- choices = _get_tui_choices_from_scan(
209
- repos=hf_cache_info.repos,
210
- preselected=preselected,
211
- sort_by=sort_by,
212
- )
213
- checkbox = inquirer.checkbox(
214
- message="Select revisions to delete:",
215
- choices=choices, # List of revisions with some pre-selection
216
- cycle=False, # No loop between top and bottom
217
- height=100, # Large list if possible
218
- # We use the instruction to display to the user the expected effect of the
219
- # deletion.
220
- instruction=_get_expectations_str(
221
- hf_cache_info,
222
- selected_hashes=[c.value for c in choices if isinstance(c, Choice) and c.enabled],
223
- ),
224
- # We use the long instruction to should keybindings instructions to the user
225
- long_instruction="Press <space> to select, <enter> to validate and <ctrl+c> to quit without modification.",
226
- # Message that is displayed once the user validates its selection.
227
- transformer=lambda result: f"{len(result)} revision(s) selected.",
228
- )
229
-
230
- # Add a callback to update the information line when a revision is
231
- # selected/unselected
232
- def _update_expectations(_) -> None:
233
- # Hacky way to dynamically set an instruction message to the checkbox when
234
- # a revision hash is selected/unselected.
235
- checkbox._instruction = _get_expectations_str(
236
- hf_cache_info,
237
- selected_hashes=[choice["value"] for choice in checkbox.content_control.choices if choice["enabled"]],
238
- )
239
-
240
- checkbox.kb_func_lookup["toggle"].append({"func": _update_expectations})
241
-
242
- # Finally display the form to the user.
243
- try:
244
- return checkbox.execute()
245
- except KeyboardInterrupt:
246
- return [] # Quit without deletion
247
-
248
-
249
- @require_inquirer_py
250
- def _ask_for_confirmation_tui(message: str, default: bool = True) -> bool:
251
- """Ask for confirmation using Inquirer."""
252
- return inquirer.confirm(message, default=default).execute()
253
-
254
-
255
- def _get_tui_choices_from_scan(
256
- repos: Iterable[CachedRepoInfo],
257
- preselected: list[str],
258
- sort_by: Optional[SortingOption_T] = None,
259
- ) -> list:
260
- """Build a list of choices from the scanned repos.
261
-
262
- Args:
263
- repos (*Iterable[`CachedRepoInfo`]*):
264
- List of scanned repos on which we want to delete revisions.
265
- preselected (*list[`str`]*):
266
- List of revision hashes that will be preselected.
267
- sort_by (*Optional[SortingOption_T]*):
268
- Sorting direction. Choices: "alphabetical", "lastUpdated", "lastUsed", "size".
269
-
270
- Return:
271
- The list of choices to pass to `inquirer.checkbox`.
272
- """
273
- choices: list[Union[Choice, Separator]] = []
274
-
275
- # First choice is to cancel the deletion
276
- choices.append(
277
- Choice(
278
- _CANCEL_DELETION_STR,
279
- name="None of the following (if selected, nothing will be deleted).",
280
- enabled=False,
281
- )
282
- )
283
-
284
- # Sort repos based on specified criteria
285
- sorted_repos = sorted(repos, key=lambda repo: _get_repo_sorting_key(repo, sort_by))
286
-
287
- for repo in sorted_repos:
288
- # Repo as separator
289
- choices.append(
290
- Separator(
291
- f"\n{repo.repo_type.capitalize()} {repo.repo_id} ({repo.size_on_disk_str},"
292
- f" used {repo.last_accessed_str})"
293
- )
294
- )
295
- for revision in sorted(repo.revisions, key=_revision_sorting_order):
296
- # Revision as choice
297
- choices.append(
298
- Choice(
299
- revision.commit_hash,
300
- name=(
301
- f"{revision.commit_hash[:8]}:"
302
- f" {', '.join(sorted(revision.refs)) or '(detached)'} #"
303
- f" modified {revision.last_modified_str}"
304
- ),
305
- enabled=revision.commit_hash in preselected,
306
- )
307
- )
308
-
309
- # Return choices
310
- return choices
311
-
312
-
313
- def _manual_review_no_tui(
314
- hf_cache_info: HFCacheInfo,
315
- preselected: list[str],
316
- sort_by: Optional[SortingOption_T] = None,
317
- ) -> list[str]:
318
- """Ask the user for a manual review of the revisions to delete.
319
-
320
- Used when TUI is disabled. Manual review happens in a separate tmp file that the
321
- user can manually edit.
322
- """
323
- # 1. Generate temporary file with delete commands.
324
- fd, tmp_path = mkstemp(suffix=".txt") # suffix to make it easier to find by editors
325
- os.close(fd)
326
-
327
- lines = []
328
-
329
- sorted_repos = sorted(hf_cache_info.repos, key=lambda repo: _get_repo_sorting_key(repo, sort_by))
330
-
331
- for repo in sorted_repos:
332
- lines.append(
333
- f"\n# {repo.repo_type.capitalize()} {repo.repo_id} ({repo.size_on_disk_str},"
334
- f" used {repo.last_accessed_str})"
335
- )
336
- for revision in sorted(repo.revisions, key=_revision_sorting_order):
337
- lines.append(
338
- # Deselect by prepending a '#'
339
- f"{'' if revision.commit_hash in preselected else '#'} "
340
- f" {revision.commit_hash} # Refs:"
341
- # Print `refs` as comment on same line
342
- f" {', '.join(sorted(revision.refs)) or '(detached)'} # modified"
343
- # Print `last_modified` as comment on same line
344
- f" {revision.last_modified_str}"
345
- )
346
-
347
- with open(tmp_path, "w") as f:
348
- f.write(_MANUAL_REVIEW_NO_TUI_INSTRUCTIONS)
349
- f.write("\n".join(lines))
350
-
351
- # 2. Prompt instructions to user.
352
- instructions = f"""
353
- TUI is disabled. In order to select which revisions you want to delete, please edit
354
- the following file using the text editor of your choice. Instructions for manual
355
- editing are located at the beginning of the file. Edit the file, save it and confirm
356
- to continue.
357
- File to edit: {ANSI.bold(tmp_path)}
358
- """
359
- print("\n".join(line.strip() for line in instructions.strip().split("\n")))
360
-
361
- # 3. Wait for user confirmation.
362
- while True:
363
- selected_hashes = _read_manual_review_tmp_file(tmp_path)
364
- if _ask_for_confirmation_no_tui(
365
- _get_expectations_str(hf_cache_info, selected_hashes) + " Continue ?",
366
- default=False,
367
- ):
368
- break
369
-
370
- # 4. Return selected_hashes sorted to maintain stable order
371
- os.remove(tmp_path)
372
- return sorted(selected_hashes) # Sort to maintain stable order
373
-
374
-
375
- def _ask_for_confirmation_no_tui(message: str, default: bool = True) -> bool:
376
- """Ask for confirmation using pure-python."""
377
- YES = ("y", "yes", "1")
378
- NO = ("n", "no", "0")
379
- DEFAULT = ""
380
- ALL = YES + NO + (DEFAULT,)
381
- full_message = message + (" (Y/n) " if default else " (y/N) ")
382
- while True:
383
- answer = input(full_message).lower()
384
- if answer == DEFAULT:
385
- return default
386
- if answer in YES:
387
- return True
388
- if answer in NO:
389
- return False
390
- print(f"Invalid input. Must be one of {ALL}")
391
-
392
-
393
- def _get_expectations_str(hf_cache_info: HFCacheInfo, selected_hashes: list[str]) -> str:
394
- """Format a string to display to the user how much space would be saved.
395
-
396
- Example:
397
- ```
398
- >>> _get_expectations_str(hf_cache_info, selected_hashes)
399
- '7 revisions selected counting for 4.3G.'
400
- ```
401
- """
402
- if _CANCEL_DELETION_STR in selected_hashes:
403
- return "Nothing will be deleted."
404
- strategy = hf_cache_info.delete_revisions(*selected_hashes)
405
- return f"{len(selected_hashes)} revisions selected counting for {strategy.expected_freed_size_str}."
406
-
407
-
408
- def _read_manual_review_tmp_file(tmp_path: str) -> list[str]:
409
- """Read the manually reviewed instruction file and return a list of revision hash.
410
-
411
- Example:
412
- ```txt
413
- # This is the tmp file content
414
- ###
415
-
416
- # Commented out line
417
- 123456789 # revision hash
418
-
419
- # Something else
420
- # a_newer_hash # 2 days ago
421
- an_older_hash # 3 days ago
422
- ```
423
-
424
- ```py
425
- >>> _read_manual_review_tmp_file(tmp_path)
426
- ['123456789', 'an_older_hash']
427
- ```
428
- """
429
- with open(tmp_path) as f:
430
- content = f.read()
431
-
432
- # Split lines
433
- lines = [line.strip() for line in content.split("\n")]
434
-
435
- # Filter commented lines
436
- selected_lines = [line for line in lines if not line.startswith("#")]
437
-
438
- # Select only before comment
439
- selected_hashes = [line.split("#")[0].strip() for line in selected_lines]
440
-
441
- # Return revision hashes
442
- return [hash for hash in selected_hashes if len(hash) > 0]
443
-
444
-
445
- _MANUAL_REVIEW_NO_TUI_INSTRUCTIONS = f"""
446
- # INSTRUCTIONS
447
- # ------------
448
- # This is a temporary file created by running `huggingface-cli delete-cache` with the
449
- # `--disable-tui` option. It contains a set of revisions that can be deleted from your
450
- # local cache directory.
451
- #
452
- # Please manually review the revisions you want to delete:
453
- # - Revision hashes can be commented out with '#'.
454
- # - Only non-commented revisions in this file will be deleted.
455
- # - Revision hashes that are removed from this file are ignored as well.
456
- # - If `{_CANCEL_DELETION_STR}` line is uncommented, the all cache deletion is cancelled and
457
- # no changes will be applied.
458
- #
459
- # Once you've manually reviewed this file, please confirm deletion in the terminal. This
460
- # file will be automatically removed once done.
461
- # ------------
462
-
463
- # KILL SWITCH
464
- # ------------
465
- # Un-comment following line to completely cancel the deletion process
466
- # {_CANCEL_DELETION_STR}
467
- # ------------
468
-
469
- # REVISIONS
470
- # ------------
471
- """.strip()
472
-
473
-
474
- def _revision_sorting_order(revision: CachedRevisionInfo) -> Any:
475
- # Sort by last modified (oldest first)
476
- return revision.last_modified
@@ -1,195 +0,0 @@
1
- # coding=utf-8
2
- # Copyright 2023-present, the HuggingFace Inc. team.
3
- #
4
- # Licensed under the Apache License, Version 2.0 (the "License");
5
- # you may not use this file except in compliance with the License.
6
- # You may obtain a copy of the License at
7
- #
8
- # http://www.apache.org/licenses/LICENSE-2.0
9
- #
10
- # Unless required by applicable law or agreed to in writing, software
11
- # distributed under the License is distributed on an "AS IS" BASIS,
12
- # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
- # See the License for the specific language governing permissions and
14
- # limitations under the License.
15
- """Contains command to download files from the Hub with the CLI.
16
-
17
- Usage:
18
- huggingface-cli download --help
19
-
20
- # Download file
21
- huggingface-cli download gpt2 config.json
22
-
23
- # Download entire repo
24
- huggingface-cli download fffiloni/zeroscope --repo-type=space --revision=refs/pr/78
25
-
26
- # Download repo with filters
27
- huggingface-cli download gpt2 --include="*.safetensors"
28
-
29
- # Download with token
30
- huggingface-cli download Wauplin/private-model --token=hf_***
31
-
32
- # Download quietly (no progress bar, no warnings, only the returned path)
33
- huggingface-cli download gpt2 config.json --quiet
34
-
35
- # Download to local dir
36
- huggingface-cli download gpt2 --local-dir=./models/gpt2
37
- """
38
-
39
- import warnings
40
- from argparse import Namespace, _SubParsersAction
41
- from typing import Optional
42
-
43
- from huggingface_hub import logging
44
- from huggingface_hub._snapshot_download import snapshot_download
45
- from huggingface_hub.commands import BaseHuggingfaceCLICommand
46
- from huggingface_hub.file_download import hf_hub_download
47
- from huggingface_hub.utils import disable_progress_bars, enable_progress_bars
48
-
49
- from ._cli_utils import show_deprecation_warning
50
-
51
-
52
- logger = logging.get_logger(__name__)
53
-
54
-
55
- class DownloadCommand(BaseHuggingfaceCLICommand):
56
- @staticmethod
57
- def register_subcommand(parser: _SubParsersAction):
58
- download_parser = parser.add_parser("download", help="Download files from the Hub")
59
- download_parser.add_argument(
60
- "repo_id", type=str, help="ID of the repo to download from (e.g. `username/repo-name`)."
61
- )
62
- download_parser.add_argument(
63
- "filenames", type=str, nargs="*", help="Files to download (e.g. `config.json`, `data/metadata.jsonl`)."
64
- )
65
- download_parser.add_argument(
66
- "--repo-type",
67
- choices=["model", "dataset", "space"],
68
- default="model",
69
- help="Type of repo to download from (defaults to 'model').",
70
- )
71
- download_parser.add_argument(
72
- "--revision",
73
- type=str,
74
- help="An optional Git revision id which can be a branch name, a tag, or a commit hash.",
75
- )
76
- download_parser.add_argument(
77
- "--include", nargs="*", type=str, help="Glob patterns to match files to download."
78
- )
79
- download_parser.add_argument(
80
- "--exclude", nargs="*", type=str, help="Glob patterns to exclude from files to download."
81
- )
82
- download_parser.add_argument(
83
- "--cache-dir", type=str, help="Path to the directory where to save the downloaded files."
84
- )
85
- download_parser.add_argument(
86
- "--local-dir",
87
- type=str,
88
- help=(
89
- "If set, the downloaded file will be placed under this directory. Check out"
90
- " https://huggingface.co/docs/huggingface_hub/guides/download#download-files-to-local-folder for more"
91
- " details."
92
- ),
93
- )
94
- download_parser.add_argument(
95
- "--local-dir-use-symlinks",
96
- choices=["auto", "True", "False"],
97
- help=("Deprecated and ignored. Downloading to a local directory does not use symlinks anymore."),
98
- )
99
- download_parser.add_argument(
100
- "--force-download",
101
- action="store_true",
102
- help="If True, the files will be downloaded even if they are already cached.",
103
- )
104
- download_parser.add_argument(
105
- "--resume-download",
106
- action="store_true",
107
- help="Deprecated and ignored. Downloading a file to local dir always attempts to resume previously interrupted downloads (unless hf-transfer is enabled).",
108
- )
109
- download_parser.add_argument(
110
- "--token", type=str, help="A User Access Token generated from https://huggingface.co/settings/tokens"
111
- )
112
- download_parser.add_argument(
113
- "--quiet",
114
- action="store_true",
115
- help="If True, progress bars are disabled and only the path to the download files is printed.",
116
- )
117
- download_parser.add_argument(
118
- "--max-workers",
119
- type=int,
120
- default=8,
121
- help="Maximum number of workers to use for downloading files. Default is 8.",
122
- )
123
- download_parser.set_defaults(func=DownloadCommand)
124
-
125
- def __init__(self, args: Namespace) -> None:
126
- self.token = args.token
127
- self.repo_id: str = args.repo_id
128
- self.filenames: list[str] = args.filenames
129
- self.repo_type: str = args.repo_type
130
- self.revision: Optional[str] = args.revision
131
- self.include: Optional[list[str]] = args.include
132
- self.exclude: Optional[list[str]] = args.exclude
133
- self.cache_dir: Optional[str] = args.cache_dir
134
- self.local_dir: Optional[str] = args.local_dir
135
- self.force_download: bool = args.force_download
136
- self.quiet: bool = args.quiet
137
- self.max_workers: int = args.max_workers
138
-
139
- def run(self) -> None:
140
- show_deprecation_warning("huggingface-cli download", "hf download")
141
-
142
- if self.quiet:
143
- disable_progress_bars()
144
- with warnings.catch_warnings():
145
- warnings.simplefilter("ignore")
146
- print(self._download()) # Print path to downloaded files
147
- enable_progress_bars()
148
- else:
149
- logging.set_verbosity_info()
150
- print(self._download()) # Print path to downloaded files
151
- logging.set_verbosity_warning()
152
-
153
- def _download(self) -> str:
154
- # Warn user if patterns are ignored
155
- if len(self.filenames) > 0:
156
- if self.include is not None and len(self.include) > 0:
157
- warnings.warn("Ignoring `--include` since filenames have being explicitly set.")
158
- if self.exclude is not None and len(self.exclude) > 0:
159
- warnings.warn("Ignoring `--exclude` since filenames have being explicitly set.")
160
-
161
- # Single file to download: use `hf_hub_download`
162
- if len(self.filenames) == 1:
163
- return hf_hub_download(
164
- repo_id=self.repo_id,
165
- repo_type=self.repo_type,
166
- revision=self.revision,
167
- filename=self.filenames[0],
168
- cache_dir=self.cache_dir,
169
- force_download=self.force_download,
170
- token=self.token,
171
- local_dir=self.local_dir,
172
- library_name="huggingface-cli",
173
- )
174
-
175
- # Otherwise: use `snapshot_download` to ensure all files comes from same revision
176
- elif len(self.filenames) == 0:
177
- allow_patterns = self.include
178
- ignore_patterns = self.exclude
179
- else:
180
- allow_patterns = self.filenames
181
- ignore_patterns = None
182
-
183
- return snapshot_download(
184
- repo_id=self.repo_id,
185
- repo_type=self.repo_type,
186
- revision=self.revision,
187
- allow_patterns=allow_patterns,
188
- ignore_patterns=ignore_patterns,
189
- force_download=self.force_download,
190
- cache_dir=self.cache_dir,
191
- token=self.token,
192
- local_dir=self.local_dir,
193
- library_name="huggingface-cli",
194
- max_workers=self.max_workers,
195
- )