huggingface-hub 0.33.4__py3-none-any.whl → 0.34.0rc0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of huggingface-hub might be problematic. Click here for more details.
- huggingface_hub/__init__.py +47 -1
- huggingface_hub/_commit_api.py +21 -28
- huggingface_hub/_jobs_api.py +145 -0
- huggingface_hub/_local_folder.py +7 -1
- huggingface_hub/_login.py +5 -5
- huggingface_hub/_oauth.py +1 -1
- huggingface_hub/_snapshot_download.py +11 -6
- huggingface_hub/_upload_large_folder.py +46 -23
- huggingface_hub/cli/__init__.py +27 -0
- huggingface_hub/cli/_cli_utils.py +69 -0
- huggingface_hub/cli/auth.py +210 -0
- huggingface_hub/cli/cache.py +405 -0
- huggingface_hub/cli/download.py +181 -0
- huggingface_hub/cli/hf.py +66 -0
- huggingface_hub/cli/jobs.py +522 -0
- huggingface_hub/cli/lfs.py +198 -0
- huggingface_hub/cli/repo.py +243 -0
- huggingface_hub/cli/repo_files.py +128 -0
- huggingface_hub/cli/system.py +52 -0
- huggingface_hub/cli/upload.py +316 -0
- huggingface_hub/cli/upload_large_folder.py +132 -0
- huggingface_hub/commands/_cli_utils.py +5 -0
- huggingface_hub/commands/delete_cache.py +3 -1
- huggingface_hub/commands/download.py +4 -0
- huggingface_hub/commands/env.py +3 -0
- huggingface_hub/commands/huggingface_cli.py +2 -0
- huggingface_hub/commands/repo.py +4 -0
- huggingface_hub/commands/repo_files.py +4 -0
- huggingface_hub/commands/scan_cache.py +3 -1
- huggingface_hub/commands/tag.py +3 -1
- huggingface_hub/commands/upload.py +4 -0
- huggingface_hub/commands/upload_large_folder.py +3 -1
- huggingface_hub/commands/user.py +11 -1
- huggingface_hub/commands/version.py +3 -0
- huggingface_hub/constants.py +1 -0
- huggingface_hub/file_download.py +16 -5
- huggingface_hub/hf_api.py +519 -7
- huggingface_hub/hf_file_system.py +8 -16
- huggingface_hub/hub_mixin.py +3 -3
- huggingface_hub/inference/_client.py +38 -39
- huggingface_hub/inference/_common.py +44 -14
- huggingface_hub/inference/_generated/_async_client.py +50 -51
- huggingface_hub/inference/_generated/types/__init__.py +1 -0
- huggingface_hub/inference/_generated/types/image_to_video.py +60 -0
- huggingface_hub/inference/_mcp/cli.py +36 -18
- huggingface_hub/inference/_mcp/constants.py +8 -0
- huggingface_hub/inference/_mcp/types.py +3 -0
- huggingface_hub/inference/_providers/__init__.py +4 -1
- huggingface_hub/inference/_providers/_common.py +3 -6
- huggingface_hub/inference/_providers/fal_ai.py +85 -42
- huggingface_hub/inference/_providers/hf_inference.py +17 -9
- huggingface_hub/inference/_providers/replicate.py +19 -1
- huggingface_hub/keras_mixin.py +2 -2
- huggingface_hub/repocard.py +1 -1
- huggingface_hub/repository.py +2 -2
- huggingface_hub/utils/_auth.py +1 -1
- huggingface_hub/utils/_cache_manager.py +2 -2
- huggingface_hub/utils/_dotenv.py +51 -0
- huggingface_hub/utils/_headers.py +1 -1
- huggingface_hub/utils/_runtime.py +1 -1
- huggingface_hub/utils/_xet.py +6 -2
- huggingface_hub/utils/_xet_progress_reporting.py +141 -0
- {huggingface_hub-0.33.4.dist-info → huggingface_hub-0.34.0rc0.dist-info}/METADATA +7 -8
- {huggingface_hub-0.33.4.dist-info → huggingface_hub-0.34.0rc0.dist-info}/RECORD +68 -51
- {huggingface_hub-0.33.4.dist-info → huggingface_hub-0.34.0rc0.dist-info}/entry_points.txt +1 -0
- {huggingface_hub-0.33.4.dist-info → huggingface_hub-0.34.0rc0.dist-info}/LICENSE +0 -0
- {huggingface_hub-0.33.4.dist-info → huggingface_hub-0.34.0rc0.dist-info}/WHEEL +0 -0
- {huggingface_hub-0.33.4.dist-info → huggingface_hub-0.34.0rc0.dist-info}/top_level.txt +0 -0
|
@@ -0,0 +1,316 @@
|
|
|
1
|
+
# coding=utf-8
|
|
2
|
+
# Copyright 2023-present, the HuggingFace Inc. team.
|
|
3
|
+
#
|
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
5
|
+
# you may not use this file except in compliance with the License.
|
|
6
|
+
# You may obtain a copy of the License at
|
|
7
|
+
#
|
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
9
|
+
#
|
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
12
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
13
|
+
# See the License for the specific language governing permissions and
|
|
14
|
+
# limitations under the License.
|
|
15
|
+
"""Contains command to upload a repo or file with the CLI.
|
|
16
|
+
|
|
17
|
+
Usage:
|
|
18
|
+
# Upload file (implicit)
|
|
19
|
+
hf upload my-cool-model ./my-cool-model.safetensors
|
|
20
|
+
|
|
21
|
+
# Upload file (explicit)
|
|
22
|
+
hf upload my-cool-model ./my-cool-model.safetensors model.safetensors
|
|
23
|
+
|
|
24
|
+
# Upload directory (implicit). If `my-cool-model/` is a directory it will be uploaded, otherwise an exception is raised.
|
|
25
|
+
hf upload my-cool-model
|
|
26
|
+
|
|
27
|
+
# Upload directory (explicit)
|
|
28
|
+
hf upload my-cool-model ./models/my-cool-model .
|
|
29
|
+
|
|
30
|
+
# Upload filtered directory (example: tensorboard logs except for the last run)
|
|
31
|
+
hf upload my-cool-model ./model/training /logs --include "*.tfevents.*" --exclude "*20230905*"
|
|
32
|
+
|
|
33
|
+
# Upload with wildcard
|
|
34
|
+
hf upload my-cool-model "./model/training/*.safetensors"
|
|
35
|
+
|
|
36
|
+
# Upload private dataset
|
|
37
|
+
hf upload Wauplin/my-cool-dataset ./data . --repo-type=dataset --private
|
|
38
|
+
|
|
39
|
+
# Upload with token
|
|
40
|
+
hf upload Wauplin/my-cool-model --token=hf_****
|
|
41
|
+
|
|
42
|
+
# Sync local Space with Hub (upload new files, delete removed files)
|
|
43
|
+
hf upload Wauplin/space-example --repo-type=space --exclude="/logs/*" --delete="*" --commit-message="Sync local Space with Hub"
|
|
44
|
+
|
|
45
|
+
# Schedule commits every 30 minutes
|
|
46
|
+
hf upload Wauplin/my-cool-model --every=30
|
|
47
|
+
"""
|
|
48
|
+
|
|
49
|
+
import os
|
|
50
|
+
import time
|
|
51
|
+
import warnings
|
|
52
|
+
from argparse import Namespace, _SubParsersAction
|
|
53
|
+
from typing import List, Optional
|
|
54
|
+
|
|
55
|
+
from huggingface_hub import logging
|
|
56
|
+
from huggingface_hub._commit_scheduler import CommitScheduler
|
|
57
|
+
from huggingface_hub.commands import BaseHuggingfaceCLICommand
|
|
58
|
+
from huggingface_hub.constants import HF_HUB_ENABLE_HF_TRANSFER
|
|
59
|
+
from huggingface_hub.errors import RevisionNotFoundError
|
|
60
|
+
from huggingface_hub.hf_api import HfApi
|
|
61
|
+
from huggingface_hub.utils import disable_progress_bars, enable_progress_bars
|
|
62
|
+
from huggingface_hub.utils._runtime import is_xet_available
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
logger = logging.get_logger(__name__)
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
class UploadCommand(BaseHuggingfaceCLICommand):
|
|
69
|
+
@staticmethod
|
|
70
|
+
def register_subcommand(parser: _SubParsersAction):
|
|
71
|
+
upload_parser = parser.add_parser(
|
|
72
|
+
"upload", help="Upload a file or a folder to the Hub. Recommended for single-commit uploads."
|
|
73
|
+
)
|
|
74
|
+
upload_parser.add_argument(
|
|
75
|
+
"repo_id", type=str, help="The ID of the repo to upload to (e.g. `username/repo-name`)."
|
|
76
|
+
)
|
|
77
|
+
upload_parser.add_argument(
|
|
78
|
+
"local_path",
|
|
79
|
+
nargs="?",
|
|
80
|
+
help="Local path to the file or folder to upload. Wildcard patterns are supported. Defaults to current directory.",
|
|
81
|
+
)
|
|
82
|
+
upload_parser.add_argument(
|
|
83
|
+
"path_in_repo",
|
|
84
|
+
nargs="?",
|
|
85
|
+
help="Path of the file or folder in the repo. Defaults to the relative path of the file or folder.",
|
|
86
|
+
)
|
|
87
|
+
upload_parser.add_argument(
|
|
88
|
+
"--repo-type",
|
|
89
|
+
choices=["model", "dataset", "space"],
|
|
90
|
+
default="model",
|
|
91
|
+
help="Type of the repo to upload to (e.g. `dataset`).",
|
|
92
|
+
)
|
|
93
|
+
upload_parser.add_argument(
|
|
94
|
+
"--revision",
|
|
95
|
+
type=str,
|
|
96
|
+
help=(
|
|
97
|
+
"An optional Git revision to push to. It can be a branch name or a PR reference. If revision does not"
|
|
98
|
+
" exist and `--create-pr` is not set, a branch will be automatically created."
|
|
99
|
+
),
|
|
100
|
+
)
|
|
101
|
+
upload_parser.add_argument(
|
|
102
|
+
"--private",
|
|
103
|
+
action="store_true",
|
|
104
|
+
help=(
|
|
105
|
+
"Whether to create a private repo if repo doesn't exist on the Hub. Ignored if the repo already"
|
|
106
|
+
" exists."
|
|
107
|
+
),
|
|
108
|
+
)
|
|
109
|
+
upload_parser.add_argument("--include", nargs="*", type=str, help="Glob patterns to match files to upload.")
|
|
110
|
+
upload_parser.add_argument(
|
|
111
|
+
"--exclude", nargs="*", type=str, help="Glob patterns to exclude from files to upload."
|
|
112
|
+
)
|
|
113
|
+
upload_parser.add_argument(
|
|
114
|
+
"--delete",
|
|
115
|
+
nargs="*",
|
|
116
|
+
type=str,
|
|
117
|
+
help="Glob patterns for file to be deleted from the repo while committing.",
|
|
118
|
+
)
|
|
119
|
+
upload_parser.add_argument(
|
|
120
|
+
"--commit-message", type=str, help="The summary / title / first line of the generated commit."
|
|
121
|
+
)
|
|
122
|
+
upload_parser.add_argument("--commit-description", type=str, help="The description of the generated commit.")
|
|
123
|
+
upload_parser.add_argument(
|
|
124
|
+
"--create-pr", action="store_true", help="Whether to upload content as a new Pull Request."
|
|
125
|
+
)
|
|
126
|
+
upload_parser.add_argument(
|
|
127
|
+
"--every",
|
|
128
|
+
type=float,
|
|
129
|
+
help="If set, a background job is scheduled to create commits every `every` minutes.",
|
|
130
|
+
)
|
|
131
|
+
upload_parser.add_argument(
|
|
132
|
+
"--token", type=str, help="A User Access Token generated from https://huggingface.co/settings/tokens"
|
|
133
|
+
)
|
|
134
|
+
upload_parser.add_argument(
|
|
135
|
+
"--quiet",
|
|
136
|
+
action="store_true",
|
|
137
|
+
help="If True, progress bars are disabled and only the path to the uploaded files is printed.",
|
|
138
|
+
)
|
|
139
|
+
upload_parser.set_defaults(func=UploadCommand)
|
|
140
|
+
|
|
141
|
+
def __init__(self, args: Namespace) -> None:
|
|
142
|
+
self.repo_id: str = args.repo_id
|
|
143
|
+
self.repo_type: Optional[str] = args.repo_type
|
|
144
|
+
self.revision: Optional[str] = args.revision
|
|
145
|
+
self.private: bool = args.private
|
|
146
|
+
|
|
147
|
+
self.include: Optional[List[str]] = args.include
|
|
148
|
+
self.exclude: Optional[List[str]] = args.exclude
|
|
149
|
+
self.delete: Optional[List[str]] = args.delete
|
|
150
|
+
|
|
151
|
+
self.commit_message: Optional[str] = args.commit_message
|
|
152
|
+
self.commit_description: Optional[str] = args.commit_description
|
|
153
|
+
self.create_pr: bool = args.create_pr
|
|
154
|
+
self.api: HfApi = HfApi(token=args.token, library_name="hf")
|
|
155
|
+
self.quiet: bool = args.quiet # disable warnings and progress bars
|
|
156
|
+
|
|
157
|
+
# Check `--every` is valid
|
|
158
|
+
if args.every is not None and args.every <= 0:
|
|
159
|
+
raise ValueError(f"`every` must be a positive value (got '{args.every}')")
|
|
160
|
+
self.every: Optional[float] = args.every
|
|
161
|
+
|
|
162
|
+
# Resolve `local_path` and `path_in_repo`
|
|
163
|
+
repo_name: str = args.repo_id.split("/")[-1] # e.g. "Wauplin/my-cool-model" => "my-cool-model"
|
|
164
|
+
self.local_path: str
|
|
165
|
+
self.path_in_repo: str
|
|
166
|
+
|
|
167
|
+
if args.local_path is not None and any(c in args.local_path for c in ["*", "?", "["]):
|
|
168
|
+
if args.include is not None:
|
|
169
|
+
raise ValueError("Cannot set `--include` when passing a `local_path` containing a wildcard.")
|
|
170
|
+
if args.path_in_repo is not None and args.path_in_repo != ".":
|
|
171
|
+
raise ValueError("Cannot set `path_in_repo` when passing a `local_path` containing a wildcard.")
|
|
172
|
+
self.local_path = "."
|
|
173
|
+
self.include = args.local_path
|
|
174
|
+
self.path_in_repo = "."
|
|
175
|
+
elif args.local_path is None and os.path.isfile(repo_name):
|
|
176
|
+
# Implicit case 1: user provided only a repo_id which happen to be a local file as well => upload it with same name
|
|
177
|
+
self.local_path = repo_name
|
|
178
|
+
self.path_in_repo = repo_name
|
|
179
|
+
elif args.local_path is None and os.path.isdir(repo_name):
|
|
180
|
+
# Implicit case 2: user provided only a repo_id which happen to be a local folder as well => upload it at root
|
|
181
|
+
self.local_path = repo_name
|
|
182
|
+
self.path_in_repo = "."
|
|
183
|
+
elif args.local_path is None:
|
|
184
|
+
# Implicit case 3: user provided only a repo_id that does not match a local file or folder
|
|
185
|
+
# => the user must explicitly provide a local_path => raise exception
|
|
186
|
+
raise ValueError(f"'{repo_name}' is not a local file or folder. Please set `local_path` explicitly.")
|
|
187
|
+
elif args.path_in_repo is None and os.path.isfile(args.local_path):
|
|
188
|
+
# Explicit local path to file, no path in repo => upload it at root with same name
|
|
189
|
+
self.local_path = args.local_path
|
|
190
|
+
self.path_in_repo = os.path.basename(args.local_path)
|
|
191
|
+
elif args.path_in_repo is None:
|
|
192
|
+
# Explicit local path to folder, no path in repo => upload at root
|
|
193
|
+
self.local_path = args.local_path
|
|
194
|
+
self.path_in_repo = "."
|
|
195
|
+
else:
|
|
196
|
+
# Finally, if both paths are explicit
|
|
197
|
+
self.local_path = args.local_path
|
|
198
|
+
self.path_in_repo = args.path_in_repo
|
|
199
|
+
|
|
200
|
+
def run(self) -> None:
|
|
201
|
+
if self.quiet:
|
|
202
|
+
disable_progress_bars()
|
|
203
|
+
with warnings.catch_warnings():
|
|
204
|
+
warnings.simplefilter("ignore")
|
|
205
|
+
print(self._upload())
|
|
206
|
+
enable_progress_bars()
|
|
207
|
+
else:
|
|
208
|
+
logging.set_verbosity_info()
|
|
209
|
+
print(self._upload())
|
|
210
|
+
logging.set_verbosity_warning()
|
|
211
|
+
|
|
212
|
+
def _upload(self) -> str:
|
|
213
|
+
if os.path.isfile(self.local_path):
|
|
214
|
+
if self.include is not None and len(self.include) > 0:
|
|
215
|
+
warnings.warn("Ignoring `--include` since a single file is uploaded.")
|
|
216
|
+
if self.exclude is not None and len(self.exclude) > 0:
|
|
217
|
+
warnings.warn("Ignoring `--exclude` since a single file is uploaded.")
|
|
218
|
+
if self.delete is not None and len(self.delete) > 0:
|
|
219
|
+
warnings.warn("Ignoring `--delete` since a single file is uploaded.")
|
|
220
|
+
|
|
221
|
+
if not is_xet_available() and not HF_HUB_ENABLE_HF_TRANSFER:
|
|
222
|
+
logger.info(
|
|
223
|
+
"Consider using `hf_transfer` for faster uploads. This solution comes with some limitations. See"
|
|
224
|
+
" https://huggingface.co/docs/huggingface_hub/hf_transfer for more details."
|
|
225
|
+
)
|
|
226
|
+
|
|
227
|
+
# Schedule commits if `every` is set
|
|
228
|
+
if self.every is not None:
|
|
229
|
+
if os.path.isfile(self.local_path):
|
|
230
|
+
# If file => watch entire folder + use allow_patterns
|
|
231
|
+
folder_path = os.path.dirname(self.local_path)
|
|
232
|
+
path_in_repo = (
|
|
233
|
+
self.path_in_repo[: -len(self.local_path)] # remove filename from path_in_repo
|
|
234
|
+
if self.path_in_repo.endswith(self.local_path)
|
|
235
|
+
else self.path_in_repo
|
|
236
|
+
)
|
|
237
|
+
allow_patterns = [self.local_path]
|
|
238
|
+
ignore_patterns = []
|
|
239
|
+
else:
|
|
240
|
+
folder_path = self.local_path
|
|
241
|
+
path_in_repo = self.path_in_repo
|
|
242
|
+
allow_patterns = self.include or []
|
|
243
|
+
ignore_patterns = self.exclude or []
|
|
244
|
+
if self.delete is not None and len(self.delete) > 0:
|
|
245
|
+
warnings.warn("Ignoring `--delete` when uploading with scheduled commits.")
|
|
246
|
+
|
|
247
|
+
scheduler = CommitScheduler(
|
|
248
|
+
folder_path=folder_path,
|
|
249
|
+
repo_id=self.repo_id,
|
|
250
|
+
repo_type=self.repo_type,
|
|
251
|
+
revision=self.revision,
|
|
252
|
+
allow_patterns=allow_patterns,
|
|
253
|
+
ignore_patterns=ignore_patterns,
|
|
254
|
+
path_in_repo=path_in_repo,
|
|
255
|
+
private=self.private,
|
|
256
|
+
every=self.every,
|
|
257
|
+
hf_api=self.api,
|
|
258
|
+
)
|
|
259
|
+
print(f"Scheduling commits every {self.every} minutes to {scheduler.repo_id}.")
|
|
260
|
+
try: # Block main thread until KeyboardInterrupt
|
|
261
|
+
while True:
|
|
262
|
+
time.sleep(100)
|
|
263
|
+
except KeyboardInterrupt:
|
|
264
|
+
scheduler.stop()
|
|
265
|
+
return "Stopped scheduled commits."
|
|
266
|
+
|
|
267
|
+
# Otherwise, create repo and proceed with the upload
|
|
268
|
+
if not os.path.isfile(self.local_path) and not os.path.isdir(self.local_path):
|
|
269
|
+
raise FileNotFoundError(f"No such file or directory: '{self.local_path}'.")
|
|
270
|
+
repo_id = self.api.create_repo(
|
|
271
|
+
repo_id=self.repo_id,
|
|
272
|
+
repo_type=self.repo_type,
|
|
273
|
+
exist_ok=True,
|
|
274
|
+
private=self.private,
|
|
275
|
+
space_sdk="gradio" if self.repo_type == "space" else None,
|
|
276
|
+
# ^ We don't want it to fail when uploading to a Space => let's set Gradio by default.
|
|
277
|
+
# ^ I'd rather not add CLI args to set it explicitly as we already have `hf repo create` for that.
|
|
278
|
+
).repo_id
|
|
279
|
+
|
|
280
|
+
# Check if branch already exists and if not, create it
|
|
281
|
+
if self.revision is not None and not self.create_pr:
|
|
282
|
+
try:
|
|
283
|
+
self.api.repo_info(repo_id=repo_id, repo_type=self.repo_type, revision=self.revision)
|
|
284
|
+
except RevisionNotFoundError:
|
|
285
|
+
logger.info(f"Branch '{self.revision}' not found. Creating it...")
|
|
286
|
+
self.api.create_branch(repo_id=repo_id, repo_type=self.repo_type, branch=self.revision, exist_ok=True)
|
|
287
|
+
# ^ `exist_ok=True` to avoid race concurrency issues
|
|
288
|
+
|
|
289
|
+
# File-based upload
|
|
290
|
+
if os.path.isfile(self.local_path):
|
|
291
|
+
return self.api.upload_file(
|
|
292
|
+
path_or_fileobj=self.local_path,
|
|
293
|
+
path_in_repo=self.path_in_repo,
|
|
294
|
+
repo_id=repo_id,
|
|
295
|
+
repo_type=self.repo_type,
|
|
296
|
+
revision=self.revision,
|
|
297
|
+
commit_message=self.commit_message,
|
|
298
|
+
commit_description=self.commit_description,
|
|
299
|
+
create_pr=self.create_pr,
|
|
300
|
+
)
|
|
301
|
+
|
|
302
|
+
# Folder-based upload
|
|
303
|
+
else:
|
|
304
|
+
return self.api.upload_folder(
|
|
305
|
+
folder_path=self.local_path,
|
|
306
|
+
path_in_repo=self.path_in_repo,
|
|
307
|
+
repo_id=repo_id,
|
|
308
|
+
repo_type=self.repo_type,
|
|
309
|
+
revision=self.revision,
|
|
310
|
+
commit_message=self.commit_message,
|
|
311
|
+
commit_description=self.commit_description,
|
|
312
|
+
create_pr=self.create_pr,
|
|
313
|
+
allow_patterns=self.include,
|
|
314
|
+
ignore_patterns=self.exclude,
|
|
315
|
+
delete_patterns=self.delete,
|
|
316
|
+
)
|
|
@@ -0,0 +1,132 @@
|
|
|
1
|
+
# coding=utf-8
|
|
2
|
+
# Copyright 2023-present, the HuggingFace Inc. team.
|
|
3
|
+
#
|
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
5
|
+
# you may not use this file except in compliance with the License.
|
|
6
|
+
# You may obtain a copy of the License at
|
|
7
|
+
#
|
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
9
|
+
#
|
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
12
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
13
|
+
# See the License for the specific language governing permissions and
|
|
14
|
+
# limitations under the License.
|
|
15
|
+
"""Contains command to upload a large folder with the CLI."""
|
|
16
|
+
|
|
17
|
+
import os
|
|
18
|
+
from argparse import Namespace, _SubParsersAction
|
|
19
|
+
from typing import List, Optional
|
|
20
|
+
|
|
21
|
+
from huggingface_hub import logging
|
|
22
|
+
from huggingface_hub.commands import BaseHuggingfaceCLICommand
|
|
23
|
+
from huggingface_hub.hf_api import HfApi
|
|
24
|
+
from huggingface_hub.utils import disable_progress_bars
|
|
25
|
+
|
|
26
|
+
from ._cli_utils import ANSI
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
logger = logging.get_logger(__name__)
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
class UploadLargeFolderCommand(BaseHuggingfaceCLICommand):
|
|
33
|
+
@staticmethod
|
|
34
|
+
def register_subcommand(parser: _SubParsersAction):
|
|
35
|
+
subparser = parser.add_parser(
|
|
36
|
+
"upload-large-folder",
|
|
37
|
+
help="Upload a large folder to the Hub. Recommended for resumable uploads.",
|
|
38
|
+
)
|
|
39
|
+
subparser.add_argument(
|
|
40
|
+
"repo_id", type=str, help="The ID of the repo to upload to (e.g. `username/repo-name`)."
|
|
41
|
+
)
|
|
42
|
+
subparser.add_argument("local_path", type=str, help="Local path to the file or folder to upload.")
|
|
43
|
+
subparser.add_argument(
|
|
44
|
+
"--repo-type",
|
|
45
|
+
choices=["model", "dataset", "space"],
|
|
46
|
+
help="Type of the repo to upload to (e.g. `dataset`).",
|
|
47
|
+
)
|
|
48
|
+
subparser.add_argument(
|
|
49
|
+
"--revision",
|
|
50
|
+
type=str,
|
|
51
|
+
help=("An optional Git revision to push to. It can be a branch name or a PR reference."),
|
|
52
|
+
)
|
|
53
|
+
subparser.add_argument(
|
|
54
|
+
"--private",
|
|
55
|
+
action="store_true",
|
|
56
|
+
help=(
|
|
57
|
+
"Whether to create a private repo if repo doesn't exist on the Hub. Ignored if the repo already exists."
|
|
58
|
+
),
|
|
59
|
+
)
|
|
60
|
+
subparser.add_argument("--include", nargs="*", type=str, help="Glob patterns to match files to upload.")
|
|
61
|
+
subparser.add_argument("--exclude", nargs="*", type=str, help="Glob patterns to exclude from files to upload.")
|
|
62
|
+
subparser.add_argument(
|
|
63
|
+
"--token", type=str, help="A User Access Token generated from https://huggingface.co/settings/tokens"
|
|
64
|
+
)
|
|
65
|
+
subparser.add_argument(
|
|
66
|
+
"--num-workers", type=int, help="Number of workers to use to hash, upload and commit files."
|
|
67
|
+
)
|
|
68
|
+
subparser.add_argument("--no-report", action="store_true", help="Whether to disable regular status report.")
|
|
69
|
+
subparser.add_argument("--no-bars", action="store_true", help="Whether to disable progress bars.")
|
|
70
|
+
subparser.set_defaults(func=UploadLargeFolderCommand)
|
|
71
|
+
|
|
72
|
+
def __init__(self, args: Namespace) -> None:
|
|
73
|
+
self.repo_id: str = args.repo_id
|
|
74
|
+
self.local_path: str = args.local_path
|
|
75
|
+
self.repo_type: str = args.repo_type
|
|
76
|
+
self.revision: Optional[str] = args.revision
|
|
77
|
+
self.private: bool = args.private
|
|
78
|
+
|
|
79
|
+
self.include: Optional[List[str]] = args.include
|
|
80
|
+
self.exclude: Optional[List[str]] = args.exclude
|
|
81
|
+
|
|
82
|
+
self.api: HfApi = HfApi(token=args.token, library_name="hf")
|
|
83
|
+
|
|
84
|
+
self.num_workers: Optional[int] = args.num_workers
|
|
85
|
+
self.no_report: bool = args.no_report
|
|
86
|
+
self.no_bars: bool = args.no_bars
|
|
87
|
+
|
|
88
|
+
if not os.path.isdir(self.local_path):
|
|
89
|
+
raise ValueError("Large upload is only supported for folders.")
|
|
90
|
+
|
|
91
|
+
def run(self) -> None:
|
|
92
|
+
logging.set_verbosity_info()
|
|
93
|
+
|
|
94
|
+
print(
|
|
95
|
+
ANSI.yellow(
|
|
96
|
+
"You are about to upload a large folder to the Hub using `hf upload-large-folder`. "
|
|
97
|
+
"This is a new feature so feedback is very welcome!\n"
|
|
98
|
+
"\n"
|
|
99
|
+
"A few things to keep in mind:\n"
|
|
100
|
+
" - Repository limits still apply: https://huggingface.co/docs/hub/repositories-recommendations\n"
|
|
101
|
+
" - Do not start several processes in parallel.\n"
|
|
102
|
+
" - You can interrupt and resume the process at any time. "
|
|
103
|
+
"The script will pick up where it left off except for partially uploaded files that would have to be entirely reuploaded.\n"
|
|
104
|
+
" - Do not upload the same folder to several repositories. If you need to do so, you must delete the `./.cache/huggingface/` folder first.\n"
|
|
105
|
+
"\n"
|
|
106
|
+
f"Some temporary metadata will be stored under `{self.local_path}/.cache/huggingface`.\n"
|
|
107
|
+
" - You must not modify those files manually.\n"
|
|
108
|
+
" - You must not delete the `./.cache/huggingface/` folder while a process is running.\n"
|
|
109
|
+
" - You can delete the `./.cache/huggingface/` folder to reinitialize the upload state when process is not running. Files will have to be hashed and preuploaded again, except for already committed files.\n"
|
|
110
|
+
"\n"
|
|
111
|
+
"If the process output is too verbose, you can disable the progress bars with `--no-bars`. "
|
|
112
|
+
"You can also entirely disable the status report with `--no-report`.\n"
|
|
113
|
+
"\n"
|
|
114
|
+
"For more details, run `hf upload-large-folder --help` or check the documentation at "
|
|
115
|
+
"https://huggingface.co/docs/huggingface_hub/guides/upload#upload-a-large-folder."
|
|
116
|
+
)
|
|
117
|
+
)
|
|
118
|
+
|
|
119
|
+
if self.no_bars:
|
|
120
|
+
disable_progress_bars()
|
|
121
|
+
|
|
122
|
+
self.api.upload_large_folder(
|
|
123
|
+
repo_id=self.repo_id,
|
|
124
|
+
folder_path=self.local_path,
|
|
125
|
+
repo_type=self.repo_type,
|
|
126
|
+
revision=self.revision,
|
|
127
|
+
private=self.private,
|
|
128
|
+
allow_patterns=self.include,
|
|
129
|
+
ignore_patterns=self.exclude,
|
|
130
|
+
num_workers=self.num_workers,
|
|
131
|
+
print_report=not self.no_report,
|
|
132
|
+
)
|
|
@@ -67,3 +67,8 @@ def tabulate(rows: List[List[Union[str, int]]], headers: List[str]) -> str:
|
|
|
67
67
|
for row in rows:
|
|
68
68
|
lines.append(row_format.format(*row))
|
|
69
69
|
return "\n".join(lines)
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def show_deprecation_warning(old_command: str, new_command: str):
|
|
73
|
+
"""Show a yellow warning about deprecated CLI command."""
|
|
74
|
+
print(ANSI.yellow(f"⚠️ Warning: '{old_command}' is deprecated. Use '{new_command}' instead."))
|
|
@@ -64,7 +64,7 @@ from typing import Any, Callable, Iterable, List, Literal, Optional, Union
|
|
|
64
64
|
|
|
65
65
|
from ..utils import CachedRepoInfo, CachedRevisionInfo, HFCacheInfo, scan_cache_dir
|
|
66
66
|
from . import BaseHuggingfaceCLICommand
|
|
67
|
-
from ._cli_utils import ANSI
|
|
67
|
+
from ._cli_utils import ANSI, show_deprecation_warning
|
|
68
68
|
|
|
69
69
|
|
|
70
70
|
try:
|
|
@@ -144,6 +144,8 @@ class DeleteCacheCommand(BaseHuggingfaceCLICommand):
|
|
|
144
144
|
|
|
145
145
|
def run(self):
|
|
146
146
|
"""Run `delete-cache` command with or without TUI."""
|
|
147
|
+
show_deprecation_warning("huggingface-cli delete-cache", "hf cache delete")
|
|
148
|
+
|
|
147
149
|
# Scan cache directory
|
|
148
150
|
hf_cache_info = scan_cache_dir(self.cache_dir)
|
|
149
151
|
|
|
@@ -46,6 +46,8 @@ from huggingface_hub.commands import BaseHuggingfaceCLICommand
|
|
|
46
46
|
from huggingface_hub.file_download import hf_hub_download
|
|
47
47
|
from huggingface_hub.utils import disable_progress_bars, enable_progress_bars
|
|
48
48
|
|
|
49
|
+
from ._cli_utils import show_deprecation_warning
|
|
50
|
+
|
|
49
51
|
|
|
50
52
|
logger = logging.get_logger(__name__)
|
|
51
53
|
|
|
@@ -142,6 +144,8 @@ class DownloadCommand(BaseHuggingfaceCLICommand):
|
|
|
142
144
|
)
|
|
143
145
|
|
|
144
146
|
def run(self) -> None:
|
|
147
|
+
show_deprecation_warning("huggingface-cli download", "hf download")
|
|
148
|
+
|
|
145
149
|
if self.quiet:
|
|
146
150
|
disable_progress_bars()
|
|
147
151
|
with warnings.catch_warnings():
|
huggingface_hub/commands/env.py
CHANGED
|
@@ -21,6 +21,7 @@ from argparse import _SubParsersAction
|
|
|
21
21
|
|
|
22
22
|
from ..utils import dump_environment_info
|
|
23
23
|
from . import BaseHuggingfaceCLICommand
|
|
24
|
+
from ._cli_utils import show_deprecation_warning
|
|
24
25
|
|
|
25
26
|
|
|
26
27
|
class EnvironmentCommand(BaseHuggingfaceCLICommand):
|
|
@@ -33,4 +34,6 @@ class EnvironmentCommand(BaseHuggingfaceCLICommand):
|
|
|
33
34
|
env_parser.set_defaults(func=EnvironmentCommand)
|
|
34
35
|
|
|
35
36
|
def run(self) -> None:
|
|
37
|
+
show_deprecation_warning("huggingface-cli env", "hf env")
|
|
38
|
+
|
|
36
39
|
dump_environment_info()
|
|
@@ -14,6 +14,7 @@
|
|
|
14
14
|
|
|
15
15
|
from argparse import ArgumentParser
|
|
16
16
|
|
|
17
|
+
from huggingface_hub.commands._cli_utils import show_deprecation_warning
|
|
17
18
|
from huggingface_hub.commands.delete_cache import DeleteCacheCommand
|
|
18
19
|
from huggingface_hub.commands.download import DownloadCommand
|
|
19
20
|
from huggingface_hub.commands.env import EnvironmentCommand
|
|
@@ -51,6 +52,7 @@ def main():
|
|
|
51
52
|
# Let's go
|
|
52
53
|
args = parser.parse_args()
|
|
53
54
|
if not hasattr(args, "func"):
|
|
55
|
+
show_deprecation_warning("huggingface-cli", "hf")
|
|
54
56
|
parser.print_help()
|
|
55
57
|
exit(1)
|
|
56
58
|
|
huggingface_hub/commands/repo.py
CHANGED
|
@@ -31,6 +31,8 @@ from huggingface_hub.constants import SPACES_SDK_TYPES
|
|
|
31
31
|
from huggingface_hub.hf_api import HfApi
|
|
32
32
|
from huggingface_hub.utils import logging
|
|
33
33
|
|
|
34
|
+
from ._cli_utils import show_deprecation_warning
|
|
35
|
+
|
|
34
36
|
|
|
35
37
|
logger = logging.get_logger(__name__)
|
|
36
38
|
|
|
@@ -128,6 +130,8 @@ class RepoCreateCommand:
|
|
|
128
130
|
self._api = HfApi()
|
|
129
131
|
|
|
130
132
|
def run(self):
|
|
133
|
+
show_deprecation_warning("huggingface-cli repo", "hf repo")
|
|
134
|
+
|
|
131
135
|
if self.organization is not None:
|
|
132
136
|
if "/" in self.repo_id:
|
|
133
137
|
print(ANSI.red("You cannot pass both --organization and a repo_id with a namespace."))
|
|
@@ -41,6 +41,8 @@ from huggingface_hub import logging
|
|
|
41
41
|
from huggingface_hub.commands import BaseHuggingfaceCLICommand
|
|
42
42
|
from huggingface_hub.hf_api import HfApi
|
|
43
43
|
|
|
44
|
+
from ._cli_utils import show_deprecation_warning
|
|
45
|
+
|
|
44
46
|
|
|
45
47
|
logger = logging.get_logger(__name__)
|
|
46
48
|
|
|
@@ -59,6 +61,8 @@ class DeleteFilesSubCommand:
|
|
|
59
61
|
self.token: Optional[str] = args.token
|
|
60
62
|
|
|
61
63
|
def run(self) -> None:
|
|
64
|
+
show_deprecation_warning("huggingface-cli repo-files", "hf repo-files")
|
|
65
|
+
|
|
62
66
|
logging.set_verbosity_info()
|
|
63
67
|
url = self.api.delete_files(
|
|
64
68
|
delete_patterns=self.patterns,
|
|
@@ -27,7 +27,7 @@ from typing import Optional
|
|
|
27
27
|
|
|
28
28
|
from ..utils import CacheNotFound, HFCacheInfo, scan_cache_dir
|
|
29
29
|
from . import BaseHuggingfaceCLICommand
|
|
30
|
-
from ._cli_utils import ANSI, tabulate
|
|
30
|
+
from ._cli_utils import ANSI, show_deprecation_warning, tabulate
|
|
31
31
|
|
|
32
32
|
|
|
33
33
|
class ScanCacheCommand(BaseHuggingfaceCLICommand):
|
|
@@ -55,6 +55,8 @@ class ScanCacheCommand(BaseHuggingfaceCLICommand):
|
|
|
55
55
|
self.cache_dir: Optional[str] = args.dir
|
|
56
56
|
|
|
57
57
|
def run(self):
|
|
58
|
+
show_deprecation_warning("huggingface-cli scan-cache", "hf cache scan")
|
|
59
|
+
|
|
58
60
|
try:
|
|
59
61
|
t0 = time.time()
|
|
60
62
|
hf_cache_info = scan_cache_dir(self.cache_dir)
|
huggingface_hub/commands/tag.py
CHANGED
|
@@ -41,7 +41,7 @@ from huggingface_hub.constants import (
|
|
|
41
41
|
from huggingface_hub.hf_api import HfApi
|
|
42
42
|
|
|
43
43
|
from ..errors import HfHubHTTPError, RepositoryNotFoundError, RevisionNotFoundError
|
|
44
|
-
from ._cli_utils import ANSI
|
|
44
|
+
from ._cli_utils import ANSI, show_deprecation_warning
|
|
45
45
|
|
|
46
46
|
|
|
47
47
|
class TagCommands(BaseHuggingfaceCLICommand):
|
|
@@ -71,6 +71,8 @@ class TagCommands(BaseHuggingfaceCLICommand):
|
|
|
71
71
|
|
|
72
72
|
|
|
73
73
|
def handle_commands(args: Namespace):
|
|
74
|
+
show_deprecation_warning("huggingface-cli tag", "hf repo tag")
|
|
75
|
+
|
|
74
76
|
if args.list:
|
|
75
77
|
return TagListCommand(args)
|
|
76
78
|
elif args.delete:
|
|
@@ -61,6 +61,8 @@ from huggingface_hub.hf_api import HfApi
|
|
|
61
61
|
from huggingface_hub.utils import disable_progress_bars, enable_progress_bars
|
|
62
62
|
from huggingface_hub.utils._runtime import is_xet_available
|
|
63
63
|
|
|
64
|
+
from ._cli_utils import show_deprecation_warning
|
|
65
|
+
|
|
64
66
|
|
|
65
67
|
logger = logging.get_logger(__name__)
|
|
66
68
|
|
|
@@ -196,6 +198,8 @@ class UploadCommand(BaseHuggingfaceCLICommand):
|
|
|
196
198
|
self.path_in_repo = args.path_in_repo
|
|
197
199
|
|
|
198
200
|
def run(self) -> None:
|
|
201
|
+
show_deprecation_warning("huggingface-cli upload", "hf upload")
|
|
202
|
+
|
|
199
203
|
if self.quiet:
|
|
200
204
|
disable_progress_bars()
|
|
201
205
|
with warnings.catch_warnings():
|
|
@@ -23,7 +23,7 @@ from huggingface_hub.commands import BaseHuggingfaceCLICommand
|
|
|
23
23
|
from huggingface_hub.hf_api import HfApi
|
|
24
24
|
from huggingface_hub.utils import disable_progress_bars
|
|
25
25
|
|
|
26
|
-
from ._cli_utils import ANSI
|
|
26
|
+
from ._cli_utils import ANSI, show_deprecation_warning
|
|
27
27
|
|
|
28
28
|
|
|
29
29
|
logger = logging.get_logger(__name__)
|
|
@@ -86,6 +86,8 @@ class UploadLargeFolderCommand(BaseHuggingfaceCLICommand):
|
|
|
86
86
|
raise ValueError("Large upload is only supported for folders.")
|
|
87
87
|
|
|
88
88
|
def run(self) -> None:
|
|
89
|
+
show_deprecation_warning("huggingface-cli upload-large-folder", "hf upload-large-folder")
|
|
90
|
+
|
|
89
91
|
logging.set_verbosity_info()
|
|
90
92
|
|
|
91
93
|
print(
|