huggingface-hub 0.31.3__py3-none-any.whl → 0.32.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of huggingface-hub might be problematic. Click here for more details.
- huggingface_hub/__init__.py +42 -4
- huggingface_hub/_local_folder.py +8 -0
- huggingface_hub/_oauth.py +464 -0
- huggingface_hub/_snapshot_download.py +11 -3
- huggingface_hub/_upload_large_folder.py +16 -36
- huggingface_hub/commands/huggingface_cli.py +2 -0
- huggingface_hub/commands/repo.py +147 -0
- huggingface_hub/commands/user.py +2 -108
- huggingface_hub/constants.py +9 -1
- huggingface_hub/dataclasses.py +2 -2
- huggingface_hub/file_download.py +13 -11
- huggingface_hub/hf_api.py +48 -19
- huggingface_hub/hub_mixin.py +2 -2
- huggingface_hub/inference/_client.py +8 -7
- huggingface_hub/inference/_generated/_async_client.py +8 -7
- huggingface_hub/inference/_generated/types/__init__.py +4 -1
- huggingface_hub/inference/_generated/types/chat_completion.py +43 -9
- huggingface_hub/inference/_mcp/__init__.py +0 -0
- huggingface_hub/inference/_mcp/agent.py +99 -0
- huggingface_hub/inference/_mcp/cli.py +154 -0
- huggingface_hub/inference/_mcp/constants.py +80 -0
- huggingface_hub/inference/_mcp/mcp_client.py +322 -0
- huggingface_hub/inference/_mcp/utils.py +123 -0
- huggingface_hub/inference/_providers/__init__.py +13 -1
- huggingface_hub/inference/_providers/_common.py +1 -0
- huggingface_hub/inference/_providers/cerebras.py +1 -1
- huggingface_hub/inference/_providers/cohere.py +20 -3
- huggingface_hub/inference/_providers/fireworks_ai.py +18 -0
- huggingface_hub/inference/_providers/hf_inference.py +8 -1
- huggingface_hub/inference/_providers/nebius.py +28 -0
- huggingface_hub/inference/_providers/nscale.py +44 -0
- huggingface_hub/inference/_providers/sambanova.py +14 -0
- huggingface_hub/inference/_providers/together.py +15 -0
- huggingface_hub/utils/_experimental.py +7 -5
- huggingface_hub/utils/insecure_hashlib.py +8 -4
- {huggingface_hub-0.31.3.dist-info → huggingface_hub-0.32.0.dist-info}/METADATA +30 -8
- {huggingface_hub-0.31.3.dist-info → huggingface_hub-0.32.0.dist-info}/RECORD +41 -32
- {huggingface_hub-0.31.3.dist-info → huggingface_hub-0.32.0.dist-info}/entry_points.txt +1 -0
- {huggingface_hub-0.31.3.dist-info → huggingface_hub-0.32.0.dist-info}/LICENSE +0 -0
- {huggingface_hub-0.31.3.dist-info → huggingface_hub-0.32.0.dist-info}/WHEEL +0 -0
- {huggingface_hub-0.31.3.dist-info → huggingface_hub-0.32.0.dist-info}/top_level.txt +0 -0
|
@@ -42,8 +42,7 @@ if TYPE_CHECKING:
|
|
|
42
42
|
logger = logging.getLogger(__name__)
|
|
43
43
|
|
|
44
44
|
WAITING_TIME_IF_NO_TASKS = 10 # seconds
|
|
45
|
-
|
|
46
|
-
MAX_NB_LFS_FILES_PER_COMMIT = 150
|
|
45
|
+
MAX_NB_FILES_FETCH_UPLOAD_MODE = 100
|
|
47
46
|
COMMIT_SIZE_SCALE: List[int] = [20, 50, 75, 100, 125, 200, 250, 400, 600, 1000]
|
|
48
47
|
|
|
49
48
|
|
|
@@ -404,19 +403,19 @@ def _determine_next_job(status: LargeUploadStatus) -> Optional[Tuple[WorkerJob,
|
|
|
404
403
|
):
|
|
405
404
|
status.nb_workers_commit += 1
|
|
406
405
|
logger.debug("Job: commit (more than 5 minutes since last commit attempt)")
|
|
407
|
-
return (WorkerJob.COMMIT,
|
|
406
|
+
return (WorkerJob.COMMIT, _get_n(status.queue_commit, status.target_chunk()))
|
|
408
407
|
|
|
409
408
|
# 2. Commit if at least 100 files are ready to commit
|
|
410
409
|
elif status.nb_workers_commit == 0 and status.queue_commit.qsize() >= 150:
|
|
411
410
|
status.nb_workers_commit += 1
|
|
412
411
|
logger.debug("Job: commit (>100 files ready)")
|
|
413
|
-
return (WorkerJob.COMMIT,
|
|
412
|
+
return (WorkerJob.COMMIT, _get_n(status.queue_commit, status.target_chunk()))
|
|
414
413
|
|
|
415
|
-
# 3. Get upload mode if at least
|
|
416
|
-
elif status.queue_get_upload_mode.qsize() >=
|
|
414
|
+
# 3. Get upload mode if at least 100 files
|
|
415
|
+
elif status.queue_get_upload_mode.qsize() >= MAX_NB_FILES_FETCH_UPLOAD_MODE:
|
|
417
416
|
status.nb_workers_get_upload_mode += 1
|
|
418
|
-
logger.debug("Job: get upload mode (>
|
|
419
|
-
return (WorkerJob.GET_UPLOAD_MODE, _get_n(status.queue_get_upload_mode,
|
|
417
|
+
logger.debug(f"Job: get upload mode (>{MAX_NB_FILES_FETCH_UPLOAD_MODE} files ready)")
|
|
418
|
+
return (WorkerJob.GET_UPLOAD_MODE, _get_n(status.queue_get_upload_mode, MAX_NB_FILES_FETCH_UPLOAD_MODE))
|
|
420
419
|
|
|
421
420
|
# 4. Preupload LFS file if at least 1 file and no worker is preuploading LFS
|
|
422
421
|
elif status.queue_preupload_lfs.qsize() > 0 and status.nb_workers_preupload_lfs == 0:
|
|
@@ -434,7 +433,7 @@ def _determine_next_job(status: LargeUploadStatus) -> Optional[Tuple[WorkerJob,
|
|
|
434
433
|
elif status.queue_get_upload_mode.qsize() > 0 and status.nb_workers_get_upload_mode == 0:
|
|
435
434
|
status.nb_workers_get_upload_mode += 1
|
|
436
435
|
logger.debug("Job: get upload mode (no other worker getting upload mode)")
|
|
437
|
-
return (WorkerJob.GET_UPLOAD_MODE, _get_n(status.queue_get_upload_mode,
|
|
436
|
+
return (WorkerJob.GET_UPLOAD_MODE, _get_n(status.queue_get_upload_mode, MAX_NB_FILES_FETCH_UPLOAD_MODE))
|
|
438
437
|
|
|
439
438
|
# 7. Preupload LFS file if at least 1 file
|
|
440
439
|
# Skip if hf_transfer is enabled and there is already a worker preuploading LFS
|
|
@@ -455,7 +454,7 @@ def _determine_next_job(status: LargeUploadStatus) -> Optional[Tuple[WorkerJob,
|
|
|
455
454
|
elif status.queue_get_upload_mode.qsize() > 0:
|
|
456
455
|
status.nb_workers_get_upload_mode += 1
|
|
457
456
|
logger.debug("Job: get upload mode")
|
|
458
|
-
return (WorkerJob.GET_UPLOAD_MODE, _get_n(status.queue_get_upload_mode,
|
|
457
|
+
return (WorkerJob.GET_UPLOAD_MODE, _get_n(status.queue_get_upload_mode, MAX_NB_FILES_FETCH_UPLOAD_MODE))
|
|
459
458
|
|
|
460
459
|
# 10. Commit if at least 1 file and 1 min since last commit attempt
|
|
461
460
|
elif (
|
|
@@ -466,7 +465,7 @@ def _determine_next_job(status: LargeUploadStatus) -> Optional[Tuple[WorkerJob,
|
|
|
466
465
|
):
|
|
467
466
|
status.nb_workers_commit += 1
|
|
468
467
|
logger.debug("Job: commit (1 min since last commit attempt)")
|
|
469
|
-
return (WorkerJob.COMMIT,
|
|
468
|
+
return (WorkerJob.COMMIT, _get_n(status.queue_commit, status.target_chunk()))
|
|
470
469
|
|
|
471
470
|
# 11. Commit if at least 1 file all other queues are empty and all workers are waiting
|
|
472
471
|
# e.g. when it's the last commit
|
|
@@ -482,7 +481,7 @@ def _determine_next_job(status: LargeUploadStatus) -> Optional[Tuple[WorkerJob,
|
|
|
482
481
|
):
|
|
483
482
|
status.nb_workers_commit += 1
|
|
484
483
|
logger.debug("Job: commit")
|
|
485
|
-
return (WorkerJob.COMMIT,
|
|
484
|
+
return (WorkerJob.COMMIT, _get_n(status.queue_commit, status.target_chunk()))
|
|
486
485
|
|
|
487
486
|
# 12. If all queues are empty, exit
|
|
488
487
|
elif all(metadata.is_committed or metadata.should_ignore for _, metadata in status.items):
|
|
@@ -522,11 +521,13 @@ def _get_upload_mode(items: List[JOB_ITEM_T], api: "HfApi", repo_id: str, repo_t
|
|
|
522
521
|
repo_id=repo_id,
|
|
523
522
|
headers=api._build_hf_headers(),
|
|
524
523
|
revision=quote(revision, safe=""),
|
|
524
|
+
endpoint=api.endpoint,
|
|
525
525
|
)
|
|
526
526
|
for item, addition in zip(items, additions):
|
|
527
527
|
paths, metadata = item
|
|
528
528
|
metadata.upload_mode = addition._upload_mode
|
|
529
529
|
metadata.should_ignore = addition._should_ignore
|
|
530
|
+
metadata.remote_oid = addition._remote_oid
|
|
530
531
|
metadata.save(paths)
|
|
531
532
|
|
|
532
533
|
|
|
@@ -579,6 +580,9 @@ def _build_hacky_operation(item: JOB_ITEM_T) -> HackyCommitOperationAdd:
|
|
|
579
580
|
if metadata.sha256 is None:
|
|
580
581
|
raise ValueError("sha256 must have been computed by now!")
|
|
581
582
|
operation.upload_info = UploadInfo(sha256=bytes.fromhex(metadata.sha256), size=metadata.size, sample=sample)
|
|
583
|
+
operation._upload_mode = metadata.upload_mode # type: ignore[assignment]
|
|
584
|
+
operation._should_ignore = metadata.should_ignore
|
|
585
|
+
operation._remote_oid = metadata.remote_oid
|
|
582
586
|
return operation
|
|
583
587
|
|
|
584
588
|
|
|
@@ -595,30 +599,6 @@ def _get_n(queue: "queue.Queue[JOB_ITEM_T]", n: int) -> List[JOB_ITEM_T]:
|
|
|
595
599
|
return [queue.get() for _ in range(min(queue.qsize(), n))]
|
|
596
600
|
|
|
597
601
|
|
|
598
|
-
def _get_items_to_commit(queue: "queue.Queue[JOB_ITEM_T]") -> List[JOB_ITEM_T]:
|
|
599
|
-
"""Special case for commit job: the number of items to commit depends on the type of files."""
|
|
600
|
-
# Can take at most 50 regular files and/or 100 LFS files in a single commit
|
|
601
|
-
items: List[JOB_ITEM_T] = []
|
|
602
|
-
nb_lfs, nb_regular = 0, 0
|
|
603
|
-
while True:
|
|
604
|
-
# If empty queue => commit everything
|
|
605
|
-
if queue.qsize() == 0:
|
|
606
|
-
return items
|
|
607
|
-
|
|
608
|
-
# If we have enough items => commit them
|
|
609
|
-
if nb_lfs >= MAX_NB_LFS_FILES_PER_COMMIT or nb_regular >= MAX_NB_REGULAR_FILES_PER_COMMIT:
|
|
610
|
-
return items
|
|
611
|
-
|
|
612
|
-
# Else, get a new item and increase counter
|
|
613
|
-
item = queue.get()
|
|
614
|
-
items.append(item)
|
|
615
|
-
_, metadata = item
|
|
616
|
-
if metadata.upload_mode == "lfs":
|
|
617
|
-
nb_lfs += 1
|
|
618
|
-
else:
|
|
619
|
-
nb_regular += 1
|
|
620
|
-
|
|
621
|
-
|
|
622
602
|
def _print_overwrite(report: str) -> None:
|
|
623
603
|
"""Print a report, overwriting the previous lines.
|
|
624
604
|
|
|
@@ -18,6 +18,7 @@ from huggingface_hub.commands.delete_cache import DeleteCacheCommand
|
|
|
18
18
|
from huggingface_hub.commands.download import DownloadCommand
|
|
19
19
|
from huggingface_hub.commands.env import EnvironmentCommand
|
|
20
20
|
from huggingface_hub.commands.lfs import LfsCommands
|
|
21
|
+
from huggingface_hub.commands.repo import RepoCommands
|
|
21
22
|
from huggingface_hub.commands.repo_files import RepoFilesCommand
|
|
22
23
|
from huggingface_hub.commands.scan_cache import ScanCacheCommand
|
|
23
24
|
from huggingface_hub.commands.tag import TagCommands
|
|
@@ -37,6 +38,7 @@ def main():
|
|
|
37
38
|
RepoFilesCommand.register_subcommand(commands_parser)
|
|
38
39
|
EnvironmentCommand.register_subcommand(commands_parser)
|
|
39
40
|
UserCommands.register_subcommand(commands_parser)
|
|
41
|
+
RepoCommands.register_subcommand(commands_parser)
|
|
40
42
|
LfsCommands.register_subcommand(commands_parser)
|
|
41
43
|
ScanCacheCommand.register_subcommand(commands_parser)
|
|
42
44
|
DeleteCacheCommand.register_subcommand(commands_parser)
|
|
@@ -0,0 +1,147 @@
|
|
|
1
|
+
# Copyright 2025 The HuggingFace Team. All rights reserved.
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
"""Contains commands to interact with repositories on the Hugging Face Hub.
|
|
15
|
+
|
|
16
|
+
Usage:
|
|
17
|
+
# create a new dataset repo on the Hub
|
|
18
|
+
huggingface-cli repo create my-cool-dataset --repo-type=dataset
|
|
19
|
+
|
|
20
|
+
# create a private model repo on the Hub
|
|
21
|
+
huggingface-cli repo create my-cool-model --private
|
|
22
|
+
"""
|
|
23
|
+
|
|
24
|
+
import argparse
|
|
25
|
+
from argparse import _SubParsersAction
|
|
26
|
+
from typing import Optional
|
|
27
|
+
|
|
28
|
+
from huggingface_hub.commands import BaseHuggingfaceCLICommand
|
|
29
|
+
from huggingface_hub.commands._cli_utils import ANSI
|
|
30
|
+
from huggingface_hub.constants import SPACES_SDK_TYPES
|
|
31
|
+
from huggingface_hub.hf_api import HfApi
|
|
32
|
+
from huggingface_hub.utils import logging
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
logger = logging.get_logger(__name__)
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
class RepoCommands(BaseHuggingfaceCLICommand):
|
|
39
|
+
@staticmethod
|
|
40
|
+
def register_subcommand(parser: _SubParsersAction):
|
|
41
|
+
repo_parser = parser.add_parser("repo", help="{create} Commands to interact with your huggingface.co repos.")
|
|
42
|
+
repo_subparsers = repo_parser.add_subparsers(help="huggingface.co repos related commands")
|
|
43
|
+
repo_create_parser = repo_subparsers.add_parser("create", help="Create a new repo on huggingface.co")
|
|
44
|
+
repo_create_parser.add_argument(
|
|
45
|
+
"repo_id",
|
|
46
|
+
type=str,
|
|
47
|
+
help="The ID of the repo to create to (e.g. `username/repo-name`). The username is optional and will be set to your username if not provided.",
|
|
48
|
+
)
|
|
49
|
+
repo_create_parser.add_argument(
|
|
50
|
+
"--repo-type",
|
|
51
|
+
type=str,
|
|
52
|
+
help='Optional: set to "dataset" or "space" if creating a dataset or space, default is model.',
|
|
53
|
+
)
|
|
54
|
+
repo_create_parser.add_argument(
|
|
55
|
+
"--space_sdk",
|
|
56
|
+
type=str,
|
|
57
|
+
help='Optional: Hugging Face Spaces SDK type. Required when --type is set to "space".',
|
|
58
|
+
choices=SPACES_SDK_TYPES,
|
|
59
|
+
)
|
|
60
|
+
repo_create_parser.add_argument(
|
|
61
|
+
"--private",
|
|
62
|
+
action="store_true",
|
|
63
|
+
help="Whether to create a private repository. Defaults to public unless the organization's default is private.",
|
|
64
|
+
)
|
|
65
|
+
repo_create_parser.add_argument(
|
|
66
|
+
"--token",
|
|
67
|
+
type=str,
|
|
68
|
+
help="Hugging Face token. Will default to the locally saved token if not provided.",
|
|
69
|
+
)
|
|
70
|
+
repo_create_parser.add_argument(
|
|
71
|
+
"--exist-ok",
|
|
72
|
+
action="store_true",
|
|
73
|
+
help="Do not raise an error if repo already exists.",
|
|
74
|
+
)
|
|
75
|
+
repo_create_parser.add_argument(
|
|
76
|
+
"--resource-group-id",
|
|
77
|
+
type=str,
|
|
78
|
+
help="Resource group in which to create the repo. Resource groups is only available for Enterprise Hub organizations.",
|
|
79
|
+
)
|
|
80
|
+
repo_create_parser.add_argument(
|
|
81
|
+
"--type",
|
|
82
|
+
type=str,
|
|
83
|
+
help="[Deprecated]: use --repo-type instead.",
|
|
84
|
+
)
|
|
85
|
+
repo_create_parser.add_argument(
|
|
86
|
+
"-y",
|
|
87
|
+
"--yes",
|
|
88
|
+
action="store_true",
|
|
89
|
+
help="[Deprecated] no effect.",
|
|
90
|
+
)
|
|
91
|
+
repo_create_parser.add_argument(
|
|
92
|
+
"--organization", type=str, help="[Deprecated] Pass the organization namespace directly in the repo_id."
|
|
93
|
+
)
|
|
94
|
+
repo_create_parser.set_defaults(func=lambda args: RepoCreateCommand(args))
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
class RepoCreateCommand:
|
|
98
|
+
def __init__(self, args: argparse.Namespace):
|
|
99
|
+
self.repo_id: str = args.repo_id
|
|
100
|
+
self.repo_type: Optional[str] = args.repo_type or args.type
|
|
101
|
+
self.space_sdk: Optional[str] = args.space_sdk
|
|
102
|
+
self.organization: Optional[str] = args.organization
|
|
103
|
+
self.yes: bool = args.yes
|
|
104
|
+
self.private: bool = args.private
|
|
105
|
+
self.token: Optional[str] = args.token
|
|
106
|
+
self.exist_ok: bool = args.exist_ok
|
|
107
|
+
self.resource_group_id: Optional[str] = args.resource_group_id
|
|
108
|
+
|
|
109
|
+
if args.type is not None:
|
|
110
|
+
print(
|
|
111
|
+
ANSI.yellow(
|
|
112
|
+
"The --type argument is deprecated and will be removed in a future version. Use --repo-type instead."
|
|
113
|
+
)
|
|
114
|
+
)
|
|
115
|
+
if self.organization is not None:
|
|
116
|
+
print(
|
|
117
|
+
ANSI.yellow(
|
|
118
|
+
"The --organization argument is deprecated and will be removed in a future version. Pass the organization namespace directly in the repo_id."
|
|
119
|
+
)
|
|
120
|
+
)
|
|
121
|
+
if self.yes:
|
|
122
|
+
print(
|
|
123
|
+
ANSI.yellow(
|
|
124
|
+
"The --yes argument is deprecated and will be removed in a future version. It does not have any effect."
|
|
125
|
+
)
|
|
126
|
+
)
|
|
127
|
+
|
|
128
|
+
self._api = HfApi()
|
|
129
|
+
|
|
130
|
+
def run(self):
|
|
131
|
+
if self.organization is not None:
|
|
132
|
+
if "/" in self.repo_id:
|
|
133
|
+
print(ANSI.red("You cannot pass both --organization and a repo_id with a namespace."))
|
|
134
|
+
exit(1)
|
|
135
|
+
self.repo_id = f"{self.organization}/{self.repo_id}"
|
|
136
|
+
|
|
137
|
+
repo_url = self._api.create_repo(
|
|
138
|
+
repo_id=self.repo_id,
|
|
139
|
+
repo_type=self.repo_type,
|
|
140
|
+
private=self.private,
|
|
141
|
+
token=self.token,
|
|
142
|
+
exist_ok=self.exist_ok,
|
|
143
|
+
resource_group_id=self.resource_group_id,
|
|
144
|
+
space_sdk=self.space_sdk,
|
|
145
|
+
)
|
|
146
|
+
print(f"Successfully created {ANSI.bold(repo_url.repo_id)} on the Hub.")
|
|
147
|
+
print(f"Your repo is now available at {ANSI.bold(repo_url)}")
|
huggingface_hub/commands/user.py
CHANGED
|
@@ -28,32 +28,18 @@ Usage:
|
|
|
28
28
|
|
|
29
29
|
# find out which huggingface.co account you are logged in as
|
|
30
30
|
huggingface-cli whoami
|
|
31
|
-
|
|
32
|
-
# create a new dataset repo on the Hub
|
|
33
|
-
huggingface-cli repo create mydataset --type=dataset
|
|
34
|
-
|
|
35
31
|
"""
|
|
36
32
|
|
|
37
|
-
import subprocess
|
|
38
33
|
from argparse import _SubParsersAction
|
|
39
34
|
from typing import List, Optional
|
|
40
35
|
|
|
41
36
|
from requests.exceptions import HTTPError
|
|
42
37
|
|
|
43
38
|
from huggingface_hub.commands import BaseHuggingfaceCLICommand
|
|
44
|
-
from huggingface_hub.constants import ENDPOINT
|
|
39
|
+
from huggingface_hub.constants import ENDPOINT
|
|
45
40
|
from huggingface_hub.hf_api import HfApi
|
|
46
41
|
|
|
47
|
-
from .._login import
|
|
48
|
-
NOTEBOOK_LOGIN_PASSWORD_HTML,
|
|
49
|
-
NOTEBOOK_LOGIN_TOKEN_HTML_END,
|
|
50
|
-
NOTEBOOK_LOGIN_TOKEN_HTML_START,
|
|
51
|
-
auth_list,
|
|
52
|
-
auth_switch,
|
|
53
|
-
login,
|
|
54
|
-
logout,
|
|
55
|
-
notebook_login,
|
|
56
|
-
)
|
|
42
|
+
from .._login import auth_list, auth_switch, login, logout
|
|
57
43
|
from ..utils import get_stored_tokens, get_token, logging
|
|
58
44
|
from ._cli_utils import ANSI
|
|
59
45
|
|
|
@@ -111,34 +97,6 @@ class UserCommands(BaseHuggingfaceCLICommand):
|
|
|
111
97
|
auth_switch_parser.set_defaults(func=lambda args: AuthSwitchCommand(args))
|
|
112
98
|
auth_list_parser = auth_subparsers.add_parser("list", help="List all stored access tokens")
|
|
113
99
|
auth_list_parser.set_defaults(func=lambda args: AuthListCommand(args))
|
|
114
|
-
# new system: git-based repo system
|
|
115
|
-
repo_parser = parser.add_parser("repo", help="{create} Commands to interact with your huggingface.co repos.")
|
|
116
|
-
repo_subparsers = repo_parser.add_subparsers(help="huggingface.co repos related commands")
|
|
117
|
-
repo_create_parser = repo_subparsers.add_parser("create", help="Create a new repo on huggingface.co")
|
|
118
|
-
repo_create_parser.add_argument(
|
|
119
|
-
"name",
|
|
120
|
-
type=str,
|
|
121
|
-
help="Name for your repo. Will be namespaced under your username to build the repo id.",
|
|
122
|
-
)
|
|
123
|
-
repo_create_parser.add_argument(
|
|
124
|
-
"--type",
|
|
125
|
-
type=str,
|
|
126
|
-
help='Optional: repo_type: set to "dataset" or "space" if creating a dataset or space, default is model.',
|
|
127
|
-
)
|
|
128
|
-
repo_create_parser.add_argument("--organization", type=str, help="Optional: organization namespace.")
|
|
129
|
-
repo_create_parser.add_argument(
|
|
130
|
-
"--space_sdk",
|
|
131
|
-
type=str,
|
|
132
|
-
help='Optional: Hugging Face Spaces SDK type. Required when --type is set to "space".',
|
|
133
|
-
choices=SPACES_SDK_TYPES,
|
|
134
|
-
)
|
|
135
|
-
repo_create_parser.add_argument(
|
|
136
|
-
"-y",
|
|
137
|
-
"--yes",
|
|
138
|
-
action="store_true",
|
|
139
|
-
help="Optional: answer Yes to the prompt",
|
|
140
|
-
)
|
|
141
|
-
repo_create_parser.set_defaults(func=lambda args: RepoCreateCommand(args))
|
|
142
100
|
|
|
143
101
|
|
|
144
102
|
class BaseUserCommand:
|
|
@@ -238,67 +196,3 @@ class WhoamiCommand(BaseUserCommand):
|
|
|
238
196
|
print(e)
|
|
239
197
|
print(ANSI.red(e.response.text))
|
|
240
198
|
exit(1)
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
class RepoCreateCommand(BaseUserCommand):
|
|
244
|
-
def run(self):
|
|
245
|
-
token = get_token()
|
|
246
|
-
if token is None:
|
|
247
|
-
print("Not logged in")
|
|
248
|
-
exit(1)
|
|
249
|
-
try:
|
|
250
|
-
stdout = subprocess.check_output(["git", "--version"]).decode("utf-8")
|
|
251
|
-
print(ANSI.gray(stdout.strip()))
|
|
252
|
-
except FileNotFoundError:
|
|
253
|
-
print("Looks like you do not have git installed, please install.")
|
|
254
|
-
|
|
255
|
-
try:
|
|
256
|
-
stdout = subprocess.check_output(["git-lfs", "--version"]).decode("utf-8")
|
|
257
|
-
print(ANSI.gray(stdout.strip()))
|
|
258
|
-
except FileNotFoundError:
|
|
259
|
-
print(
|
|
260
|
-
ANSI.red(
|
|
261
|
-
"Looks like you do not have git-lfs installed, please install."
|
|
262
|
-
" You can install from https://git-lfs.github.com/."
|
|
263
|
-
" Then run `git lfs install` (you only have to do this once)."
|
|
264
|
-
)
|
|
265
|
-
)
|
|
266
|
-
print("")
|
|
267
|
-
|
|
268
|
-
user = self._api.whoami(token)["name"]
|
|
269
|
-
namespace = self.args.organization if self.args.organization is not None else user
|
|
270
|
-
|
|
271
|
-
repo_id = f"{namespace}/{self.args.name}"
|
|
272
|
-
|
|
273
|
-
if self.args.type not in REPO_TYPES:
|
|
274
|
-
print("Invalid repo --type")
|
|
275
|
-
exit(1)
|
|
276
|
-
|
|
277
|
-
if self.args.type in REPO_TYPES_URL_PREFIXES:
|
|
278
|
-
prefixed_repo_id = REPO_TYPES_URL_PREFIXES[self.args.type] + repo_id
|
|
279
|
-
else:
|
|
280
|
-
prefixed_repo_id = repo_id
|
|
281
|
-
|
|
282
|
-
print(f"You are about to create {ANSI.bold(prefixed_repo_id)}")
|
|
283
|
-
|
|
284
|
-
if not self.args.yes:
|
|
285
|
-
choice = input("Proceed? [Y/n] ").lower()
|
|
286
|
-
if not (choice == "" or choice == "y" or choice == "yes"):
|
|
287
|
-
print("Abort")
|
|
288
|
-
exit()
|
|
289
|
-
try:
|
|
290
|
-
url = self._api.create_repo(
|
|
291
|
-
repo_id=repo_id,
|
|
292
|
-
token=token,
|
|
293
|
-
repo_type=self.args.type,
|
|
294
|
-
space_sdk=self.args.space_sdk,
|
|
295
|
-
)
|
|
296
|
-
except HTTPError as e:
|
|
297
|
-
print(e)
|
|
298
|
-
print(ANSI.red(e.response.text))
|
|
299
|
-
exit(1)
|
|
300
|
-
print("\nYour repo now lives at:")
|
|
301
|
-
print(f" {ANSI.bold(url)}")
|
|
302
|
-
print("\nYou can clone it locally with the command below, and commit/push as usual.")
|
|
303
|
-
print(f"\n git clone {url}")
|
|
304
|
-
print("")
|
huggingface_hub/constants.py
CHANGED
|
@@ -271,9 +271,17 @@ ALL_INFERENCE_API_FRAMEWORKS = MAIN_INFERENCE_API_FRAMEWORKS + [
|
|
|
271
271
|
"timm",
|
|
272
272
|
]
|
|
273
273
|
|
|
274
|
-
#
|
|
274
|
+
# If OAuth didn't work after 2 redirects, there's likely a third-party cookie issue in the Space iframe view.
|
|
275
|
+
# In this case, we redirect the user to the non-iframe view.
|
|
276
|
+
OAUTH_MAX_REDIRECTS = 2
|
|
275
277
|
|
|
278
|
+
# OAuth-related environment variables injected by the Space
|
|
279
|
+
OAUTH_CLIENT_ID = os.environ.get("OAUTH_CLIENT_ID")
|
|
280
|
+
OAUTH_CLIENT_SECRET = os.environ.get("OAUTH_CLIENT_SECRET")
|
|
281
|
+
OAUTH_SCOPES = os.environ.get("OAUTH_SCOPES")
|
|
282
|
+
OPENID_PROVIDER_URL = os.environ.get("OPENID_PROVIDER_URL")
|
|
276
283
|
|
|
284
|
+
# Xet constants
|
|
277
285
|
HUGGINGFACE_HEADER_X_XET_ENDPOINT = "X-Xet-Cas-Url"
|
|
278
286
|
HUGGINGFACE_HEADER_X_XET_ACCESS_TOKEN = "X-Xet-Access-Token"
|
|
279
287
|
HUGGINGFACE_HEADER_X_XET_EXPIRATION = "X-Xet-Token-Expiration"
|
huggingface_hub/dataclasses.py
CHANGED
|
@@ -269,8 +269,8 @@ def validated_field(
|
|
|
269
269
|
metadata = {}
|
|
270
270
|
metadata["validator"] = validator
|
|
271
271
|
return field( # type: ignore
|
|
272
|
-
default=default,
|
|
273
|
-
default_factory=default_factory,
|
|
272
|
+
default=default, # type: ignore [arg-type]
|
|
273
|
+
default_factory=default_factory, # type: ignore [arg-type]
|
|
274
274
|
init=init,
|
|
275
275
|
repr=repr,
|
|
276
276
|
hash=hash,
|
huggingface_hub/file_download.py
CHANGED
|
@@ -1130,16 +1130,6 @@ def _hf_hub_download_to_cache_dir(
|
|
|
1130
1130
|
# In that case store a ref.
|
|
1131
1131
|
_cache_commit_hash_for_specific_revision(storage_folder, revision, commit_hash)
|
|
1132
1132
|
|
|
1133
|
-
# If file already exists, return it (except if force_download=True)
|
|
1134
|
-
if not force_download:
|
|
1135
|
-
if os.path.exists(pointer_path):
|
|
1136
|
-
return pointer_path
|
|
1137
|
-
|
|
1138
|
-
if os.path.exists(blob_path):
|
|
1139
|
-
# we have the blob already, but not the pointer
|
|
1140
|
-
_create_symlink(blob_path, pointer_path, new_blob=False)
|
|
1141
|
-
return pointer_path
|
|
1142
|
-
|
|
1143
1133
|
# Prevent parallel downloads of the same file with a lock.
|
|
1144
1134
|
# etag could be duplicated across repos,
|
|
1145
1135
|
lock_path = os.path.join(locks_dir, repo_folder_name(repo_id=repo_id, repo_type=repo_type), f"{etag}.lock")
|
|
@@ -1152,9 +1142,21 @@ def _hf_hub_download_to_cache_dir(
|
|
|
1152
1142
|
if os.name == "nt" and len(os.path.abspath(blob_path)) > 255:
|
|
1153
1143
|
blob_path = "\\\\?\\" + os.path.abspath(blob_path)
|
|
1154
1144
|
|
|
1145
|
+
Path(lock_path).parent.mkdir(parents=True, exist_ok=True)
|
|
1146
|
+
|
|
1147
|
+
# pointer already exists -> immediate return
|
|
1148
|
+
if not force_download and os.path.exists(pointer_path):
|
|
1149
|
+
return pointer_path
|
|
1150
|
+
|
|
1151
|
+
# Blob exists but pointer must be (safely) created -> take the lock
|
|
1152
|
+
if not force_download and os.path.exists(blob_path):
|
|
1153
|
+
with WeakFileLock(lock_path):
|
|
1154
|
+
if not os.path.exists(pointer_path):
|
|
1155
|
+
_create_symlink(blob_path, pointer_path, new_blob=False)
|
|
1156
|
+
return pointer_path
|
|
1157
|
+
|
|
1155
1158
|
# Local file doesn't exist or etag isn't a match => retrieve file from remote (or cache)
|
|
1156
1159
|
|
|
1157
|
-
Path(lock_path).parent.mkdir(parents=True, exist_ok=True)
|
|
1158
1160
|
with WeakFileLock(lock_path):
|
|
1159
1161
|
_download_to_tmp_and_move(
|
|
1160
1162
|
incomplete_path=Path(blob_path + ".incomplete"),
|
huggingface_hub/hf_api.py
CHANGED
|
@@ -3623,7 +3623,7 @@ class HfApi:
|
|
|
3623
3623
|
exist_ok (`bool`, *optional*, defaults to `False`):
|
|
3624
3624
|
If `True`, do not raise an error if repo already exists.
|
|
3625
3625
|
resource_group_id (`str`, *optional*):
|
|
3626
|
-
Resource group in which to create the repo. Resource groups is only available for organizations and
|
|
3626
|
+
Resource group in which to create the repo. Resource groups is only available for Enterprise Hub organizations and
|
|
3627
3627
|
allow to define which members of the organization can access the resource. The ID of a resource group
|
|
3628
3628
|
can be found in the URL of the resource's page on the Hub (e.g. `"66670e5163145ca562cb1988"`).
|
|
3629
3629
|
To learn more about resource groups, see https://huggingface.co/docs/hub/en/security-resource-groups.
|
|
@@ -4421,20 +4421,23 @@ class HfApi:
|
|
|
4421
4421
|
new_additions = [addition for addition in additions if not addition._is_uploaded]
|
|
4422
4422
|
|
|
4423
4423
|
# Check which new files are LFS
|
|
4424
|
-
|
|
4425
|
-
|
|
4426
|
-
|
|
4427
|
-
|
|
4428
|
-
|
|
4429
|
-
|
|
4430
|
-
|
|
4431
|
-
|
|
4432
|
-
|
|
4433
|
-
|
|
4434
|
-
|
|
4435
|
-
|
|
4436
|
-
|
|
4437
|
-
|
|
4424
|
+
# For some items, we might have already fetched the upload mode (in case of upload_large_folder)
|
|
4425
|
+
additions_no_upload_mode = [addition for addition in new_additions if addition._upload_mode is None]
|
|
4426
|
+
if len(additions_no_upload_mode) > 0:
|
|
4427
|
+
try:
|
|
4428
|
+
_fetch_upload_modes(
|
|
4429
|
+
additions=additions_no_upload_mode,
|
|
4430
|
+
repo_type=repo_type,
|
|
4431
|
+
repo_id=repo_id,
|
|
4432
|
+
headers=headers,
|
|
4433
|
+
revision=revision,
|
|
4434
|
+
endpoint=self.endpoint,
|
|
4435
|
+
create_pr=create_pr or False,
|
|
4436
|
+
gitignore_content=gitignore_content,
|
|
4437
|
+
)
|
|
4438
|
+
except RepositoryNotFoundError as e:
|
|
4439
|
+
e.append_to_message(_CREATE_COMMIT_NO_REPO_ERROR_MESSAGE)
|
|
4440
|
+
raise
|
|
4438
4441
|
|
|
4439
4442
|
# Filter out regular files
|
|
4440
4443
|
new_lfs_additions = [addition for addition in new_additions if addition._upload_mode == "lfs"]
|
|
@@ -7566,9 +7569,9 @@ class HfApi:
|
|
|
7566
7569
|
region: str,
|
|
7567
7570
|
vendor: str,
|
|
7568
7571
|
account_id: Optional[str] = None,
|
|
7569
|
-
min_replica: int =
|
|
7572
|
+
min_replica: int = 1,
|
|
7570
7573
|
max_replica: int = 1,
|
|
7571
|
-
scale_to_zero_timeout: int =
|
|
7574
|
+
scale_to_zero_timeout: Optional[int] = None,
|
|
7572
7575
|
revision: Optional[str] = None,
|
|
7573
7576
|
task: Optional[str] = None,
|
|
7574
7577
|
custom_image: Optional[Dict] = None,
|
|
@@ -7604,11 +7607,13 @@ class HfApi:
|
|
|
7604
7607
|
account_id (`str`, *optional*):
|
|
7605
7608
|
The account ID used to link a VPC to a private Inference Endpoint (if applicable).
|
|
7606
7609
|
min_replica (`int`, *optional*):
|
|
7607
|
-
The minimum number of replicas (instances) to keep running for the Inference Endpoint.
|
|
7610
|
+
The minimum number of replicas (instances) to keep running for the Inference Endpoint. To enable
|
|
7611
|
+
scaling to zero, set this value to 0 and adjust `scale_to_zero_timeout` accordingly. Defaults to 1.
|
|
7608
7612
|
max_replica (`int`, *optional*):
|
|
7609
7613
|
The maximum number of replicas (instances) to scale to for the Inference Endpoint. Defaults to 1.
|
|
7610
7614
|
scale_to_zero_timeout (`int`, *optional*):
|
|
7611
|
-
The duration in minutes before an inactive endpoint is scaled to zero
|
|
7615
|
+
The duration in minutes before an inactive endpoint is scaled to zero, or no scaling to zero if
|
|
7616
|
+
set to None and `min_replica` is not 0. Defaults to None.
|
|
7612
7617
|
revision (`str`, *optional*):
|
|
7613
7618
|
The specific model revision to deploy on the Inference Endpoint (e.g. `"6c0e6080953db56375760c0471a8c5f2929baf11"`).
|
|
7614
7619
|
task (`str`, *optional*):
|
|
@@ -7693,8 +7698,32 @@ class HfApi:
|
|
|
7693
7698
|
... secrets={"MY_SECRET_KEY": "secret_value"},
|
|
7694
7699
|
... tags=["dev", "text-generation"],
|
|
7695
7700
|
... )
|
|
7701
|
+
```
|
|
7696
7702
|
|
|
7703
|
+
```python
|
|
7704
|
+
# Start an Inference Endpoint running ProsusAI/finbert while scaling to zero in 15 minutes
|
|
7705
|
+
>>> from huggingface_hub import HfApi
|
|
7706
|
+
>>> api = HfApi()
|
|
7707
|
+
>>> endpoint = api.create_inference_endpoint(
|
|
7708
|
+
... "finbert-classifier",
|
|
7709
|
+
... repository="ProsusAI/finbert",
|
|
7710
|
+
... framework="pytorch",
|
|
7711
|
+
... task="text-classification",
|
|
7712
|
+
... min_replica=0,
|
|
7713
|
+
... scale_to_zero_timeout=15,
|
|
7714
|
+
... accelerator="cpu",
|
|
7715
|
+
... vendor="aws",
|
|
7716
|
+
... region="us-east-1",
|
|
7717
|
+
... type="protected",
|
|
7718
|
+
... instance_size="x2",
|
|
7719
|
+
... instance_type="intel-icl",
|
|
7720
|
+
... )
|
|
7721
|
+
>>> endpoint.wait(timeout=300)
|
|
7722
|
+
# Run inference on the endpoint
|
|
7723
|
+
>>> endpoint.client.text_generation(...)
|
|
7724
|
+
TextClassificationOutputElement(label='positive', score=0.8983615040779114)
|
|
7697
7725
|
```
|
|
7726
|
+
|
|
7698
7727
|
"""
|
|
7699
7728
|
namespace = namespace or self._get_namespace(token=token)
|
|
7700
7729
|
|
huggingface_hub/hub_mixin.py
CHANGED
|
@@ -353,7 +353,7 @@ class ModelHubMixin:
|
|
|
353
353
|
def _encode_arg(cls, arg: Any) -> Any:
|
|
354
354
|
"""Encode an argument into a JSON serializable format."""
|
|
355
355
|
if is_dataclass(arg):
|
|
356
|
-
return asdict(arg)
|
|
356
|
+
return asdict(arg) # type: ignore[arg-type]
|
|
357
357
|
for type_, (encoder, _) in cls._hub_mixin_coders.items():
|
|
358
358
|
if isinstance(arg, type_):
|
|
359
359
|
if arg is None:
|
|
@@ -767,7 +767,7 @@ class PyTorchModelHubMixin(ModelHubMixin):
|
|
|
767
767
|
def _save_pretrained(self, save_directory: Path) -> None:
|
|
768
768
|
"""Save weights from a Pytorch model to a local directory."""
|
|
769
769
|
model_to_save = self.module if hasattr(self, "module") else self # type: ignore
|
|
770
|
-
save_model_as_safetensor(model_to_save, str(save_directory / constants.SAFETENSORS_SINGLE_FILE))
|
|
770
|
+
save_model_as_safetensor(model_to_save, str(save_directory / constants.SAFETENSORS_SINGLE_FILE)) # type: ignore [arg-type]
|
|
771
771
|
|
|
772
772
|
@classmethod
|
|
773
773
|
def _from_pretrained(
|