huggingface-hub 0.24.7__py3-none-any.whl → 0.25.0rc0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of huggingface-hub might be problematic. Click here for more details.
- huggingface_hub/__init__.py +21 -1
- huggingface_hub/_commit_api.py +4 -4
- huggingface_hub/_inference_endpoints.py +13 -1
- huggingface_hub/_local_folder.py +191 -4
- huggingface_hub/_login.py +6 -6
- huggingface_hub/_snapshot_download.py +8 -17
- huggingface_hub/_space_api.py +5 -0
- huggingface_hub/_tensorboard_logger.py +29 -13
- huggingface_hub/_upload_large_folder.py +573 -0
- huggingface_hub/_webhooks_server.py +1 -1
- huggingface_hub/commands/_cli_utils.py +5 -0
- huggingface_hub/commands/download.py +8 -0
- huggingface_hub/commands/huggingface_cli.py +6 -1
- huggingface_hub/commands/lfs.py +2 -1
- huggingface_hub/commands/repo_files.py +2 -2
- huggingface_hub/commands/scan_cache.py +99 -57
- huggingface_hub/commands/tag.py +1 -1
- huggingface_hub/commands/upload.py +2 -1
- huggingface_hub/commands/upload_large_folder.py +129 -0
- huggingface_hub/commands/version.py +37 -0
- huggingface_hub/community.py +2 -2
- huggingface_hub/errors.py +218 -1
- huggingface_hub/fastai_utils.py +2 -3
- huggingface_hub/file_download.py +61 -62
- huggingface_hub/hf_api.py +758 -314
- huggingface_hub/hf_file_system.py +15 -23
- huggingface_hub/hub_mixin.py +27 -25
- huggingface_hub/inference/_client.py +78 -127
- huggingface_hub/inference/_generated/_async_client.py +169 -144
- huggingface_hub/inference/_generated/types/base.py +0 -9
- huggingface_hub/inference/_templating.py +2 -3
- huggingface_hub/inference_api.py +2 -2
- huggingface_hub/keras_mixin.py +2 -2
- huggingface_hub/lfs.py +7 -98
- huggingface_hub/repocard.py +6 -5
- huggingface_hub/repository.py +5 -5
- huggingface_hub/serialization/_torch.py +64 -11
- huggingface_hub/utils/__init__.py +13 -14
- huggingface_hub/utils/_cache_manager.py +97 -14
- huggingface_hub/utils/_fixes.py +18 -2
- huggingface_hub/utils/_http.py +228 -2
- huggingface_hub/utils/_lfs.py +110 -0
- huggingface_hub/utils/_runtime.py +7 -1
- huggingface_hub/utils/_token.py +3 -2
- {huggingface_hub-0.24.7.dist-info → huggingface_hub-0.25.0rc0.dist-info}/METADATA +2 -2
- {huggingface_hub-0.24.7.dist-info → huggingface_hub-0.25.0rc0.dist-info}/RECORD +50 -48
- huggingface_hub/inference/_types.py +0 -52
- huggingface_hub/utils/_errors.py +0 -397
- {huggingface_hub-0.24.7.dist-info → huggingface_hub-0.25.0rc0.dist-info}/LICENSE +0 -0
- {huggingface_hub-0.24.7.dist-info → huggingface_hub-0.25.0rc0.dist-info}/WHEEL +0 -0
- {huggingface_hub-0.24.7.dist-info → huggingface_hub-0.25.0rc0.dist-info}/entry_points.txt +0 -0
- {huggingface_hub-0.24.7.dist-info → huggingface_hub-0.25.0rc0.dist-info}/top_level.txt +0 -0
huggingface_hub/hf_api.py
CHANGED
|
@@ -19,6 +19,7 @@ import json
|
|
|
19
19
|
import re
|
|
20
20
|
import struct
|
|
21
21
|
import warnings
|
|
22
|
+
from collections import defaultdict
|
|
22
23
|
from concurrent.futures import Future, ThreadPoolExecutor
|
|
23
24
|
from dataclasses import asdict, dataclass, field
|
|
24
25
|
from datetime import datetime
|
|
@@ -47,6 +48,7 @@ from requests.exceptions import HTTPError
|
|
|
47
48
|
from tqdm.auto import tqdm as base_tqdm
|
|
48
49
|
from tqdm.contrib.concurrent import thread_map
|
|
49
50
|
|
|
51
|
+
from . import constants
|
|
50
52
|
from ._commit_api import (
|
|
51
53
|
CommitOperation,
|
|
52
54
|
CommitOperationAdd,
|
|
@@ -73,6 +75,7 @@ from ._multi_commits import (
|
|
|
73
75
|
plan_multi_commits,
|
|
74
76
|
)
|
|
75
77
|
from ._space_api import SpaceHardware, SpaceRuntime, SpaceStorage, SpaceVariable
|
|
78
|
+
from ._upload_large_folder import upload_large_folder_internal
|
|
76
79
|
from .community import (
|
|
77
80
|
Discussion,
|
|
78
81
|
DiscussionComment,
|
|
@@ -82,39 +85,41 @@ from .community import (
|
|
|
82
85
|
deserialize_event,
|
|
83
86
|
)
|
|
84
87
|
from .constants import (
|
|
85
|
-
DEFAULT_ETAG_TIMEOUT,
|
|
86
|
-
DEFAULT_REQUEST_TIMEOUT,
|
|
87
|
-
DEFAULT_REVISION,
|
|
88
|
-
DISCUSSION_STATUS,
|
|
89
|
-
DISCUSSION_TYPES,
|
|
90
|
-
ENDPOINT,
|
|
91
|
-
INFERENCE_ENDPOINTS_ENDPOINT,
|
|
92
|
-
REGEX_COMMIT_OID,
|
|
93
|
-
REPO_TYPE_MODEL,
|
|
94
|
-
REPO_TYPES,
|
|
95
|
-
REPO_TYPES_MAPPING,
|
|
96
|
-
REPO_TYPES_URL_PREFIXES,
|
|
97
|
-
SAFETENSORS_INDEX_FILE,
|
|
98
|
-
SAFETENSORS_MAX_HEADER_LENGTH,
|
|
99
|
-
SAFETENSORS_SINGLE_FILE,
|
|
100
|
-
SPACES_SDK_TYPES,
|
|
101
|
-
WEBHOOK_DOMAIN_T,
|
|
102
|
-
DiscussionStatusFilter,
|
|
103
|
-
DiscussionTypeFilter,
|
|
88
|
+
DEFAULT_ETAG_TIMEOUT, # noqa: F401 # kept for backward compatibility
|
|
89
|
+
DEFAULT_REQUEST_TIMEOUT, # noqa: F401 # kept for backward compatibility
|
|
90
|
+
DEFAULT_REVISION, # noqa: F401 # kept for backward compatibility
|
|
91
|
+
DISCUSSION_STATUS, # noqa: F401 # kept for backward compatibility
|
|
92
|
+
DISCUSSION_TYPES, # noqa: F401 # kept for backward compatibility
|
|
93
|
+
ENDPOINT, # noqa: F401 # kept for backward compatibility
|
|
94
|
+
INFERENCE_ENDPOINTS_ENDPOINT, # noqa: F401 # kept for backward compatibility
|
|
95
|
+
REGEX_COMMIT_OID, # noqa: F401 # kept for backward compatibility
|
|
96
|
+
REPO_TYPE_MODEL, # noqa: F401 # kept for backward compatibility
|
|
97
|
+
REPO_TYPES, # noqa: F401 # kept for backward compatibility
|
|
98
|
+
REPO_TYPES_MAPPING, # noqa: F401 # kept for backward compatibility
|
|
99
|
+
REPO_TYPES_URL_PREFIXES, # noqa: F401 # kept for backward compatibility
|
|
100
|
+
SAFETENSORS_INDEX_FILE, # noqa: F401 # kept for backward compatibility
|
|
101
|
+
SAFETENSORS_MAX_HEADER_LENGTH, # noqa: F401 # kept for backward compatibility
|
|
102
|
+
SAFETENSORS_SINGLE_FILE, # noqa: F401 # kept for backward compatibility
|
|
103
|
+
SPACES_SDK_TYPES, # noqa: F401 # kept for backward compatibility
|
|
104
|
+
WEBHOOK_DOMAIN_T, # noqa: F401 # kept for backward compatibility
|
|
105
|
+
DiscussionStatusFilter, # noqa: F401 # kept for backward compatibility
|
|
106
|
+
DiscussionTypeFilter, # noqa: F401 # kept for backward compatibility
|
|
107
|
+
)
|
|
108
|
+
from .errors import (
|
|
109
|
+
BadRequestError,
|
|
110
|
+
EntryNotFoundError,
|
|
111
|
+
GatedRepoError,
|
|
112
|
+
HfHubHTTPError,
|
|
113
|
+
RepositoryNotFoundError,
|
|
114
|
+
RevisionNotFoundError,
|
|
104
115
|
)
|
|
105
116
|
from .file_download import HfFileMetadata, get_hf_file_metadata, hf_hub_url
|
|
106
117
|
from .repocard_data import DatasetCardData, ModelCardData, SpaceCardData
|
|
107
118
|
from .utils import (
|
|
108
119
|
DEFAULT_IGNORE_PATTERNS,
|
|
109
|
-
BadRequestError,
|
|
110
|
-
EntryNotFoundError,
|
|
111
|
-
GatedRepoError,
|
|
112
120
|
HfFolder, # noqa: F401 # kept for backward compatibility
|
|
113
|
-
HfHubHTTPError,
|
|
114
121
|
LocalTokenNotFoundError,
|
|
115
122
|
NotASafetensorsRepoError,
|
|
116
|
-
RepositoryNotFoundError,
|
|
117
|
-
RevisionNotFoundError,
|
|
118
123
|
SafetensorsFileMetadata,
|
|
119
124
|
SafetensorsParsingError,
|
|
120
125
|
SafetensorsRepoMetadata,
|
|
@@ -132,9 +137,7 @@ from .utils import (
|
|
|
132
137
|
)
|
|
133
138
|
from .utils import tqdm as hf_tqdm
|
|
134
139
|
from .utils._typing import CallableT
|
|
135
|
-
from .utils.endpoint_helpers import
|
|
136
|
-
_is_emission_within_threshold,
|
|
137
|
-
)
|
|
140
|
+
from .utils.endpoint_helpers import _is_emission_within_threshold
|
|
138
141
|
|
|
139
142
|
|
|
140
143
|
R = TypeVar("R") # Return type
|
|
@@ -142,13 +145,16 @@ CollectionItemType_T = Literal["model", "dataset", "space", "paper"]
|
|
|
142
145
|
|
|
143
146
|
ExpandModelProperty_T = Literal[
|
|
144
147
|
"author",
|
|
148
|
+
"baseModels",
|
|
145
149
|
"cardData",
|
|
150
|
+
"childrenModelCount",
|
|
146
151
|
"config",
|
|
147
152
|
"createdAt",
|
|
148
153
|
"disabled",
|
|
149
154
|
"downloads",
|
|
150
155
|
"downloadsAllTime",
|
|
151
156
|
"gated",
|
|
157
|
+
"gguf",
|
|
152
158
|
"inference",
|
|
153
159
|
"lastModified",
|
|
154
160
|
"library_name",
|
|
@@ -163,6 +169,7 @@ ExpandModelProperty_T = Literal[
|
|
|
163
169
|
"spaces",
|
|
164
170
|
"tags",
|
|
165
171
|
"transformersInfo",
|
|
172
|
+
"trendingScore",
|
|
166
173
|
"widgetData",
|
|
167
174
|
]
|
|
168
175
|
|
|
@@ -182,17 +189,19 @@ ExpandDatasetProperty_T = Literal[
|
|
|
182
189
|
"private",
|
|
183
190
|
"siblings",
|
|
184
191
|
"sha",
|
|
192
|
+
"trendingScore",
|
|
185
193
|
"tags",
|
|
186
194
|
]
|
|
187
195
|
|
|
188
196
|
ExpandSpaceProperty_T = Literal[
|
|
189
197
|
"author",
|
|
190
198
|
"cardData",
|
|
199
|
+
"createdAt",
|
|
191
200
|
"datasets",
|
|
192
201
|
"disabled",
|
|
193
202
|
"lastModified",
|
|
194
|
-
"createdAt",
|
|
195
203
|
"likes",
|
|
204
|
+
"models",
|
|
196
205
|
"private",
|
|
197
206
|
"runtime",
|
|
198
207
|
"sdk",
|
|
@@ -200,7 +209,7 @@ ExpandSpaceProperty_T = Literal[
|
|
|
200
209
|
"sha",
|
|
201
210
|
"subdomain",
|
|
202
211
|
"tags",
|
|
203
|
-
"
|
|
212
|
+
"trendingScore",
|
|
204
213
|
]
|
|
205
214
|
|
|
206
215
|
USERNAME_PLACEHOLDER = "hf_user"
|
|
@@ -210,7 +219,11 @@ _CREATE_COMMIT_NO_REPO_ERROR_MESSAGE = (
|
|
|
210
219
|
"\nNote: Creating a commit assumes that the repo already exists on the"
|
|
211
220
|
" Huggingface Hub. Please use `create_repo` if it's not the case."
|
|
212
221
|
)
|
|
213
|
-
|
|
222
|
+
_AUTH_CHECK_NO_REPO_ERROR_MESSAGE = (
|
|
223
|
+
"\nNote: The repository either does not exist or you do not have access rights."
|
|
224
|
+
" Please check the repository ID and your access permissions."
|
|
225
|
+
" If this is a private repository, ensure that your token is correct."
|
|
226
|
+
)
|
|
214
227
|
logger = logging.get_logger(__name__)
|
|
215
228
|
|
|
216
229
|
|
|
@@ -245,7 +258,7 @@ def repo_type_and_id_from_hf_id(hf_id: str, hub_url: Optional[str] = None) -> Tu
|
|
|
245
258
|
"""
|
|
246
259
|
input_hf_id = hf_id
|
|
247
260
|
|
|
248
|
-
hub_url = re.sub(r"https?://", "", hub_url if hub_url is not None else ENDPOINT)
|
|
261
|
+
hub_url = re.sub(r"https?://", "", hub_url if hub_url is not None else constants.ENDPOINT)
|
|
249
262
|
is_hf_url = hub_url in hf_id and "@" not in hf_id
|
|
250
263
|
|
|
251
264
|
HFFS_PREFIX = "hf://"
|
|
@@ -262,9 +275,9 @@ def repo_type_and_id_from_hf_id(hf_id: str, hub_url: Optional[str] = None) -> Tu
|
|
|
262
275
|
namespace = None
|
|
263
276
|
if len(url_segments) > 2 and hub_url not in url_segments[-3]:
|
|
264
277
|
repo_type = url_segments[-3]
|
|
265
|
-
elif namespace in REPO_TYPES_MAPPING:
|
|
278
|
+
elif namespace in constants.REPO_TYPES_MAPPING:
|
|
266
279
|
# Mean canonical dataset or model
|
|
267
|
-
repo_type = REPO_TYPES_MAPPING[namespace]
|
|
280
|
+
repo_type = constants.REPO_TYPES_MAPPING[namespace]
|
|
268
281
|
namespace = None
|
|
269
282
|
else:
|
|
270
283
|
repo_type = None
|
|
@@ -273,9 +286,9 @@ def repo_type_and_id_from_hf_id(hf_id: str, hub_url: Optional[str] = None) -> Tu
|
|
|
273
286
|
# Passed <repo_type>/<user>/<model_id> or <repo_type>/<org>/<model_id>
|
|
274
287
|
repo_type, namespace, repo_id = url_segments[-3:]
|
|
275
288
|
elif len(url_segments) == 2:
|
|
276
|
-
if url_segments[0] in REPO_TYPES_MAPPING:
|
|
289
|
+
if url_segments[0] in constants.REPO_TYPES_MAPPING:
|
|
277
290
|
# Passed '<model_id>' or 'datasets/<dataset_id>' for a canonical model or dataset
|
|
278
|
-
repo_type = REPO_TYPES_MAPPING[url_segments[0]]
|
|
291
|
+
repo_type = constants.REPO_TYPES_MAPPING[url_segments[0]]
|
|
279
292
|
namespace = None
|
|
280
293
|
repo_id = hf_id.split("/")[-1]
|
|
281
294
|
else:
|
|
@@ -290,11 +303,11 @@ def repo_type_and_id_from_hf_id(hf_id: str, hub_url: Optional[str] = None) -> Tu
|
|
|
290
303
|
raise ValueError(f"Unable to retrieve user and repo ID from the passed HF ID: {hf_id}")
|
|
291
304
|
|
|
292
305
|
# Check if repo type is known (mapping "spaces" => "space" + empty value => `None`)
|
|
293
|
-
if repo_type in REPO_TYPES_MAPPING:
|
|
294
|
-
repo_type = REPO_TYPES_MAPPING[repo_type]
|
|
306
|
+
if repo_type in constants.REPO_TYPES_MAPPING:
|
|
307
|
+
repo_type = constants.REPO_TYPES_MAPPING[repo_type]
|
|
295
308
|
if repo_type == "":
|
|
296
309
|
repo_type = None
|
|
297
|
-
if repo_type not in REPO_TYPES:
|
|
310
|
+
if repo_type not in constants.REPO_TYPES:
|
|
298
311
|
raise ValueError(f"Unknown `repo_type`: '{repo_type}' ('{input_hf_id}')")
|
|
299
312
|
|
|
300
313
|
return repo_type, namespace, repo_id
|
|
@@ -385,6 +398,9 @@ class CommitInfo(str):
|
|
|
385
398
|
`create_pr=True` is passed. Can be passed as `discussion_num` in
|
|
386
399
|
[`get_discussion_details`]. Example: `1`.
|
|
387
400
|
|
|
401
|
+
repo_url (`RepoUrl`):
|
|
402
|
+
Repo URL of the commit containing info like repo_id, repo_type, etc.
|
|
403
|
+
|
|
388
404
|
_url (`str`, *optional*):
|
|
389
405
|
Legacy url for `str` compatibility. Can be the url to the uploaded file on the Hub (if returned by
|
|
390
406
|
[`upload_file`]), to the uploaded folder on the Hub (if returned by [`upload_folder`]) or to the commit on
|
|
@@ -398,6 +414,9 @@ class CommitInfo(str):
|
|
|
398
414
|
oid: str
|
|
399
415
|
pr_url: Optional[str] = None
|
|
400
416
|
|
|
417
|
+
# Computed from `commit_url` in `__post_init__`
|
|
418
|
+
repo_url: RepoUrl = field(init=False)
|
|
419
|
+
|
|
401
420
|
# Computed from `pr_url` in `__post_init__`
|
|
402
421
|
pr_revision: Optional[str] = field(init=False)
|
|
403
422
|
pr_num: Optional[str] = field(init=False)
|
|
@@ -413,6 +432,10 @@ class CommitInfo(str):
|
|
|
413
432
|
|
|
414
433
|
See https://docs.python.org/3.10/library/dataclasses.html#post-init-processing.
|
|
415
434
|
"""
|
|
435
|
+
# Repo info
|
|
436
|
+
self.repo_url = RepoUrl(self.commit_url.split("/commit/")[0])
|
|
437
|
+
|
|
438
|
+
# PR info
|
|
416
439
|
if self.pr_url is not None:
|
|
417
440
|
self.pr_revision = _parse_revision_from_pr_url(self.pr_url)
|
|
418
441
|
self.pr_num = int(self.pr_revision.split("/")[-1])
|
|
@@ -488,7 +511,7 @@ class WebhookInfo:
|
|
|
488
511
|
id: str
|
|
489
512
|
url: str
|
|
490
513
|
watched: List[WebhookWatchedItem]
|
|
491
|
-
domains: List[WEBHOOK_DOMAIN_T]
|
|
514
|
+
domains: List[constants.WEBHOOK_DOMAIN_T]
|
|
492
515
|
secret: Optional[str]
|
|
493
516
|
disabled: bool
|
|
494
517
|
|
|
@@ -540,14 +563,14 @@ class RepoUrl(str):
|
|
|
540
563
|
def __init__(self, url: Any, endpoint: Optional[str] = None) -> None:
|
|
541
564
|
super().__init__()
|
|
542
565
|
# Parse URL
|
|
543
|
-
self.endpoint = endpoint or ENDPOINT
|
|
566
|
+
self.endpoint = endpoint or constants.ENDPOINT
|
|
544
567
|
repo_type, namespace, repo_name = repo_type_and_id_from_hf_id(self, hub_url=self.endpoint)
|
|
545
568
|
|
|
546
569
|
# Populate fields
|
|
547
570
|
self.namespace = namespace
|
|
548
571
|
self.repo_name = repo_name
|
|
549
572
|
self.repo_id = repo_name if namespace is None else f"{namespace}/{repo_name}"
|
|
550
|
-
self.repo_type = repo_type or REPO_TYPE_MODEL
|
|
573
|
+
self.repo_type = repo_type or constants.REPO_TYPE_MODEL
|
|
551
574
|
self.url = str(self) # just in case it's needed
|
|
552
575
|
|
|
553
576
|
def __repr__(self) -> str:
|
|
@@ -703,13 +726,17 @@ class ModelInfo:
|
|
|
703
726
|
Is the repo private.
|
|
704
727
|
disabled (`bool`, *optional*):
|
|
705
728
|
Is the repo disabled.
|
|
706
|
-
gated (`Literal["auto", "manual", False]`, *optional*):
|
|
707
|
-
Is the repo gated.
|
|
708
|
-
If so, whether there is manual or automatic approval.
|
|
709
729
|
downloads (`int`):
|
|
710
730
|
Number of downloads of the model over the last 30 days.
|
|
711
731
|
downloads_all_time (`int`):
|
|
712
732
|
Cumulated number of downloads of the model since its creation.
|
|
733
|
+
gated (`Literal["auto", "manual", False]`, *optional*):
|
|
734
|
+
Is the repo gated.
|
|
735
|
+
If so, whether there is manual or automatic approval.
|
|
736
|
+
inference (`Literal["cold", "frozen", "warm"]`, *optional*):
|
|
737
|
+
Status of the model on the inference API.
|
|
738
|
+
Warm models are available for immediate use. Cold models will be loaded on first inference call.
|
|
739
|
+
Frozen models are not available in Inference API.
|
|
713
740
|
likes (`int`):
|
|
714
741
|
Number of likes of the model.
|
|
715
742
|
library_name (`str`, *optional*):
|
|
@@ -729,6 +756,8 @@ class ModelInfo:
|
|
|
729
756
|
Model configuration.
|
|
730
757
|
transformers_info (`TransformersInfo`, *optional*):
|
|
731
758
|
Transformers-specific info (auto class, processor, etc.) associated with the model.
|
|
759
|
+
trending_score (`int`, *optional*):
|
|
760
|
+
Trending score of the model.
|
|
732
761
|
card_data (`ModelCardData`, *optional*):
|
|
733
762
|
Model Card Metadata as a [`huggingface_hub.repocard_data.ModelCardData`] object.
|
|
734
763
|
siblings (`List[RepoSibling]`):
|
|
@@ -745,10 +774,12 @@ class ModelInfo:
|
|
|
745
774
|
created_at: Optional[datetime]
|
|
746
775
|
last_modified: Optional[datetime]
|
|
747
776
|
private: Optional[bool]
|
|
748
|
-
gated: Optional[Literal["auto", "manual", False]]
|
|
749
777
|
disabled: Optional[bool]
|
|
750
778
|
downloads: Optional[int]
|
|
751
779
|
downloads_all_time: Optional[int]
|
|
780
|
+
gated: Optional[Literal["auto", "manual", False]]
|
|
781
|
+
gguf: Optional[Dict]
|
|
782
|
+
inference: Optional[Literal["warm", "cold", "frozen"]]
|
|
752
783
|
likes: Optional[int]
|
|
753
784
|
library_name: Optional[str]
|
|
754
785
|
tags: Optional[List[str]]
|
|
@@ -759,6 +790,7 @@ class ModelInfo:
|
|
|
759
790
|
model_index: Optional[Dict]
|
|
760
791
|
config: Optional[Dict]
|
|
761
792
|
transformers_info: Optional[TransformersInfo]
|
|
793
|
+
trending_score: Optional[int]
|
|
762
794
|
siblings: Optional[List[RepoSibling]]
|
|
763
795
|
spaces: Optional[List[str]]
|
|
764
796
|
safetensors: Optional[SafeTensorsInfo]
|
|
@@ -778,9 +810,13 @@ class ModelInfo:
|
|
|
778
810
|
self.downloads_all_time = kwargs.pop("downloadsAllTime", None)
|
|
779
811
|
self.likes = kwargs.pop("likes", None)
|
|
780
812
|
self.library_name = kwargs.pop("library_name", None)
|
|
813
|
+
self.gguf = kwargs.pop("gguf", None)
|
|
814
|
+
self.inference = kwargs.pop("inference", None)
|
|
781
815
|
self.tags = kwargs.pop("tags", None)
|
|
782
816
|
self.pipeline_tag = kwargs.pop("pipeline_tag", None)
|
|
783
817
|
self.mask_token = kwargs.pop("mask_token", None)
|
|
818
|
+
self.trending_score = kwargs.pop("trendingScore", None)
|
|
819
|
+
|
|
784
820
|
card_data = kwargs.pop("cardData", None) or kwargs.pop("card_data", None)
|
|
785
821
|
self.card_data = (
|
|
786
822
|
ModelCardData(**card_data, ignore_metadata_errors=True) if isinstance(card_data, dict) else card_data
|
|
@@ -810,7 +846,7 @@ class ModelInfo:
|
|
|
810
846
|
)
|
|
811
847
|
for sibling in siblings
|
|
812
848
|
]
|
|
813
|
-
if siblings
|
|
849
|
+
if siblings is not None
|
|
814
850
|
else None
|
|
815
851
|
)
|
|
816
852
|
self.spaces = kwargs.pop("spaces", None)
|
|
@@ -875,6 +911,10 @@ class DatasetInfo:
|
|
|
875
911
|
Model Card Metadata as a [`huggingface_hub.repocard_data.DatasetCardData`] object.
|
|
876
912
|
siblings (`List[RepoSibling]`):
|
|
877
913
|
List of [`huggingface_hub.hf_api.RepoSibling`] objects that constitute the dataset.
|
|
914
|
+
paperswithcode_id (`str`, *optional*):
|
|
915
|
+
Papers with code ID of the dataset.
|
|
916
|
+
trending_score (`int`, *optional*):
|
|
917
|
+
Trending score of the dataset.
|
|
878
918
|
"""
|
|
879
919
|
|
|
880
920
|
id: str
|
|
@@ -890,6 +930,7 @@ class DatasetInfo:
|
|
|
890
930
|
likes: Optional[int]
|
|
891
931
|
paperswithcode_id: Optional[str]
|
|
892
932
|
tags: Optional[List[str]]
|
|
933
|
+
trending_score: Optional[int]
|
|
893
934
|
card_data: Optional[DatasetCardData]
|
|
894
935
|
siblings: Optional[List[RepoSibling]]
|
|
895
936
|
|
|
@@ -909,6 +950,8 @@ class DatasetInfo:
|
|
|
909
950
|
self.likes = kwargs.pop("likes", None)
|
|
910
951
|
self.paperswithcode_id = kwargs.pop("paperswithcode_id", None)
|
|
911
952
|
self.tags = kwargs.pop("tags", None)
|
|
953
|
+
self.trending_score = kwargs.pop("trendingScore", None)
|
|
954
|
+
|
|
912
955
|
card_data = kwargs.pop("cardData", None) or kwargs.pop("card_data", None)
|
|
913
956
|
self.card_data = (
|
|
914
957
|
DatasetCardData(**card_data, ignore_metadata_errors=True) if isinstance(card_data, dict) else card_data
|
|
@@ -932,7 +975,7 @@ class DatasetInfo:
|
|
|
932
975
|
)
|
|
933
976
|
for sibling in siblings
|
|
934
977
|
]
|
|
935
|
-
if siblings
|
|
978
|
+
if siblings is not None
|
|
936
979
|
else None
|
|
937
980
|
)
|
|
938
981
|
|
|
@@ -994,6 +1037,8 @@ class SpaceInfo:
|
|
|
994
1037
|
List of models used by the Space.
|
|
995
1038
|
datasets (`List[str]`, *optional*):
|
|
996
1039
|
List of datasets used by the Space.
|
|
1040
|
+
trending_score (`int`, *optional*):
|
|
1041
|
+
Trending score of the Space.
|
|
997
1042
|
"""
|
|
998
1043
|
|
|
999
1044
|
id: str
|
|
@@ -1010,6 +1055,7 @@ class SpaceInfo:
|
|
|
1010
1055
|
sdk: Optional[str]
|
|
1011
1056
|
tags: Optional[List[str]]
|
|
1012
1057
|
siblings: Optional[List[RepoSibling]]
|
|
1058
|
+
trending_score: Optional[int]
|
|
1013
1059
|
card_data: Optional[SpaceCardData]
|
|
1014
1060
|
runtime: Optional[SpaceRuntime]
|
|
1015
1061
|
models: Optional[List[str]]
|
|
@@ -1031,6 +1077,7 @@ class SpaceInfo:
|
|
|
1031
1077
|
self.likes = kwargs.pop("likes", None)
|
|
1032
1078
|
self.sdk = kwargs.pop("sdk", None)
|
|
1033
1079
|
self.tags = kwargs.pop("tags", None)
|
|
1080
|
+
self.trending_score = kwargs.pop("trendingScore", None)
|
|
1034
1081
|
card_data = kwargs.pop("cardData", None) or kwargs.pop("card_data", None)
|
|
1035
1082
|
self.card_data = (
|
|
1036
1083
|
SpaceCardData(**card_data, ignore_metadata_errors=True) if isinstance(card_data, dict) else card_data
|
|
@@ -1054,7 +1101,7 @@ class SpaceInfo:
|
|
|
1054
1101
|
)
|
|
1055
1102
|
for sibling in siblings
|
|
1056
1103
|
]
|
|
1057
|
-
if siblings
|
|
1104
|
+
if siblings is not None
|
|
1058
1105
|
else None
|
|
1059
1106
|
)
|
|
1060
1107
|
runtime = kwargs.pop("runtime", None)
|
|
@@ -1184,7 +1231,7 @@ class Collection:
|
|
|
1184
1231
|
self.description = kwargs.pop("description", None)
|
|
1185
1232
|
endpoint = kwargs.pop("endpoint", None)
|
|
1186
1233
|
if endpoint is None:
|
|
1187
|
-
endpoint = ENDPOINT
|
|
1234
|
+
endpoint = constants.ENDPOINT
|
|
1188
1235
|
self._url = f"{endpoint}/collections/{self.slug}"
|
|
1189
1236
|
|
|
1190
1237
|
@property
|
|
@@ -1299,18 +1346,49 @@ class UserLikes:
|
|
|
1299
1346
|
spaces: List[str]
|
|
1300
1347
|
|
|
1301
1348
|
|
|
1349
|
+
@dataclass
|
|
1350
|
+
class Organization:
|
|
1351
|
+
"""
|
|
1352
|
+
Contains information about an organization on the Hub.
|
|
1353
|
+
|
|
1354
|
+
Attributes:
|
|
1355
|
+
avatar_url (`str`):
|
|
1356
|
+
URL of the organization's avatar.
|
|
1357
|
+
name (`str`):
|
|
1358
|
+
Name of the organization on the Hub (unique).
|
|
1359
|
+
fullname (`str`):
|
|
1360
|
+
Organization's full name.
|
|
1361
|
+
"""
|
|
1362
|
+
|
|
1363
|
+
avatar_url: str
|
|
1364
|
+
name: str
|
|
1365
|
+
fullname: str
|
|
1366
|
+
|
|
1367
|
+
def __init__(self, **kwargs) -> None:
|
|
1368
|
+
self.avatar_url = kwargs.pop("avatarUrl", "")
|
|
1369
|
+
self.name = kwargs.pop("name", "")
|
|
1370
|
+
self.fullname = kwargs.pop("fullname", "")
|
|
1371
|
+
|
|
1372
|
+
# forward compatibility
|
|
1373
|
+
self.__dict__.update(**kwargs)
|
|
1374
|
+
|
|
1375
|
+
|
|
1302
1376
|
@dataclass
|
|
1303
1377
|
class User:
|
|
1304
1378
|
"""
|
|
1305
1379
|
Contains information about a user on the Hub.
|
|
1306
1380
|
|
|
1307
1381
|
Attributes:
|
|
1308
|
-
avatar_url (`str`):
|
|
1309
|
-
URL of the user's avatar.
|
|
1310
1382
|
username (`str`):
|
|
1311
1383
|
Name of the user on the Hub (unique).
|
|
1312
1384
|
fullname (`str`):
|
|
1313
1385
|
User's full name.
|
|
1386
|
+
avatar_url (`str`):
|
|
1387
|
+
URL of the user's avatar.
|
|
1388
|
+
details (`str`, *optional*):
|
|
1389
|
+
User's details.
|
|
1390
|
+
is_following (`bool`, *optional*):
|
|
1391
|
+
Whether the authenticated user is following this user.
|
|
1314
1392
|
is_pro (`bool`, *optional*):
|
|
1315
1393
|
Whether the user is a pro user.
|
|
1316
1394
|
num_models (`int`, *optional*):
|
|
@@ -1327,16 +1405,16 @@ class User:
|
|
|
1327
1405
|
Number of upvotes received by the user.
|
|
1328
1406
|
num_likes (`int`, *optional*):
|
|
1329
1407
|
Number of likes given by the user.
|
|
1330
|
-
|
|
1331
|
-
|
|
1332
|
-
details (`str`, *optional*):
|
|
1333
|
-
User's details.
|
|
1408
|
+
orgs (list of [`Organization`]):
|
|
1409
|
+
List of organizations the user is part of.
|
|
1334
1410
|
"""
|
|
1335
1411
|
|
|
1336
1412
|
# Metadata
|
|
1337
|
-
avatar_url: str
|
|
1338
1413
|
username: str
|
|
1339
1414
|
fullname: str
|
|
1415
|
+
avatar_url: str
|
|
1416
|
+
details: Optional[str] = None
|
|
1417
|
+
is_following: Optional[bool] = None
|
|
1340
1418
|
is_pro: Optional[bool] = None
|
|
1341
1419
|
num_models: Optional[int] = None
|
|
1342
1420
|
num_datasets: Optional[int] = None
|
|
@@ -1345,24 +1423,24 @@ class User:
|
|
|
1345
1423
|
num_papers: Optional[int] = None
|
|
1346
1424
|
num_upvotes: Optional[int] = None
|
|
1347
1425
|
num_likes: Optional[int] = None
|
|
1348
|
-
|
|
1349
|
-
details: Optional[str] = None
|
|
1426
|
+
orgs: List[Organization] = field(default_factory=list)
|
|
1350
1427
|
|
|
1351
1428
|
def __init__(self, **kwargs) -> None:
|
|
1352
|
-
self.
|
|
1353
|
-
self.
|
|
1354
|
-
self.
|
|
1355
|
-
self.
|
|
1356
|
-
self.
|
|
1357
|
-
self.
|
|
1358
|
-
self.
|
|
1359
|
-
self.
|
|
1360
|
-
self.
|
|
1361
|
-
self.
|
|
1362
|
-
self.
|
|
1363
|
-
self.
|
|
1364
|
-
self.
|
|
1365
|
-
self.
|
|
1429
|
+
self.username = kwargs.pop("user", "")
|
|
1430
|
+
self.fullname = kwargs.pop("fullname", "")
|
|
1431
|
+
self.avatar_url = kwargs.pop("avatarUrl", "")
|
|
1432
|
+
self.is_following = kwargs.pop("isFollowing", None)
|
|
1433
|
+
self.is_pro = kwargs.pop("isPro", None)
|
|
1434
|
+
self.details = kwargs.pop("details", None)
|
|
1435
|
+
self.num_models = kwargs.pop("numModels", None)
|
|
1436
|
+
self.num_datasets = kwargs.pop("numDatasets", None)
|
|
1437
|
+
self.num_spaces = kwargs.pop("numSpaces", None)
|
|
1438
|
+
self.num_discussions = kwargs.pop("numDiscussions", None)
|
|
1439
|
+
self.num_papers = kwargs.pop("numPapers", None)
|
|
1440
|
+
self.num_upvotes = kwargs.pop("numUpvotes", None)
|
|
1441
|
+
self.num_likes = kwargs.pop("numLikes", None)
|
|
1442
|
+
self.user_type = kwargs.pop("type", None)
|
|
1443
|
+
self.orgs = [Organization(**org) for org in kwargs.pop("orgs", [])]
|
|
1366
1444
|
|
|
1367
1445
|
# forward compatibility
|
|
1368
1446
|
self.__dict__.update(**kwargs)
|
|
@@ -1437,7 +1515,7 @@ class HfApi:
|
|
|
1437
1515
|
Additional headers to be sent with each request. Example: `{"X-My-Header": "value"}`.
|
|
1438
1516
|
Headers passed here are taking precedence over the default headers.
|
|
1439
1517
|
"""
|
|
1440
|
-
self.endpoint = endpoint if endpoint is not None else ENDPOINT
|
|
1518
|
+
self.endpoint = endpoint if endpoint is not None else constants.ENDPOINT
|
|
1441
1519
|
self.token = token
|
|
1442
1520
|
self.library_name = library_name
|
|
1443
1521
|
self.library_version = library_version
|
|
@@ -1564,6 +1642,8 @@ class HfApi:
|
|
|
1564
1642
|
# Search-query parameter
|
|
1565
1643
|
filter: Union[str, Iterable[str], None] = None,
|
|
1566
1644
|
author: Optional[str] = None,
|
|
1645
|
+
gated: Optional[bool] = None,
|
|
1646
|
+
inference: Optional[Literal["cold", "frozen", "warm"]] = None,
|
|
1567
1647
|
library: Optional[Union[str, List[str]]] = None,
|
|
1568
1648
|
language: Optional[Union[str, List[str]]] = None,
|
|
1569
1649
|
model_name: Optional[str] = None,
|
|
@@ -1592,7 +1672,15 @@ class HfApi:
|
|
|
1592
1672
|
A string or list of string to filter models on the Hub.
|
|
1593
1673
|
author (`str`, *optional*):
|
|
1594
1674
|
A string which identify the author (user or organization) of the
|
|
1595
|
-
returned models
|
|
1675
|
+
returned models.
|
|
1676
|
+
gated (`bool`, *optional*):
|
|
1677
|
+
A boolean to filter models on the Hub that are gated or not. By default, all models are returned.
|
|
1678
|
+
If `gated=True` is passed, only gated models are returned.
|
|
1679
|
+
If `gated=False` is passed, only non-gated models are returned.
|
|
1680
|
+
inference (`Literal["cold", "frozen", "warm"]`, *optional*):
|
|
1681
|
+
A string to filter models on the Hub by their state on the Inference API.
|
|
1682
|
+
Warm models are available for immediate use. Cold models will be loaded on first inference call.
|
|
1683
|
+
Frozen models are not available in Inference API.
|
|
1596
1684
|
library (`str` or `List`, *optional*):
|
|
1597
1685
|
A string or list of strings of foundational libraries models were
|
|
1598
1686
|
originally trained from, such as pytorch, tensorflow, or allennlp.
|
|
@@ -1630,7 +1718,7 @@ class HfApi:
|
|
|
1630
1718
|
expand (`List[ExpandModelProperty_T]`, *optional*):
|
|
1631
1719
|
List properties to return in the response. When used, only the properties in the list will be returned.
|
|
1632
1720
|
This parameter cannot be used if `full`, `cardData` or `fetch_config` are passed.
|
|
1633
|
-
Possible values are `"author"`, `"cardData"`, `"config"`, `"createdAt"`, `"disabled"`, `"downloads"`, `"downloadsAllTime"`, `"gated"`, `"inference"`, `"lastModified"`, `"library_name"`, `"likes"`, `"mask_token"`, `"model-index"`, `"pipeline_tag"`, `"private"`, `"safetensors"`, `"sha"`, `"siblings"`, `"spaces"`, `"tags"`, `"transformersInfo"` and `"widgetData"`.
|
|
1721
|
+
Possible values are `"author"`, `"baseModels"`, `"cardData"`, `"childrenModelCount"`, `"config"`, `"createdAt"`, `"disabled"`, `"downloads"`, `"downloadsAllTime"`, `"gated"`, `"gguf"`, `"inference"`, `"lastModified"`, `"library_name"`, `"likes"`, `"mask_token"`, `"model-index"`, `"pipeline_tag"`, `"private"`, `"safetensors"`, `"sha"`, `"siblings"`, `"spaces"`, `"tags"`, `"transformersInfo"`, `"trendingScore"` and `"widgetData"`.
|
|
1634
1722
|
full (`bool`, *optional*):
|
|
1635
1723
|
Whether to fetch all model data, including the `last_modified`,
|
|
1636
1724
|
the `sha`, the files and the `tags`. This is set to `True` by
|
|
@@ -1718,6 +1806,10 @@ class HfApi:
|
|
|
1718
1806
|
# Handle other query params
|
|
1719
1807
|
if author:
|
|
1720
1808
|
params["author"] = author
|
|
1809
|
+
if gated is not None:
|
|
1810
|
+
params["gated"] = gated
|
|
1811
|
+
if inference is not None:
|
|
1812
|
+
params["inference"] = inference
|
|
1721
1813
|
if pipeline_tag:
|
|
1722
1814
|
params["pipeline_tag"] = pipeline_tag
|
|
1723
1815
|
search_list = []
|
|
@@ -1764,6 +1856,7 @@ class HfApi:
|
|
|
1764
1856
|
author: Optional[str] = None,
|
|
1765
1857
|
benchmark: Optional[Union[str, List[str]]] = None,
|
|
1766
1858
|
dataset_name: Optional[str] = None,
|
|
1859
|
+
gated: Optional[bool] = None,
|
|
1767
1860
|
language_creators: Optional[Union[str, List[str]]] = None,
|
|
1768
1861
|
language: Optional[Union[str, List[str]]] = None,
|
|
1769
1862
|
multilinguality: Optional[Union[str, List[str]]] = None,
|
|
@@ -1795,6 +1888,10 @@ class HfApi:
|
|
|
1795
1888
|
dataset_name (`str`, *optional*):
|
|
1796
1889
|
A string or list of strings that can be used to identify datasets on
|
|
1797
1890
|
the Hub by its name, such as `SQAC` or `wikineural`
|
|
1891
|
+
gated (`bool`, *optional*):
|
|
1892
|
+
A boolean to filter datasets on the Hub that are gated or not. By default, all datasets are returned.
|
|
1893
|
+
If `gated=True` is passed, only gated datasets are returned.
|
|
1894
|
+
If `gated=False` is passed, only non-gated datasets are returned.
|
|
1798
1895
|
language_creators (`str` or `List`, *optional*):
|
|
1799
1896
|
A string or list of strings that can be used to identify datasets on
|
|
1800
1897
|
the Hub with how the data was curated, such as `crowdsourced` or
|
|
@@ -1833,7 +1930,7 @@ class HfApi:
|
|
|
1833
1930
|
expand (`List[ExpandDatasetProperty_T]`, *optional*):
|
|
1834
1931
|
List properties to return in the response. When used, only the properties in the list will be returned.
|
|
1835
1932
|
This parameter cannot be used if `full` is passed.
|
|
1836
|
-
Possible values are `"author"`, `"cardData"`, `"citation"`, `"createdAt"`, `"disabled"`, `"description"`, `"downloads"`, `"downloadsAllTime"`, `"gated"`, `"lastModified"`, `"likes"`, `"paperswithcode_id"`, `"private"`, `"siblings"`, `"sha"` and `"
|
|
1933
|
+
Possible values are `"author"`, `"cardData"`, `"citation"`, `"createdAt"`, `"disabled"`, `"description"`, `"downloads"`, `"downloadsAllTime"`, `"gated"`, `"lastModified"`, `"likes"`, `"paperswithcode_id"`, `"private"`, `"siblings"`, `"sha"`, `"tags"` and `"trendingScore"`.
|
|
1837
1934
|
full (`bool`, *optional*):
|
|
1838
1935
|
Whether to fetch all dataset data, including the `last_modified`,
|
|
1839
1936
|
the `card_data` and the files. Can contain useful information such as the
|
|
@@ -1923,6 +2020,8 @@ class HfApi:
|
|
|
1923
2020
|
# Handle other query params
|
|
1924
2021
|
if author:
|
|
1925
2022
|
params["author"] = author
|
|
2023
|
+
if gated is not None:
|
|
2024
|
+
params["gated"] = gated
|
|
1926
2025
|
search_list = []
|
|
1927
2026
|
if dataset_name:
|
|
1928
2027
|
search_list.append(dataset_name)
|
|
@@ -2014,7 +2113,7 @@ class HfApi:
|
|
|
2014
2113
|
expand (`List[ExpandSpaceProperty_T]`, *optional*):
|
|
2015
2114
|
List properties to return in the response. When used, only the properties in the list will be returned.
|
|
2016
2115
|
This parameter cannot be used if `full` is passed.
|
|
2017
|
-
Possible values are `"author"`, `"cardData"`, `"datasets"`, `"disabled"`, `"lastModified"`, `"createdAt"`, `"likes"`, `"private"`, `"runtime"`, `"sdk"`, `"siblings"`, `"sha"`, `"subdomain"`, `"tags"` and `"
|
|
2116
|
+
Possible values are `"author"`, `"cardData"`, `"datasets"`, `"disabled"`, `"lastModified"`, `"createdAt"`, `"likes"`, `"models"`, `"private"`, `"runtime"`, `"sdk"`, `"siblings"`, `"sha"`, `"subdomain"`, `"tags"` and `"trendingScore"`.
|
|
2018
2117
|
full (`bool`, *optional*):
|
|
2019
2118
|
Whether to fetch all Spaces data, including the `last_modified`, `siblings`
|
|
2020
2119
|
and `card_data` fields.
|
|
@@ -2110,7 +2209,7 @@ class HfApi:
|
|
|
2110
2209
|
```
|
|
2111
2210
|
"""
|
|
2112
2211
|
if repo_type is None:
|
|
2113
|
-
repo_type = REPO_TYPE_MODEL
|
|
2212
|
+
repo_type = constants.REPO_TYPE_MODEL
|
|
2114
2213
|
response = get_session().post(
|
|
2115
2214
|
url=f"{self.endpoint}/api/{repo_type}s/{repo_id}/like",
|
|
2116
2215
|
headers=self._build_hf_headers(token=token),
|
|
@@ -2161,7 +2260,7 @@ class HfApi:
|
|
|
2161
2260
|
```
|
|
2162
2261
|
"""
|
|
2163
2262
|
if repo_type is None:
|
|
2164
|
-
repo_type = REPO_TYPE_MODEL
|
|
2263
|
+
repo_type = constants.REPO_TYPE_MODEL
|
|
2165
2264
|
response = get_session().delete(
|
|
2166
2265
|
url=f"{self.endpoint}/api/{repo_type}s/{repo_id}/like", headers=self._build_hf_headers(token=token)
|
|
2167
2266
|
)
|
|
@@ -2250,7 +2349,7 @@ class HfApi:
|
|
|
2250
2349
|
*,
|
|
2251
2350
|
repo_type: Optional[str] = None,
|
|
2252
2351
|
token: Union[bool, str, None] = None,
|
|
2253
|
-
) ->
|
|
2352
|
+
) -> Iterable[User]:
|
|
2254
2353
|
"""
|
|
2255
2354
|
List all users who liked a given repo on the hugging Face Hub.
|
|
2256
2355
|
|
|
@@ -2272,29 +2371,15 @@ class HfApi:
|
|
|
2272
2371
|
`None`.
|
|
2273
2372
|
|
|
2274
2373
|
Returns:
|
|
2275
|
-
`
|
|
2374
|
+
`Iterable[User]`: an iterable of [`huggingface_hub.hf_api.User`] objects.
|
|
2276
2375
|
"""
|
|
2277
2376
|
|
|
2278
2377
|
# Construct the API endpoint
|
|
2279
2378
|
if repo_type is None:
|
|
2280
|
-
repo_type = REPO_TYPE_MODEL
|
|
2379
|
+
repo_type = constants.REPO_TYPE_MODEL
|
|
2281
2380
|
path = f"{self.endpoint}/api/{repo_type}s/{repo_id}/likers"
|
|
2282
|
-
|
|
2283
|
-
|
|
2284
|
-
# Make the request
|
|
2285
|
-
response = get_session().get(path, headers=headers)
|
|
2286
|
-
hf_raise_for_status(response)
|
|
2287
|
-
|
|
2288
|
-
# Parse the results into User objects
|
|
2289
|
-
likers_data = response.json()
|
|
2290
|
-
return [
|
|
2291
|
-
User(
|
|
2292
|
-
username=user_data["user"],
|
|
2293
|
-
fullname=user_data["fullname"],
|
|
2294
|
-
avatar_url=user_data["avatarUrl"],
|
|
2295
|
-
)
|
|
2296
|
-
for user_data in likers_data
|
|
2297
|
-
]
|
|
2381
|
+
for liker in paginate(path, params={}, headers=self._build_hf_headers(token=token)):
|
|
2382
|
+
yield User(username=liker["user"], fullname=liker["fullname"], avatar_url=liker["avatarUrl"])
|
|
2298
2383
|
|
|
2299
2384
|
@validate_hf_hub_args
|
|
2300
2385
|
def model_info(
|
|
@@ -2331,7 +2416,7 @@ class HfApi:
|
|
|
2331
2416
|
expand (`List[ExpandModelProperty_T]`, *optional*):
|
|
2332
2417
|
List properties to return in the response. When used, only the properties in the list will be returned.
|
|
2333
2418
|
This parameter cannot be used if `securityStatus` or `files_metadata` are passed.
|
|
2334
|
-
Possible values are `"author"`, `"cardData"`, `"config"`, `"createdAt"`, `"disabled"`, `"downloads"`, `"downloadsAllTime"`, `"gated"`, `"inference"`, `"lastModified"`, `"library_name"`, `"likes"`, `"mask_token"`, `"model-index"`, `"pipeline_tag"`, `"private"`, `"safetensors"`, `"sha"`, `"siblings"`, `"spaces"`, `"tags"`, `"transformersInfo"` and `"widgetData"`.
|
|
2419
|
+
Possible values are `"author"`, `"baseModels"`, `"cardData"`, `"childrenModelCount"`, `"config"`, `"createdAt"`, `"disabled"`, `"downloads"`, `"downloadsAllTime"`, `"gated"`, `"gguf"`, `"inference"`, `"lastModified"`, `"library_name"`, `"likes"`, `"mask_token"`, `"model-index"`, `"pipeline_tag"`, `"private"`, `"safetensors"`, `"sha"`, `"siblings"`, `"spaces"`, `"tags"`, `"transformersInfo"`, `"trendingScore"` and `"widgetData"`.
|
|
2335
2420
|
token (Union[bool, str, None], optional):
|
|
2336
2421
|
A valid user access token (string). Defaults to the locally saved
|
|
2337
2422
|
token, which is the recommended method for authentication (see
|
|
@@ -2405,7 +2490,7 @@ class HfApi:
|
|
|
2405
2490
|
expand (`List[ExpandDatasetProperty_T]`, *optional*):
|
|
2406
2491
|
List properties to return in the response. When used, only the properties in the list will be returned.
|
|
2407
2492
|
This parameter cannot be used if `files_metadata` is passed.
|
|
2408
|
-
Possible values are `"author"`, `"cardData"`, `"citation"`, `"createdAt"`, `"disabled"`, `"description"`, `"downloads"`, `"downloadsAllTime"`, `"gated"`, `"lastModified"`, `"likes"`, `"paperswithcode_id"`, `"private"`, `"siblings"`, `"sha"` and `"
|
|
2493
|
+
Possible values are `"author"`, `"cardData"`, `"citation"`, `"createdAt"`, `"disabled"`, `"description"`, `"downloads"`, `"downloadsAllTime"`, `"gated"`, `"lastModified"`, `"likes"`, `"paperswithcode_id"`, `"private"`, `"siblings"`, `"sha"`, `"tags"` and `"trendingScore"`.
|
|
2409
2494
|
token (Union[bool, str, None], optional):
|
|
2410
2495
|
A valid user access token (string). Defaults to the locally saved
|
|
2411
2496
|
token, which is the recommended method for authentication (see
|
|
@@ -2478,7 +2563,7 @@ class HfApi:
|
|
|
2478
2563
|
expand (`List[ExpandSpaceProperty_T]`, *optional*):
|
|
2479
2564
|
List properties to return in the response. When used, only the properties in the list will be returned.
|
|
2480
2565
|
This parameter cannot be used if `full` is passed.
|
|
2481
|
-
Possible values are `"author"`, `"cardData"`, `"datasets"`, `"disabled"`, `"lastModified"`, `"
|
|
2566
|
+
Possible values are `"author"`, `"cardData"`, `"createdAt"`, `"datasets"`, `"disabled"`, `"lastModified"`, `"likes"`, `"models"`, `"private"`, `"runtime"`, `"sdk"`, `"siblings"`, `"sha"`, `"subdomain"`, `"tags"` and `"trendingScore"`.
|
|
2482
2567
|
token (Union[bool, str, None], optional):
|
|
2483
2568
|
A valid user access token (string). Defaults to the locally saved
|
|
2484
2569
|
token, which is the recommended method for authentication (see
|
|
@@ -2903,8 +2988,8 @@ class HfApi:
|
|
|
2903
2988
|
]
|
|
2904
2989
|
```
|
|
2905
2990
|
"""
|
|
2906
|
-
repo_type = repo_type or REPO_TYPE_MODEL
|
|
2907
|
-
revision = quote(revision, safe="") if revision is not None else DEFAULT_REVISION
|
|
2991
|
+
repo_type = repo_type or constants.REPO_TYPE_MODEL
|
|
2992
|
+
revision = quote(revision, safe="") if revision is not None else constants.DEFAULT_REVISION
|
|
2908
2993
|
headers = self._build_hf_headers(token=token)
|
|
2909
2994
|
|
|
2910
2995
|
encoded_path_in_repo = "/" + quote(path_in_repo, safe="") if path_in_repo else ""
|
|
@@ -2963,7 +3048,7 @@ class HfApi:
|
|
|
2963
3048
|
[`GitRefs`]: object containing all information about branches and tags for a
|
|
2964
3049
|
repo on the Hub.
|
|
2965
3050
|
"""
|
|
2966
|
-
repo_type = repo_type or REPO_TYPE_MODEL
|
|
3051
|
+
repo_type = repo_type or constants.REPO_TYPE_MODEL
|
|
2967
3052
|
response = get_session().get(
|
|
2968
3053
|
f"{self.endpoint}/api/{repo_type}s/{repo_id}/refs",
|
|
2969
3054
|
headers=self._build_hf_headers(token=token),
|
|
@@ -3049,8 +3134,8 @@ class HfApi:
|
|
|
3049
3134
|
[`~utils.RevisionNotFoundError`]:
|
|
3050
3135
|
If revision is not found (error 404) on the repo.
|
|
3051
3136
|
"""
|
|
3052
|
-
repo_type = repo_type or REPO_TYPE_MODEL
|
|
3053
|
-
revision = quote(revision, safe="") if revision is not None else DEFAULT_REVISION
|
|
3137
|
+
repo_type = repo_type or constants.REPO_TYPE_MODEL
|
|
3138
|
+
revision = quote(revision, safe="") if revision is not None else constants.DEFAULT_REVISION
|
|
3054
3139
|
|
|
3055
3140
|
# Paginate over results and return the list of commits.
|
|
3056
3141
|
return [
|
|
@@ -3128,8 +3213,8 @@ class HfApi:
|
|
|
3128
3213
|
]
|
|
3129
3214
|
```
|
|
3130
3215
|
"""
|
|
3131
|
-
repo_type = repo_type or REPO_TYPE_MODEL
|
|
3132
|
-
revision = quote(revision, safe="") if revision is not None else DEFAULT_REVISION
|
|
3216
|
+
repo_type = repo_type or constants.REPO_TYPE_MODEL
|
|
3217
|
+
revision = quote(revision, safe="") if revision is not None else constants.DEFAULT_REVISION
|
|
3133
3218
|
headers = self._build_hf_headers(token=token)
|
|
3134
3219
|
|
|
3135
3220
|
response = get_session().post(
|
|
@@ -3218,11 +3303,11 @@ class HfApi:
|
|
|
3218
3303
|
```
|
|
3219
3304
|
"""
|
|
3220
3305
|
if repo_type is None:
|
|
3221
|
-
repo_type = REPO_TYPE_MODEL
|
|
3222
|
-
if repo_type not in REPO_TYPES:
|
|
3306
|
+
repo_type = constants.REPO_TYPE_MODEL
|
|
3307
|
+
if repo_type not in constants.REPO_TYPES:
|
|
3223
3308
|
raise ValueError("Invalid repo type")
|
|
3224
3309
|
if branch is None:
|
|
3225
|
-
branch = DEFAULT_REVISION
|
|
3310
|
+
branch = constants.DEFAULT_REVISION
|
|
3226
3311
|
|
|
3227
3312
|
# Prepare request
|
|
3228
3313
|
url = f"{self.endpoint}/api/{repo_type}s/{repo_id}/super-squash/{branch}"
|
|
@@ -3300,7 +3385,7 @@ class HfApi:
|
|
|
3300
3385
|
|
|
3301
3386
|
path = f"{self.endpoint}/api/repos/create"
|
|
3302
3387
|
|
|
3303
|
-
if repo_type not in REPO_TYPES:
|
|
3388
|
+
if repo_type not in constants.REPO_TYPES:
|
|
3304
3389
|
raise ValueError("Invalid repo type")
|
|
3305
3390
|
|
|
3306
3391
|
json: Dict[str, Any] = {"name": name, "organization": organization, "private": private}
|
|
@@ -3310,10 +3395,10 @@ class HfApi:
|
|
|
3310
3395
|
if space_sdk is None:
|
|
3311
3396
|
raise ValueError(
|
|
3312
3397
|
"No space_sdk provided. `create_repo` expects space_sdk to be one"
|
|
3313
|
-
f" of {SPACES_SDK_TYPES} when repo_type is 'space'`"
|
|
3398
|
+
f" of {constants.SPACES_SDK_TYPES} when repo_type is 'space'`"
|
|
3314
3399
|
)
|
|
3315
|
-
if space_sdk not in SPACES_SDK_TYPES:
|
|
3316
|
-
raise ValueError(f"Invalid space_sdk. Please choose one of {SPACES_SDK_TYPES}.")
|
|
3400
|
+
if space_sdk not in constants.SPACES_SDK_TYPES:
|
|
3401
|
+
raise ValueError(f"Invalid space_sdk. Please choose one of {constants.SPACES_SDK_TYPES}.")
|
|
3317
3402
|
json["sdk"] = space_sdk
|
|
3318
3403
|
|
|
3319
3404
|
if space_sdk is not None and repo_type != "space":
|
|
@@ -3369,7 +3454,7 @@ class HfApi:
|
|
|
3369
3454
|
# No write permission on the namespace but repo might already exist
|
|
3370
3455
|
try:
|
|
3371
3456
|
self.repo_info(repo_id=repo_id, repo_type=repo_type, token=token)
|
|
3372
|
-
if repo_type is None or repo_type == REPO_TYPE_MODEL:
|
|
3457
|
+
if repo_type is None or repo_type == constants.REPO_TYPE_MODEL:
|
|
3373
3458
|
return RepoUrl(f"{self.endpoint}/{repo_id}")
|
|
3374
3459
|
return RepoUrl(f"{self.endpoint}/{repo_type}/{repo_id}")
|
|
3375
3460
|
except HfHubHTTPError:
|
|
@@ -3415,7 +3500,7 @@ class HfApi:
|
|
|
3415
3500
|
|
|
3416
3501
|
path = f"{self.endpoint}/api/repos/delete"
|
|
3417
3502
|
|
|
3418
|
-
if repo_type not in REPO_TYPES:
|
|
3503
|
+
if repo_type not in constants.REPO_TYPES:
|
|
3419
3504
|
raise ValueError("Invalid repo type")
|
|
3420
3505
|
|
|
3421
3506
|
json = {"name": name, "organization": organization}
|
|
@@ -3469,10 +3554,10 @@ class HfApi:
|
|
|
3469
3554
|
|
|
3470
3555
|
</Tip>
|
|
3471
3556
|
"""
|
|
3472
|
-
if repo_type not in REPO_TYPES:
|
|
3473
|
-
raise ValueError(f"Invalid repo type, must be one of {REPO_TYPES}")
|
|
3557
|
+
if repo_type not in constants.REPO_TYPES:
|
|
3558
|
+
raise ValueError(f"Invalid repo type, must be one of {constants.REPO_TYPES}")
|
|
3474
3559
|
if repo_type is None:
|
|
3475
|
-
repo_type = REPO_TYPE_MODEL # default repo type
|
|
3560
|
+
repo_type = constants.REPO_TYPE_MODEL # default repo type
|
|
3476
3561
|
|
|
3477
3562
|
r = get_session().put(
|
|
3478
3563
|
url=f"{self.endpoint}/api/{repo_type}s/{repo_id}/settings",
|
|
@@ -3482,6 +3567,62 @@ class HfApi:
|
|
|
3482
3567
|
hf_raise_for_status(r)
|
|
3483
3568
|
return r.json()
|
|
3484
3569
|
|
|
3570
|
+
@validate_hf_hub_args
|
|
3571
|
+
def update_repo_settings(
|
|
3572
|
+
self,
|
|
3573
|
+
repo_id: str,
|
|
3574
|
+
*,
|
|
3575
|
+
gated: Literal["auto", "manual", False] = False,
|
|
3576
|
+
token: Union[str, bool, None] = None,
|
|
3577
|
+
repo_type: Optional[str] = None,
|
|
3578
|
+
) -> None:
|
|
3579
|
+
"""
|
|
3580
|
+
Update the gated settings of a repository.
|
|
3581
|
+
To give more control over how repos are used, the Hub allows repo authors to enable **access requests** for their repos.
|
|
3582
|
+
|
|
3583
|
+
Args:
|
|
3584
|
+
repo_id (`str`):
|
|
3585
|
+
A namespace (user or an organization) and a repo name separated by a /.
|
|
3586
|
+
gated (`Literal["auto", "manual", False]`, *optional*):
|
|
3587
|
+
The gated release status for the repository.
|
|
3588
|
+
* "auto": The repository is gated, and access requests are automatically approved or denied based on predefined criteria.
|
|
3589
|
+
* "manual": The repository is gated, and access requests require manual approval.
|
|
3590
|
+
* False (default): The repository is not gated, and anyone can access it.
|
|
3591
|
+
token (`Union[str, bool, None]`, *optional*):
|
|
3592
|
+
A valid user access token (string). Defaults to the locally saved token,
|
|
3593
|
+
which is the recommended method for authentication (see
|
|
3594
|
+
https://huggingface.co/docs/huggingface_hub/quick-start#authentication).
|
|
3595
|
+
To disable authentication, pass False.
|
|
3596
|
+
repo_type (`str`, *optional*):
|
|
3597
|
+
The type of the repository to update settings from (`"model"`, `"dataset"` or `"space"`.
|
|
3598
|
+
Defaults to `"model"`.
|
|
3599
|
+
|
|
3600
|
+
Raises:
|
|
3601
|
+
[`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError)
|
|
3602
|
+
If gated is not one of "auto", "manual", or False.
|
|
3603
|
+
[`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError)
|
|
3604
|
+
If repo_type is not one of the values in constants.REPO_TYPES.
|
|
3605
|
+
[`~utils.HfHubHTTPError`]:
|
|
3606
|
+
If the request to the Hugging Face Hub API fails.
|
|
3607
|
+
"""
|
|
3608
|
+
if gated not in ["auto", "manual", False]:
|
|
3609
|
+
raise ValueError(f"Invalid gated status, must be one of 'auto', 'manual', or False. Got '{gated}'.")
|
|
3610
|
+
|
|
3611
|
+
if repo_type not in constants.REPO_TYPES:
|
|
3612
|
+
raise ValueError(f"Invalid repo type, must be one of {constants.REPO_TYPES}")
|
|
3613
|
+
if repo_type is None:
|
|
3614
|
+
repo_type = constants.REPO_TYPE_MODEL # default repo type
|
|
3615
|
+
|
|
3616
|
+
# Build headers
|
|
3617
|
+
headers = self._build_hf_headers(token=token)
|
|
3618
|
+
|
|
3619
|
+
r = get_session().put(
|
|
3620
|
+
url=f"{self.endpoint}/api/{repo_type}s/{repo_id}/settings",
|
|
3621
|
+
headers=headers,
|
|
3622
|
+
json={"gated": gated},
|
|
3623
|
+
)
|
|
3624
|
+
hf_raise_for_status(r)
|
|
3625
|
+
|
|
3485
3626
|
def move_repo(
|
|
3486
3627
|
self,
|
|
3487
3628
|
from_id: str,
|
|
@@ -3531,7 +3672,7 @@ class HfApi:
|
|
|
3531
3672
|
raise ValueError(f"Invalid repo_id: {to_id}. It should have a namespace (:namespace:/:repo_name:)")
|
|
3532
3673
|
|
|
3533
3674
|
if repo_type is None:
|
|
3534
|
-
repo_type = REPO_TYPE_MODEL # Hub won't accept `None`.
|
|
3675
|
+
repo_type = constants.REPO_TYPE_MODEL # Hub won't accept `None`.
|
|
3535
3676
|
|
|
3536
3677
|
json = {"fromRepo": from_id, "toRepo": to_id, "type": repo_type}
|
|
3537
3678
|
|
|
@@ -3701,19 +3842,19 @@ class HfApi:
|
|
|
3701
3842
|
If repository is not found (error 404): wrong repo_id/repo_type, private
|
|
3702
3843
|
but not authenticated or repo does not exist.
|
|
3703
3844
|
"""
|
|
3704
|
-
if parent_commit is not None and not REGEX_COMMIT_OID.fullmatch(parent_commit):
|
|
3845
|
+
if parent_commit is not None and not constants.REGEX_COMMIT_OID.fullmatch(parent_commit):
|
|
3705
3846
|
raise ValueError(
|
|
3706
|
-
f"`parent_commit` is not a valid commit OID. It must match the following regex: {REGEX_COMMIT_OID}"
|
|
3847
|
+
f"`parent_commit` is not a valid commit OID. It must match the following regex: {constants.REGEX_COMMIT_OID}"
|
|
3707
3848
|
)
|
|
3708
3849
|
|
|
3709
3850
|
if commit_message is None or len(commit_message) == 0:
|
|
3710
3851
|
raise ValueError("`commit_message` can't be empty, please pass a value.")
|
|
3711
3852
|
|
|
3712
3853
|
commit_description = commit_description if commit_description is not None else ""
|
|
3713
|
-
repo_type = repo_type if repo_type is not None else REPO_TYPE_MODEL
|
|
3714
|
-
if repo_type not in REPO_TYPES:
|
|
3715
|
-
raise ValueError(f"Invalid repo type, must be one of {REPO_TYPES}")
|
|
3716
|
-
unquoted_revision = revision or DEFAULT_REVISION
|
|
3854
|
+
repo_type = repo_type if repo_type is not None else constants.REPO_TYPE_MODEL
|
|
3855
|
+
if repo_type not in constants.REPO_TYPES:
|
|
3856
|
+
raise ValueError(f"Invalid repo type, must be one of {constants.REPO_TYPES}")
|
|
3857
|
+
unquoted_revision = revision or constants.DEFAULT_REVISION
|
|
3717
3858
|
revision = quote(unquoted_revision, safe="")
|
|
3718
3859
|
create_pr = create_pr if create_pr is not None else False
|
|
3719
3860
|
|
|
@@ -3733,6 +3874,15 @@ class HfApi:
|
|
|
3733
3874
|
" new CommitOperationAdd object if you want to create a new commit."
|
|
3734
3875
|
)
|
|
3735
3876
|
|
|
3877
|
+
if repo_type != "dataset":
|
|
3878
|
+
for addition in additions:
|
|
3879
|
+
if addition.path_in_repo.endswith((".arrow", ".parquet")):
|
|
3880
|
+
warnings.warn(
|
|
3881
|
+
f"It seems that you are about to commit a data file ({addition.path_in_repo}) to a {repo_type}"
|
|
3882
|
+
" repository. You are sure this is intended? If you are trying to upload a dataset, please"
|
|
3883
|
+
" set `repo_type='dataset'` or `--repo-type=dataset` in a CLI."
|
|
3884
|
+
)
|
|
3885
|
+
|
|
3736
3886
|
logger.debug(
|
|
3737
3887
|
f"About to commit to the hub: {len(additions)} addition(s), {len(copies)} copie(s) and"
|
|
3738
3888
|
f" {nb_deletions} deletion(s)."
|
|
@@ -3743,26 +3893,10 @@ class HfApi:
|
|
|
3743
3893
|
for addition in additions:
|
|
3744
3894
|
if addition.path_in_repo == "README.md":
|
|
3745
3895
|
with addition.as_file() as file:
|
|
3746
|
-
|
|
3747
|
-
|
|
3748
|
-
|
|
3749
|
-
|
|
3750
|
-
)
|
|
3751
|
-
# Handle warnings (example: empty metadata)
|
|
3752
|
-
response_content = response.json()
|
|
3753
|
-
message = "\n".join(
|
|
3754
|
-
[f"- {warning.get('message')}" for warning in response_content.get("warnings", [])]
|
|
3755
|
-
)
|
|
3756
|
-
if message:
|
|
3757
|
-
warnings.warn(f"Warnings while validating metadata in README.md:\n{message}")
|
|
3758
|
-
|
|
3759
|
-
# Raise on errors
|
|
3760
|
-
try:
|
|
3761
|
-
hf_raise_for_status(response)
|
|
3762
|
-
except BadRequestError as e:
|
|
3763
|
-
errors = response_content.get("errors", [])
|
|
3764
|
-
message = "\n".join([f"- {error.get('message')}" for error in errors])
|
|
3765
|
-
raise ValueError(f"Invalid metadata in README.md.\n{message}") from e
|
|
3896
|
+
content = file.read().decode()
|
|
3897
|
+
self._validate_yaml(content, repo_type=repo_type, token=token)
|
|
3898
|
+
# Skip other additions after `README.md` has been processed
|
|
3899
|
+
break
|
|
3766
3900
|
|
|
3767
3901
|
# If updating twice the same file or update then delete a file in a single commit
|
|
3768
3902
|
_warn_on_overwriting_operations(operations)
|
|
@@ -3808,7 +3942,7 @@ class HfApi:
|
|
|
3808
3942
|
|
|
3809
3943
|
# Return commit info based on latest commit
|
|
3810
3944
|
url_prefix = self.endpoint
|
|
3811
|
-
if repo_type is not None and repo_type != REPO_TYPE_MODEL:
|
|
3945
|
+
if repo_type is not None and repo_type != constants.REPO_TYPE_MODEL:
|
|
3812
3946
|
url_prefix = f"{url_prefix}/{repo_type}s"
|
|
3813
3947
|
return CommitInfo(
|
|
3814
3948
|
commit_url=f"{url_prefix}/{repo_id}/commit/{info.sha}",
|
|
@@ -4043,7 +4177,7 @@ class HfApi:
|
|
|
4043
4177
|
commits_on_main_branch = {
|
|
4044
4178
|
commit.commit_id
|
|
4045
4179
|
for commit in self.list_repo_commits(
|
|
4046
|
-
repo_id=repo_id, repo_type=repo_type, token=token, revision=DEFAULT_REVISION
|
|
4180
|
+
repo_id=repo_id, repo_type=repo_type, token=token, revision=constants.DEFAULT_REVISION
|
|
4047
4181
|
)
|
|
4048
4182
|
}
|
|
4049
4183
|
pr_commits = [
|
|
@@ -4262,10 +4396,10 @@ class HfApi:
|
|
|
4262
4396
|
>>> create_commit(repo_id, operations=operations, commit_message="Commit all shards")
|
|
4263
4397
|
```
|
|
4264
4398
|
"""
|
|
4265
|
-
repo_type = repo_type if repo_type is not None else REPO_TYPE_MODEL
|
|
4266
|
-
if repo_type not in REPO_TYPES:
|
|
4267
|
-
raise ValueError(f"Invalid repo type, must be one of {REPO_TYPES}")
|
|
4268
|
-
revision = quote(revision, safe="") if revision is not None else DEFAULT_REVISION
|
|
4399
|
+
repo_type = repo_type if repo_type is not None else constants.REPO_TYPE_MODEL
|
|
4400
|
+
if repo_type not in constants.REPO_TYPES:
|
|
4401
|
+
raise ValueError(f"Invalid repo type, must be one of {constants.REPO_TYPES}")
|
|
4402
|
+
revision = quote(revision, safe="") if revision is not None else constants.DEFAULT_REVISION
|
|
4269
4403
|
create_pr = create_pr if create_pr is not None else False
|
|
4270
4404
|
headers = self._build_hf_headers(token=token)
|
|
4271
4405
|
|
|
@@ -4493,8 +4627,8 @@ class HfApi:
|
|
|
4493
4627
|
"https://huggingface.co/username/my-model/blob/refs%2Fpr%2F1/remote/file/path.h5"
|
|
4494
4628
|
```
|
|
4495
4629
|
"""
|
|
4496
|
-
if repo_type not in REPO_TYPES:
|
|
4497
|
-
raise ValueError(f"Invalid repo type, must be one of {REPO_TYPES}")
|
|
4630
|
+
if repo_type not in constants.REPO_TYPES:
|
|
4631
|
+
raise ValueError(f"Invalid repo type, must be one of {constants.REPO_TYPES}")
|
|
4498
4632
|
|
|
4499
4633
|
commit_message = (
|
|
4500
4634
|
commit_message if commit_message is not None else f"Upload {path_in_repo} with huggingface_hub"
|
|
@@ -4518,9 +4652,9 @@ class HfApi:
|
|
|
4518
4652
|
|
|
4519
4653
|
if commit_info.pr_url is not None:
|
|
4520
4654
|
revision = quote(_parse_revision_from_pr_url(commit_info.pr_url), safe="")
|
|
4521
|
-
if repo_type in REPO_TYPES_URL_PREFIXES:
|
|
4522
|
-
repo_id = REPO_TYPES_URL_PREFIXES[repo_type] + repo_id
|
|
4523
|
-
revision = revision if revision is not None else DEFAULT_REVISION
|
|
4655
|
+
if repo_type in constants.REPO_TYPES_URL_PREFIXES:
|
|
4656
|
+
repo_id = constants.REPO_TYPES_URL_PREFIXES[repo_type] + repo_id
|
|
4657
|
+
revision = revision if revision is not None else constants.DEFAULT_REVISION
|
|
4524
4658
|
|
|
4525
4659
|
return CommitInfo(
|
|
4526
4660
|
commit_url=commit_info.commit_url,
|
|
@@ -4795,11 +4929,11 @@ class HfApi:
|
|
|
4795
4929
|
|
|
4796
4930
|
```
|
|
4797
4931
|
"""
|
|
4798
|
-
if repo_type not in REPO_TYPES:
|
|
4799
|
-
raise ValueError(f"Invalid repo type, must be one of {REPO_TYPES}")
|
|
4932
|
+
if repo_type not in constants.REPO_TYPES:
|
|
4933
|
+
raise ValueError(f"Invalid repo type, must be one of {constants.REPO_TYPES}")
|
|
4800
4934
|
|
|
4801
4935
|
if multi_commits:
|
|
4802
|
-
if revision is not None and revision != DEFAULT_REVISION:
|
|
4936
|
+
if revision is not None and revision != constants.DEFAULT_REVISION:
|
|
4803
4937
|
raise ValueError("Cannot use `multi_commit` to commit changes other than the main branch.")
|
|
4804
4938
|
|
|
4805
4939
|
# By default, upload folder to the root directory in repo.
|
|
@@ -4816,16 +4950,18 @@ class HfApi:
|
|
|
4816
4950
|
delete_operations = self._prepare_folder_deletions(
|
|
4817
4951
|
repo_id=repo_id,
|
|
4818
4952
|
repo_type=repo_type,
|
|
4819
|
-
revision=DEFAULT_REVISION if create_pr else revision,
|
|
4953
|
+
revision=constants.DEFAULT_REVISION if create_pr else revision,
|
|
4820
4954
|
token=token,
|
|
4821
4955
|
path_in_repo=path_in_repo,
|
|
4822
4956
|
delete_patterns=delete_patterns,
|
|
4823
4957
|
)
|
|
4824
|
-
add_operations = _prepare_upload_folder_additions(
|
|
4958
|
+
add_operations = self._prepare_upload_folder_additions(
|
|
4825
4959
|
folder_path,
|
|
4826
4960
|
path_in_repo,
|
|
4827
4961
|
allow_patterns=allow_patterns,
|
|
4828
4962
|
ignore_patterns=ignore_patterns,
|
|
4963
|
+
token=token,
|
|
4964
|
+
repo_type=repo_type,
|
|
4829
4965
|
)
|
|
4830
4966
|
|
|
4831
4967
|
# Optimize operations: if some files will be overwritten, we don't need to delete them first
|
|
@@ -4869,9 +5005,9 @@ class HfApi:
|
|
|
4869
5005
|
# Create url to uploaded folder (for legacy return value)
|
|
4870
5006
|
if create_pr and commit_info.pr_url is not None:
|
|
4871
5007
|
revision = quote(_parse_revision_from_pr_url(commit_info.pr_url), safe="")
|
|
4872
|
-
if repo_type in REPO_TYPES_URL_PREFIXES:
|
|
4873
|
-
repo_id = REPO_TYPES_URL_PREFIXES[repo_type] + repo_id
|
|
4874
|
-
revision = revision if revision is not None else DEFAULT_REVISION
|
|
5008
|
+
if repo_type in constants.REPO_TYPES_URL_PREFIXES:
|
|
5009
|
+
repo_id = constants.REPO_TYPES_URL_PREFIXES[repo_type] + repo_id
|
|
5010
|
+
revision = revision if revision is not None else constants.DEFAULT_REVISION
|
|
4875
5011
|
|
|
4876
5012
|
return CommitInfo(
|
|
4877
5013
|
commit_url=commit_info.commit_url,
|
|
@@ -5118,6 +5254,123 @@ class HfApi:
|
|
|
5118
5254
|
parent_commit=parent_commit,
|
|
5119
5255
|
)
|
|
5120
5256
|
|
|
5257
|
+
def upload_large_folder(
|
|
5258
|
+
self,
|
|
5259
|
+
repo_id: str,
|
|
5260
|
+
folder_path: Union[str, Path],
|
|
5261
|
+
*,
|
|
5262
|
+
repo_type: str, # Repo type is required!
|
|
5263
|
+
revision: Optional[str] = None,
|
|
5264
|
+
private: bool = False,
|
|
5265
|
+
allow_patterns: Optional[Union[List[str], str]] = None,
|
|
5266
|
+
ignore_patterns: Optional[Union[List[str], str]] = None,
|
|
5267
|
+
num_workers: Optional[int] = None,
|
|
5268
|
+
print_report: bool = True,
|
|
5269
|
+
print_report_every: int = 60,
|
|
5270
|
+
) -> None:
|
|
5271
|
+
"""Upload a large folder to the Hub in the most resilient way possible.
|
|
5272
|
+
|
|
5273
|
+
Several workers are started to upload files in an optimized way. Before being committed to a repo, files must be
|
|
5274
|
+
hashed and be pre-uploaded if they are LFS files. Workers will perform these tasks for each file in the folder.
|
|
5275
|
+
At each step, some metadata information about the upload process is saved in the folder under `.cache/.huggingface/`
|
|
5276
|
+
to be able to resume the process if interrupted. The whole process might result in several commits.
|
|
5277
|
+
|
|
5278
|
+
Args:
|
|
5279
|
+
repo_id (`str`):
|
|
5280
|
+
The repository to which the file will be uploaded.
|
|
5281
|
+
E.g. `"HuggingFaceTB/smollm-corpus"`.
|
|
5282
|
+
folder_path (`str` or `Path`):
|
|
5283
|
+
Path to the folder to upload on the local file system.
|
|
5284
|
+
repo_type (`str`):
|
|
5285
|
+
Type of the repository. Must be one of `"model"`, `"dataset"` or `"space"`.
|
|
5286
|
+
Unlike in all other `HfApi` methods, `repo_type` is explicitly required here. This is to avoid
|
|
5287
|
+
any mistake when uploading a large folder to the Hub, and therefore prevent from having to re-upload
|
|
5288
|
+
everything.
|
|
5289
|
+
revision (`str`, `optional`):
|
|
5290
|
+
The branch to commit to. If not provided, the `main` branch will be used.
|
|
5291
|
+
private (`bool`, `optional`):
|
|
5292
|
+
Whether the repository should be private. Defaults to False.
|
|
5293
|
+
allow_patterns (`List[str]` or `str`, *optional*):
|
|
5294
|
+
If provided, only files matching at least one pattern are uploaded.
|
|
5295
|
+
ignore_patterns (`List[str]` or `str`, *optional*):
|
|
5296
|
+
If provided, files matching any of the patterns are not uploaded.
|
|
5297
|
+
num_workers (`int`, *optional*):
|
|
5298
|
+
Number of workers to start. Defaults to `os.cpu_count() - 2` (minimum 2).
|
|
5299
|
+
A higher number of workers may speed up the process if your machine allows it. However, on machines with a
|
|
5300
|
+
slower connection, it is recommended to keep the number of workers low to ensure better resumability.
|
|
5301
|
+
Indeed, partially uploaded files will have to be completely re-uploaded if the process is interrupted.
|
|
5302
|
+
print_report (`bool`, *optional*):
|
|
5303
|
+
Whether to print a report of the upload progress. Defaults to True.
|
|
5304
|
+
Report is printed to `sys.stdout` every X seconds (60 by defaults) and overwrites the previous report.
|
|
5305
|
+
print_report_every (`int`, *optional*):
|
|
5306
|
+
Frequency at which the report is printed. Defaults to 60 seconds.
|
|
5307
|
+
|
|
5308
|
+
<Tip>
|
|
5309
|
+
|
|
5310
|
+
A few things to keep in mind:
|
|
5311
|
+
- Repository limits still apply: https://huggingface.co/docs/hub/repositories-recommendations
|
|
5312
|
+
- Do not start several processes in parallel.
|
|
5313
|
+
- You can interrupt and resume the process at any time.
|
|
5314
|
+
- Do not upload the same folder to several repositories. If you need to do so, you must delete the local `.cache/.huggingface/` folder first.
|
|
5315
|
+
|
|
5316
|
+
</Tip>
|
|
5317
|
+
|
|
5318
|
+
<Tip warning={true}>
|
|
5319
|
+
|
|
5320
|
+
While being much more robust to upload large folders, `upload_large_folder` is more limited than [`upload_folder`] feature-wise. In practice:
|
|
5321
|
+
- you cannot set a custom `path_in_repo`. If you want to upload to a subfolder, you need to set the proper structure locally.
|
|
5322
|
+
- you cannot set a custom `commit_message` and `commit_description` since multiple commits are created.
|
|
5323
|
+
- you cannot delete from the repo while uploading. Please make a separate commit first.
|
|
5324
|
+
- you cannot create a PR directly. Please create a PR first (from the UI or using [`create_pull_request`]) and then commit to it by passing `revision`.
|
|
5325
|
+
|
|
5326
|
+
</Tip>
|
|
5327
|
+
|
|
5328
|
+
**Technical details:**
|
|
5329
|
+
|
|
5330
|
+
`upload_large_folder` process is as follow:
|
|
5331
|
+
1. (Check parameters and setup.)
|
|
5332
|
+
2. Create repo if missing.
|
|
5333
|
+
3. List local files to upload.
|
|
5334
|
+
4. Start workers. Workers can perform the following tasks:
|
|
5335
|
+
- Hash a file.
|
|
5336
|
+
- Get upload mode (regular or LFS) for a list of files.
|
|
5337
|
+
- Pre-upload an LFS file.
|
|
5338
|
+
- Commit a bunch of files.
|
|
5339
|
+
Once a worker finishes a task, it will move on to the next task based on the priority list (see below) until
|
|
5340
|
+
all files are uploaded and committed.
|
|
5341
|
+
5. While workers are up, regularly print a report to sys.stdout.
|
|
5342
|
+
|
|
5343
|
+
Order of priority:
|
|
5344
|
+
1. Commit if more than 5 minutes since last commit attempt (and at least 1 file).
|
|
5345
|
+
2. Commit if at least 25 files are ready to commit.
|
|
5346
|
+
3. Get upload mode if at least 10 files have been hashed.
|
|
5347
|
+
4. Pre-upload LFS file if at least 1 file and no worker is pre-uploading.
|
|
5348
|
+
5. Hash file if at least 1 file and no worker is hashing.
|
|
5349
|
+
6. Get upload mode if at least 1 file and no worker is getting upload mode.
|
|
5350
|
+
7. Pre-upload LFS file if at least 1 file (exception: if hf_transfer is enabled, only 1 worker can preupload LFS at a time).
|
|
5351
|
+
8. Hash file if at least 1 file to hash.
|
|
5352
|
+
9. Get upload mode if at least 1 file to get upload mode.
|
|
5353
|
+
10. Commit if at least 1 file to commit.
|
|
5354
|
+
|
|
5355
|
+
Special rules:
|
|
5356
|
+
- If `hf_transfer` is enabled, only 1 LFS uploader at a time. Otherwise the CPU would be bloated by `hf_transfer`.
|
|
5357
|
+
- Only one worker can commit at a time.
|
|
5358
|
+
- If no tasks are available, the worker waits for 10 seconds before checking again.
|
|
5359
|
+
"""
|
|
5360
|
+
return upload_large_folder_internal(
|
|
5361
|
+
self,
|
|
5362
|
+
repo_id=repo_id,
|
|
5363
|
+
folder_path=folder_path,
|
|
5364
|
+
repo_type=repo_type,
|
|
5365
|
+
revision=revision,
|
|
5366
|
+
private=private,
|
|
5367
|
+
allow_patterns=allow_patterns,
|
|
5368
|
+
ignore_patterns=ignore_patterns,
|
|
5369
|
+
num_workers=num_workers,
|
|
5370
|
+
print_report=print_report,
|
|
5371
|
+
print_report_every=print_report_every,
|
|
5372
|
+
)
|
|
5373
|
+
|
|
5121
5374
|
@validate_hf_hub_args
|
|
5122
5375
|
def get_hf_file_metadata(
|
|
5123
5376
|
self,
|
|
@@ -5125,7 +5378,7 @@ class HfApi:
|
|
|
5125
5378
|
url: str,
|
|
5126
5379
|
token: Union[bool, str, None] = None,
|
|
5127
5380
|
proxies: Optional[Dict] = None,
|
|
5128
|
-
timeout: Optional[float] = DEFAULT_REQUEST_TIMEOUT,
|
|
5381
|
+
timeout: Optional[float] = constants.DEFAULT_REQUEST_TIMEOUT,
|
|
5129
5382
|
) -> HfFileMetadata:
|
|
5130
5383
|
"""Fetch metadata of a file versioned on the Hub for a given url.
|
|
5131
5384
|
|
|
@@ -5172,7 +5425,7 @@ class HfApi:
|
|
|
5172
5425
|
local_dir: Union[str, Path, None] = None,
|
|
5173
5426
|
force_download: bool = False,
|
|
5174
5427
|
proxies: Optional[Dict] = None,
|
|
5175
|
-
etag_timeout: float = DEFAULT_ETAG_TIMEOUT,
|
|
5428
|
+
etag_timeout: float = constants.DEFAULT_ETAG_TIMEOUT,
|
|
5176
5429
|
token: Union[bool, str, None] = None,
|
|
5177
5430
|
local_files_only: bool = False,
|
|
5178
5431
|
# Deprecated args
|
|
@@ -5311,7 +5564,7 @@ class HfApi:
|
|
|
5311
5564
|
cache_dir: Union[str, Path, None] = None,
|
|
5312
5565
|
local_dir: Union[str, Path, None] = None,
|
|
5313
5566
|
proxies: Optional[Dict] = None,
|
|
5314
|
-
etag_timeout: float = DEFAULT_ETAG_TIMEOUT,
|
|
5567
|
+
etag_timeout: float = constants.DEFAULT_ETAG_TIMEOUT,
|
|
5315
5568
|
force_download: bool = False,
|
|
5316
5569
|
token: Union[bool, str, None] = None,
|
|
5317
5570
|
local_files_only: bool = False,
|
|
@@ -5500,23 +5753,41 @@ class HfApi:
|
|
|
5500
5753
|
```
|
|
5501
5754
|
"""
|
|
5502
5755
|
if self.file_exists( # Single safetensors file => non-sharded model
|
|
5503
|
-
repo_id=repo_id,
|
|
5756
|
+
repo_id=repo_id,
|
|
5757
|
+
filename=constants.SAFETENSORS_SINGLE_FILE,
|
|
5758
|
+
repo_type=repo_type,
|
|
5759
|
+
revision=revision,
|
|
5760
|
+
token=token,
|
|
5504
5761
|
):
|
|
5505
5762
|
file_metadata = self.parse_safetensors_file_metadata(
|
|
5506
|
-
repo_id=repo_id,
|
|
5763
|
+
repo_id=repo_id,
|
|
5764
|
+
filename=constants.SAFETENSORS_SINGLE_FILE,
|
|
5765
|
+
repo_type=repo_type,
|
|
5766
|
+
revision=revision,
|
|
5767
|
+
token=token,
|
|
5507
5768
|
)
|
|
5508
5769
|
return SafetensorsRepoMetadata(
|
|
5509
5770
|
metadata=None,
|
|
5510
5771
|
sharded=False,
|
|
5511
|
-
weight_map={
|
|
5512
|
-
|
|
5772
|
+
weight_map={
|
|
5773
|
+
tensor_name: constants.SAFETENSORS_SINGLE_FILE for tensor_name in file_metadata.tensors.keys()
|
|
5774
|
+
},
|
|
5775
|
+
files_metadata={constants.SAFETENSORS_SINGLE_FILE: file_metadata},
|
|
5513
5776
|
)
|
|
5514
5777
|
elif self.file_exists( # Multiple safetensors files => sharded with index
|
|
5515
|
-
repo_id=repo_id,
|
|
5778
|
+
repo_id=repo_id,
|
|
5779
|
+
filename=constants.SAFETENSORS_INDEX_FILE,
|
|
5780
|
+
repo_type=repo_type,
|
|
5781
|
+
revision=revision,
|
|
5782
|
+
token=token,
|
|
5516
5783
|
):
|
|
5517
5784
|
# Fetch index
|
|
5518
5785
|
index_file = self.hf_hub_download(
|
|
5519
|
-
repo_id=repo_id,
|
|
5786
|
+
repo_id=repo_id,
|
|
5787
|
+
filename=constants.SAFETENSORS_INDEX_FILE,
|
|
5788
|
+
repo_type=repo_type,
|
|
5789
|
+
revision=revision,
|
|
5790
|
+
token=token,
|
|
5520
5791
|
)
|
|
5521
5792
|
with open(index_file) as f:
|
|
5522
5793
|
index = json.load(f)
|
|
@@ -5547,7 +5818,7 @@ class HfApi:
|
|
|
5547
5818
|
else:
|
|
5548
5819
|
# Not a safetensors repo
|
|
5549
5820
|
raise NotASafetensorsRepoError(
|
|
5550
|
-
f"'{repo_id}' is not a safetensors repo. Couldn't find '{SAFETENSORS_INDEX_FILE}' or '{SAFETENSORS_SINGLE_FILE}' files."
|
|
5821
|
+
f"'{repo_id}' is not a safetensors repo. Couldn't find '{constants.SAFETENSORS_INDEX_FILE}' or '{constants.SAFETENSORS_SINGLE_FILE}' files."
|
|
5551
5822
|
)
|
|
5552
5823
|
|
|
5553
5824
|
def parse_safetensors_file_metadata(
|
|
@@ -5608,11 +5879,11 @@ class HfApi:
|
|
|
5608
5879
|
|
|
5609
5880
|
# 2. Parse metadata size
|
|
5610
5881
|
metadata_size = struct.unpack("<Q", response.content[:8])[0]
|
|
5611
|
-
if metadata_size > SAFETENSORS_MAX_HEADER_LENGTH:
|
|
5882
|
+
if metadata_size > constants.SAFETENSORS_MAX_HEADER_LENGTH:
|
|
5612
5883
|
raise SafetensorsParsingError(
|
|
5613
5884
|
f"Failed to parse safetensors header for '{filename}' (repo '{repo_id}', revision "
|
|
5614
|
-
f"'{revision or DEFAULT_REVISION}'): safetensors header is too big. Maximum supported size is "
|
|
5615
|
-
f"{SAFETENSORS_MAX_HEADER_LENGTH} bytes (got {metadata_size})."
|
|
5885
|
+
f"'{revision or constants.DEFAULT_REVISION}'): safetensors header is too big. Maximum supported size is "
|
|
5886
|
+
f"{constants.SAFETENSORS_MAX_HEADER_LENGTH} bytes (got {metadata_size})."
|
|
5616
5887
|
)
|
|
5617
5888
|
|
|
5618
5889
|
# 3.a. Get metadata from payload
|
|
@@ -5629,7 +5900,7 @@ class HfApi:
|
|
|
5629
5900
|
except json.JSONDecodeError as e:
|
|
5630
5901
|
raise SafetensorsParsingError(
|
|
5631
5902
|
f"Failed to parse safetensors header for '{filename}' (repo '{repo_id}', revision "
|
|
5632
|
-
f"'{revision or DEFAULT_REVISION}'): header is not json-encoded string. Please make sure this is a "
|
|
5903
|
+
f"'{revision or constants.DEFAULT_REVISION}'): header is not json-encoded string. Please make sure this is a "
|
|
5633
5904
|
"correctly formatted safetensors file."
|
|
5634
5905
|
) from e
|
|
5635
5906
|
|
|
@@ -5649,7 +5920,7 @@ class HfApi:
|
|
|
5649
5920
|
except (KeyError, IndexError) as e:
|
|
5650
5921
|
raise SafetensorsParsingError(
|
|
5651
5922
|
f"Failed to parse safetensors header for '{filename}' (repo '{repo_id}', revision "
|
|
5652
|
-
f"'{revision or DEFAULT_REVISION}'): header format not recognized. Please make sure this is a correctly"
|
|
5923
|
+
f"'{revision or constants.DEFAULT_REVISION}'): header format not recognized. Please make sure this is a correctly"
|
|
5653
5924
|
" formatted safetensors file."
|
|
5654
5925
|
) from e
|
|
5655
5926
|
|
|
@@ -5705,7 +5976,7 @@ class HfApi:
|
|
|
5705
5976
|
set to `False`.
|
|
5706
5977
|
"""
|
|
5707
5978
|
if repo_type is None:
|
|
5708
|
-
repo_type = REPO_TYPE_MODEL
|
|
5979
|
+
repo_type = constants.REPO_TYPE_MODEL
|
|
5709
5980
|
branch = quote(branch, safe="")
|
|
5710
5981
|
|
|
5711
5982
|
# Prepare request
|
|
@@ -5720,8 +5991,18 @@ class HfApi:
|
|
|
5720
5991
|
try:
|
|
5721
5992
|
hf_raise_for_status(response)
|
|
5722
5993
|
except HfHubHTTPError as e:
|
|
5723
|
-
if
|
|
5724
|
-
|
|
5994
|
+
if exist_ok and e.response.status_code == 409:
|
|
5995
|
+
return
|
|
5996
|
+
elif exist_ok and e.response.status_code == 403:
|
|
5997
|
+
# No write permission on the namespace but branch might already exist
|
|
5998
|
+
try:
|
|
5999
|
+
refs = self.list_repo_refs(repo_id=repo_id, repo_type=repo_type, token=token)
|
|
6000
|
+
for branch_ref in refs.branches:
|
|
6001
|
+
if branch_ref.name == branch:
|
|
6002
|
+
return # Branch already exists => do not raise
|
|
6003
|
+
except HfHubHTTPError:
|
|
6004
|
+
pass # We raise the original error if the branch does not exist
|
|
6005
|
+
raise
|
|
5725
6006
|
|
|
5726
6007
|
@validate_hf_hub_args
|
|
5727
6008
|
def delete_branch(
|
|
@@ -5764,7 +6045,7 @@ class HfApi:
|
|
|
5764
6045
|
|
|
5765
6046
|
"""
|
|
5766
6047
|
if repo_type is None:
|
|
5767
|
-
repo_type = REPO_TYPE_MODEL
|
|
6048
|
+
repo_type = constants.REPO_TYPE_MODEL
|
|
5768
6049
|
branch = quote(branch, safe="")
|
|
5769
6050
|
|
|
5770
6051
|
# Prepare request
|
|
@@ -5831,8 +6112,8 @@ class HfApi:
|
|
|
5831
6112
|
set to `False`.
|
|
5832
6113
|
"""
|
|
5833
6114
|
if repo_type is None:
|
|
5834
|
-
repo_type = REPO_TYPE_MODEL
|
|
5835
|
-
revision = quote(revision, safe="") if revision is not None else DEFAULT_REVISION
|
|
6115
|
+
repo_type = constants.REPO_TYPE_MODEL
|
|
6116
|
+
revision = quote(revision, safe="") if revision is not None else constants.DEFAULT_REVISION
|
|
5836
6117
|
|
|
5837
6118
|
# Prepare request
|
|
5838
6119
|
tag_url = f"{self.endpoint}/api/{repo_type}s/{repo_id}/tag/{revision}"
|
|
@@ -5887,7 +6168,7 @@ class HfApi:
|
|
|
5887
6168
|
If tag is not found.
|
|
5888
6169
|
"""
|
|
5889
6170
|
if repo_type is None:
|
|
5890
|
-
repo_type = REPO_TYPE_MODEL
|
|
6171
|
+
repo_type = constants.REPO_TYPE_MODEL
|
|
5891
6172
|
tag = quote(tag, safe="")
|
|
5892
6173
|
|
|
5893
6174
|
# Prepare request
|
|
@@ -5942,8 +6223,8 @@ class HfApi:
|
|
|
5942
6223
|
repo_id: str,
|
|
5943
6224
|
*,
|
|
5944
6225
|
author: Optional[str] = None,
|
|
5945
|
-
discussion_type: Optional[DiscussionTypeFilter] = None,
|
|
5946
|
-
discussion_status: Optional[DiscussionStatusFilter] = None,
|
|
6226
|
+
discussion_type: Optional[constants.DiscussionTypeFilter] = None,
|
|
6227
|
+
discussion_status: Optional[constants.DiscussionStatusFilter] = None,
|
|
5947
6228
|
repo_type: Optional[str] = None,
|
|
5948
6229
|
token: Union[bool, str, None] = None,
|
|
5949
6230
|
) -> Iterator[Discussion]:
|
|
@@ -5995,14 +6276,14 @@ class HfApi:
|
|
|
5995
6276
|
... print(discussion.num, discussion.title)
|
|
5996
6277
|
```
|
|
5997
6278
|
"""
|
|
5998
|
-
if repo_type not in REPO_TYPES:
|
|
5999
|
-
raise ValueError(f"Invalid repo type, must be one of {REPO_TYPES}")
|
|
6279
|
+
if repo_type not in constants.REPO_TYPES:
|
|
6280
|
+
raise ValueError(f"Invalid repo type, must be one of {constants.REPO_TYPES}")
|
|
6000
6281
|
if repo_type is None:
|
|
6001
|
-
repo_type = REPO_TYPE_MODEL
|
|
6002
|
-
if discussion_type is not None and discussion_type not in DISCUSSION_TYPES:
|
|
6003
|
-
raise ValueError(f"Invalid discussion_type, must be one of {DISCUSSION_TYPES}")
|
|
6004
|
-
if discussion_status is not None and discussion_status not in DISCUSSION_STATUS:
|
|
6005
|
-
raise ValueError(f"Invalid discussion_status, must be one of {DISCUSSION_STATUS}")
|
|
6282
|
+
repo_type = constants.REPO_TYPE_MODEL
|
|
6283
|
+
if discussion_type is not None and discussion_type not in constants.DISCUSSION_TYPES:
|
|
6284
|
+
raise ValueError(f"Invalid discussion_type, must be one of {constants.DISCUSSION_TYPES}")
|
|
6285
|
+
if discussion_status is not None and discussion_status not in constants.DISCUSSION_STATUS:
|
|
6286
|
+
raise ValueError(f"Invalid discussion_status, must be one of {constants.DISCUSSION_STATUS}")
|
|
6006
6287
|
|
|
6007
6288
|
headers = self._build_hf_headers(token=token)
|
|
6008
6289
|
path = f"{self.endpoint}/api/{repo_type}s/{repo_id}/discussions"
|
|
@@ -6089,10 +6370,10 @@ class HfApi:
|
|
|
6089
6370
|
"""
|
|
6090
6371
|
if not isinstance(discussion_num, int) or discussion_num <= 0:
|
|
6091
6372
|
raise ValueError("Invalid discussion_num, must be a positive integer")
|
|
6092
|
-
if repo_type not in REPO_TYPES:
|
|
6093
|
-
raise ValueError(f"Invalid repo type, must be one of {REPO_TYPES}")
|
|
6373
|
+
if repo_type not in constants.REPO_TYPES:
|
|
6374
|
+
raise ValueError(f"Invalid repo type, must be one of {constants.REPO_TYPES}")
|
|
6094
6375
|
if repo_type is None:
|
|
6095
|
-
repo_type = REPO_TYPE_MODEL
|
|
6376
|
+
repo_type = constants.REPO_TYPE_MODEL
|
|
6096
6377
|
|
|
6097
6378
|
path = f"{self.endpoint}/api/{repo_type}s/{repo_id}/discussions/{discussion_num}"
|
|
6098
6379
|
headers = self._build_hf_headers(token=token)
|
|
@@ -6179,10 +6460,10 @@ class HfApi:
|
|
|
6179
6460
|
or because it is set to `private` and you do not have access.
|
|
6180
6461
|
|
|
6181
6462
|
</Tip>"""
|
|
6182
|
-
if repo_type not in REPO_TYPES:
|
|
6183
|
-
raise ValueError(f"Invalid repo type, must be one of {REPO_TYPES}")
|
|
6463
|
+
if repo_type not in constants.REPO_TYPES:
|
|
6464
|
+
raise ValueError(f"Invalid repo type, must be one of {constants.REPO_TYPES}")
|
|
6184
6465
|
if repo_type is None:
|
|
6185
|
-
repo_type = REPO_TYPE_MODEL
|
|
6466
|
+
repo_type = constants.REPO_TYPE_MODEL
|
|
6186
6467
|
|
|
6187
6468
|
if description is not None:
|
|
6188
6469
|
description = description.strip()
|
|
@@ -6289,10 +6570,10 @@ class HfApi:
|
|
|
6289
6570
|
"""Internal utility to POST changes to a Discussion or Pull Request"""
|
|
6290
6571
|
if not isinstance(discussion_num, int) or discussion_num <= 0:
|
|
6291
6572
|
raise ValueError("Invalid discussion_num, must be a positive integer")
|
|
6292
|
-
if repo_type not in REPO_TYPES:
|
|
6293
|
-
raise ValueError(f"Invalid repo type, must be one of {REPO_TYPES}")
|
|
6573
|
+
if repo_type not in constants.REPO_TYPES:
|
|
6574
|
+
raise ValueError(f"Invalid repo type, must be one of {constants.REPO_TYPES}")
|
|
6294
6575
|
if repo_type is None:
|
|
6295
|
-
repo_type = REPO_TYPE_MODEL
|
|
6576
|
+
repo_type = constants.REPO_TYPE_MODEL
|
|
6296
6577
|
repo_id = f"{repo_type}s/{repo_id}"
|
|
6297
6578
|
|
|
6298
6579
|
path = f"{self.endpoint}/api/{repo_id}/discussions/{discussion_num}/{resource}"
|
|
@@ -7316,7 +7597,7 @@ class HfApi:
|
|
|
7316
7597
|
namespace = namespace or self._get_namespace(token=token)
|
|
7317
7598
|
|
|
7318
7599
|
response = get_session().get(
|
|
7319
|
-
f"{INFERENCE_ENDPOINTS_ENDPOINT}/endpoint/{namespace}",
|
|
7600
|
+
f"{constants.INFERENCE_ENDPOINTS_ENDPOINT}/endpoint/{namespace}",
|
|
7320
7601
|
headers=self._build_hf_headers(token=token),
|
|
7321
7602
|
)
|
|
7322
7603
|
hf_raise_for_status(response)
|
|
@@ -7340,9 +7621,11 @@ class HfApi:
|
|
|
7340
7621
|
account_id: Optional[str] = None,
|
|
7341
7622
|
min_replica: int = 0,
|
|
7342
7623
|
max_replica: int = 1,
|
|
7624
|
+
scale_to_zero_timeout: int = 15,
|
|
7343
7625
|
revision: Optional[str] = None,
|
|
7344
7626
|
task: Optional[str] = None,
|
|
7345
7627
|
custom_image: Optional[Dict] = None,
|
|
7628
|
+
secrets: Optional[Dict[str, str]] = None,
|
|
7346
7629
|
type: InferenceEndpointType = InferenceEndpointType.PROTECTED,
|
|
7347
7630
|
namespace: Optional[str] = None,
|
|
7348
7631
|
token: Union[bool, str, None] = None,
|
|
@@ -7372,6 +7655,8 @@ class HfApi:
|
|
|
7372
7655
|
The minimum number of replicas (instances) to keep running for the Inference Endpoint. Defaults to 0.
|
|
7373
7656
|
max_replica (`int`, *optional*):
|
|
7374
7657
|
The maximum number of replicas (instances) to scale to for the Inference Endpoint. Defaults to 1.
|
|
7658
|
+
scale_to_zero_timeout (`int`, *optional*):
|
|
7659
|
+
The duration in minutes before an inactive endpoint is scaled to zero. Defaults to 15.
|
|
7375
7660
|
revision (`str`, *optional*):
|
|
7376
7661
|
The specific model revision to deploy on the Inference Endpoint (e.g. `"6c0e6080953db56375760c0471a8c5f2929baf11"`).
|
|
7377
7662
|
task (`str`, *optional*):
|
|
@@ -7379,6 +7664,8 @@ class HfApi:
|
|
|
7379
7664
|
custom_image (`Dict`, *optional*):
|
|
7380
7665
|
A custom Docker image to use for the Inference Endpoint. This is useful if you want to deploy an
|
|
7381
7666
|
Inference Endpoint running on the `text-generation-inference` (TGI) framework (see examples).
|
|
7667
|
+
secrets (`Dict[str, str]`, *optional*):
|
|
7668
|
+
Secret values to inject in the container environment.
|
|
7382
7669
|
type ([`InferenceEndpointType]`, *optional*):
|
|
7383
7670
|
The type of the Inference Endpoint, which can be `"protected"` (default), `"public"` or `"private"`.
|
|
7384
7671
|
namespace (`str`, *optional*):
|
|
@@ -7441,6 +7728,7 @@ class HfApi:
|
|
|
7441
7728
|
... },
|
|
7442
7729
|
... "url": "ghcr.io/huggingface/text-generation-inference:1.1.0",
|
|
7443
7730
|
... },
|
|
7731
|
+
... secrets={"MY_SECRET_KEY": "secret_value"},
|
|
7444
7732
|
... )
|
|
7445
7733
|
|
|
7446
7734
|
```
|
|
@@ -7457,6 +7745,7 @@ class HfApi:
|
|
|
7457
7745
|
"scaling": {
|
|
7458
7746
|
"maxReplica": max_replica,
|
|
7459
7747
|
"minReplica": min_replica,
|
|
7748
|
+
"scaleToZeroTimeout": scale_to_zero_timeout,
|
|
7460
7749
|
},
|
|
7461
7750
|
},
|
|
7462
7751
|
"model": {
|
|
@@ -7473,9 +7762,10 @@ class HfApi:
|
|
|
7473
7762
|
},
|
|
7474
7763
|
"type": type,
|
|
7475
7764
|
}
|
|
7476
|
-
|
|
7765
|
+
if secrets:
|
|
7766
|
+
payload["model"]["secrets"] = secrets
|
|
7477
7767
|
response = get_session().post(
|
|
7478
|
-
f"{INFERENCE_ENDPOINTS_ENDPOINT}/endpoint/{namespace}",
|
|
7768
|
+
f"{constants.INFERENCE_ENDPOINTS_ENDPOINT}/endpoint/{namespace}",
|
|
7479
7769
|
headers=self._build_hf_headers(token=token),
|
|
7480
7770
|
json=payload,
|
|
7481
7771
|
)
|
|
@@ -7523,7 +7813,7 @@ class HfApi:
|
|
|
7523
7813
|
namespace = namespace or self._get_namespace(token=token)
|
|
7524
7814
|
|
|
7525
7815
|
response = get_session().get(
|
|
7526
|
-
f"{INFERENCE_ENDPOINTS_ENDPOINT}/endpoint/{namespace}/{name}",
|
|
7816
|
+
f"{constants.INFERENCE_ENDPOINTS_ENDPOINT}/endpoint/{namespace}/{name}",
|
|
7527
7817
|
headers=self._build_hf_headers(token=token),
|
|
7528
7818
|
)
|
|
7529
7819
|
hf_raise_for_status(response)
|
|
@@ -7540,12 +7830,14 @@ class HfApi:
|
|
|
7540
7830
|
instance_type: Optional[str] = None,
|
|
7541
7831
|
min_replica: Optional[int] = None,
|
|
7542
7832
|
max_replica: Optional[int] = None,
|
|
7833
|
+
scale_to_zero_timeout: Optional[int] = None,
|
|
7543
7834
|
# Model update
|
|
7544
7835
|
repository: Optional[str] = None,
|
|
7545
7836
|
framework: Optional[str] = None,
|
|
7546
7837
|
revision: Optional[str] = None,
|
|
7547
7838
|
task: Optional[str] = None,
|
|
7548
7839
|
custom_image: Optional[Dict] = None,
|
|
7840
|
+
secrets: Optional[Dict[str, str]] = None,
|
|
7549
7841
|
# Other
|
|
7550
7842
|
namespace: Optional[str] = None,
|
|
7551
7843
|
token: Union[bool, str, None] = None,
|
|
@@ -7571,6 +7863,8 @@ class HfApi:
|
|
|
7571
7863
|
The minimum number of replicas (instances) to keep running for the Inference Endpoint.
|
|
7572
7864
|
max_replica (`int`, *optional*):
|
|
7573
7865
|
The maximum number of replicas (instances) to scale to for the Inference Endpoint.
|
|
7866
|
+
scale_to_zero_timeout (`int`, *optional*):
|
|
7867
|
+
The duration in minutes before an inactive endpoint is scaled to zero.
|
|
7574
7868
|
|
|
7575
7869
|
repository (`str`, *optional*):
|
|
7576
7870
|
The name of the model repository associated with the Inference Endpoint (e.g. `"gpt2"`).
|
|
@@ -7583,7 +7877,8 @@ class HfApi:
|
|
|
7583
7877
|
custom_image (`Dict`, *optional*):
|
|
7584
7878
|
A custom Docker image to use for the Inference Endpoint. This is useful if you want to deploy an
|
|
7585
7879
|
Inference Endpoint running on the `text-generation-inference` (TGI) framework (see examples).
|
|
7586
|
-
|
|
7880
|
+
secrets (`Dict[str, str]`, *optional*):
|
|
7881
|
+
Secret values to inject in the container environment.
|
|
7587
7882
|
namespace (`str`, *optional*):
|
|
7588
7883
|
The namespace where the Inference Endpoint will be updated. Defaults to the current user's namespace.
|
|
7589
7884
|
token (Union[bool, str, None], optional):
|
|
@@ -7597,29 +7892,35 @@ class HfApi:
|
|
|
7597
7892
|
"""
|
|
7598
7893
|
namespace = namespace or self._get_namespace(token=token)
|
|
7599
7894
|
|
|
7600
|
-
|
|
7601
|
-
|
|
7602
|
-
|
|
7603
|
-
|
|
7604
|
-
|
|
7605
|
-
|
|
7606
|
-
|
|
7607
|
-
|
|
7608
|
-
|
|
7609
|
-
|
|
7610
|
-
|
|
7611
|
-
|
|
7612
|
-
|
|
7613
|
-
payload["
|
|
7614
|
-
|
|
7615
|
-
|
|
7616
|
-
|
|
7617
|
-
|
|
7618
|
-
|
|
7619
|
-
|
|
7895
|
+
# Populate only the fields that are not None
|
|
7896
|
+
payload: Dict = defaultdict(lambda: defaultdict(dict))
|
|
7897
|
+
if accelerator is not None:
|
|
7898
|
+
payload["compute"]["accelerator"] = accelerator
|
|
7899
|
+
if instance_size is not None:
|
|
7900
|
+
payload["compute"]["instanceSize"] = instance_size
|
|
7901
|
+
if instance_type is not None:
|
|
7902
|
+
payload["compute"]["instanceType"] = instance_type
|
|
7903
|
+
if max_replica is not None:
|
|
7904
|
+
payload["compute"]["scaling"]["maxReplica"] = max_replica
|
|
7905
|
+
if min_replica is not None:
|
|
7906
|
+
payload["compute"]["scaling"]["minReplica"] = min_replica
|
|
7907
|
+
if scale_to_zero_timeout is not None:
|
|
7908
|
+
payload["compute"]["scaling"]["scaleToZeroTimeout"] = scale_to_zero_timeout
|
|
7909
|
+
if repository is not None:
|
|
7910
|
+
payload["model"]["repository"] = repository
|
|
7911
|
+
if framework is not None:
|
|
7912
|
+
payload["model"]["framework"] = framework
|
|
7913
|
+
if revision is not None:
|
|
7914
|
+
payload["model"]["revision"] = revision
|
|
7915
|
+
if task is not None:
|
|
7916
|
+
payload["model"]["task"] = task
|
|
7917
|
+
if custom_image is not None:
|
|
7918
|
+
payload["model"]["image"] = {"custom": custom_image}
|
|
7919
|
+
if secrets is not None:
|
|
7920
|
+
payload["model"]["secrets"] = secrets
|
|
7620
7921
|
|
|
7621
7922
|
response = get_session().put(
|
|
7622
|
-
f"{INFERENCE_ENDPOINTS_ENDPOINT}/endpoint/{namespace}/{name}",
|
|
7923
|
+
f"{constants.INFERENCE_ENDPOINTS_ENDPOINT}/endpoint/{namespace}/{name}",
|
|
7623
7924
|
headers=self._build_hf_headers(token=token),
|
|
7624
7925
|
json=payload,
|
|
7625
7926
|
)
|
|
@@ -7650,7 +7951,7 @@ class HfApi:
|
|
|
7650
7951
|
"""
|
|
7651
7952
|
namespace = namespace or self._get_namespace(token=token)
|
|
7652
7953
|
response = get_session().delete(
|
|
7653
|
-
f"{INFERENCE_ENDPOINTS_ENDPOINT}/endpoint/{namespace}/{name}",
|
|
7954
|
+
f"{constants.INFERENCE_ENDPOINTS_ENDPOINT}/endpoint/{namespace}/{name}",
|
|
7654
7955
|
headers=self._build_hf_headers(token=token),
|
|
7655
7956
|
)
|
|
7656
7957
|
hf_raise_for_status(response)
|
|
@@ -7683,7 +7984,7 @@ class HfApi:
|
|
|
7683
7984
|
namespace = namespace or self._get_namespace(token=token)
|
|
7684
7985
|
|
|
7685
7986
|
response = get_session().post(
|
|
7686
|
-
f"{INFERENCE_ENDPOINTS_ENDPOINT}/endpoint/{namespace}/{name}/pause",
|
|
7987
|
+
f"{constants.INFERENCE_ENDPOINTS_ENDPOINT}/endpoint/{namespace}/{name}/pause",
|
|
7687
7988
|
headers=self._build_hf_headers(token=token),
|
|
7688
7989
|
)
|
|
7689
7990
|
hf_raise_for_status(response)
|
|
@@ -7722,7 +8023,7 @@ class HfApi:
|
|
|
7722
8023
|
namespace = namespace or self._get_namespace(token=token)
|
|
7723
8024
|
|
|
7724
8025
|
response = get_session().post(
|
|
7725
|
-
f"{INFERENCE_ENDPOINTS_ENDPOINT}/endpoint/{namespace}/{name}/resume",
|
|
8026
|
+
f"{constants.INFERENCE_ENDPOINTS_ENDPOINT}/endpoint/{namespace}/{name}/resume",
|
|
7726
8027
|
headers=self._build_hf_headers(token=token),
|
|
7727
8028
|
)
|
|
7728
8029
|
try:
|
|
@@ -7764,7 +8065,7 @@ class HfApi:
|
|
|
7764
8065
|
namespace = namespace or self._get_namespace(token=token)
|
|
7765
8066
|
|
|
7766
8067
|
response = get_session().post(
|
|
7767
|
-
f"{INFERENCE_ENDPOINTS_ENDPOINT}/endpoint/{namespace}/{name}/scale-to-zero",
|
|
8068
|
+
f"{constants.INFERENCE_ENDPOINTS_ENDPOINT}/endpoint/{namespace}/{name}/scale-to-zero",
|
|
7768
8069
|
headers=self._build_hf_headers(token=token),
|
|
7769
8070
|
)
|
|
7770
8071
|
hf_raise_for_status(response)
|
|
@@ -8446,13 +8747,13 @@ class HfApi:
|
|
|
8446
8747
|
repo_type: Optional[str] = None,
|
|
8447
8748
|
token: Union[bool, str, None] = None,
|
|
8448
8749
|
) -> List[AccessRequest]:
|
|
8449
|
-
if repo_type not in REPO_TYPES:
|
|
8450
|
-
raise ValueError(f"Invalid repo type, must be one of {REPO_TYPES}")
|
|
8750
|
+
if repo_type not in constants.REPO_TYPES:
|
|
8751
|
+
raise ValueError(f"Invalid repo type, must be one of {constants.REPO_TYPES}")
|
|
8451
8752
|
if repo_type is None:
|
|
8452
|
-
repo_type = REPO_TYPE_MODEL
|
|
8753
|
+
repo_type = constants.REPO_TYPE_MODEL
|
|
8453
8754
|
|
|
8454
8755
|
response = get_session().get(
|
|
8455
|
-
f"{ENDPOINT}/api/{repo_type}s/{repo_id}/user-access-request/{status}",
|
|
8756
|
+
f"{constants.ENDPOINT}/api/{repo_type}s/{repo_id}/user-access-request/{status}",
|
|
8456
8757
|
headers=self._build_hf_headers(token=token),
|
|
8457
8758
|
)
|
|
8458
8759
|
hf_raise_for_status(response)
|
|
@@ -8601,13 +8902,13 @@ class HfApi:
|
|
|
8601
8902
|
repo_type: Optional[str] = None,
|
|
8602
8903
|
token: Union[bool, str, None] = None,
|
|
8603
8904
|
) -> None:
|
|
8604
|
-
if repo_type not in REPO_TYPES:
|
|
8605
|
-
raise ValueError(f"Invalid repo type, must be one of {REPO_TYPES}")
|
|
8905
|
+
if repo_type not in constants.REPO_TYPES:
|
|
8906
|
+
raise ValueError(f"Invalid repo type, must be one of {constants.REPO_TYPES}")
|
|
8606
8907
|
if repo_type is None:
|
|
8607
|
-
repo_type = REPO_TYPE_MODEL
|
|
8908
|
+
repo_type = constants.REPO_TYPE_MODEL
|
|
8608
8909
|
|
|
8609
8910
|
response = get_session().post(
|
|
8610
|
-
f"{ENDPOINT}/api/{repo_type}s/{repo_id}/user-access-request/handle",
|
|
8911
|
+
f"{constants.ENDPOINT}/api/{repo_type}s/{repo_id}/user-access-request/handle",
|
|
8611
8912
|
headers=self._build_hf_headers(token=token),
|
|
8612
8913
|
json={"user": user, "status": status},
|
|
8613
8914
|
)
|
|
@@ -8651,13 +8952,13 @@ class HfApi:
|
|
|
8651
8952
|
[`HTTPError`](https://requests.readthedocs.io/en/latest/api/#requests.HTTPError):
|
|
8652
8953
|
HTTP 404 if the user does not exist on the Hub.
|
|
8653
8954
|
"""
|
|
8654
|
-
if repo_type not in REPO_TYPES:
|
|
8655
|
-
raise ValueError(f"Invalid repo type, must be one of {REPO_TYPES}")
|
|
8955
|
+
if repo_type not in constants.REPO_TYPES:
|
|
8956
|
+
raise ValueError(f"Invalid repo type, must be one of {constants.REPO_TYPES}")
|
|
8656
8957
|
if repo_type is None:
|
|
8657
|
-
repo_type = REPO_TYPE_MODEL
|
|
8958
|
+
repo_type = constants.REPO_TYPE_MODEL
|
|
8658
8959
|
|
|
8659
8960
|
response = get_session().post(
|
|
8660
|
-
f"{ENDPOINT}/api/models/{repo_id}/user-access-request/grant",
|
|
8961
|
+
f"{constants.ENDPOINT}/api/models/{repo_id}/user-access-request/grant",
|
|
8661
8962
|
headers=self._build_hf_headers(token=token),
|
|
8662
8963
|
json={"user": user},
|
|
8663
8964
|
)
|
|
@@ -8700,7 +9001,7 @@ class HfApi:
|
|
|
8700
9001
|
```
|
|
8701
9002
|
"""
|
|
8702
9003
|
response = get_session().get(
|
|
8703
|
-
f"{ENDPOINT}/api/settings/webhooks/{webhook_id}",
|
|
9004
|
+
f"{constants.ENDPOINT}/api/settings/webhooks/{webhook_id}",
|
|
8704
9005
|
headers=self._build_hf_headers(token=token),
|
|
8705
9006
|
)
|
|
8706
9007
|
hf_raise_for_status(response)
|
|
@@ -8751,7 +9052,7 @@ class HfApi:
|
|
|
8751
9052
|
```
|
|
8752
9053
|
"""
|
|
8753
9054
|
response = get_session().get(
|
|
8754
|
-
f"{ENDPOINT}/api/settings/webhooks",
|
|
9055
|
+
f"{constants.ENDPOINT}/api/settings/webhooks",
|
|
8755
9056
|
headers=self._build_hf_headers(token=token),
|
|
8756
9057
|
)
|
|
8757
9058
|
hf_raise_for_status(response)
|
|
@@ -8775,7 +9076,7 @@ class HfApi:
|
|
|
8775
9076
|
*,
|
|
8776
9077
|
url: str,
|
|
8777
9078
|
watched: List[Union[Dict, WebhookWatchedItem]],
|
|
8778
|
-
domains: Optional[List[WEBHOOK_DOMAIN_T]] = None,
|
|
9079
|
+
domains: Optional[List[constants.WEBHOOK_DOMAIN_T]] = None,
|
|
8779
9080
|
secret: Optional[str] = None,
|
|
8780
9081
|
token: Union[bool, str, None] = None,
|
|
8781
9082
|
) -> WebhookInfo:
|
|
@@ -8823,7 +9124,7 @@ class HfApi:
|
|
|
8823
9124
|
watched_dicts = [asdict(item) if isinstance(item, WebhookWatchedItem) else item for item in watched]
|
|
8824
9125
|
|
|
8825
9126
|
response = get_session().post(
|
|
8826
|
-
f"{ENDPOINT}/api/settings/webhooks",
|
|
9127
|
+
f"{constants.ENDPOINT}/api/settings/webhooks",
|
|
8827
9128
|
json={"watched": watched_dicts, "url": url, "domains": domains, "secret": secret},
|
|
8828
9129
|
headers=self._build_hf_headers(token=token),
|
|
8829
9130
|
)
|
|
@@ -8849,7 +9150,7 @@ class HfApi:
|
|
|
8849
9150
|
*,
|
|
8850
9151
|
url: Optional[str] = None,
|
|
8851
9152
|
watched: Optional[List[Union[Dict, WebhookWatchedItem]]] = None,
|
|
8852
|
-
domains: Optional[List[WEBHOOK_DOMAIN_T]] = None,
|
|
9153
|
+
domains: Optional[List[constants.WEBHOOK_DOMAIN_T]] = None,
|
|
8853
9154
|
secret: Optional[str] = None,
|
|
8854
9155
|
token: Union[bool, str, None] = None,
|
|
8855
9156
|
) -> WebhookInfo:
|
|
@@ -8901,7 +9202,7 @@ class HfApi:
|
|
|
8901
9202
|
watched_dicts = [asdict(item) if isinstance(item, WebhookWatchedItem) else item for item in watched]
|
|
8902
9203
|
|
|
8903
9204
|
response = get_session().post(
|
|
8904
|
-
f"{ENDPOINT}/api/settings/webhooks/{webhook_id}",
|
|
9205
|
+
f"{constants.ENDPOINT}/api/settings/webhooks/{webhook_id}",
|
|
8905
9206
|
json={"watched": watched_dicts, "url": url, "domains": domains, "secret": secret},
|
|
8906
9207
|
headers=self._build_hf_headers(token=token),
|
|
8907
9208
|
)
|
|
@@ -8953,7 +9254,7 @@ class HfApi:
|
|
|
8953
9254
|
```
|
|
8954
9255
|
"""
|
|
8955
9256
|
response = get_session().post(
|
|
8956
|
-
f"{ENDPOINT}/api/settings/webhooks/{webhook_id}/enable",
|
|
9257
|
+
f"{constants.ENDPOINT}/api/settings/webhooks/{webhook_id}/enable",
|
|
8957
9258
|
headers=self._build_hf_headers(token=token),
|
|
8958
9259
|
)
|
|
8959
9260
|
hf_raise_for_status(response)
|
|
@@ -9004,7 +9305,7 @@ class HfApi:
|
|
|
9004
9305
|
```
|
|
9005
9306
|
"""
|
|
9006
9307
|
response = get_session().post(
|
|
9007
|
-
f"{ENDPOINT}/api/settings/webhooks/{webhook_id}/disable",
|
|
9308
|
+
f"{constants.ENDPOINT}/api/settings/webhooks/{webhook_id}/disable",
|
|
9008
9309
|
headers=self._build_hf_headers(token=token),
|
|
9009
9310
|
)
|
|
9010
9311
|
hf_raise_for_status(response)
|
|
@@ -9045,7 +9346,7 @@ class HfApi:
|
|
|
9045
9346
|
```
|
|
9046
9347
|
"""
|
|
9047
9348
|
response = get_session().delete(
|
|
9048
|
-
f"{ENDPOINT}/api/settings/webhooks/{webhook_id}",
|
|
9349
|
+
f"{constants.ENDPOINT}/api/settings/webhooks/{webhook_id}",
|
|
9049
9350
|
headers=self._build_hf_headers(token=token),
|
|
9050
9351
|
)
|
|
9051
9352
|
hf_raise_for_status(response)
|
|
@@ -9118,13 +9419,113 @@ class HfApi:
|
|
|
9118
9419
|
if relpath_to_abspath[relpath] != ".gitattributes"
|
|
9119
9420
|
]
|
|
9120
9421
|
|
|
9121
|
-
def
|
|
9422
|
+
def _prepare_upload_folder_additions(
|
|
9423
|
+
self,
|
|
9424
|
+
folder_path: Union[str, Path],
|
|
9425
|
+
path_in_repo: str,
|
|
9426
|
+
allow_patterns: Optional[Union[List[str], str]] = None,
|
|
9427
|
+
ignore_patterns: Optional[Union[List[str], str]] = None,
|
|
9428
|
+
repo_type: Optional[str] = None,
|
|
9429
|
+
token: Union[bool, str, None] = None,
|
|
9430
|
+
) -> List[CommitOperationAdd]:
|
|
9431
|
+
"""Generate the list of Add operations for a commit to upload a folder.
|
|
9432
|
+
|
|
9433
|
+
Files not matching the `allow_patterns` (allowlist) and `ignore_patterns` (denylist)
|
|
9434
|
+
constraints are discarded.
|
|
9435
|
+
"""
|
|
9436
|
+
|
|
9437
|
+
folder_path = Path(folder_path).expanduser().resolve()
|
|
9438
|
+
if not folder_path.is_dir():
|
|
9439
|
+
raise ValueError(f"Provided path: '{folder_path}' is not a directory")
|
|
9440
|
+
|
|
9441
|
+
# List files from folder
|
|
9442
|
+
relpath_to_abspath = {
|
|
9443
|
+
path.relative_to(folder_path).as_posix(): path
|
|
9444
|
+
for path in sorted(folder_path.glob("**/*")) # sorted to be deterministic
|
|
9445
|
+
if path.is_file()
|
|
9446
|
+
}
|
|
9447
|
+
|
|
9448
|
+
# Filter files
|
|
9449
|
+
# Patterns are applied on the path relative to `folder_path`. `path_in_repo` is prefixed after the filtering.
|
|
9450
|
+
filtered_repo_objects = list(
|
|
9451
|
+
filter_repo_objects(
|
|
9452
|
+
relpath_to_abspath.keys(), allow_patterns=allow_patterns, ignore_patterns=ignore_patterns
|
|
9453
|
+
)
|
|
9454
|
+
)
|
|
9455
|
+
|
|
9456
|
+
prefix = f"{path_in_repo.strip('/')}/" if path_in_repo else ""
|
|
9457
|
+
|
|
9458
|
+
# If updating a README.md file, make sure the metadata format is valid
|
|
9459
|
+
# It's better to fail early than to fail after all the files have been hashed.
|
|
9460
|
+
if "README.md" in filtered_repo_objects:
|
|
9461
|
+
self._validate_yaml(
|
|
9462
|
+
content=relpath_to_abspath["README.md"].read_text(),
|
|
9463
|
+
repo_type=repo_type,
|
|
9464
|
+
token=token,
|
|
9465
|
+
)
|
|
9466
|
+
|
|
9467
|
+
return [
|
|
9468
|
+
CommitOperationAdd(
|
|
9469
|
+
path_or_fileobj=relpath_to_abspath[relpath], # absolute path on disk
|
|
9470
|
+
path_in_repo=prefix + relpath, # "absolute" path in repo
|
|
9471
|
+
)
|
|
9472
|
+
for relpath in filtered_repo_objects
|
|
9473
|
+
]
|
|
9474
|
+
|
|
9475
|
+
def _validate_yaml(self, content: str, *, repo_type: Optional[str] = None, token: Union[bool, str, None] = None):
|
|
9476
|
+
"""
|
|
9477
|
+
Validate YAML from `README.md`, used before file hashing and upload.
|
|
9478
|
+
|
|
9479
|
+
Args:
|
|
9480
|
+
content (`str`):
|
|
9481
|
+
Content of `README.md` to validate.
|
|
9482
|
+
repo_type (`str`, *optional*):
|
|
9483
|
+
The type of the repo to grant access to. Must be one of `model`, `dataset` or `space`.
|
|
9484
|
+
Defaults to `model`.
|
|
9485
|
+
token (Union[bool, str, None], optional):
|
|
9486
|
+
A valid user access token (string). Defaults to the locally saved
|
|
9487
|
+
token, which is the recommended method for authentication (see
|
|
9488
|
+
https://huggingface.co/docs/huggingface_hub/quick-start#authentication).
|
|
9489
|
+
To disable authentication, pass `False`.
|
|
9490
|
+
|
|
9491
|
+
Raises:
|
|
9492
|
+
- [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError)
|
|
9493
|
+
if YAML is invalid
|
|
9494
|
+
"""
|
|
9495
|
+
repo_type = repo_type if repo_type is not None else constants.REPO_TYPE_MODEL
|
|
9496
|
+
headers = self._build_hf_headers(token=token)
|
|
9497
|
+
|
|
9498
|
+
response = get_session().post(
|
|
9499
|
+
f"{self.endpoint}/api/validate-yaml",
|
|
9500
|
+
json={"content": content, "repoType": repo_type},
|
|
9501
|
+
headers=headers,
|
|
9502
|
+
)
|
|
9503
|
+
# Handle warnings (example: empty metadata)
|
|
9504
|
+
response_content = response.json()
|
|
9505
|
+
message = "\n".join([f"- {warning.get('message')}" for warning in response_content.get("warnings", [])])
|
|
9506
|
+
if message:
|
|
9507
|
+
warnings.warn(f"Warnings while validating metadata in README.md:\n{message}")
|
|
9508
|
+
|
|
9509
|
+
# Raise on errors
|
|
9510
|
+
try:
|
|
9511
|
+
hf_raise_for_status(response)
|
|
9512
|
+
except BadRequestError as e:
|
|
9513
|
+
errors = response_content.get("errors", [])
|
|
9514
|
+
message = "\n".join([f"- {error.get('message')}" for error in errors])
|
|
9515
|
+
raise ValueError(f"Invalid metadata in README.md.\n{message}") from e
|
|
9516
|
+
|
|
9517
|
+
def get_user_overview(self, username: str, token: Union[bool, str, None] = None) -> User:
|
|
9122
9518
|
"""
|
|
9123
9519
|
Get an overview of a user on the Hub.
|
|
9124
9520
|
|
|
9125
9521
|
Args:
|
|
9126
9522
|
username (`str`):
|
|
9127
9523
|
Username of the user to get an overview of.
|
|
9524
|
+
token (Union[bool, str, None], optional):
|
|
9525
|
+
A valid user access token (string). Defaults to the locally saved
|
|
9526
|
+
token, which is the recommended method for authentication (see
|
|
9527
|
+
https://huggingface.co/docs/huggingface_hub/quick-start#authentication).
|
|
9528
|
+
To disable authentication, pass `False`.
|
|
9128
9529
|
|
|
9129
9530
|
Returns:
|
|
9130
9531
|
`User`: A [`User`] object with the user's overview.
|
|
@@ -9133,18 +9534,24 @@ class HfApi:
|
|
|
9133
9534
|
[`HTTPError`](https://requests.readthedocs.io/en/latest/api/#requests.HTTPError):
|
|
9134
9535
|
HTTP 404 If the user does not exist on the Hub.
|
|
9135
9536
|
"""
|
|
9136
|
-
r = get_session().get(
|
|
9137
|
-
|
|
9537
|
+
r = get_session().get(
|
|
9538
|
+
f"{constants.ENDPOINT}/api/users/{username}/overview", headers=self._build_hf_headers(token=token)
|
|
9539
|
+
)
|
|
9138
9540
|
hf_raise_for_status(r)
|
|
9139
9541
|
return User(**r.json())
|
|
9140
9542
|
|
|
9141
|
-
def list_organization_members(self, organization: str) -> Iterable[User]:
|
|
9543
|
+
def list_organization_members(self, organization: str, token: Union[bool, str, None] = None) -> Iterable[User]:
|
|
9142
9544
|
"""
|
|
9143
9545
|
List of members of an organization on the Hub.
|
|
9144
9546
|
|
|
9145
9547
|
Args:
|
|
9146
9548
|
organization (`str`):
|
|
9147
9549
|
Name of the organization to get the members of.
|
|
9550
|
+
token (Union[bool, str, None], optional):
|
|
9551
|
+
A valid user access token (string). Defaults to the locally saved
|
|
9552
|
+
token, which is the recommended method for authentication (see
|
|
9553
|
+
https://huggingface.co/docs/huggingface_hub/quick-start#authentication).
|
|
9554
|
+
To disable authentication, pass `False`.
|
|
9148
9555
|
|
|
9149
9556
|
Returns:
|
|
9150
9557
|
`Iterable[User]`: A list of [`User`] objects with the members of the organization.
|
|
@@ -9154,21 +9561,25 @@ class HfApi:
|
|
|
9154
9561
|
HTTP 404 If the organization does not exist on the Hub.
|
|
9155
9562
|
|
|
9156
9563
|
"""
|
|
9157
|
-
|
|
9158
|
-
|
|
9159
|
-
|
|
9160
|
-
|
|
9161
|
-
|
|
9162
|
-
for member in r.json():
|
|
9564
|
+
for member in paginate(
|
|
9565
|
+
path=f"{constants.ENDPOINT}/api/organizations/{organization}/members",
|
|
9566
|
+
params={},
|
|
9567
|
+
headers=self._build_hf_headers(token=token),
|
|
9568
|
+
):
|
|
9163
9569
|
yield User(**member)
|
|
9164
9570
|
|
|
9165
|
-
def list_user_followers(self, username: str) -> Iterable[User]:
|
|
9571
|
+
def list_user_followers(self, username: str, token: Union[bool, str, None] = None) -> Iterable[User]:
|
|
9166
9572
|
"""
|
|
9167
9573
|
Get the list of followers of a user on the Hub.
|
|
9168
9574
|
|
|
9169
9575
|
Args:
|
|
9170
9576
|
username (`str`):
|
|
9171
9577
|
Username of the user to get the followers of.
|
|
9578
|
+
token (Union[bool, str, None], optional):
|
|
9579
|
+
A valid user access token (string). Defaults to the locally saved
|
|
9580
|
+
token, which is the recommended method for authentication (see
|
|
9581
|
+
https://huggingface.co/docs/huggingface_hub/quick-start#authentication).
|
|
9582
|
+
To disable authentication, pass `False`.
|
|
9172
9583
|
|
|
9173
9584
|
Returns:
|
|
9174
9585
|
`Iterable[User]`: A list of [`User`] objects with the followers of the user.
|
|
@@ -9178,21 +9589,25 @@ class HfApi:
|
|
|
9178
9589
|
HTTP 404 If the user does not exist on the Hub.
|
|
9179
9590
|
|
|
9180
9591
|
"""
|
|
9181
|
-
|
|
9182
|
-
|
|
9183
|
-
|
|
9184
|
-
|
|
9185
|
-
|
|
9186
|
-
for follower in r.json():
|
|
9592
|
+
for follower in paginate(
|
|
9593
|
+
path=f"{constants.ENDPOINT}/api/users/{username}/followers",
|
|
9594
|
+
params={},
|
|
9595
|
+
headers=self._build_hf_headers(token=token),
|
|
9596
|
+
):
|
|
9187
9597
|
yield User(**follower)
|
|
9188
9598
|
|
|
9189
|
-
def list_user_following(self, username: str) -> Iterable[User]:
|
|
9599
|
+
def list_user_following(self, username: str, token: Union[bool, str, None] = None) -> Iterable[User]:
|
|
9190
9600
|
"""
|
|
9191
9601
|
Get the list of users followed by a user on the Hub.
|
|
9192
9602
|
|
|
9193
9603
|
Args:
|
|
9194
9604
|
username (`str`):
|
|
9195
9605
|
Username of the user to get the users followed by.
|
|
9606
|
+
token (Union[bool, str, None], optional):
|
|
9607
|
+
A valid user access token (string). Defaults to the locally saved
|
|
9608
|
+
token, which is the recommended method for authentication (see
|
|
9609
|
+
https://huggingface.co/docs/huggingface_hub/quick-start#authentication).
|
|
9610
|
+
To disable authentication, pass `False`.
|
|
9196
9611
|
|
|
9197
9612
|
Returns:
|
|
9198
9613
|
`Iterable[User]`: A list of [`User`] objects with the users followed by the user.
|
|
@@ -9202,49 +9617,76 @@ class HfApi:
|
|
|
9202
9617
|
HTTP 404 If the user does not exist on the Hub.
|
|
9203
9618
|
|
|
9204
9619
|
"""
|
|
9620
|
+
for followed_user in paginate(
|
|
9621
|
+
path=f"{constants.ENDPOINT}/api/users/{username}/following",
|
|
9622
|
+
params={},
|
|
9623
|
+
headers=self._build_hf_headers(token=token),
|
|
9624
|
+
):
|
|
9625
|
+
yield User(**followed_user)
|
|
9205
9626
|
|
|
9206
|
-
|
|
9627
|
+
def auth_check(
|
|
9628
|
+
self, repo_id: str, *, repo_type: Optional[str] = None, token: Union[bool, str, None] = None
|
|
9629
|
+
) -> None:
|
|
9630
|
+
"""
|
|
9631
|
+
Check if the provided user token has access to a specific repository on the Hugging Face Hub.
|
|
9207
9632
|
|
|
9208
|
-
|
|
9633
|
+
This method verifies whether the user, authenticated via the provided token, has access to the specified
|
|
9634
|
+
repository. If the repository is not found or if the user lacks the required permissions to access it,
|
|
9635
|
+
the method raises an appropriate exception.
|
|
9209
9636
|
|
|
9210
|
-
|
|
9211
|
-
|
|
9637
|
+
Args:
|
|
9638
|
+
repo_id (`str`):
|
|
9639
|
+
The repository to check for access. Format should be `"user/repo_name"`.
|
|
9640
|
+
Example: `"user/my-cool-model"`.
|
|
9641
|
+
|
|
9642
|
+
repo_type (`str`, *optional*):
|
|
9643
|
+
The type of the repository. Should be one of `"model"`, `"dataset"`, or `"space"`.
|
|
9644
|
+
If not specified, the default is `"model"`.
|
|
9212
9645
|
|
|
9646
|
+
token `(Union[bool, str, None]`, *optional*):
|
|
9647
|
+
A valid user access token. If not provided, the locally saved token will be used, which is the
|
|
9648
|
+
recommended authentication method. Set to `False` to disable authentication.
|
|
9649
|
+
Refer to: https://huggingface.co/docs/huggingface_hub/quick-start#authentication.
|
|
9213
9650
|
|
|
9214
|
-
|
|
9215
|
-
|
|
9216
|
-
|
|
9217
|
-
|
|
9218
|
-
|
|
9219
|
-
) -> List[CommitOperationAdd]:
|
|
9220
|
-
"""Generate the list of Add operations for a commit to upload a folder.
|
|
9651
|
+
Raises:
|
|
9652
|
+
[`~utils.RepositoryNotFoundError`]:
|
|
9653
|
+
Raised if the repository does not exist, is private, or the user does not have access. This can
|
|
9654
|
+
occur if the `repo_id` or `repo_type` is incorrect or if the repository is private but the user
|
|
9655
|
+
is not authenticated.
|
|
9221
9656
|
|
|
9222
|
-
|
|
9223
|
-
|
|
9224
|
-
|
|
9225
|
-
|
|
9226
|
-
|
|
9227
|
-
|
|
9228
|
-
|
|
9229
|
-
|
|
9230
|
-
|
|
9231
|
-
|
|
9232
|
-
|
|
9233
|
-
|
|
9234
|
-
|
|
9235
|
-
|
|
9236
|
-
|
|
9237
|
-
|
|
9238
|
-
|
|
9239
|
-
|
|
9240
|
-
|
|
9241
|
-
|
|
9242
|
-
|
|
9243
|
-
|
|
9244
|
-
|
|
9245
|
-
|
|
9246
|
-
|
|
9247
|
-
|
|
9657
|
+
[`~utils.GatedRepoError`]:
|
|
9658
|
+
Raised if the repository exists but is gated and the user is not authorized to access it.
|
|
9659
|
+
|
|
9660
|
+
Example:
|
|
9661
|
+
Check if the user has access to a repository:
|
|
9662
|
+
|
|
9663
|
+
```python
|
|
9664
|
+
>>> from huggingface_hub import auth_check
|
|
9665
|
+
>>> from huggingface_hub.utils import GatedRepoError, RepositoryNotFoundError
|
|
9666
|
+
|
|
9667
|
+
try:
|
|
9668
|
+
auth_check("user/my-cool-model")
|
|
9669
|
+
except GatedRepoError:
|
|
9670
|
+
# Handle gated repository error
|
|
9671
|
+
print("You do not have permission to access this gated repository.")
|
|
9672
|
+
except RepositoryNotFoundError:
|
|
9673
|
+
# Handle repository not found error
|
|
9674
|
+
print("The repository was not found or you do not have access.")
|
|
9675
|
+
```
|
|
9676
|
+
|
|
9677
|
+
In this example:
|
|
9678
|
+
- If the user has access, the method completes successfully.
|
|
9679
|
+
- If the repository is gated or does not exist, appropriate exceptions are raised, allowing the user
|
|
9680
|
+
to handle them accordingly.
|
|
9681
|
+
"""
|
|
9682
|
+
headers = self._build_hf_headers(token=token)
|
|
9683
|
+
if repo_type is None:
|
|
9684
|
+
repo_type = constants.REPO_TYPE_MODEL
|
|
9685
|
+
if repo_type not in constants.REPO_TYPES:
|
|
9686
|
+
raise ValueError(f"Invalid repo type, must be one of {constants.REPO_TYPES}")
|
|
9687
|
+
path = f"{self.endpoint}/api/{repo_type}s/{repo_id}/auth-check"
|
|
9688
|
+
r = get_session().get(path, headers=headers)
|
|
9689
|
+
hf_raise_for_status(r)
|
|
9248
9690
|
|
|
9249
9691
|
|
|
9250
9692
|
def _parse_revision_from_pr_url(pr_url: str) -> str:
|
|
@@ -9265,6 +9707,7 @@ def _parse_revision_from_pr_url(pr_url: str) -> str:
|
|
|
9265
9707
|
api = HfApi()
|
|
9266
9708
|
|
|
9267
9709
|
whoami = api.whoami
|
|
9710
|
+
auth_check = api.auth_check
|
|
9268
9711
|
get_token_permission = api.get_token_permission
|
|
9269
9712
|
|
|
9270
9713
|
list_models = api.list_models
|
|
@@ -9285,7 +9728,6 @@ list_repo_refs = api.list_repo_refs
|
|
|
9285
9728
|
list_repo_commits = api.list_repo_commits
|
|
9286
9729
|
list_repo_tree = api.list_repo_tree
|
|
9287
9730
|
get_paths_info = api.get_paths_info
|
|
9288
|
-
|
|
9289
9731
|
list_metrics = api.list_metrics
|
|
9290
9732
|
|
|
9291
9733
|
get_model_tags = api.get_model_tags
|
|
@@ -9295,6 +9737,7 @@ create_commit = api.create_commit
|
|
|
9295
9737
|
create_repo = api.create_repo
|
|
9296
9738
|
delete_repo = api.delete_repo
|
|
9297
9739
|
update_repo_visibility = api.update_repo_visibility
|
|
9740
|
+
update_repo_settings = api.update_repo_settings
|
|
9298
9741
|
super_squash_history = api.super_squash_history
|
|
9299
9742
|
move_repo = api.move_repo
|
|
9300
9743
|
upload_file = api.upload_file
|
|
@@ -9303,6 +9746,7 @@ delete_file = api.delete_file
|
|
|
9303
9746
|
delete_folder = api.delete_folder
|
|
9304
9747
|
delete_files = api.delete_files
|
|
9305
9748
|
create_commits_on_pr = api.create_commits_on_pr
|
|
9749
|
+
upload_large_folder = api.upload_large_folder
|
|
9306
9750
|
preupload_lfs_files = api.preupload_lfs_files
|
|
9307
9751
|
create_branch = api.create_branch
|
|
9308
9752
|
delete_branch = api.delete_branch
|