huggingface-hub 0.35.1__py3-none-any.whl → 1.0.0rc1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of huggingface-hub might be problematic. Click here for more details.
- huggingface_hub/__init__.py +28 -45
- huggingface_hub/_commit_api.py +28 -28
- huggingface_hub/_commit_scheduler.py +11 -8
- huggingface_hub/_inference_endpoints.py +8 -8
- huggingface_hub/_jobs_api.py +20 -20
- huggingface_hub/_login.py +13 -39
- huggingface_hub/_oauth.py +8 -8
- huggingface_hub/_snapshot_download.py +14 -28
- huggingface_hub/_space_api.py +4 -4
- huggingface_hub/_tensorboard_logger.py +5 -5
- huggingface_hub/_upload_large_folder.py +15 -15
- huggingface_hub/_webhooks_payload.py +3 -3
- huggingface_hub/_webhooks_server.py +2 -2
- huggingface_hub/cli/__init__.py +0 -14
- huggingface_hub/cli/_cli_utils.py +80 -3
- huggingface_hub/cli/auth.py +104 -150
- huggingface_hub/cli/cache.py +102 -126
- huggingface_hub/cli/download.py +93 -110
- huggingface_hub/cli/hf.py +37 -41
- huggingface_hub/cli/jobs.py +689 -1017
- huggingface_hub/cli/lfs.py +120 -143
- huggingface_hub/cli/repo.py +158 -216
- huggingface_hub/cli/repo_files.py +50 -84
- huggingface_hub/cli/system.py +6 -25
- huggingface_hub/cli/upload.py +198 -212
- huggingface_hub/cli/upload_large_folder.py +90 -105
- huggingface_hub/commands/_cli_utils.py +2 -2
- huggingface_hub/commands/delete_cache.py +11 -11
- huggingface_hub/commands/download.py +4 -13
- huggingface_hub/commands/lfs.py +4 -4
- huggingface_hub/commands/repo_files.py +2 -2
- huggingface_hub/commands/tag.py +1 -3
- huggingface_hub/commands/upload.py +4 -4
- huggingface_hub/commands/upload_large_folder.py +3 -3
- huggingface_hub/commands/user.py +4 -5
- huggingface_hub/community.py +5 -5
- huggingface_hub/constants.py +3 -41
- huggingface_hub/dataclasses.py +16 -22
- huggingface_hub/errors.py +43 -30
- huggingface_hub/fastai_utils.py +8 -9
- huggingface_hub/file_download.py +154 -253
- huggingface_hub/hf_api.py +329 -558
- huggingface_hub/hf_file_system.py +104 -62
- huggingface_hub/hub_mixin.py +32 -54
- huggingface_hub/inference/_client.py +178 -163
- huggingface_hub/inference/_common.py +38 -54
- huggingface_hub/inference/_generated/_async_client.py +219 -259
- huggingface_hub/inference/_generated/types/automatic_speech_recognition.py +3 -3
- huggingface_hub/inference/_generated/types/base.py +10 -7
- huggingface_hub/inference/_generated/types/chat_completion.py +16 -16
- huggingface_hub/inference/_generated/types/depth_estimation.py +2 -2
- huggingface_hub/inference/_generated/types/document_question_answering.py +2 -2
- huggingface_hub/inference/_generated/types/feature_extraction.py +2 -2
- huggingface_hub/inference/_generated/types/fill_mask.py +2 -2
- huggingface_hub/inference/_generated/types/sentence_similarity.py +3 -3
- huggingface_hub/inference/_generated/types/summarization.py +2 -2
- huggingface_hub/inference/_generated/types/table_question_answering.py +4 -4
- huggingface_hub/inference/_generated/types/text2text_generation.py +2 -2
- huggingface_hub/inference/_generated/types/text_generation.py +10 -10
- huggingface_hub/inference/_generated/types/text_to_video.py +2 -2
- huggingface_hub/inference/_generated/types/token_classification.py +2 -2
- huggingface_hub/inference/_generated/types/translation.py +2 -2
- huggingface_hub/inference/_generated/types/zero_shot_classification.py +2 -2
- huggingface_hub/inference/_generated/types/zero_shot_image_classification.py +2 -2
- huggingface_hub/inference/_generated/types/zero_shot_object_detection.py +1 -3
- huggingface_hub/inference/_mcp/agent.py +3 -3
- huggingface_hub/inference/_mcp/constants.py +1 -2
- huggingface_hub/inference/_mcp/mcp_client.py +33 -22
- huggingface_hub/inference/_mcp/types.py +10 -10
- huggingface_hub/inference/_mcp/utils.py +4 -4
- huggingface_hub/inference/_providers/__init__.py +2 -13
- huggingface_hub/inference/_providers/_common.py +24 -25
- huggingface_hub/inference/_providers/black_forest_labs.py +6 -6
- huggingface_hub/inference/_providers/cohere.py +3 -3
- huggingface_hub/inference/_providers/fal_ai.py +25 -25
- huggingface_hub/inference/_providers/featherless_ai.py +4 -4
- huggingface_hub/inference/_providers/fireworks_ai.py +3 -3
- huggingface_hub/inference/_providers/hf_inference.py +13 -13
- huggingface_hub/inference/_providers/hyperbolic.py +4 -4
- huggingface_hub/inference/_providers/nebius.py +10 -10
- huggingface_hub/inference/_providers/novita.py +5 -5
- huggingface_hub/inference/_providers/nscale.py +4 -4
- huggingface_hub/inference/_providers/replicate.py +15 -15
- huggingface_hub/inference/_providers/sambanova.py +6 -6
- huggingface_hub/inference/_providers/together.py +7 -7
- huggingface_hub/lfs.py +24 -33
- huggingface_hub/repocard.py +16 -17
- huggingface_hub/repocard_data.py +56 -56
- huggingface_hub/serialization/__init__.py +0 -1
- huggingface_hub/serialization/_base.py +9 -9
- huggingface_hub/serialization/_dduf.py +7 -7
- huggingface_hub/serialization/_torch.py +28 -28
- huggingface_hub/utils/__init__.py +10 -4
- huggingface_hub/utils/_auth.py +5 -5
- huggingface_hub/utils/_cache_manager.py +31 -31
- huggingface_hub/utils/_deprecation.py +1 -1
- huggingface_hub/utils/_dotenv.py +3 -3
- huggingface_hub/utils/_fixes.py +0 -10
- huggingface_hub/utils/_git_credential.py +3 -3
- huggingface_hub/utils/_headers.py +7 -29
- huggingface_hub/utils/_http.py +369 -209
- huggingface_hub/utils/_pagination.py +4 -4
- huggingface_hub/utils/_paths.py +5 -5
- huggingface_hub/utils/_runtime.py +15 -13
- huggingface_hub/utils/_safetensors.py +21 -21
- huggingface_hub/utils/_subprocess.py +9 -9
- huggingface_hub/utils/_telemetry.py +3 -3
- huggingface_hub/utils/_typing.py +3 -3
- huggingface_hub/utils/_validators.py +53 -72
- huggingface_hub/utils/_xet.py +16 -16
- huggingface_hub/utils/_xet_progress_reporting.py +1 -1
- huggingface_hub/utils/insecure_hashlib.py +3 -9
- huggingface_hub/utils/tqdm.py +3 -3
- {huggingface_hub-0.35.1.dist-info → huggingface_hub-1.0.0rc1.dist-info}/METADATA +17 -26
- huggingface_hub-1.0.0rc1.dist-info/RECORD +161 -0
- huggingface_hub/inference/_providers/publicai.py +0 -6
- huggingface_hub/inference/_providers/scaleway.py +0 -28
- huggingface_hub/inference_api.py +0 -217
- huggingface_hub/keras_mixin.py +0 -500
- huggingface_hub/repository.py +0 -1477
- huggingface_hub/serialization/_tensorflow.py +0 -95
- huggingface_hub/utils/_hf_folder.py +0 -68
- huggingface_hub-0.35.1.dist-info/RECORD +0 -168
- {huggingface_hub-0.35.1.dist-info → huggingface_hub-1.0.0rc1.dist-info}/LICENSE +0 -0
- {huggingface_hub-0.35.1.dist-info → huggingface_hub-1.0.0rc1.dist-info}/WHEEL +0 -0
- {huggingface_hub-0.35.1.dist-info → huggingface_hub-1.0.0rc1.dist-info}/entry_points.txt +0 -0
- {huggingface_hub-0.35.1.dist-info → huggingface_hub-1.0.0rc1.dist-info}/top_level.txt +0 -0
huggingface_hub/_jobs_api.py
CHANGED
|
@@ -15,7 +15,7 @@
|
|
|
15
15
|
from dataclasses import dataclass
|
|
16
16
|
from datetime import datetime
|
|
17
17
|
from enum import Enum
|
|
18
|
-
from typing import Any,
|
|
18
|
+
from typing import Any, Optional, Union
|
|
19
19
|
|
|
20
20
|
from huggingface_hub import constants
|
|
21
21
|
from huggingface_hub._space_api import SpaceHardware
|
|
@@ -71,13 +71,13 @@ class JobInfo:
|
|
|
71
71
|
space_id (`str` or `None`):
|
|
72
72
|
The Docker image from Hugging Face Spaces used for the Job.
|
|
73
73
|
Can be None if docker_image is present instead.
|
|
74
|
-
command (`
|
|
74
|
+
command (`list[str]` or `None`):
|
|
75
75
|
Command of the Job, e.g. `["python", "-c", "print('hello world')"]`
|
|
76
|
-
arguments (`
|
|
76
|
+
arguments (`list[str]` or `None`):
|
|
77
77
|
Arguments passed to the command
|
|
78
|
-
environment (`
|
|
78
|
+
environment (`dict[str]` or `None`):
|
|
79
79
|
Environment variables of the Job as a dictionary.
|
|
80
|
-
secrets (`
|
|
80
|
+
secrets (`dict[str]` or `None`):
|
|
81
81
|
Secret environment variables of the Job (encrypted).
|
|
82
82
|
flavor (`str` or `None`):
|
|
83
83
|
Flavor for the hardware, as in Hugging Face Spaces. See [`SpaceHardware`] for possible values.
|
|
@@ -111,10 +111,10 @@ class JobInfo:
|
|
|
111
111
|
created_at: Optional[datetime]
|
|
112
112
|
docker_image: Optional[str]
|
|
113
113
|
space_id: Optional[str]
|
|
114
|
-
command: Optional[
|
|
115
|
-
arguments: Optional[
|
|
116
|
-
environment: Optional[
|
|
117
|
-
secrets: Optional[
|
|
114
|
+
command: Optional[list[str]]
|
|
115
|
+
arguments: Optional[list[str]]
|
|
116
|
+
environment: Optional[dict[str, Any]]
|
|
117
|
+
secrets: Optional[dict[str, Any]]
|
|
118
118
|
flavor: Optional[SpaceHardware]
|
|
119
119
|
status: JobStatus
|
|
120
120
|
owner: JobOwner
|
|
@@ -148,13 +148,13 @@ class JobInfo:
|
|
|
148
148
|
class JobSpec:
|
|
149
149
|
docker_image: Optional[str]
|
|
150
150
|
space_id: Optional[str]
|
|
151
|
-
command: Optional[
|
|
152
|
-
arguments: Optional[
|
|
153
|
-
environment: Optional[
|
|
154
|
-
secrets: Optional[
|
|
151
|
+
command: Optional[list[str]]
|
|
152
|
+
arguments: Optional[list[str]]
|
|
153
|
+
environment: Optional[dict[str, Any]]
|
|
154
|
+
secrets: Optional[dict[str, Any]]
|
|
155
155
|
flavor: Optional[SpaceHardware]
|
|
156
156
|
timeout: Optional[int]
|
|
157
|
-
tags: Optional[
|
|
157
|
+
tags: Optional[list[str]]
|
|
158
158
|
arch: Optional[str]
|
|
159
159
|
|
|
160
160
|
def __init__(self, **kwargs) -> None:
|
|
@@ -202,7 +202,7 @@ class ScheduledJobInfo:
|
|
|
202
202
|
Scheduled Job ID.
|
|
203
203
|
created_at (`datetime` or `None`):
|
|
204
204
|
When the scheduled Job was created.
|
|
205
|
-
tags (`
|
|
205
|
+
tags (`list[str]` or `None`):
|
|
206
206
|
The tags of the scheduled Job.
|
|
207
207
|
schedule (`str` or `None`):
|
|
208
208
|
One of "@annually", "@yearly", "@monthly", "@weekly", "@daily", "@hourly", or a
|
|
@@ -263,14 +263,14 @@ class ScheduledJobInfo:
|
|
|
263
263
|
def _create_job_spec(
|
|
264
264
|
*,
|
|
265
265
|
image: str,
|
|
266
|
-
command:
|
|
267
|
-
env: Optional[
|
|
268
|
-
secrets: Optional[
|
|
266
|
+
command: list[str],
|
|
267
|
+
env: Optional[dict[str, Any]],
|
|
268
|
+
secrets: Optional[dict[str, Any]],
|
|
269
269
|
flavor: Optional[SpaceHardware],
|
|
270
270
|
timeout: Optional[Union[int, float, str]],
|
|
271
|
-
) ->
|
|
271
|
+
) -> dict[str, Any]:
|
|
272
272
|
# prepare job spec to send to HF Jobs API
|
|
273
|
-
job_spec:
|
|
273
|
+
job_spec: dict[str, Any] = {
|
|
274
274
|
"command": command,
|
|
275
275
|
"arguments": [],
|
|
276
276
|
"environment": env or {},
|
huggingface_hub/_login.py
CHANGED
|
@@ -41,7 +41,6 @@ from .utils._auth import (
|
|
|
41
41
|
_save_token,
|
|
42
42
|
get_stored_tokens,
|
|
43
43
|
)
|
|
44
|
-
from .utils._deprecation import _deprecate_arguments, _deprecate_positional_args
|
|
45
44
|
|
|
46
45
|
|
|
47
46
|
logger = logging.get_logger(__name__)
|
|
@@ -55,18 +54,11 @@ _HF_LOGO_ASCII = """
|
|
|
55
54
|
"""
|
|
56
55
|
|
|
57
56
|
|
|
58
|
-
@_deprecate_arguments(
|
|
59
|
-
version="1.0",
|
|
60
|
-
deprecated_args="write_permission",
|
|
61
|
-
custom_message="Fine-grained tokens added complexity to the permissions, making it irrelevant to check if a token has 'write' access.",
|
|
62
|
-
)
|
|
63
|
-
@_deprecate_positional_args(version="1.0")
|
|
64
57
|
def login(
|
|
65
58
|
token: Optional[str] = None,
|
|
66
59
|
*,
|
|
67
60
|
add_to_git_credential: bool = False,
|
|
68
|
-
|
|
69
|
-
write_permission: bool = False,
|
|
61
|
+
skip_if_logged_in: bool = False,
|
|
70
62
|
) -> None:
|
|
71
63
|
"""Login the machine to access the Hub.
|
|
72
64
|
|
|
@@ -102,10 +94,8 @@ def login(
|
|
|
102
94
|
is configured, a warning will be displayed to the user. If `token` is `None`,
|
|
103
95
|
the value of `add_to_git_credential` is ignored and will be prompted again
|
|
104
96
|
to the end user.
|
|
105
|
-
|
|
106
|
-
If `True`,
|
|
107
|
-
write_permission (`bool`):
|
|
108
|
-
Ignored and deprecated argument.
|
|
97
|
+
skip_if_logged_in (`bool`, defaults to `False`):
|
|
98
|
+
If `True`, do not prompt for token if user is already logged in.
|
|
109
99
|
Raises:
|
|
110
100
|
[`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError)
|
|
111
101
|
If an organization token is passed. Only personal account tokens are valid
|
|
@@ -125,9 +115,9 @@ def login(
|
|
|
125
115
|
)
|
|
126
116
|
_login(token, add_to_git_credential=add_to_git_credential)
|
|
127
117
|
elif is_notebook():
|
|
128
|
-
notebook_login(
|
|
118
|
+
notebook_login(skip_if_logged_in=skip_if_logged_in)
|
|
129
119
|
else:
|
|
130
|
-
interpreter_login(
|
|
120
|
+
interpreter_login(skip_if_logged_in=skip_if_logged_in)
|
|
131
121
|
|
|
132
122
|
|
|
133
123
|
def logout(token_name: Optional[str] = None) -> None:
|
|
@@ -242,13 +232,7 @@ def auth_list() -> None:
|
|
|
242
232
|
###
|
|
243
233
|
|
|
244
234
|
|
|
245
|
-
|
|
246
|
-
version="1.0",
|
|
247
|
-
deprecated_args="write_permission",
|
|
248
|
-
custom_message="Fine-grained tokens added complexity to the permissions, making it irrelevant to check if a token has 'write' access.",
|
|
249
|
-
)
|
|
250
|
-
@_deprecate_positional_args(version="1.0")
|
|
251
|
-
def interpreter_login(*, new_session: bool = True, write_permission: bool = False) -> None:
|
|
235
|
+
def interpreter_login(*, skip_if_logged_in: bool = False) -> None:
|
|
252
236
|
"""
|
|
253
237
|
Displays a prompt to log in to the HF website and store the token.
|
|
254
238
|
|
|
@@ -259,12 +243,10 @@ def interpreter_login(*, new_session: bool = True, write_permission: bool = Fals
|
|
|
259
243
|
For more details, see [`login`].
|
|
260
244
|
|
|
261
245
|
Args:
|
|
262
|
-
|
|
263
|
-
If `True`,
|
|
264
|
-
write_permission (`bool`):
|
|
265
|
-
Ignored and deprecated argument.
|
|
246
|
+
skip_if_logged_in (`bool`, defaults to `False`):
|
|
247
|
+
If `True`, do not prompt for token if user is already logged in.
|
|
266
248
|
"""
|
|
267
|
-
if not
|
|
249
|
+
if not skip_if_logged_in and get_token() is not None:
|
|
268
250
|
logger.info("User is already logged in.")
|
|
269
251
|
return
|
|
270
252
|
|
|
@@ -314,13 +296,7 @@ NOTEBOOK_LOGIN_TOKEN_HTML_END = """
|
|
|
314
296
|
notebooks. </center>"""
|
|
315
297
|
|
|
316
298
|
|
|
317
|
-
|
|
318
|
-
version="1.0",
|
|
319
|
-
deprecated_args="write_permission",
|
|
320
|
-
custom_message="Fine-grained tokens added complexity to the permissions, making it irrelevant to check if a token has 'write' access.",
|
|
321
|
-
)
|
|
322
|
-
@_deprecate_positional_args(version="1.0")
|
|
323
|
-
def notebook_login(*, new_session: bool = True, write_permission: bool = False) -> None:
|
|
299
|
+
def notebook_login(*, skip_if_logged_in: bool = False) -> None:
|
|
324
300
|
"""
|
|
325
301
|
Displays a widget to log in to the HF website and store the token.
|
|
326
302
|
|
|
@@ -331,10 +307,8 @@ def notebook_login(*, new_session: bool = True, write_permission: bool = False)
|
|
|
331
307
|
For more details, see [`login`].
|
|
332
308
|
|
|
333
309
|
Args:
|
|
334
|
-
|
|
335
|
-
If `True`,
|
|
336
|
-
write_permission (`bool`):
|
|
337
|
-
Ignored and deprecated argument.
|
|
310
|
+
skip_if_logged_in (`bool`, defaults to `False`):
|
|
311
|
+
If `True`, do not prompt for token if user is already logged in.
|
|
338
312
|
"""
|
|
339
313
|
try:
|
|
340
314
|
import ipywidgets.widgets as widgets # type: ignore
|
|
@@ -344,7 +318,7 @@ def notebook_login(*, new_session: bool = True, write_permission: bool = False)
|
|
|
344
318
|
"The `notebook_login` function can only be used in a notebook (Jupyter or"
|
|
345
319
|
" Colab) and you need the `ipywidgets` module: `pip install ipywidgets`."
|
|
346
320
|
)
|
|
347
|
-
if not
|
|
321
|
+
if not skip_if_logged_in and get_token() is not None:
|
|
348
322
|
logger.info("User is already logged in.")
|
|
349
323
|
return
|
|
350
324
|
|
huggingface_hub/_oauth.py
CHANGED
|
@@ -6,7 +6,7 @@ import time
|
|
|
6
6
|
import urllib.parse
|
|
7
7
|
import warnings
|
|
8
8
|
from dataclasses import dataclass
|
|
9
|
-
from typing import TYPE_CHECKING,
|
|
9
|
+
from typing import TYPE_CHECKING, Literal, Optional, Union
|
|
10
10
|
|
|
11
11
|
from . import constants
|
|
12
12
|
from .hf_api import whoami
|
|
@@ -39,7 +39,7 @@ class OAuthOrgInfo:
|
|
|
39
39
|
Whether the org has a payment method set up. Hugging Face field.
|
|
40
40
|
role_in_org (`Optional[str]`, *optional*):
|
|
41
41
|
The user's role in the org. Hugging Face field.
|
|
42
|
-
security_restrictions (`Optional[
|
|
42
|
+
security_restrictions (`Optional[list[Literal["ip", "token-policy", "mfa", "sso"]]]`, *optional*):
|
|
43
43
|
Array of security restrictions that the user hasn't completed for this org. Possible values: "ip", "token-policy", "mfa", "sso". Hugging Face field.
|
|
44
44
|
"""
|
|
45
45
|
|
|
@@ -50,7 +50,7 @@ class OAuthOrgInfo:
|
|
|
50
50
|
is_enterprise: bool
|
|
51
51
|
can_pay: Optional[bool] = None
|
|
52
52
|
role_in_org: Optional[str] = None
|
|
53
|
-
security_restrictions: Optional[
|
|
53
|
+
security_restrictions: Optional[list[Literal["ip", "token-policy", "mfa", "sso"]]] = None
|
|
54
54
|
|
|
55
55
|
|
|
56
56
|
@dataclass
|
|
@@ -79,7 +79,7 @@ class OAuthUserInfo:
|
|
|
79
79
|
Whether the user is a pro user. Hugging Face field.
|
|
80
80
|
can_pay (`Optional[bool]`, *optional*):
|
|
81
81
|
Whether the user has a payment method set up. Hugging Face field.
|
|
82
|
-
orgs (`Optional[
|
|
82
|
+
orgs (`Optional[list[OrgInfo]]`, *optional*):
|
|
83
83
|
List of organizations the user is part of. Hugging Face field.
|
|
84
84
|
"""
|
|
85
85
|
|
|
@@ -93,7 +93,7 @@ class OAuthUserInfo:
|
|
|
93
93
|
website: Optional[str]
|
|
94
94
|
is_pro: bool
|
|
95
95
|
can_pay: Optional[bool]
|
|
96
|
-
orgs: Optional[
|
|
96
|
+
orgs: Optional[list[OAuthOrgInfo]]
|
|
97
97
|
|
|
98
98
|
|
|
99
99
|
@dataclass
|
|
@@ -306,7 +306,7 @@ def _add_oauth_routes(app: "fastapi.FastAPI", route_prefix: str) -> None:
|
|
|
306
306
|
target_url = request.query_params.get("_target_url")
|
|
307
307
|
|
|
308
308
|
# Build redirect URI with the same query params as before and bump nb_redirects count
|
|
309
|
-
query_params:
|
|
309
|
+
query_params: dict[str, Union[int, str]] = {"_nb_redirects": nb_redirects + 1}
|
|
310
310
|
if target_url:
|
|
311
311
|
query_params["_target_url"] = target_url
|
|
312
312
|
|
|
@@ -406,7 +406,7 @@ def _get_redirect_target(request: "fastapi.Request", default_target: str = "/")
|
|
|
406
406
|
return request.query_params.get("_target_url", default_target)
|
|
407
407
|
|
|
408
408
|
|
|
409
|
-
def _get_mocked_oauth_info() ->
|
|
409
|
+
def _get_mocked_oauth_info() -> dict:
|
|
410
410
|
token = get_token()
|
|
411
411
|
if token is None:
|
|
412
412
|
raise ValueError(
|
|
@@ -449,7 +449,7 @@ def _get_mocked_oauth_info() -> Dict:
|
|
|
449
449
|
}
|
|
450
450
|
|
|
451
451
|
|
|
452
|
-
def _get_oauth_uris(route_prefix: str = "/") ->
|
|
452
|
+
def _get_oauth_uris(route_prefix: str = "/") -> tuple[str, str, str]:
|
|
453
453
|
route_prefix = route_prefix.strip("/")
|
|
454
454
|
if route_prefix:
|
|
455
455
|
route_prefix = f"/{route_prefix}"
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
import os
|
|
2
2
|
from pathlib import Path
|
|
3
|
-
from typing import
|
|
3
|
+
from typing import Iterable, Optional, Union
|
|
4
4
|
|
|
5
|
-
import
|
|
5
|
+
import httpx
|
|
6
6
|
from tqdm.auto import tqdm as base_tqdm
|
|
7
7
|
from tqdm.contrib.concurrent import thread_map
|
|
8
8
|
|
|
@@ -35,21 +35,17 @@ def snapshot_download(
|
|
|
35
35
|
local_dir: Union[str, Path, None] = None,
|
|
36
36
|
library_name: Optional[str] = None,
|
|
37
37
|
library_version: Optional[str] = None,
|
|
38
|
-
user_agent: Optional[Union[
|
|
39
|
-
proxies: Optional[Dict] = None,
|
|
38
|
+
user_agent: Optional[Union[dict, str]] = None,
|
|
40
39
|
etag_timeout: float = constants.DEFAULT_ETAG_TIMEOUT,
|
|
41
40
|
force_download: bool = False,
|
|
42
41
|
token: Optional[Union[bool, str]] = None,
|
|
43
42
|
local_files_only: bool = False,
|
|
44
|
-
allow_patterns: Optional[Union[
|
|
45
|
-
ignore_patterns: Optional[Union[
|
|
43
|
+
allow_patterns: Optional[Union[list[str], str]] = None,
|
|
44
|
+
ignore_patterns: Optional[Union[list[str], str]] = None,
|
|
46
45
|
max_workers: int = 8,
|
|
47
|
-
tqdm_class: Optional[
|
|
48
|
-
headers: Optional[
|
|
46
|
+
tqdm_class: Optional[type[base_tqdm]] = None,
|
|
47
|
+
headers: Optional[dict[str, str]] = None,
|
|
49
48
|
endpoint: Optional[str] = None,
|
|
50
|
-
# Deprecated args
|
|
51
|
-
local_dir_use_symlinks: Union[bool, Literal["auto"]] = "auto",
|
|
52
|
-
resume_download: Optional[bool] = None,
|
|
53
49
|
) -> str:
|
|
54
50
|
"""Download repo files.
|
|
55
51
|
|
|
@@ -85,12 +81,9 @@ def snapshot_download(
|
|
|
85
81
|
The version of the library.
|
|
86
82
|
user_agent (`str`, `dict`, *optional*):
|
|
87
83
|
The user-agent info in the form of a dictionary or a string.
|
|
88
|
-
proxies (`dict`, *optional*):
|
|
89
|
-
Dictionary mapping protocol to the URL of the proxy passed to
|
|
90
|
-
`requests.request`.
|
|
91
84
|
etag_timeout (`float`, *optional*, defaults to `10`):
|
|
92
85
|
When fetching ETag, how many seconds to wait for the server to send
|
|
93
|
-
data before giving up which is passed to `
|
|
86
|
+
data before giving up which is passed to `httpx.request`.
|
|
94
87
|
force_download (`bool`, *optional*, defaults to `False`):
|
|
95
88
|
Whether the file should be downloaded even if it already exists in the local cache.
|
|
96
89
|
token (`str`, `bool`, *optional*):
|
|
@@ -103,9 +96,9 @@ def snapshot_download(
|
|
|
103
96
|
local_files_only (`bool`, *optional*, defaults to `False`):
|
|
104
97
|
If `True`, avoid downloading the file and return the path to the
|
|
105
98
|
local cached file if it exists.
|
|
106
|
-
allow_patterns (`
|
|
99
|
+
allow_patterns (`list[str]` or `str`, *optional*):
|
|
107
100
|
If provided, only files matching at least one pattern are downloaded.
|
|
108
|
-
ignore_patterns (`
|
|
101
|
+
ignore_patterns (`list[str]` or `str`, *optional*):
|
|
109
102
|
If provided, files matching any of the patterns are not downloaded.
|
|
110
103
|
max_workers (`int`, *optional*):
|
|
111
104
|
Number of concurrent threads to download files (1 thread = 1 file download).
|
|
@@ -163,14 +156,10 @@ def snapshot_download(
|
|
|
163
156
|
try:
|
|
164
157
|
# if we have internet connection we want to list files to download
|
|
165
158
|
repo_info = api.repo_info(repo_id=repo_id, repo_type=repo_type, revision=revision)
|
|
166
|
-
except
|
|
167
|
-
# Actually raise
|
|
159
|
+
except httpx.ProxyError:
|
|
160
|
+
# Actually raise on proxy error
|
|
168
161
|
raise
|
|
169
|
-
except (
|
|
170
|
-
requests.exceptions.ConnectionError,
|
|
171
|
-
requests.exceptions.Timeout,
|
|
172
|
-
OfflineModeIsEnabled,
|
|
173
|
-
) as error:
|
|
162
|
+
except (httpx.ConnectError, httpx.TimeoutException, OfflineModeIsEnabled) as error:
|
|
174
163
|
# Internet connection is down
|
|
175
164
|
# => will try to use local files only
|
|
176
165
|
api_call_error = error
|
|
@@ -178,7 +167,7 @@ def snapshot_download(
|
|
|
178
167
|
except RevisionNotFoundError:
|
|
179
168
|
# The repo was found but the revision doesn't exist on the Hub (never existed or got deleted)
|
|
180
169
|
raise
|
|
181
|
-
except
|
|
170
|
+
except HfHubHTTPError as error:
|
|
182
171
|
# Multiple reasons for an http error:
|
|
183
172
|
# - Repository is private and invalid/missing token sent
|
|
184
173
|
# - Repository is gated and invalid/missing token sent
|
|
@@ -311,13 +300,10 @@ def snapshot_download(
|
|
|
311
300
|
endpoint=endpoint,
|
|
312
301
|
cache_dir=cache_dir,
|
|
313
302
|
local_dir=local_dir,
|
|
314
|
-
local_dir_use_symlinks=local_dir_use_symlinks,
|
|
315
303
|
library_name=library_name,
|
|
316
304
|
library_version=library_version,
|
|
317
305
|
user_agent=user_agent,
|
|
318
|
-
proxies=proxies,
|
|
319
306
|
etag_timeout=etag_timeout,
|
|
320
|
-
resume_download=resume_download,
|
|
321
307
|
force_download=force_download,
|
|
322
308
|
token=token,
|
|
323
309
|
headers=headers,
|
huggingface_hub/_space_api.py
CHANGED
|
@@ -15,7 +15,7 @@
|
|
|
15
15
|
from dataclasses import dataclass
|
|
16
16
|
from datetime import datetime
|
|
17
17
|
from enum import Enum
|
|
18
|
-
from typing import
|
|
18
|
+
from typing import Optional
|
|
19
19
|
|
|
20
20
|
from huggingface_hub.utils import parse_datetime
|
|
21
21
|
|
|
@@ -128,9 +128,9 @@ class SpaceRuntime:
|
|
|
128
128
|
requested_hardware: Optional[SpaceHardware]
|
|
129
129
|
sleep_time: Optional[int]
|
|
130
130
|
storage: Optional[SpaceStorage]
|
|
131
|
-
raw:
|
|
131
|
+
raw: dict
|
|
132
132
|
|
|
133
|
-
def __init__(self, data:
|
|
133
|
+
def __init__(self, data: dict) -> None:
|
|
134
134
|
self.stage = data["stage"]
|
|
135
135
|
self.hardware = data.get("hardware", {}).get("current")
|
|
136
136
|
self.requested_hardware = data.get("hardware", {}).get("requested")
|
|
@@ -160,7 +160,7 @@ class SpaceVariable:
|
|
|
160
160
|
description: Optional[str]
|
|
161
161
|
updated_at: Optional[datetime]
|
|
162
162
|
|
|
163
|
-
def __init__(self, key: str, values:
|
|
163
|
+
def __init__(self, key: str, values: dict) -> None:
|
|
164
164
|
self.key = key
|
|
165
165
|
self.value = values["value"]
|
|
166
166
|
self.description = values.get("description")
|
|
@@ -14,7 +14,7 @@
|
|
|
14
14
|
"""Contains a logger to push training logs to the Hub, using Tensorboard."""
|
|
15
15
|
|
|
16
16
|
from pathlib import Path
|
|
17
|
-
from typing import
|
|
17
|
+
from typing import Optional, Union
|
|
18
18
|
|
|
19
19
|
from ._commit_scheduler import CommitScheduler
|
|
20
20
|
from .errors import EntryNotFoundError
|
|
@@ -77,10 +77,10 @@ class HFSummaryWriter(_RuntimeSummaryWriter):
|
|
|
77
77
|
Whether to make the repo private. If `None` (default), the repo will be public unless the organization's default is private. This value is ignored if the repo already exists.
|
|
78
78
|
path_in_repo (`str`, *optional*):
|
|
79
79
|
The path to the folder in the repo where the logs will be pushed. Defaults to "tensorboard/".
|
|
80
|
-
repo_allow_patterns (`
|
|
80
|
+
repo_allow_patterns (`list[str]` or `str`, *optional*):
|
|
81
81
|
A list of patterns to include in the upload. Defaults to `"*.tfevents.*"`. Check out the
|
|
82
82
|
[upload guide](https://huggingface.co/docs/huggingface_hub/guides/upload#upload-a-folder) for more details.
|
|
83
|
-
repo_ignore_patterns (`
|
|
83
|
+
repo_ignore_patterns (`list[str]` or `str`, *optional*):
|
|
84
84
|
A list of patterns to exclude in the upload. Check out the
|
|
85
85
|
[upload guide](https://huggingface.co/docs/huggingface_hub/guides/upload#upload-a-folder) for more details.
|
|
86
86
|
token (`str`, *optional*):
|
|
@@ -137,8 +137,8 @@ class HFSummaryWriter(_RuntimeSummaryWriter):
|
|
|
137
137
|
repo_revision: Optional[str] = None,
|
|
138
138
|
repo_private: Optional[bool] = None,
|
|
139
139
|
path_in_repo: Optional[str] = "tensorboard",
|
|
140
|
-
repo_allow_patterns: Optional[Union[
|
|
141
|
-
repo_ignore_patterns: Optional[Union[
|
|
140
|
+
repo_allow_patterns: Optional[Union[list[str], str]] = "*.tfevents.*",
|
|
141
|
+
repo_ignore_patterns: Optional[Union[list[str], str]] = None,
|
|
142
142
|
token: Optional[str] = None,
|
|
143
143
|
**kwargs,
|
|
144
144
|
):
|
|
@@ -24,7 +24,7 @@ import traceback
|
|
|
24
24
|
from datetime import datetime
|
|
25
25
|
from pathlib import Path
|
|
26
26
|
from threading import Lock
|
|
27
|
-
from typing import TYPE_CHECKING, Any,
|
|
27
|
+
from typing import TYPE_CHECKING, Any, Optional, Union
|
|
28
28
|
from urllib.parse import quote
|
|
29
29
|
|
|
30
30
|
from . import constants
|
|
@@ -44,7 +44,7 @@ logger = logging.getLogger(__name__)
|
|
|
44
44
|
|
|
45
45
|
WAITING_TIME_IF_NO_TASKS = 10 # seconds
|
|
46
46
|
MAX_NB_FILES_FETCH_UPLOAD_MODE = 100
|
|
47
|
-
COMMIT_SIZE_SCALE:
|
|
47
|
+
COMMIT_SIZE_SCALE: list[int] = [20, 50, 75, 100, 125, 200, 250, 400, 600, 1000]
|
|
48
48
|
|
|
49
49
|
UPLOAD_BATCH_SIZE_XET = 256 # Max 256 files per upload batch for XET-enabled repos
|
|
50
50
|
UPLOAD_BATCH_SIZE_LFS = 1 # Otherwise, batches of 1 for regular LFS upload
|
|
@@ -56,7 +56,7 @@ MAX_FILE_SIZE_GB = 50 # Hard limit for individual file size
|
|
|
56
56
|
RECOMMENDED_FILE_SIZE_GB = 20 # Recommended maximum for individual file size
|
|
57
57
|
|
|
58
58
|
|
|
59
|
-
def _validate_upload_limits(paths_list:
|
|
59
|
+
def _validate_upload_limits(paths_list: list[LocalUploadFilePaths]) -> None:
|
|
60
60
|
"""
|
|
61
61
|
Validate upload against repository limits and warn about potential issues.
|
|
62
62
|
|
|
@@ -85,7 +85,7 @@ def _validate_upload_limits(paths_list: List[LocalUploadFilePaths]) -> None:
|
|
|
85
85
|
# Track immediate children (files and subdirs) for each folder
|
|
86
86
|
from collections import defaultdict
|
|
87
87
|
|
|
88
|
-
entries_per_folder:
|
|
88
|
+
entries_per_folder: dict[str, Any] = defaultdict(lambda: {"files": 0, "subdirs": set()})
|
|
89
89
|
|
|
90
90
|
for paths in paths_list:
|
|
91
91
|
path = Path(paths.path_in_repo)
|
|
@@ -160,8 +160,8 @@ def upload_large_folder_internal(
|
|
|
160
160
|
repo_type: str, # Repo type is required!
|
|
161
161
|
revision: Optional[str] = None,
|
|
162
162
|
private: Optional[bool] = None,
|
|
163
|
-
allow_patterns: Optional[Union[
|
|
164
|
-
ignore_patterns: Optional[Union[
|
|
163
|
+
allow_patterns: Optional[Union[list[str], str]] = None,
|
|
164
|
+
ignore_patterns: Optional[Union[list[str], str]] = None,
|
|
165
165
|
num_workers: Optional[int] = None,
|
|
166
166
|
print_report: bool = True,
|
|
167
167
|
print_report_every: int = 60,
|
|
@@ -284,13 +284,13 @@ class WorkerJob(enum.Enum):
|
|
|
284
284
|
WAIT = enum.auto() # if no tasks are available but we don't want to exit
|
|
285
285
|
|
|
286
286
|
|
|
287
|
-
JOB_ITEM_T =
|
|
287
|
+
JOB_ITEM_T = tuple[LocalUploadFilePaths, LocalUploadFileMetadata]
|
|
288
288
|
|
|
289
289
|
|
|
290
290
|
class LargeUploadStatus:
|
|
291
291
|
"""Contains information, queues and tasks for a large upload process."""
|
|
292
292
|
|
|
293
|
-
def __init__(self, items:
|
|
293
|
+
def __init__(self, items: list[JOB_ITEM_T], upload_batch_size: int = 1):
|
|
294
294
|
self.items = items
|
|
295
295
|
self.queue_sha256: "queue.Queue[JOB_ITEM_T]" = queue.Queue()
|
|
296
296
|
self.queue_get_upload_mode: "queue.Queue[JOB_ITEM_T]" = queue.Queue()
|
|
@@ -423,7 +423,7 @@ def _worker_job(
|
|
|
423
423
|
Read `upload_large_folder` docstring for more information on how tasks are prioritized.
|
|
424
424
|
"""
|
|
425
425
|
while True:
|
|
426
|
-
next_job: Optional[
|
|
426
|
+
next_job: Optional[tuple[WorkerJob, list[JOB_ITEM_T]]] = None
|
|
427
427
|
|
|
428
428
|
# Determine next task
|
|
429
429
|
next_job = _determine_next_job(status)
|
|
@@ -516,7 +516,7 @@ def _worker_job(
|
|
|
516
516
|
status.nb_workers_waiting -= 1
|
|
517
517
|
|
|
518
518
|
|
|
519
|
-
def _determine_next_job(status: LargeUploadStatus) -> Optional[
|
|
519
|
+
def _determine_next_job(status: LargeUploadStatus) -> Optional[tuple[WorkerJob, list[JOB_ITEM_T]]]:
|
|
520
520
|
with status.lock:
|
|
521
521
|
# 1. Commit if more than 5 minutes since last commit attempt (and at least 1 file)
|
|
522
522
|
if (
|
|
@@ -639,7 +639,7 @@ def _compute_sha256(item: JOB_ITEM_T) -> None:
|
|
|
639
639
|
metadata.save(paths)
|
|
640
640
|
|
|
641
641
|
|
|
642
|
-
def _get_upload_mode(items:
|
|
642
|
+
def _get_upload_mode(items: list[JOB_ITEM_T], api: "HfApi", repo_id: str, repo_type: str, revision: str) -> None:
|
|
643
643
|
"""Get upload mode for each file and update metadata.
|
|
644
644
|
|
|
645
645
|
Also receive info if the file should be ignored.
|
|
@@ -661,7 +661,7 @@ def _get_upload_mode(items: List[JOB_ITEM_T], api: "HfApi", repo_id: str, repo_t
|
|
|
661
661
|
metadata.save(paths)
|
|
662
662
|
|
|
663
663
|
|
|
664
|
-
def _preupload_lfs(items:
|
|
664
|
+
def _preupload_lfs(items: list[JOB_ITEM_T], api: "HfApi", repo_id: str, repo_type: str, revision: str) -> None:
|
|
665
665
|
"""Preupload LFS files and update metadata."""
|
|
666
666
|
additions = [_build_hacky_operation(item) for item in items]
|
|
667
667
|
api.preupload_lfs_files(
|
|
@@ -676,7 +676,7 @@ def _preupload_lfs(items: List[JOB_ITEM_T], api: "HfApi", repo_id: str, repo_typ
|
|
|
676
676
|
metadata.save(paths)
|
|
677
677
|
|
|
678
678
|
|
|
679
|
-
def _commit(items:
|
|
679
|
+
def _commit(items: list[JOB_ITEM_T], api: "HfApi", repo_id: str, repo_type: str, revision: str) -> None:
|
|
680
680
|
"""Commit files to the repo."""
|
|
681
681
|
additions = [_build_hacky_operation(item) for item in items]
|
|
682
682
|
api.create_commit(
|
|
@@ -721,11 +721,11 @@ def _build_hacky_operation(item: JOB_ITEM_T) -> HackyCommitOperationAdd:
|
|
|
721
721
|
####################
|
|
722
722
|
|
|
723
723
|
|
|
724
|
-
def _get_one(queue: "queue.Queue[JOB_ITEM_T]") ->
|
|
724
|
+
def _get_one(queue: "queue.Queue[JOB_ITEM_T]") -> list[JOB_ITEM_T]:
|
|
725
725
|
return [queue.get()]
|
|
726
726
|
|
|
727
727
|
|
|
728
|
-
def _get_n(queue: "queue.Queue[JOB_ITEM_T]", n: int) ->
|
|
728
|
+
def _get_n(queue: "queue.Queue[JOB_ITEM_T]", n: int) -> list[JOB_ITEM_T]:
|
|
729
729
|
return [queue.get() for _ in range(min(queue.qsize(), n))]
|
|
730
730
|
|
|
731
731
|
|
|
@@ -14,7 +14,7 @@
|
|
|
14
14
|
# limitations under the License.
|
|
15
15
|
"""Contains data structures to parse the webhooks payload."""
|
|
16
16
|
|
|
17
|
-
from typing import
|
|
17
|
+
from typing import Literal, Optional
|
|
18
18
|
|
|
19
19
|
from .utils import is_pydantic_available
|
|
20
20
|
|
|
@@ -116,7 +116,7 @@ class WebhookPayloadRepo(ObjectId):
|
|
|
116
116
|
name: str
|
|
117
117
|
private: bool
|
|
118
118
|
subdomain: Optional[str] = None
|
|
119
|
-
tags: Optional[
|
|
119
|
+
tags: Optional[list[str]] = None
|
|
120
120
|
type: Literal["dataset", "model", "space"]
|
|
121
121
|
url: WebhookPayloadUrl
|
|
122
122
|
|
|
@@ -134,4 +134,4 @@ class WebhookPayload(BaseModel):
|
|
|
134
134
|
comment: Optional[WebhookPayloadComment] = None
|
|
135
135
|
webhook: WebhookPayloadWebhook
|
|
136
136
|
movedTo: Optional[WebhookPayloadMovedTo] = None
|
|
137
|
-
updatedRefs: Optional[
|
|
137
|
+
updatedRefs: Optional[list[WebhookPayloadUpdatedRef]] = None
|
|
@@ -18,7 +18,7 @@ import atexit
|
|
|
18
18
|
import inspect
|
|
19
19
|
import os
|
|
20
20
|
from functools import wraps
|
|
21
|
-
from typing import TYPE_CHECKING, Any, Callable,
|
|
21
|
+
from typing import TYPE_CHECKING, Any, Callable, Optional
|
|
22
22
|
|
|
23
23
|
from .utils import experimental, is_fastapi_available, is_gradio_available
|
|
24
24
|
|
|
@@ -115,7 +115,7 @@ class WebhooksServer:
|
|
|
115
115
|
self._ui = ui
|
|
116
116
|
|
|
117
117
|
self.webhook_secret = webhook_secret or os.getenv("WEBHOOK_SECRET")
|
|
118
|
-
self.registered_webhooks:
|
|
118
|
+
self.registered_webhooks: dict[str, Callable] = {}
|
|
119
119
|
_warn_on_empty_secret(self.webhook_secret)
|
|
120
120
|
|
|
121
121
|
def add_webhook(self, path: Optional[str] = None) -> Callable:
|
huggingface_hub/cli/__init__.py
CHANGED
|
@@ -11,17 +11,3 @@
|
|
|
11
11
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
12
|
# See the License for the specific language governing permissions and
|
|
13
13
|
# limitations under the License.
|
|
14
|
-
|
|
15
|
-
from abc import ABC, abstractmethod
|
|
16
|
-
from argparse import _SubParsersAction
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
class BaseHuggingfaceCLICommand(ABC):
|
|
20
|
-
@staticmethod
|
|
21
|
-
@abstractmethod
|
|
22
|
-
def register_subcommand(parser: _SubParsersAction):
|
|
23
|
-
raise NotImplementedError()
|
|
24
|
-
|
|
25
|
-
@abstractmethod
|
|
26
|
-
def run(self):
|
|
27
|
-
raise NotImplementedError()
|