huggingface-hub 0.35.0rc0__py3-none-any.whl → 0.35.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of huggingface-hub might be problematic. Click here for more details.
- huggingface_hub/__init__.py +19 -1
- huggingface_hub/_jobs_api.py +168 -12
- huggingface_hub/_local_folder.py +1 -1
- huggingface_hub/_oauth.py +5 -9
- huggingface_hub/_tensorboard_logger.py +9 -10
- huggingface_hub/_upload_large_folder.py +108 -1
- huggingface_hub/cli/auth.py +4 -1
- huggingface_hub/cli/cache.py +7 -9
- huggingface_hub/cli/hf.py +2 -5
- huggingface_hub/cli/jobs.py +591 -13
- huggingface_hub/cli/repo.py +10 -4
- huggingface_hub/commands/delete_cache.py +2 -2
- huggingface_hub/commands/scan_cache.py +1 -1
- huggingface_hub/dataclasses.py +3 -0
- huggingface_hub/file_download.py +12 -10
- huggingface_hub/hf_api.py +549 -95
- huggingface_hub/hf_file_system.py +4 -10
- huggingface_hub/hub_mixin.py +5 -3
- huggingface_hub/inference/_client.py +98 -181
- huggingface_hub/inference/_common.py +72 -70
- huggingface_hub/inference/_generated/_async_client.py +116 -201
- huggingface_hub/inference/_generated/types/chat_completion.py +2 -0
- huggingface_hub/inference/_mcp/_cli_hacks.py +3 -3
- huggingface_hub/inference/_mcp/cli.py +1 -1
- huggingface_hub/inference/_mcp/constants.py +1 -1
- huggingface_hub/inference/_mcp/mcp_client.py +28 -11
- huggingface_hub/inference/_mcp/types.py +3 -0
- huggingface_hub/inference/_mcp/utils.py +7 -3
- huggingface_hub/inference/_providers/__init__.py +13 -0
- huggingface_hub/inference/_providers/_common.py +29 -4
- huggingface_hub/inference/_providers/black_forest_labs.py +1 -1
- huggingface_hub/inference/_providers/fal_ai.py +33 -2
- huggingface_hub/inference/_providers/hf_inference.py +15 -7
- huggingface_hub/inference/_providers/publicai.py +6 -0
- huggingface_hub/inference/_providers/replicate.py +1 -1
- huggingface_hub/inference/_providers/scaleway.py +28 -0
- huggingface_hub/lfs.py +2 -4
- huggingface_hub/repocard.py +2 -1
- huggingface_hub/utils/_dotenv.py +24 -20
- huggingface_hub/utils/_git_credential.py +1 -1
- huggingface_hub/utils/_http.py +3 -5
- huggingface_hub/utils/_runtime.py +1 -0
- huggingface_hub/utils/_typing.py +24 -4
- huggingface_hub/utils/_xet_progress_reporting.py +31 -10
- {huggingface_hub-0.35.0rc0.dist-info → huggingface_hub-0.35.1.dist-info}/METADATA +7 -4
- {huggingface_hub-0.35.0rc0.dist-info → huggingface_hub-0.35.1.dist-info}/RECORD +50 -48
- {huggingface_hub-0.35.0rc0.dist-info → huggingface_hub-0.35.1.dist-info}/LICENSE +0 -0
- {huggingface_hub-0.35.0rc0.dist-info → huggingface_hub-0.35.1.dist-info}/WHEEL +0 -0
- {huggingface_hub-0.35.0rc0.dist-info → huggingface_hub-0.35.1.dist-info}/entry_points.txt +0 -0
- {huggingface_hub-0.35.0rc0.dist-info → huggingface_hub-0.35.1.dist-info}/top_level.txt +0 -0
|
@@ -3,7 +3,7 @@ from typing import Any, Dict, List, Optional, Union, overload
|
|
|
3
3
|
|
|
4
4
|
from huggingface_hub import constants
|
|
5
5
|
from huggingface_hub.hf_api import InferenceProviderMapping
|
|
6
|
-
from huggingface_hub.inference._common import RequestParameters
|
|
6
|
+
from huggingface_hub.inference._common import MimeBytes, RequestParameters
|
|
7
7
|
from huggingface_hub.inference._generated.types.chat_completion import ChatCompletionInputMessage
|
|
8
8
|
from huggingface_hub.utils import build_hf_headers, get_token, logging
|
|
9
9
|
|
|
@@ -33,6 +33,7 @@ HARDCODED_MODEL_INFERENCE_MAPPING: Dict[str, Dict[str, InferenceProviderMapping]
|
|
|
33
33
|
"nscale": {},
|
|
34
34
|
"replicate": {},
|
|
35
35
|
"sambanova": {},
|
|
36
|
+
"scaleway": {},
|
|
36
37
|
"together": {},
|
|
37
38
|
}
|
|
38
39
|
|
|
@@ -108,8 +109,17 @@ class TaskProviderHelper:
|
|
|
108
109
|
raise ValueError("Both payload and data cannot be set in the same request.")
|
|
109
110
|
if payload is None and data is None:
|
|
110
111
|
raise ValueError("Either payload or data must be set in the request.")
|
|
112
|
+
|
|
113
|
+
# normalize headers to lowercase and add content-type if not present
|
|
114
|
+
normalized_headers = self._normalize_headers(headers, payload, data)
|
|
115
|
+
|
|
111
116
|
return RequestParameters(
|
|
112
|
-
url=url,
|
|
117
|
+
url=url,
|
|
118
|
+
task=self.task,
|
|
119
|
+
model=provider_mapping_info.provider_id,
|
|
120
|
+
json=payload,
|
|
121
|
+
data=data,
|
|
122
|
+
headers=normalized_headers,
|
|
113
123
|
)
|
|
114
124
|
|
|
115
125
|
def get_response(
|
|
@@ -172,7 +182,22 @@ class TaskProviderHelper:
|
|
|
172
182
|
)
|
|
173
183
|
return provider_mapping
|
|
174
184
|
|
|
175
|
-
def
|
|
185
|
+
def _normalize_headers(
|
|
186
|
+
self, headers: Dict[str, Any], payload: Optional[Dict[str, Any]], data: Optional[MimeBytes]
|
|
187
|
+
) -> Dict[str, Any]:
|
|
188
|
+
"""Normalize the headers to use for the request.
|
|
189
|
+
|
|
190
|
+
Override this method in subclasses for customized headers.
|
|
191
|
+
"""
|
|
192
|
+
normalized_headers = {key.lower(): value for key, value in headers.items() if value is not None}
|
|
193
|
+
if normalized_headers.get("content-type") is None:
|
|
194
|
+
if data is not None and data.mime_type is not None:
|
|
195
|
+
normalized_headers["content-type"] = data.mime_type
|
|
196
|
+
elif payload is not None:
|
|
197
|
+
normalized_headers["content-type"] = "application/json"
|
|
198
|
+
return normalized_headers
|
|
199
|
+
|
|
200
|
+
def _prepare_headers(self, headers: Dict, api_key: str) -> Dict[str, Any]:
|
|
176
201
|
"""Return the headers to use for the request.
|
|
177
202
|
|
|
178
203
|
Override this method in subclasses for customized headers.
|
|
@@ -222,7 +247,7 @@ class TaskProviderHelper:
|
|
|
222
247
|
parameters: Dict,
|
|
223
248
|
provider_mapping_info: InferenceProviderMapping,
|
|
224
249
|
extra_payload: Optional[Dict],
|
|
225
|
-
) -> Optional[
|
|
250
|
+
) -> Optional[MimeBytes]:
|
|
226
251
|
"""Return the body to use for the request, as bytes.
|
|
227
252
|
|
|
228
253
|
Override this method in subclasses for customized body data.
|
|
@@ -18,7 +18,7 @@ class BlackForestLabsTextToImageTask(TaskProviderHelper):
|
|
|
18
18
|
def __init__(self):
|
|
19
19
|
super().__init__(provider="black-forest-labs", base_url="https://api.us1.bfl.ai", task="text-to-image")
|
|
20
20
|
|
|
21
|
-
def _prepare_headers(self, headers: Dict, api_key: str) -> Dict:
|
|
21
|
+
def _prepare_headers(self, headers: Dict, api_key: str) -> Dict[str, Any]:
|
|
22
22
|
headers = super()._prepare_headers(headers, api_key)
|
|
23
23
|
if not api_key.startswith("hf_"):
|
|
24
24
|
_ = headers.pop("authorization")
|
|
@@ -22,7 +22,7 @@ class FalAITask(TaskProviderHelper, ABC):
|
|
|
22
22
|
def __init__(self, task: str):
|
|
23
23
|
super().__init__(provider="fal-ai", base_url="https://fal.run", task=task)
|
|
24
24
|
|
|
25
|
-
def _prepare_headers(self, headers: Dict, api_key: str) -> Dict:
|
|
25
|
+
def _prepare_headers(self, headers: Dict, api_key: str) -> Dict[str, Any]:
|
|
26
26
|
headers = super()._prepare_headers(headers, api_key)
|
|
27
27
|
if not api_key.startswith("hf_"):
|
|
28
28
|
headers["authorization"] = f"Key {api_key}"
|
|
@@ -36,7 +36,7 @@ class FalAIQueueTask(TaskProviderHelper, ABC):
|
|
|
36
36
|
def __init__(self, task: str):
|
|
37
37
|
super().__init__(provider="fal-ai", base_url="https://queue.fal.run", task=task)
|
|
38
38
|
|
|
39
|
-
def _prepare_headers(self, headers: Dict, api_key: str) -> Dict:
|
|
39
|
+
def _prepare_headers(self, headers: Dict, api_key: str) -> Dict[str, Any]:
|
|
40
40
|
headers = super()._prepare_headers(headers, api_key)
|
|
41
41
|
if not api_key.startswith("hf_"):
|
|
42
42
|
headers["authorization"] = f"Key {api_key}"
|
|
@@ -213,3 +213,34 @@ class FalAIImageToImageTask(FalAIQueueTask):
|
|
|
213
213
|
output = super().get_response(response, request_params)
|
|
214
214
|
url = _as_dict(output)["images"][0]["url"]
|
|
215
215
|
return get_session().get(url).content
|
|
216
|
+
|
|
217
|
+
|
|
218
|
+
class FalAIImageToVideoTask(FalAIQueueTask):
|
|
219
|
+
def __init__(self):
|
|
220
|
+
super().__init__("image-to-video")
|
|
221
|
+
|
|
222
|
+
def _prepare_payload_as_dict(
|
|
223
|
+
self, inputs: Any, parameters: Dict, provider_mapping_info: InferenceProviderMapping
|
|
224
|
+
) -> Optional[Dict]:
|
|
225
|
+
image_url = _as_url(inputs, default_mime_type="image/jpeg")
|
|
226
|
+
payload: Dict[str, Any] = {
|
|
227
|
+
"image_url": image_url,
|
|
228
|
+
**filter_none(parameters),
|
|
229
|
+
}
|
|
230
|
+
if provider_mapping_info.adapter_weights_path is not None:
|
|
231
|
+
lora_path = constants.HUGGINGFACE_CO_URL_TEMPLATE.format(
|
|
232
|
+
repo_id=provider_mapping_info.hf_model_id,
|
|
233
|
+
revision="main",
|
|
234
|
+
filename=provider_mapping_info.adapter_weights_path,
|
|
235
|
+
)
|
|
236
|
+
payload["loras"] = [{"path": lora_path, "scale": 1}]
|
|
237
|
+
return payload
|
|
238
|
+
|
|
239
|
+
def get_response(
|
|
240
|
+
self,
|
|
241
|
+
response: Union[bytes, Dict],
|
|
242
|
+
request_params: Optional[RequestParameters] = None,
|
|
243
|
+
) -> Any:
|
|
244
|
+
output = super().get_response(response, request_params)
|
|
245
|
+
url = _as_dict(output)["video"]["url"]
|
|
246
|
+
return get_session().get(url).content
|
|
@@ -6,7 +6,13 @@ from urllib.parse import urlparse, urlunparse
|
|
|
6
6
|
|
|
7
7
|
from huggingface_hub import constants
|
|
8
8
|
from huggingface_hub.hf_api import InferenceProviderMapping
|
|
9
|
-
from huggingface_hub.inference._common import
|
|
9
|
+
from huggingface_hub.inference._common import (
|
|
10
|
+
MimeBytes,
|
|
11
|
+
RequestParameters,
|
|
12
|
+
_b64_encode,
|
|
13
|
+
_bytes_to_dict,
|
|
14
|
+
_open_as_mime_bytes,
|
|
15
|
+
)
|
|
10
16
|
from huggingface_hub.inference._providers._common import TaskProviderHelper, filter_none
|
|
11
17
|
from huggingface_hub.utils import build_hf_headers, get_session, get_token, hf_raise_for_status
|
|
12
18
|
|
|
@@ -75,7 +81,7 @@ class HFInferenceBinaryInputTask(HFInferenceTask):
|
|
|
75
81
|
parameters: Dict,
|
|
76
82
|
provider_mapping_info: InferenceProviderMapping,
|
|
77
83
|
extra_payload: Optional[Dict],
|
|
78
|
-
) -> Optional[
|
|
84
|
+
) -> Optional[MimeBytes]:
|
|
79
85
|
parameters = filter_none(parameters)
|
|
80
86
|
extra_payload = extra_payload or {}
|
|
81
87
|
has_parameters = len(parameters) > 0 or len(extra_payload) > 0
|
|
@@ -86,12 +92,13 @@ class HFInferenceBinaryInputTask(HFInferenceTask):
|
|
|
86
92
|
|
|
87
93
|
# Send inputs as raw content when no parameters are provided
|
|
88
94
|
if not has_parameters:
|
|
89
|
-
|
|
90
|
-
data_as_bytes = data if isinstance(data, bytes) else data.read()
|
|
91
|
-
return data_as_bytes
|
|
95
|
+
return _open_as_mime_bytes(inputs)
|
|
92
96
|
|
|
93
97
|
# Otherwise encode as b64
|
|
94
|
-
return
|
|
98
|
+
return MimeBytes(
|
|
99
|
+
json.dumps({"inputs": _b64_encode(inputs), "parameters": parameters, **extra_payload}).encode("utf-8"),
|
|
100
|
+
mime_type="application/json",
|
|
101
|
+
)
|
|
95
102
|
|
|
96
103
|
|
|
97
104
|
class HFInferenceConversational(HFInferenceTask):
|
|
@@ -144,7 +151,8 @@ def _build_chat_completion_url(model_url: str) -> str:
|
|
|
144
151
|
new_path = path + "/v1/chat/completions"
|
|
145
152
|
|
|
146
153
|
# Reconstruct the URL with the new path and original query parameters.
|
|
147
|
-
|
|
154
|
+
new_parsed = parsed._replace(path=new_path)
|
|
155
|
+
return str(urlunparse(new_parsed))
|
|
148
156
|
|
|
149
157
|
|
|
150
158
|
@lru_cache(maxsize=1)
|
|
@@ -14,7 +14,7 @@ class ReplicateTask(TaskProviderHelper):
|
|
|
14
14
|
def __init__(self, task: str):
|
|
15
15
|
super().__init__(provider=_PROVIDER, base_url=_BASE_URL, task=task)
|
|
16
16
|
|
|
17
|
-
def _prepare_headers(self, headers: Dict, api_key: str) -> Dict:
|
|
17
|
+
def _prepare_headers(self, headers: Dict, api_key: str) -> Dict[str, Any]:
|
|
18
18
|
headers = super()._prepare_headers(headers, api_key)
|
|
19
19
|
headers["Prefer"] = "wait"
|
|
20
20
|
return headers
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
from typing import Any, Dict, Optional, Union
|
|
2
|
+
|
|
3
|
+
from huggingface_hub.inference._common import RequestParameters, _as_dict
|
|
4
|
+
|
|
5
|
+
from ._common import BaseConversationalTask, InferenceProviderMapping, TaskProviderHelper, filter_none
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class ScalewayConversationalTask(BaseConversationalTask):
|
|
9
|
+
def __init__(self):
|
|
10
|
+
super().__init__(provider="scaleway", base_url="https://api.scaleway.ai")
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class ScalewayFeatureExtractionTask(TaskProviderHelper):
|
|
14
|
+
def __init__(self):
|
|
15
|
+
super().__init__(provider="scaleway", base_url="https://api.scaleway.ai", task="feature-extraction")
|
|
16
|
+
|
|
17
|
+
def _prepare_route(self, mapped_model: str, api_key: str) -> str:
|
|
18
|
+
return "/v1/embeddings"
|
|
19
|
+
|
|
20
|
+
def _prepare_payload_as_dict(
|
|
21
|
+
self, inputs: Any, parameters: Dict, provider_mapping_info: InferenceProviderMapping
|
|
22
|
+
) -> Optional[Dict]:
|
|
23
|
+
parameters = filter_none(parameters)
|
|
24
|
+
return {"input": inputs, "model": provider_mapping_info.provider_id, **parameters}
|
|
25
|
+
|
|
26
|
+
def get_response(self, response: Union[bytes, Dict], request_params: Optional[RequestParameters] = None) -> Any:
|
|
27
|
+
embeddings = _as_dict(response)["data"]
|
|
28
|
+
return [embedding["embedding"] for embedding in embeddings]
|
huggingface_hub/lfs.py
CHANGED
|
@@ -316,7 +316,7 @@ def _upload_single_part(operation: "CommitOperationAdd", upload_url: str) -> Non
|
|
|
316
316
|
"""
|
|
317
317
|
with operation.as_file(with_tqdm=True) as fileobj:
|
|
318
318
|
# S3 might raise a transient 500 error -> let's retry if that happens
|
|
319
|
-
response = http_backoff("PUT", upload_url, data=fileobj
|
|
319
|
+
response = http_backoff("PUT", upload_url, data=fileobj)
|
|
320
320
|
hf_raise_for_status(response)
|
|
321
321
|
|
|
322
322
|
|
|
@@ -400,9 +400,7 @@ def _upload_parts_iteratively(
|
|
|
400
400
|
read_limit=chunk_size,
|
|
401
401
|
) as fileobj_slice:
|
|
402
402
|
# S3 might raise a transient 500 error -> let's retry if that happens
|
|
403
|
-
part_upload_res = http_backoff(
|
|
404
|
-
"PUT", part_upload_url, data=fileobj_slice, retry_on_status_codes=(500, 502, 503, 504)
|
|
405
|
-
)
|
|
403
|
+
part_upload_res = http_backoff("PUT", part_upload_url, data=fileobj_slice)
|
|
406
404
|
hf_raise_for_status(part_upload_res)
|
|
407
405
|
headers.append(part_upload_res.headers)
|
|
408
406
|
return headers # type: ignore
|
huggingface_hub/repocard.py
CHANGED
|
@@ -771,7 +771,8 @@ def metadata_update(
|
|
|
771
771
|
raise ValueError("Cannot update metadata on a Space that doesn't contain a `README.md` file.")
|
|
772
772
|
|
|
773
773
|
# Initialize a ModelCard or DatasetCard from default template and no data.
|
|
774
|
-
card
|
|
774
|
+
# Cast to the concrete expected card type to satisfy type checkers.
|
|
775
|
+
card = card_class.from_template(CardData()) # type: ignore[return-value]
|
|
775
776
|
|
|
776
777
|
for key, value in metadata.items():
|
|
777
778
|
if key == "model-index":
|
huggingface_hub/utils/_dotenv.py
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
# AI-generated module (ChatGPT)
|
|
2
2
|
import re
|
|
3
|
-
from typing import Dict
|
|
3
|
+
from typing import Dict, Optional
|
|
4
4
|
|
|
5
5
|
|
|
6
|
-
def load_dotenv(dotenv_str: str) -> Dict[str, str]:
|
|
6
|
+
def load_dotenv(dotenv_str: str, environ: Optional[Dict[str, str]] = None) -> Dict[str, str]:
|
|
7
7
|
"""
|
|
8
8
|
Parse a DOTENV-format string and return a dictionary of key-value pairs.
|
|
9
9
|
Handles quoted values, comments, export keyword, and blank lines.
|
|
@@ -12,17 +12,17 @@ def load_dotenv(dotenv_str: str) -> Dict[str, str]:
|
|
|
12
12
|
line_pattern = re.compile(
|
|
13
13
|
r"""
|
|
14
14
|
^\s*
|
|
15
|
-
(?:export\
|
|
15
|
+
(?:export[^\S\n]+)? # optional export
|
|
16
16
|
([A-Za-z_][A-Za-z0-9_]*) # key
|
|
17
|
-
\
|
|
17
|
+
[^\S\n]*(=)?[^\S\n]*
|
|
18
18
|
( # value group
|
|
19
19
|
(?:
|
|
20
20
|
'(?:\\'|[^'])*' # single-quoted value
|
|
21
|
-
| "(
|
|
21
|
+
| \"(?:\\\"|[^\"])*\" # double-quoted value
|
|
22
22
|
| [^#\n\r]+? # unquoted value
|
|
23
23
|
)
|
|
24
24
|
)?
|
|
25
|
-
\
|
|
25
|
+
[^\S\n]*(?:\#.*)?$ # optional inline comment
|
|
26
26
|
""",
|
|
27
27
|
re.VERBOSE,
|
|
28
28
|
)
|
|
@@ -33,19 +33,23 @@ def load_dotenv(dotenv_str: str) -> Dict[str, str]:
|
|
|
33
33
|
continue # Skip comments and empty lines
|
|
34
34
|
|
|
35
35
|
match = line_pattern.match(line)
|
|
36
|
-
if
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
36
|
+
if match:
|
|
37
|
+
key = match.group(1)
|
|
38
|
+
val = None
|
|
39
|
+
if match.group(2): # if there is '='
|
|
40
|
+
raw_val = match.group(3) or ""
|
|
41
|
+
val = raw_val.strip()
|
|
42
|
+
# Remove surrounding quotes if quoted
|
|
43
|
+
if (val.startswith('"') and val.endswith('"')) or (val.startswith("'") and val.endswith("'")):
|
|
44
|
+
val = val[1:-1]
|
|
45
|
+
val = val.replace(r"\n", "\n").replace(r"\t", "\t").replace(r"\"", '"').replace(r"\\", "\\")
|
|
46
|
+
if raw_val.startswith('"'):
|
|
47
|
+
val = val.replace(r"\$", "$") # only in double quotes
|
|
48
|
+
elif environ is not None:
|
|
49
|
+
# Get it from the current environment
|
|
50
|
+
val = environ.get(key)
|
|
51
|
+
|
|
52
|
+
if val is not None:
|
|
53
|
+
env[key] = val
|
|
50
54
|
|
|
51
55
|
return env
|
|
@@ -27,7 +27,7 @@ GIT_CREDENTIAL_REGEX = re.compile(
|
|
|
27
27
|
^\s* # start of line
|
|
28
28
|
credential\.helper # credential.helper value
|
|
29
29
|
\s*=\s* # separator
|
|
30
|
-
(\w+) # the helper name (group 1)
|
|
30
|
+
([\w\-\/]+) # the helper name or absolute path (group 1)
|
|
31
31
|
(\s|$) # whitespace or end of line
|
|
32
32
|
""",
|
|
33
33
|
flags=re.MULTILINE | re.IGNORECASE | re.VERBOSE,
|
huggingface_hub/utils/_http.py
CHANGED
|
@@ -21,7 +21,6 @@ import threading
|
|
|
21
21
|
import time
|
|
22
22
|
import uuid
|
|
23
23
|
from functools import lru_cache
|
|
24
|
-
from http import HTTPStatus
|
|
25
24
|
from shlex import quote
|
|
26
25
|
from typing import Any, Callable, List, Optional, Tuple, Type, Union
|
|
27
26
|
|
|
@@ -221,7 +220,7 @@ def http_backoff(
|
|
|
221
220
|
requests.Timeout,
|
|
222
221
|
requests.ConnectionError,
|
|
223
222
|
),
|
|
224
|
-
retry_on_status_codes: Union[int, Tuple[int, ...]] =
|
|
223
|
+
retry_on_status_codes: Union[int, Tuple[int, ...]] = (500, 502, 503, 504),
|
|
225
224
|
**kwargs,
|
|
226
225
|
) -> Response:
|
|
227
226
|
"""Wrapper around requests to retry calls on an endpoint, with exponential backoff.
|
|
@@ -250,9 +249,8 @@ def http_backoff(
|
|
|
250
249
|
retry_on_exceptions (`Type[Exception]` or `Tuple[Type[Exception]]`, *optional*):
|
|
251
250
|
Define which exceptions must be caught to retry the request. Can be a single type or a tuple of types.
|
|
252
251
|
By default, retry on `requests.Timeout` and `requests.ConnectionError`.
|
|
253
|
-
retry_on_status_codes (`int` or `Tuple[int]`, *optional*, defaults to `503`):
|
|
254
|
-
Define on which status codes the request must be retried. By default,
|
|
255
|
-
HTTP 503 Service Unavailable is retried.
|
|
252
|
+
retry_on_status_codes (`int` or `Tuple[int]`, *optional*, defaults to `(500, 502, 503, 504)`):
|
|
253
|
+
Define on which status codes the request must be retried. By default, 5xx errors are retried.
|
|
256
254
|
**kwargs (`dict`, *optional*):
|
|
257
255
|
kwargs to pass to `requests.request`.
|
|
258
256
|
|
|
@@ -385,6 +385,7 @@ def dump_environment_info() -> Dict[str, Any]:
|
|
|
385
385
|
info["HF_HUB_DISABLE_SYMLINKS_WARNING"] = constants.HF_HUB_DISABLE_SYMLINKS_WARNING
|
|
386
386
|
info["HF_HUB_DISABLE_EXPERIMENTAL_WARNING"] = constants.HF_HUB_DISABLE_EXPERIMENTAL_WARNING
|
|
387
387
|
info["HF_HUB_DISABLE_IMPLICIT_TOKEN"] = constants.HF_HUB_DISABLE_IMPLICIT_TOKEN
|
|
388
|
+
info["HF_HUB_DISABLE_XET"] = constants.HF_HUB_DISABLE_XET
|
|
388
389
|
info["HF_HUB_ENABLE_HF_TRANSFER"] = constants.HF_HUB_ENABLE_HF_TRANSFER
|
|
389
390
|
info["HF_HUB_ETAG_TIMEOUT"] = constants.HF_HUB_ETAG_TIMEOUT
|
|
390
391
|
info["HF_HUB_DOWNLOAD_TIMEOUT"] = constants.HF_HUB_DOWNLOAD_TIMEOUT
|
huggingface_hub/utils/_typing.py
CHANGED
|
@@ -15,7 +15,7 @@
|
|
|
15
15
|
"""Handle typing imports based on system compatibility."""
|
|
16
16
|
|
|
17
17
|
import sys
|
|
18
|
-
from typing import Any, Callable, List, Literal, Type, TypeVar, Union, get_args, get_origin
|
|
18
|
+
from typing import Any, Callable, List, Literal, Optional, Set, Type, TypeVar, Union, get_args, get_origin
|
|
19
19
|
|
|
20
20
|
|
|
21
21
|
UNION_TYPES: List[Any] = [Union]
|
|
@@ -33,7 +33,7 @@ CallableT = TypeVar("CallableT", bound=Callable)
|
|
|
33
33
|
_JSON_SERIALIZABLE_TYPES = (int, float, str, bool, type(None))
|
|
34
34
|
|
|
35
35
|
|
|
36
|
-
def is_jsonable(obj: Any) -> bool:
|
|
36
|
+
def is_jsonable(obj: Any, _visited: Optional[Set[int]] = None) -> bool:
|
|
37
37
|
"""Check if an object is JSON serializable.
|
|
38
38
|
|
|
39
39
|
This is a weak check, as it does not check for the actual JSON serialization, but only for the types of the object.
|
|
@@ -43,19 +43,39 @@ def is_jsonable(obj: Any) -> bool:
|
|
|
43
43
|
- it is an instance of int, float, str, bool, or NoneType
|
|
44
44
|
- it is a list or tuple and all its items are json serializable
|
|
45
45
|
- it is a dict and all its keys are strings and all its values are json serializable
|
|
46
|
+
|
|
47
|
+
Uses a visited set to avoid infinite recursion on circular references. If object has already been visited, it is
|
|
48
|
+
considered not json serializable.
|
|
46
49
|
"""
|
|
50
|
+
# Initialize visited set to track object ids and detect circular references
|
|
51
|
+
if _visited is None:
|
|
52
|
+
_visited = set()
|
|
53
|
+
|
|
54
|
+
# Detect circular reference
|
|
55
|
+
obj_id = id(obj)
|
|
56
|
+
if obj_id in _visited:
|
|
57
|
+
return False
|
|
58
|
+
|
|
59
|
+
# Add current object to visited before recursive checks
|
|
60
|
+
_visited.add(obj_id)
|
|
47
61
|
try:
|
|
48
62
|
if isinstance(obj, _JSON_SERIALIZABLE_TYPES):
|
|
49
63
|
return True
|
|
50
64
|
if isinstance(obj, (list, tuple)):
|
|
51
|
-
return all(is_jsonable(item) for item in obj)
|
|
65
|
+
return all(is_jsonable(item, _visited) for item in obj)
|
|
52
66
|
if isinstance(obj, dict):
|
|
53
|
-
return all(
|
|
67
|
+
return all(
|
|
68
|
+
isinstance(key, _JSON_SERIALIZABLE_TYPES) and is_jsonable(value, _visited)
|
|
69
|
+
for key, value in obj.items()
|
|
70
|
+
)
|
|
54
71
|
if hasattr(obj, "__json__"):
|
|
55
72
|
return True
|
|
56
73
|
return False
|
|
57
74
|
except RecursionError:
|
|
58
75
|
return False
|
|
76
|
+
finally:
|
|
77
|
+
# Remove the object id from visited to avoid side‑effects for other branches
|
|
78
|
+
_visited.discard(obj_id)
|
|
59
79
|
|
|
60
80
|
|
|
61
81
|
def is_simple_optional_type(type_: Type) -> bool:
|
|
@@ -3,20 +3,29 @@ from typing import List
|
|
|
3
3
|
|
|
4
4
|
from hf_xet import PyItemProgressUpdate, PyTotalProgressUpdate
|
|
5
5
|
|
|
6
|
+
from . import is_google_colab, is_notebook
|
|
6
7
|
from .tqdm import tqdm
|
|
7
8
|
|
|
8
9
|
|
|
9
10
|
class XetProgressReporter:
|
|
10
|
-
|
|
11
|
+
"""
|
|
12
|
+
Reports on progress for Xet uploads.
|
|
13
|
+
|
|
14
|
+
Shows summary progress bars when running in notebooks or GUIs, and detailed per-file progress in console environments.
|
|
15
|
+
"""
|
|
16
|
+
|
|
17
|
+
def __init__(self, n_lines: int = 10, description_width: int = 30):
|
|
11
18
|
self.n_lines = n_lines
|
|
12
19
|
self.description_width = description_width
|
|
13
20
|
|
|
21
|
+
self.per_file_progress = is_google_colab() or not is_notebook()
|
|
22
|
+
|
|
14
23
|
self.tqdm_settings = {
|
|
15
24
|
"unit": "B",
|
|
16
25
|
"unit_scale": True,
|
|
17
26
|
"leave": True,
|
|
18
27
|
"unit_divisor": 1000,
|
|
19
|
-
"nrows": n_lines + 3,
|
|
28
|
+
"nrows": n_lines + 3 if self.per_file_progress else 3,
|
|
20
29
|
"miniters": 1,
|
|
21
30
|
"bar_format": "{l_bar}{bar}| {n_fmt:>5}B / {total_fmt:>5}B{postfix:>12}",
|
|
22
31
|
}
|
|
@@ -40,8 +49,13 @@ class XetProgressReporter:
|
|
|
40
49
|
def format_desc(self, name: str, indent: bool) -> str:
|
|
41
50
|
"""
|
|
42
51
|
if name is longer than width characters, prints ... at the start and then the last width-3 characters of the name, otherwise
|
|
43
|
-
the whole name right justified into
|
|
52
|
+
the whole name right justified into description_width characters. Also adds some padding.
|
|
44
53
|
"""
|
|
54
|
+
|
|
55
|
+
if not self.per_file_progress:
|
|
56
|
+
# Here we just use the defaults.
|
|
57
|
+
return name
|
|
58
|
+
|
|
45
59
|
padding = " " if indent else ""
|
|
46
60
|
width = self.description_width - len(padding)
|
|
47
61
|
|
|
@@ -74,6 +88,10 @@ class XetProgressReporter:
|
|
|
74
88
|
self.completed_items.add(name)
|
|
75
89
|
new_completed.append(name)
|
|
76
90
|
|
|
91
|
+
# If we're only showing summary information, then don't update the individual bars
|
|
92
|
+
if not self.per_file_progress:
|
|
93
|
+
continue
|
|
94
|
+
|
|
77
95
|
# If we've run out of bars to use, then collapse the last ones together.
|
|
78
96
|
if bar_idx >= len(self.current_bars):
|
|
79
97
|
bar = self.current_bars[-1]
|
|
@@ -111,10 +129,11 @@ class XetProgressReporter:
|
|
|
111
129
|
|
|
112
130
|
del self.item_state[name]
|
|
113
131
|
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
bar
|
|
132
|
+
if self.per_file_progress:
|
|
133
|
+
# Now manually refresh each of the bars
|
|
134
|
+
for bar in self.current_bars:
|
|
135
|
+
if bar:
|
|
136
|
+
bar.refresh()
|
|
118
137
|
|
|
119
138
|
# Update overall bars
|
|
120
139
|
def postfix(speed):
|
|
@@ -136,6 +155,8 @@ class XetProgressReporter:
|
|
|
136
155
|
def close(self, _success):
|
|
137
156
|
self.data_processing_bar.close()
|
|
138
157
|
self.upload_bar.close()
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
158
|
+
|
|
159
|
+
if self.per_file_progress:
|
|
160
|
+
for bar in self.current_bars:
|
|
161
|
+
if bar:
|
|
162
|
+
bar.close()
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: huggingface-hub
|
|
3
|
-
Version: 0.35.
|
|
3
|
+
Version: 0.35.1
|
|
4
4
|
Summary: Client library to download and publish models, datasets and other repos on the huggingface.co hub
|
|
5
5
|
Home-page: https://github.com/huggingface/huggingface_hub
|
|
6
6
|
Author: Hugging Face, Inc.
|
|
@@ -48,7 +48,7 @@ Requires-Dist: pytest-env; extra == "all"
|
|
|
48
48
|
Requires-Dist: pytest-xdist; extra == "all"
|
|
49
49
|
Requires-Dist: pytest-vcr; extra == "all"
|
|
50
50
|
Requires-Dist: pytest-asyncio; extra == "all"
|
|
51
|
-
Requires-Dist: pytest-rerunfailures; extra == "all"
|
|
51
|
+
Requires-Dist: pytest-rerunfailures<16.0; extra == "all"
|
|
52
52
|
Requires-Dist: pytest-mock; extra == "all"
|
|
53
53
|
Requires-Dist: urllib3<2.0; extra == "all"
|
|
54
54
|
Requires-Dist: soundfile; extra == "all"
|
|
@@ -57,6 +57,7 @@ Requires-Dist: gradio>=4.0.0; extra == "all"
|
|
|
57
57
|
Requires-Dist: numpy; extra == "all"
|
|
58
58
|
Requires-Dist: ruff>=0.9.0; extra == "all"
|
|
59
59
|
Requires-Dist: libcst>=1.4.0; extra == "all"
|
|
60
|
+
Requires-Dist: ty; extra == "all"
|
|
60
61
|
Requires-Dist: typing-extensions>=4.8.0; extra == "all"
|
|
61
62
|
Requires-Dist: types-PyYAML; extra == "all"
|
|
62
63
|
Requires-Dist: types-requests; extra == "all"
|
|
@@ -83,7 +84,7 @@ Requires-Dist: pytest-env; extra == "dev"
|
|
|
83
84
|
Requires-Dist: pytest-xdist; extra == "dev"
|
|
84
85
|
Requires-Dist: pytest-vcr; extra == "dev"
|
|
85
86
|
Requires-Dist: pytest-asyncio; extra == "dev"
|
|
86
|
-
Requires-Dist: pytest-rerunfailures; extra == "dev"
|
|
87
|
+
Requires-Dist: pytest-rerunfailures<16.0; extra == "dev"
|
|
87
88
|
Requires-Dist: pytest-mock; extra == "dev"
|
|
88
89
|
Requires-Dist: urllib3<2.0; extra == "dev"
|
|
89
90
|
Requires-Dist: soundfile; extra == "dev"
|
|
@@ -92,6 +93,7 @@ Requires-Dist: gradio>=4.0.0; extra == "dev"
|
|
|
92
93
|
Requires-Dist: numpy; extra == "dev"
|
|
93
94
|
Requires-Dist: ruff>=0.9.0; extra == "dev"
|
|
94
95
|
Requires-Dist: libcst>=1.4.0; extra == "dev"
|
|
96
|
+
Requires-Dist: ty; extra == "dev"
|
|
95
97
|
Requires-Dist: typing-extensions>=4.8.0; extra == "dev"
|
|
96
98
|
Requires-Dist: types-PyYAML; extra == "dev"
|
|
97
99
|
Requires-Dist: types-requests; extra == "dev"
|
|
@@ -123,6 +125,7 @@ Requires-Dist: itsdangerous; extra == "oauth"
|
|
|
123
125
|
Provides-Extra: quality
|
|
124
126
|
Requires-Dist: ruff>=0.9.0; extra == "quality"
|
|
125
127
|
Requires-Dist: libcst>=1.4.0; extra == "quality"
|
|
128
|
+
Requires-Dist: ty; extra == "quality"
|
|
126
129
|
Requires-Dist: mypy<1.15.0,>=1.14.1; python_version == "3.8" and extra == "quality"
|
|
127
130
|
Requires-Dist: mypy==1.15.0; python_version >= "3.9" and extra == "quality"
|
|
128
131
|
Provides-Extra: tensorflow
|
|
@@ -147,7 +150,7 @@ Requires-Dist: pytest-env; extra == "testing"
|
|
|
147
150
|
Requires-Dist: pytest-xdist; extra == "testing"
|
|
148
151
|
Requires-Dist: pytest-vcr; extra == "testing"
|
|
149
152
|
Requires-Dist: pytest-asyncio; extra == "testing"
|
|
150
|
-
Requires-Dist: pytest-rerunfailures; extra == "testing"
|
|
153
|
+
Requires-Dist: pytest-rerunfailures<16.0; extra == "testing"
|
|
151
154
|
Requires-Dist: pytest-mock; extra == "testing"
|
|
152
155
|
Requires-Dist: urllib3<2.0; extra == "testing"
|
|
153
156
|
Requires-Dist: soundfile; extra == "testing"
|