huggingface-hub 0.18.0__py3-none-any.whl → 0.19.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of huggingface-hub might be problematic. Click here for more details.

Files changed (43) hide show
  1. huggingface_hub/__init__.py +31 -5
  2. huggingface_hub/_inference_endpoints.py +348 -0
  3. huggingface_hub/_login.py +9 -7
  4. huggingface_hub/_multi_commits.py +1 -1
  5. huggingface_hub/_snapshot_download.py +6 -7
  6. huggingface_hub/_space_api.py +7 -4
  7. huggingface_hub/_tensorboard_logger.py +1 -0
  8. huggingface_hub/_webhooks_payload.py +7 -7
  9. huggingface_hub/commands/lfs.py +3 -6
  10. huggingface_hub/commands/user.py +1 -4
  11. huggingface_hub/constants.py +27 -0
  12. huggingface_hub/file_download.py +142 -134
  13. huggingface_hub/hf_api.py +1036 -501
  14. huggingface_hub/hf_file_system.py +57 -12
  15. huggingface_hub/hub_mixin.py +3 -5
  16. huggingface_hub/inference/_client.py +43 -8
  17. huggingface_hub/inference/_common.py +8 -16
  18. huggingface_hub/inference/_generated/_async_client.py +41 -8
  19. huggingface_hub/inference/_text_generation.py +43 -0
  20. huggingface_hub/inference_api.py +1 -1
  21. huggingface_hub/lfs.py +32 -14
  22. huggingface_hub/repocard_data.py +7 -0
  23. huggingface_hub/repository.py +19 -3
  24. huggingface_hub/templates/modelcard_template.md +1 -1
  25. huggingface_hub/utils/__init__.py +1 -1
  26. huggingface_hub/utils/_cache_assets.py +3 -3
  27. huggingface_hub/utils/_cache_manager.py +6 -7
  28. huggingface_hub/utils/_datetime.py +3 -1
  29. huggingface_hub/utils/_errors.py +10 -0
  30. huggingface_hub/utils/_hf_folder.py +4 -2
  31. huggingface_hub/utils/_http.py +10 -1
  32. huggingface_hub/utils/_runtime.py +4 -2
  33. huggingface_hub/utils/endpoint_helpers.py +27 -175
  34. huggingface_hub/utils/insecure_hashlib.py +34 -0
  35. huggingface_hub/utils/logging.py +4 -6
  36. huggingface_hub/utils/sha.py +2 -1
  37. {huggingface_hub-0.18.0.dist-info → huggingface_hub-0.19.0.dist-info}/METADATA +16 -15
  38. huggingface_hub-0.19.0.dist-info/RECORD +74 -0
  39. {huggingface_hub-0.18.0.dist-info → huggingface_hub-0.19.0.dist-info}/WHEEL +1 -1
  40. huggingface_hub-0.18.0.dist-info/RECORD +0 -72
  41. {huggingface_hub-0.18.0.dist-info → huggingface_hub-0.19.0.dist-info}/LICENSE +0 -0
  42. {huggingface_hub-0.18.0.dist-info → huggingface_hub-0.19.0.dist-info}/entry_points.txt +0 -0
  43. {huggingface_hub-0.18.0.dist-info → huggingface_hub-0.19.0.dist-info}/top_level.txt +0 -0
@@ -4,13 +4,14 @@ import re
4
4
  import tempfile
5
5
  from dataclasses import dataclass
6
6
  from datetime import datetime
7
- from typing import Any, Dict, List, Optional, Tuple, Union
7
+ from typing import Any, Dict, List, NoReturn, Optional, Tuple, Union
8
8
  from urllib.parse import quote, unquote
9
9
 
10
10
  import fsspec
11
11
 
12
12
  from ._commit_api import CommitOperationCopy, CommitOperationDelete
13
13
  from .constants import DEFAULT_REVISION, ENDPOINT, REPO_TYPE_MODEL, REPO_TYPES_MAPPING, REPO_TYPES_URL_PREFIXES
14
+ from .file_download import hf_hub_url
14
15
  from .hf_api import HfApi
15
16
  from .utils import (
16
17
  EntryNotFoundError,
@@ -45,10 +46,8 @@ class HfFileSystemResolvedPath:
45
46
  path_in_repo: str
46
47
 
47
48
  def unresolve(self) -> str:
48
- return (
49
- f"{REPO_TYPES_URL_PREFIXES.get(self.repo_type, '') + self.repo_id}@{safe_revision(self.revision)}/{self.path_in_repo}"
50
- .rstrip("/")
51
- )
49
+ repo_path = REPO_TYPES_URL_PREFIXES.get(self.repo_type, "") + self.repo_id
50
+ return f"{repo_path}@{safe_revision(self.revision)}/{self.path_in_repo}".rstrip("/")
52
51
 
53
52
 
54
53
  class HfFileSystem(fsspec.AbstractFileSystem):
@@ -181,7 +180,7 @@ class HfFileSystem(fsspec.AbstractFileSystem):
181
180
  revision = _align_revision_in_path_with_revision(revision_in_path, revision)
182
181
  repo_and_revision_exist, err = self._repo_and_revision_exist(repo_type, repo_id, revision)
183
182
  if not repo_and_revision_exist:
184
- raise FileNotFoundError(path) from err
183
+ _raise_file_not_found(path, err)
185
184
  else:
186
185
  repo_id_with_namespace = "/".join(path.split("/")[:2])
187
186
  path_in_repo_with_namespace = "/".join(path.split("/")[2:])
@@ -196,9 +195,9 @@ class HfFileSystem(fsspec.AbstractFileSystem):
196
195
  path_in_repo = path_in_repo_without_namespace
197
196
  repo_and_revision_exist, _ = self._repo_and_revision_exist(repo_type, repo_id, revision)
198
197
  if not repo_and_revision_exist:
199
- raise FileNotFoundError(path) from err
198
+ _raise_file_not_found(path, err)
200
199
  else:
201
- raise FileNotFoundError(path) from err
200
+ _raise_file_not_found(path, err)
202
201
  else:
203
202
  repo_id = path
204
203
  path_in_repo = ""
@@ -230,7 +229,7 @@ class HfFileSystem(fsspec.AbstractFileSystem):
230
229
  revision: Optional[str] = None,
231
230
  **kwargs,
232
231
  ) -> "HfFileSystemFile":
233
- if mode == "ab":
232
+ if "a" in mode:
234
233
  raise NotImplementedError("Appending to remote files is not yet supported.")
235
234
  return HfFileSystemFile(self, path, mode=mode, revision=revision, **kwargs)
236
235
 
@@ -392,20 +391,55 @@ class HfFileSystem(fsspec.AbstractFileSystem):
392
391
  return {"name": name, "size": 0, "type": "directory"}
393
392
  return super().info(path, **kwargs)
394
393
 
394
+ @property
395
+ def transaction(self):
396
+ """A context within which files are committed together upon exit
397
+
398
+ Requires the file class to implement `.commit()` and `.discard()`
399
+ for the normal and exception cases.
400
+ """
401
+ # Taken from https://github.com/fsspec/filesystem_spec/blob/3fbb6fee33b46cccb015607630843dea049d3243/fsspec/spec.py#L231
402
+ # See https://github.com/huggingface/huggingface_hub/issues/1733
403
+ raise NotImplementedError("Transactional commits are not supported.")
404
+
405
+ def start_transaction(self):
406
+ """Begin write transaction for deferring files, non-context version"""
407
+ # Taken from https://github.com/fsspec/filesystem_spec/blob/3fbb6fee33b46cccb015607630843dea049d3243/fsspec/spec.py#L241
408
+ # See https://github.com/huggingface/huggingface_hub/issues/1733
409
+ raise NotImplementedError("Transactional commits are not supported.")
410
+
395
411
 
396
412
  class HfFileSystemFile(fsspec.spec.AbstractBufferedFile):
397
413
  def __init__(self, fs: HfFileSystem, path: str, revision: Optional[str] = None, **kwargs):
398
414
  super().__init__(fs, path, **kwargs)
399
415
  self.fs: HfFileSystem
400
- self.resolved_path = fs.resolve_path(path, revision=revision)
416
+
417
+ mode = kwargs.get("mode", "r")
418
+ try:
419
+ self.resolved_path = fs.resolve_path(path, revision=revision)
420
+ except FileNotFoundError as e:
421
+ if "w" in mode:
422
+ raise FileNotFoundError(
423
+ f"{e}.\nMake sure the repository and revision exist before writing data."
424
+ ) from e
425
+
426
+ def __del__(self):
427
+ if not hasattr(self, "resolved_path"):
428
+ # Means that the constructor failed. Nothing to do.
429
+ return
430
+ return super().__del__()
401
431
 
402
432
  def _fetch_range(self, start: int, end: int) -> bytes:
403
433
  headers = {
404
434
  "range": f"bytes={start}-{end - 1}",
405
435
  **self.fs._api._build_hf_headers(),
406
436
  }
407
- url = (
408
- f"{self.fs.endpoint}/{REPO_TYPES_URL_PREFIXES.get(self.resolved_path.repo_type, '') + self.resolved_path.repo_id}/resolve/{safe_quote(self.resolved_path.revision)}/{safe_quote(self.resolved_path.path_in_repo)}"
437
+ url = hf_hub_url(
438
+ repo_id=self.resolved_path.repo_id,
439
+ revision=self.resolved_path.revision,
440
+ filename=self.resolved_path.path_in_repo,
441
+ repo_type=self.resolved_path.repo_type,
442
+ endpoint=self.fs.endpoint,
409
443
  )
410
444
  r = http_backoff("GET", url, headers=headers)
411
445
  hf_raise_for_status(r)
@@ -442,3 +476,14 @@ def safe_revision(revision: str) -> str:
442
476
 
443
477
  def safe_quote(s: str) -> str:
444
478
  return quote(s, safe="")
479
+
480
+
481
+ def _raise_file_not_found(path: str, err: Optional[Exception]) -> NoReturn:
482
+ msg = path
483
+ if isinstance(err, RepositoryNotFoundError):
484
+ msg = f"{path} (repository not found)"
485
+ elif isinstance(err, RevisionNotFoundError):
486
+ msg = f"{path} (revision not found)"
487
+ elif isinstance(err, HFValidationError):
488
+ msg = f"{path} (invalid repository id)"
489
+ raise FileNotFoundError(msg) from err
@@ -3,12 +3,10 @@ import os
3
3
  from pathlib import Path
4
4
  from typing import Dict, List, Optional, Type, TypeVar, Union
5
5
 
6
- import requests
7
-
8
6
  from .constants import CONFIG_NAME, PYTORCH_WEIGHTS_NAME
9
7
  from .file_download import hf_hub_download, is_torch_available
10
8
  from .hf_api import HfApi
11
- from .utils import SoftTemporaryDirectory, logging, validate_hf_hub_args
9
+ from .utils import HfHubHTTPError, SoftTemporaryDirectory, logging, validate_hf_hub_args
12
10
 
13
11
 
14
12
  if is_torch_available():
@@ -148,8 +146,8 @@ class ModelHubMixin:
148
146
  token=token,
149
147
  local_files_only=local_files_only,
150
148
  )
151
- except requests.exceptions.RequestException:
152
- logger.warning(f"{CONFIG_NAME} not found in HuggingFace Hub.")
149
+ except HfHubHTTPError:
150
+ logger.info(f"{CONFIG_NAME} not found in HuggingFace Hub.")
153
151
 
154
152
  if config_file is not None:
155
153
  with open(config_file, "r", encoding="utf-8") as f:
@@ -63,7 +63,7 @@ from huggingface_hub.inference._common import (
63
63
  _bytes_to_dict,
64
64
  _bytes_to_image,
65
65
  _bytes_to_list,
66
- _get_recommended_model,
66
+ _fetch_recommended_models,
67
67
  _import_numpy,
68
68
  _is_tgi_server,
69
69
  _open_as_binary,
@@ -146,7 +146,7 @@ class InferenceClient:
146
146
  return f"<InferenceClient(model='{self.model if self.model else ''}', timeout={self.timeout})>"
147
147
 
148
148
  @overload
149
- def post( # type: ignore
149
+ def post( # type: ignore[misc]
150
150
  self,
151
151
  *,
152
152
  json: Optional[Union[str, Dict, List]] = None,
@@ -158,7 +158,7 @@ class InferenceClient:
158
158
  pass
159
159
 
160
160
  @overload
161
- def post( # type: ignore
161
+ def post(
162
162
  self,
163
163
  *,
164
164
  json: Optional[Union[str, Dict, List]] = None,
@@ -183,16 +183,18 @@ class InferenceClient:
183
183
 
184
184
  Args:
185
185
  json (`Union[str, Dict, List]`, *optional*):
186
- The JSON data to send in the request body. Defaults to None.
186
+ The JSON data to send in the request body, specific to each task. Defaults to None.
187
187
  data (`Union[str, Path, bytes, BinaryIO]`, *optional*):
188
- The content to send in the request body. It can be raw bytes, a pointer to an opened file, a local file
189
- path, or a URL to an online resource (image, audio file,...). If both `json` and `data` are passed,
188
+ The content to send in the request body, specific to each task.
189
+ It can be raw bytes, a pointer to an opened file, a local file path,
190
+ or a URL to an online resource (image, audio file,...). If both `json` and `data` are passed,
190
191
  `data` will take precedence. At least `json` or `data` must be provided. Defaults to None.
191
192
  model (`str`, *optional*):
192
193
  The model to use for inference. Can be a model ID hosted on the Hugging Face Hub or a URL to a deployed
193
194
  Inference Endpoint. Will override the model defined at the instance level. Defaults to None.
194
195
  task (`str`, *optional*):
195
- The task to perform on the inference. Used only to default to a recommended model if `model` is not
196
+ The task to perform on the inference. All available tasks can be found
197
+ [here](https://huggingface.co/tasks). Used only to default to a recommended model if `model` is not
196
198
  provided. At least `model` or `task` must be provided. Defaults to None.
197
199
  stream (`bool`, *optional*):
198
200
  Whether to iterate over streaming APIs.
@@ -238,6 +240,10 @@ class InferenceClient:
238
240
  hf_raise_for_status(response)
239
241
  return response.iter_lines() if stream else response.content
240
242
  except HTTPError as error:
243
+ if error.response.status_code == 422 and task is not None:
244
+ error.args = (
245
+ f"{error.args[0]}\nMake sure '{task}' task is supported by the model.",
246
+ ) + error.args[1:]
241
247
  if error.response.status_code == 503:
242
248
  # If Model is unavailable, either raise a TimeoutError...
243
249
  if timeout is not None and time.time() - t0 > timeout:
@@ -1864,7 +1870,12 @@ class InferenceClient:
1864
1870
  "You must specify at least a model (repo_id or URL) or a task, either when instantiating"
1865
1871
  " `InferenceClient` or when making a request."
1866
1872
  )
1867
- model = _get_recommended_model(task)
1873
+ model = self.get_recommended_model(task)
1874
+ logger.info(
1875
+ f"Using recommended model {model} for task {task}. Note that it is"
1876
+ f" encouraged to explicitly set `model='{model}'` as the recommended"
1877
+ " models list might get updated without prior notice."
1878
+ )
1868
1879
 
1869
1880
  # Compute InferenceAPI url
1870
1881
  return (
@@ -1875,6 +1886,30 @@ class InferenceClient:
1875
1886
  else f"{INFERENCE_ENDPOINT}/models/{model}"
1876
1887
  )
1877
1888
 
1889
+ @staticmethod
1890
+ def get_recommended_model(task: str) -> str:
1891
+ """
1892
+ Get the model Hugging Face recommends for the input task.
1893
+
1894
+ Args:
1895
+ task (`str`):
1896
+ The Hugging Face task to get which model Hugging Face recommends.
1897
+ All available tasks can be found [here](https://huggingface.co/tasks).
1898
+
1899
+ Returns:
1900
+ `str`: Name of the model recommended for the input task.
1901
+
1902
+ Raises:
1903
+ `ValueError`: If Hugging Face has no recommendation for the input task.
1904
+ """
1905
+ model = _fetch_recommended_models().get(task)
1906
+ if model is None:
1907
+ raise ValueError(
1908
+ f"Task {task} has no recommended model. Please specify a model"
1909
+ " explicitly. Visit https://huggingface.co/tasks for more info."
1910
+ )
1911
+ return model
1912
+
1878
1913
  def get_model_status(self, model: Optional[str] = None) -> ModelStatus:
1879
1914
  """
1880
1915
  Get the status of a model hosted on the Inference API.
@@ -75,9 +75,15 @@ class ModelStatus:
75
75
 
76
76
  Args:
77
77
  loaded (`bool`):
78
- If the model is currently loaded.
78
+ If the model is currently loaded into Hugging Face's InferenceAPI. Models
79
+ are loaded on-demand, leading to the user's first request taking longer.
80
+ If a model is loaded, you can be assured that it is in a healthy state.
79
81
  state (`str`):
80
- The current state of the model. This can be 'Loaded', 'Loadable', 'TooBig'
82
+ The current state of the model. This can be 'Loaded', 'Loadable', 'TooBig'.
83
+ If a model's state is 'Loadable', it's not too big and has a supported
84
+ backend. Loadable models are automatically loaded when the user first
85
+ requests inference on the endpoint. This means it is transparent for the
86
+ user to load a model, except that the first call takes longer to complete.
81
87
  compute_type (`str`):
82
88
  The type of compute resource the model is using or will use, such as 'gpu' or 'cpu'.
83
89
  framework (`str`):
@@ -134,20 +140,6 @@ def _import_pil_image():
134
140
  _RECOMMENDED_MODELS: Optional[Dict[str, Optional[str]]] = None
135
141
 
136
142
 
137
- def _get_recommended_model(task: str) -> str:
138
- model = _fetch_recommended_models().get(task)
139
- if model is None:
140
- raise ValueError(
141
- f"Task {task} has no recommended task. Please specify a model explicitly. Visit"
142
- " https://huggingface.co/tasks for more info."
143
- )
144
- logger.info(
145
- f"Using recommended model {model} for task {task}. Note that it is encouraged to explicitly set"
146
- f" `model='{model}'` as the recommended models list might get updated without prior notice."
147
- )
148
- return model
149
-
150
-
151
143
  def _fetch_recommended_models() -> Dict[str, Optional[str]]:
152
144
  global _RECOMMENDED_MODELS
153
145
  if _RECOMMENDED_MODELS is None:
@@ -49,7 +49,7 @@ from huggingface_hub.inference._common import (
49
49
  _bytes_to_dict,
50
50
  _bytes_to_image,
51
51
  _bytes_to_list,
52
- _get_recommended_model,
52
+ _fetch_recommended_models,
53
53
  _import_numpy,
54
54
  _is_tgi_server,
55
55
  _open_as_binary,
@@ -130,7 +130,7 @@ class AsyncInferenceClient:
130
130
  return f"<InferenceClient(model='{self.model if self.model else ''}', timeout={self.timeout})>"
131
131
 
132
132
  @overload
133
- async def post( # type: ignore
133
+ async def post( # type: ignore[misc]
134
134
  self,
135
135
  *,
136
136
  json: Optional[Union[str, Dict, List]] = None,
@@ -142,7 +142,7 @@ class AsyncInferenceClient:
142
142
  pass
143
143
 
144
144
  @overload
145
- async def post( # type: ignore
145
+ async def post(
146
146
  self,
147
147
  *,
148
148
  json: Optional[Union[str, Dict, List]] = None,
@@ -167,16 +167,18 @@ class AsyncInferenceClient:
167
167
 
168
168
  Args:
169
169
  json (`Union[str, Dict, List]`, *optional*):
170
- The JSON data to send in the request body. Defaults to None.
170
+ The JSON data to send in the request body, specific to each task. Defaults to None.
171
171
  data (`Union[str, Path, bytes, BinaryIO]`, *optional*):
172
- The content to send in the request body. It can be raw bytes, a pointer to an opened file, a local file
173
- path, or a URL to an online resource (image, audio file,...). If both `json` and `data` are passed,
172
+ The content to send in the request body, specific to each task.
173
+ It can be raw bytes, a pointer to an opened file, a local file path,
174
+ or a URL to an online resource (image, audio file,...). If both `json` and `data` are passed,
174
175
  `data` will take precedence. At least `json` or `data` must be provided. Defaults to None.
175
176
  model (`str`, *optional*):
176
177
  The model to use for inference. Can be a model ID hosted on the Hugging Face Hub or a URL to a deployed
177
178
  Inference Endpoint. Will override the model defined at the instance level. Defaults to None.
178
179
  task (`str`, *optional*):
179
- The task to perform on the inference. Used only to default to a recommended model if `model` is not
180
+ The task to perform on the inference. All available tasks can be found
181
+ [here](https://huggingface.co/tasks). Used only to default to a recommended model if `model` is not
180
182
  provided. At least `model` or `task` must be provided. Defaults to None.
181
183
  stream (`bool`, *optional*):
182
184
  Whether to iterate over streaming APIs.
@@ -235,6 +237,8 @@ class AsyncInferenceClient:
235
237
  except aiohttp.ClientResponseError as error:
236
238
  error.response_error_payload = response_error_payload
237
239
  await client.close()
240
+ if response.status == 422 and task is not None:
241
+ error.message += f". Make sure '{task}' task is supported by the model."
238
242
  if response.status == 503:
239
243
  # If Model is unavailable, either raise a TimeoutError...
240
244
  if timeout is not None and time.time() - t0 > timeout:
@@ -1894,7 +1898,12 @@ class AsyncInferenceClient:
1894
1898
  "You must specify at least a model (repo_id or URL) or a task, either when instantiating"
1895
1899
  " `InferenceClient` or when making a request."
1896
1900
  )
1897
- model = _get_recommended_model(task)
1901
+ model = self.get_recommended_model(task)
1902
+ logger.info(
1903
+ f"Using recommended model {model} for task {task}. Note that it is"
1904
+ f" encouraged to explicitly set `model='{model}'` as the recommended"
1905
+ " models list might get updated without prior notice."
1906
+ )
1898
1907
 
1899
1908
  # Compute InferenceAPI url
1900
1909
  return (
@@ -1905,6 +1914,30 @@ class AsyncInferenceClient:
1905
1914
  else f"{INFERENCE_ENDPOINT}/models/{model}"
1906
1915
  )
1907
1916
 
1917
+ @staticmethod
1918
+ def get_recommended_model(task: str) -> str:
1919
+ """
1920
+ Get the model Hugging Face recommends for the input task.
1921
+
1922
+ Args:
1923
+ task (`str`):
1924
+ The Hugging Face task to get which model Hugging Face recommends.
1925
+ All available tasks can be found [here](https://huggingface.co/tasks).
1926
+
1927
+ Returns:
1928
+ `str`: Name of the model recommended for the input task.
1929
+
1930
+ Raises:
1931
+ `ValueError`: If Hugging Face has no recommendation for the input task.
1932
+ """
1933
+ model = _fetch_recommended_models().get(task)
1934
+ if model is None:
1935
+ raise ValueError(
1936
+ f"Task {task} has no recommended model. Please specify a model"
1937
+ " explicitly. Visit https://huggingface.co/tasks for more info."
1938
+ )
1939
+ return model
1940
+
1908
1941
  async def get_model_status(self, model: Optional[str] = None) -> ModelStatus:
1909
1942
  """
1910
1943
  Get the status of a model hosted on the Inference API.
@@ -214,6 +214,12 @@ class TextGenerationRequest:
214
214
  raise ValueError("`best_of` != 1 is not supported when `stream` == True")
215
215
  return field_value
216
216
 
217
+ def __post_init__(self):
218
+ if not is_pydantic_available():
219
+ # If pydantic is not installed, we need to instantiate the nested dataclasses manually
220
+ if self.parameters is not None and isinstance(self.parameters, dict):
221
+ self.parameters = TextGenerationParameters(**self.parameters)
222
+
217
223
 
218
224
  # Decoder input tokens
219
225
  @dataclass
@@ -312,6 +318,15 @@ class BestOfSequence:
312
318
  # Generated tokens
313
319
  tokens: List[Token] = field(default_factory=lambda: [])
314
320
 
321
+ def __post_init__(self):
322
+ if not is_pydantic_available():
323
+ # If pydantic is not installed, we need to instantiate the nested dataclasses manually
324
+ self.prefill = [
325
+ InputToken(**input_token) if isinstance(input_token, dict) else input_token
326
+ for input_token in self.prefill
327
+ ]
328
+ self.tokens = [Token(**token) if isinstance(token, dict) else token for token in self.tokens]
329
+
315
330
 
316
331
  # `generate` details
317
332
  @dataclass
@@ -347,6 +362,20 @@ class Details:
347
362
  # Additional sequences when using the `best_of` parameter
348
363
  best_of_sequences: Optional[List[BestOfSequence]] = None
349
364
 
365
+ def __post_init__(self):
366
+ if not is_pydantic_available():
367
+ # If pydantic is not installed, we need to instantiate the nested dataclasses manually
368
+ self.prefill = [
369
+ InputToken(**input_token) if isinstance(input_token, dict) else input_token
370
+ for input_token in self.prefill
371
+ ]
372
+ self.tokens = [Token(**token) if isinstance(token, dict) else token for token in self.tokens]
373
+ if self.best_of_sequences is not None:
374
+ self.best_of_sequences = [
375
+ BestOfSequence(**best_of_sequence) if isinstance(best_of_sequence, dict) else best_of_sequence
376
+ for best_of_sequence in self.best_of_sequences
377
+ ]
378
+
350
379
 
351
380
  # `generate` return value
352
381
  @dataclass
@@ -368,6 +397,12 @@ class TextGenerationResponse:
368
397
  # Generation details
369
398
  details: Optional[Details] = None
370
399
 
400
+ def __post_init__(self):
401
+ if not is_pydantic_available():
402
+ # If pydantic is not installed, we need to instantiate the nested dataclasses manually
403
+ if self.details is not None and isinstance(self.details, dict):
404
+ self.details = Details(**self.details)
405
+
371
406
 
372
407
  # `generate_stream` details
373
408
  @dataclass
@@ -418,6 +453,14 @@ class TextGenerationStreamResponse:
418
453
  # Only available when the generation is finished
419
454
  details: Optional[StreamDetails] = None
420
455
 
456
+ def __post_init__(self):
457
+ if not is_pydantic_available():
458
+ # If pydantic is not installed, we need to instantiate the nested dataclasses manually
459
+ if isinstance(self.token, dict):
460
+ self.token = Token(**self.token)
461
+ if self.details is not None and isinstance(self.details, dict):
462
+ self.details = StreamDetails(**self.details)
463
+
421
464
 
422
465
  # TEXT GENERATION ERRORS
423
466
  # ----------------------
@@ -92,7 +92,7 @@ class InferenceApi:
92
92
 
93
93
  @validate_hf_hub_args
94
94
  @_deprecate_method(
95
- version="0.19.0",
95
+ version="1.0",
96
96
  message=(
97
97
  "`InferenceApi` client is deprecated in favor of the more feature-complete `InferenceClient`. Check out"
98
98
  " this guide to learn how to convert your script to use it:"
huggingface_hub/lfs.py CHANGED
@@ -13,6 +13,7 @@
13
13
  # See the License for the specific language governing permissions and
14
14
  # limitations under the License.
15
15
  """Git LFS related type definitions and utilities"""
16
+ import inspect
16
17
  import io
17
18
  import os
18
19
  import re
@@ -29,7 +30,7 @@ from requests.auth import HTTPBasicAuth
29
30
  from huggingface_hub.constants import ENDPOINT, HF_HUB_ENABLE_HF_TRANSFER, REPO_TYPES_URL_PREFIXES
30
31
  from huggingface_hub.utils import get_session
31
32
 
32
- from .utils import get_token_to_send, hf_raise_for_status, http_backoff, logging, validate_hf_hub_args
33
+ from .utils import get_token_to_send, hf_raise_for_status, http_backoff, logging, tqdm, validate_hf_hub_args
33
34
  from .utils.sha import sha256, sha_fileobj
34
35
 
35
36
 
@@ -389,20 +390,37 @@ def _upload_parts_hf_transfer(
389
390
  " not available in your environment. Try `pip install hf_transfer`."
390
391
  )
391
392
 
392
- try:
393
- return multipart_upload(
394
- file_path=operation.path_or_fileobj,
395
- parts_urls=sorted_parts_urls,
396
- chunk_size=chunk_size,
397
- max_files=128,
398
- parallel_failures=127, # could be removed
399
- max_retries=5,
393
+ supports_callback = "callback" in inspect.signature(multipart_upload).parameters
394
+ if not supports_callback:
395
+ warnings.warn(
396
+ "You are using an outdated version of `hf_transfer`. Consider upgrading to latest version to enable progress bars using `pip install -U hf_transfer`."
400
397
  )
401
- except Exception as e:
402
- raise RuntimeError(
403
- "An error occurred while uploading using `hf_transfer`. Consider disabling HF_HUB_ENABLE_HF_TRANSFER for"
404
- " better error handling."
405
- ) from e
398
+
399
+ total = operation.upload_info.size
400
+ desc = operation.path_in_repo
401
+ if len(desc) > 40:
402
+ desc = f"(…){desc[-40:]}"
403
+ disable = bool(logger.getEffectiveLevel() == logging.NOTSET)
404
+
405
+ with tqdm(unit="B", unit_scale=True, total=total, initial=0, desc=desc, disable=disable) as progress:
406
+ try:
407
+ output = multipart_upload(
408
+ file_path=operation.path_or_fileobj,
409
+ parts_urls=sorted_parts_urls,
410
+ chunk_size=chunk_size,
411
+ max_files=128,
412
+ parallel_failures=127, # could be removed
413
+ max_retries=5,
414
+ **({"callback": progress.update} if supports_callback else {}),
415
+ )
416
+ except Exception as e:
417
+ raise RuntimeError(
418
+ "An error occurred while uploading using `hf_transfer`. Consider disabling HF_HUB_ENABLE_HF_TRANSFER for"
419
+ " better error handling."
420
+ ) from e
421
+ if not supports_callback:
422
+ progress.update(total)
423
+ return output
406
424
 
407
425
 
408
426
  class SliceFileObj(AbstractContextManager):
@@ -194,6 +194,9 @@ class CardData:
194
194
  return yaml_dump(self.to_dict(), sort_keys=False, line_break=line_break).strip()
195
195
 
196
196
  def __repr__(self):
197
+ return repr(self.__dict__)
198
+
199
+ def __str__(self):
197
200
  return self.to_yaml()
198
201
 
199
202
  def get(self, key: str, default: Any = None) -> Any:
@@ -216,6 +219,10 @@ class CardData:
216
219
  """Check if a given metadata key is set."""
217
220
  return key in self.__dict__
218
221
 
222
+ def __len__(self) -> int:
223
+ """Return the number of metadata keys set."""
224
+ return len(self.__dict__)
225
+
219
226
 
220
227
  class ModelCardData(CardData):
221
228
  """Model Card Metadata that is used by Hugging Face Hub when included at the top of your README.md
@@ -22,6 +22,7 @@ from .utils import (
22
22
  tqdm,
23
23
  validate_hf_hub_args,
24
24
  )
25
+ from .utils._deprecation import _deprecate_method
25
26
 
26
27
 
27
28
  logger = logging.get_logger(__name__)
@@ -421,6 +422,14 @@ def _lfs_log_progress():
421
422
  os.environ["GIT_LFS_PROGRESS"] = current_lfs_progress_value
422
423
 
423
424
 
425
+ @_deprecate_method(
426
+ version="1.0",
427
+ message=(
428
+ "Please prefer the http-based alternatives instead. Given its large adoption in legacy code, the complete"
429
+ " removal is only planned on next major release.\nFor more details, please read"
430
+ " https://huggingface.co/docs/huggingface_hub/concepts/git_vs_http."
431
+ ),
432
+ )
424
433
  class Repository:
425
434
  """
426
435
  Helper class to wrap the git and git-lfs commands.
@@ -428,6 +437,15 @@ class Repository:
428
437
  The aim is to facilitate interacting with huggingface.co hosted model or
429
438
  dataset repos, though not a lot here (if any) is actually specific to
430
439
  huggingface.co.
440
+
441
+ <Tip warning={true}>
442
+
443
+ [`Repository`] is deprecated in favor of the http-based alternatives implemented in
444
+ [`HfApi`]. Given its large adoption in legacy code, the complete removal of
445
+ [`Repository`] will only happen in release `v1.0`. For more details, please read
446
+ https://huggingface.co/docs/huggingface_hub/concepts/git_vs_http.
447
+
448
+ </Tip>
431
449
  """
432
450
 
433
451
  command_queue: List[CommandInProgress]
@@ -1367,9 +1385,7 @@ class Repository:
1367
1385
  files_to_stage = files_to_be_staged(".", folder=self.local_dir)
1368
1386
 
1369
1387
  if len(files_to_stage):
1370
- if len(files_to_stage) > 5:
1371
- files_in_msg = str(files_to_stage[:5])[:-1] + ", ...]"
1372
-
1388
+ files_in_msg = str(files_to_stage[:5])[:-1] + ", ...]" if len(files_to_stage) > 5 else str(files_to_stage)
1373
1389
  logger.error(
1374
1390
  "There exists some updated files in the local repository that are not"
1375
1391
  f" committed: {files_in_msg}. This may lead to errors if checking out"
@@ -24,7 +24,7 @@
24
24
  - **Model type:** {{ model_type | default("[More Information Needed]", true)}}
25
25
  - **Language(s) (NLP):** {{ language | default("[More Information Needed]", true)}}
26
26
  - **License:** {{ license | default("[More Information Needed]", true)}}
27
- - **Finetuned from model [optional]:** {{ finetuned_from | default("[More Information Needed]", true)}}
27
+ - **Finetuned from model [optional]:** {{ base_model | default("[More Information Needed]", true)}}
28
28
 
29
29
  ### Model Sources [optional]
30
30
 
@@ -44,7 +44,7 @@ from ._fixes import SoftTemporaryDirectory, yaml_dump
44
44
  from ._git_credential import list_credential_helpers, set_git_credential, unset_git_credential
45
45
  from ._headers import build_hf_headers, get_token_to_send, LocalTokenNotFoundError
46
46
  from ._hf_folder import HfFolder
47
- from ._http import configure_http_backend, get_session, http_backoff
47
+ from ._http import configure_http_backend, get_session, http_backoff, reset_sessions
48
48
  from ._pagination import paginate
49
49
  from ._paths import filter_repo_objects, IGNORE_GIT_FOLDER_PATTERNS
50
50
  from ._experimental import experimental