huggingface-hub 0.20.3__py3-none-any.whl → 0.21.0__py3-none-any.whl

huggingface_hub/hf_api.py

@@ -20,7 +20,7 @@ import re
 import struct
 import warnings
 from concurrent.futures import Future, ThreadPoolExecutor
-from dataclasses import dataclass, field
+from dataclasses import asdict, dataclass, field
 from datetime import datetime
 from functools import wraps
 from itertools import islice
@@ -36,12 +36,11 @@ from typing import (
     Literal,
     Optional,
     Tuple,
-    TypedDict,
     TypeVar,
     Union,
     overload,
 )
-from urllib.parse import quote, urlencode
+from urllib.parse import quote
 
 import requests
 from requests.exceptions import HTTPError
@@ -53,7 +52,7 @@ from ._commit_api import (
     CommitOperationAdd,
     CommitOperationCopy,
     CommitOperationDelete,
-    _fetch_lfs_files_to_copy,
+    _fetch_files_to_copy,
     _fetch_upload_modes,
     _prepare_commit_payload,
     _upload_lfs_files,
@@ -130,7 +129,7 @@ from .utils import (  # noqa: F401 # imported for backward compatibility
     validate_hf_hub_args,
 )
 from .utils import tqdm as hf_tqdm
-from .utils._deprecation import _deprecate_method
+from .utils._deprecation import _deprecate_arguments, _deprecate_method
 from .utils._typing import CallableT
 from .utils.endpoint_helpers import (
     DatasetFilter,
@@ -238,36 +237,56 @@ def repo_type_and_id_from_hf_id(hf_id: str, hub_url: Optional[str] = None) -> Tu
     return repo_type, namespace, repo_id
 
 
-class LastCommitInfo(TypedDict, total=False):
+@dataclass
+class LastCommitInfo(dict):
     oid: str
     title: str
     date: datetime
 
+    def __post_init__(self):  # hack to make LastCommitInfo backward compatible
+        self.update(asdict(self))
+
 
-class BlobLfsInfo(TypedDict, total=False):
+@dataclass
+class BlobLfsInfo(dict):
     size: int
     sha256: str
     pointer_size: int
 
+    def __post_init__(self):  # hack to make BlobLfsInfo backward compatible
+        self.update(asdict(self))
 
-class BlobSecurityInfo(TypedDict, total=False):
+
+@dataclass
+class BlobSecurityInfo(dict):
     safe: bool
     av_scan: Optional[Dict]
     pickle_import_scan: Optional[Dict]
 
+    def __post_init__(self):  # hack to make BlogSecurityInfo backward compatible
+        self.update(asdict(self))
+
 
-class TransformersInfo(TypedDict, total=False):
+@dataclass
+class TransformersInfo(dict):
     auto_model: str
-    custom_class: Optional[str]
+    custom_class: Optional[str] = None
     # possible `pipeline_tag` values: https://github.com/huggingface/huggingface.js/blob/3ee32554b8620644a6287e786b2a83bf5caf559c/packages/tasks/src/pipelines.ts#L72
-    pipeline_tag: Optional[str]
-    processor: Optional[str]
+    pipeline_tag: Optional[str] = None
+    processor: Optional[str] = None
+
+    def __post_init__(self):  # hack to make TransformersInfo backward compatible
+        self.update(asdict(self))
 
 
-class SafeTensorsInfo(TypedDict, total=False):
+@dataclass
+class SafeTensorsInfo(dict):
     parameters: List[Dict[str, int]]
     total: int
 
+    def __post_init__(self):  # hack to make SafeTensorsInfo backward compatible
+        self.update(asdict(self))
+
 
 @dataclass
 class CommitInfo(str):
@@ -656,7 +675,7 @@ class ModelInfo:
             ModelCardData(**card_data, ignore_metadata_errors=True) if isinstance(card_data, dict) else card_data
         )
 
-        self.widget_data = kwargs.pop("widget_data", None)
+        self.widget_data = kwargs.pop("widgetData", None)
         self.model_index = kwargs.pop("model-index", None) or kwargs.pop("model_index", None)
         self.config = kwargs.pop("config", None)
         transformers_info = kwargs.pop("transformersInfo", None) or kwargs.pop("transformers_info", None)
@@ -1094,7 +1113,7 @@ class GitRefs:
     branches: List[GitRefInfo]
     converts: List[GitRefInfo]
     tags: List[GitRefInfo]
-    pull_requests: Optional[List[GitRefInfo]]
+    pull_requests: Optional[List[GitRefInfo]] = None
 
 
 @dataclass
@@ -1361,6 +1380,12 @@ class HfApi:
         *,
         filter: Union[ModelFilter, str, Iterable[str], None] = None,
         author: Optional[str] = None,
+        library: Optional[Union[str, List[str]]] = None,
+        language: Optional[Union[str, List[str]]] = None,
+        model_name: Optional[str] = None,
+        task: Optional[Union[str, List[str]]] = None,
+        trained_dataset: Optional[Union[str, List[str]]] = None,
+        tags: Optional[Union[str, List[str]]] = None,
         search: Optional[str] = None,
         emissions_thresholds: Optional[Tuple[float, float]] = None,
         sort: Union[Literal["last_modified"], str, None] = None,
@@ -1370,6 +1395,7 @@ class HfApi:
         cardData: bool = False,
         fetch_config: bool = False,
         token: Optional[Union[bool, str]] = None,
+        pipeline_tag: Optional[str] = None,
     ) -> Iterable[ModelInfo]:
         """
         List models hosted on the Huggingface Hub, given some filters.
@@ -1381,6 +1407,24 @@ class HfApi:
             author (`str`, *optional*):
                 A string which identify the author (user or organization) of the
                 returned models
+            library (`str` or `List`, *optional*):
+                A string or list of strings of foundational libraries models were
+                originally trained from, such as pytorch, tensorflow, or allennlp.
+            language (`str` or `List`, *optional*):
+                A string or list of strings of languages, both by name and country
+                code, such as "en" or "English"
+            model_name (`str`, *optional*):
+                A string that contain complete or partial names for models on the
+                Hub, such as "bert" or "bert-base-cased"
+            task (`str` or `List`, *optional*):
+                A string or list of strings of tasks models were designed for, such
+                as: "fill-mask" or "automatic-speech-recognition"
+            trained_dataset (`str` or `List`, *optional*):
+                A string tag or a list of string tags of the trained dataset for a
+                model on the Hub.
+            tags (`str` or `List`, *optional*):
+                A string tag or a list of tags to filter models on the Hub by, such
+                as `text-generation` or `spacy`.
             search (`str`, *optional*):
                 A string that will be contained in the returned model ids.
             emissions_thresholds (`Tuple`, *optional*):
@@ -1411,6 +1455,9 @@ class HfApi:
                 If `None` or `True` and machine is logged in (through `huggingface-cli login`
                 or [`~huggingface_hub.login`]), token will be retrieved from the cache.
                 If `False`, token is not sent in the request header.
+            pipeline_tag (`str`, *optional*):
+                A string pipeline tag to filter models on the Hub by, such as `summarization`
+
 
         Returns:
             `Iterable[ModelInfo]`: an iterable of [`huggingface_hub.hf_api.ModelInfo`] objects.
@@ -1427,9 +1474,6 @@ class HfApi:
 
         >>> # List only the text classification models
         >>> api.list_models(filter="text-classification")
-        >>> # Using the `ModelFilter`
-        >>> filt = ModelFilter(task="text-classification")
-
 
         >>> # List only models from the AllenNLP library
         >>> api.list_models(filter="allennlp")
@@ -1455,15 +1499,38 @@ class HfApi:
         path = f"{self.endpoint}/api/models"
         headers = self._build_hf_headers(token=token)
         params = {}
+        filter_list = []
+
         if filter is not None:
             if isinstance(filter, ModelFilter):
                 params = self._unpack_model_filter(filter)
             else:
                 params.update({"filter": filter})
+
             params.update({"full": True})
-        if author is not None:
+
+        # Build the filter list
+        if author:
             params.update({"author": author})
-        if search is not None:
+        if model_name:
+            params.update({"search": model_name})
+        if library:
+            filter_list.extend([library] if isinstance(library, str) else library)
+        if task:
+            filter_list.extend([task] if isinstance(task, str) else task)
+        if trained_dataset:
+            if not isinstance(trained_dataset, (list, tuple)):
+                trained_dataset = [trained_dataset]
+            for dataset in trained_dataset:
+                if not dataset.startswith("dataset:"):
+                    dataset = f"dataset:{dataset}"
+                filter_list.append(dataset)
+        if language:
+            filter_list.extend([language] if isinstance(language, str) else language)
+        if tags:
+            filter_list.extend([tags] if isinstance(tags, str) else tags)
+
+        if search:
             params.update({"search": search})
         if sort is not None:
             params.update({"sort": "lastModified" if sort == "last_modified" else sort})
@@ -1480,6 +1547,13 @@ class HfApi:
             params.update({"config": True})
         if cardData:
             params.update({"cardData": True})
+        if pipeline_tag:
+            params.update({"pipeline_tag": pipeline_tag})
+
+        filter_value = params.get("filter", [])
+        if filter_value:
+            filter_list.extend([filter_value] if isinstance(filter_value, str) else list(filter_value))
+        params.update({"filter": filter_list})
 
         # `items` is a generator
         items = paginate(path, params=params, headers=headers)
@@ -1499,21 +1573,21 @@ class HfApi:
         model_str = ""
 
         # Handling author
-        if model_filter.author is not None:
+        if model_filter.author:
             model_str = f"{model_filter.author}/"
 
         # Handling model_name
-        if model_filter.model_name is not None:
+        if model_filter.model_name:
             model_str += model_filter.model_name
 
         filter_list: List[str] = []
 
         # Handling tasks
-        if model_filter.task is not None:
+        if model_filter.task:
             filter_list.extend([model_filter.task] if isinstance(model_filter.task, str) else model_filter.task)
 
         # Handling dataset
-        if model_filter.trained_dataset is not None:
+        if model_filter.trained_dataset:
             if not isinstance(model_filter.trained_dataset, (list, tuple)):
                 model_filter.trained_dataset = [model_filter.trained_dataset]
             for dataset in model_filter.trained_dataset:
@@ -1532,7 +1606,7 @@ class HfApi:
             filter_list.extend([model_filter.tags] if isinstance(model_filter.tags, str) else model_filter.tags)
 
         query_dict: Dict[str, Any] = {}
-        if model_str is not None:
+        if model_str:
             query_dict["search"] = model_str
         if isinstance(model_filter.language, list):
             filter_list.extend(model_filter.language)
@@ -1547,8 +1621,16 @@ class HfApi:
         *,
         filter: Union[DatasetFilter, str, Iterable[str], None] = None,
         author: Optional[str] = None,
+        benchmark: Optional[Union[str, List[str]]] = None,
+        dataset_name: Optional[str] = None,
+        language_creators: Optional[Union[str, List[str]]] = None,
+        language: Optional[Union[str, List[str]]] = None,
+        multilinguality: Optional[Union[str, List[str]]] = None,
+        size_categories: Optional[Union[str, List[str]]] = None,
+        task_categories: Optional[Union[str, List[str]]] = None,
+        task_ids: Optional[Union[str, List[str]]] = None,
         search: Optional[str] = None,
-        sort: Union[Literal["last_modified"], str, None] = None,
+        sort: Optional[Union[Literal["last_modified"], str]] = None,
         direction: Optional[Literal[-1]] = None,
         limit: Optional[int] = None,
         full: Optional[bool] = None,
@@ -1563,6 +1645,34 @@ class HfApi:
                 datasets on the hub.
             author (`str`, *optional*):
                 A string which identify the author of the returned datasets.
+            benchmark (`str` or `List`, *optional*):
+                A string or list of strings that can be used to identify datasets on
+                the Hub by their official benchmark.
+            dataset_name (`str`, *optional*):
+                A string or list of strings that can be used to identify datasets on
+                the Hub by its name, such as `SQAC` or `wikineural`
+            language_creators (`str` or `List`, *optional*):
+                A string or list of strings that can be used to identify datasets on
+                the Hub with how the data was curated, such as `crowdsourced` or
+                `machine_generated`.
+            language (`str` or `List`, *optional*):
+                A string or list of strings representing a two-character language to
+                filter datasets by on the Hub.
+            multilinguality (`str` or `List`, *optional*):
+                A string or list of strings representing a filter for datasets that
+                contain multiple languages.
+            size_categories (`str` or `List`, *optional*):
+                A string or list of strings that can be used to identify datasets on
+                the Hub by the size of the dataset such as `100K<n<1M` or
+                `1M<n<10M`.
+            task_categories (`str` or `List`, *optional*):
+                A string or list of strings that can be used to identify datasets on
+                the Hub by the designed task, such as `audio_classification` or
+                `named_entity_recognition`.
+            task_ids (`str` or `List`, *optional*):
+                A string or list of strings that can be used to identify datasets on
+                the Hub by the specific task such as `speech_emotion_recognition` or
+                `paraphrase`.
             search (`str`, *optional*):
                 A string that will be contained in the returned datasets.
             sort (`Literal["last_modified"]` or `str`, *optional*):
@@ -1600,16 +1710,12 @@ class HfApi:
 
         >>> # List only the text classification datasets
         >>> api.list_datasets(filter="task_categories:text-classification")
-        >>> # Using the `DatasetFilter`
-        >>> filt = DatasetFilter(task_categories="text-classification")
 
 
         >>> # List only the datasets in russian for language modeling
         >>> api.list_datasets(
         ...     filter=("language:ru", "task_ids:language-modeling")
         ... )
-        >>> # Using the `DatasetFilter`
-        >>> filt = DatasetFilter(language="ru", task_ids="language-modeling")
 
         >>> api.list_datasets(filter=filt)
         ```
@@ -1631,14 +1737,38 @@ class HfApi:
         path = f"{self.endpoint}/api/datasets"
         headers = self._build_hf_headers(token=token)
         params = {}
+        filter_list = []
+
         if filter is not None:
             if isinstance(filter, DatasetFilter):
                 params = self._unpack_dataset_filter(filter)
             else:
                 params.update({"filter": filter})
-        if author is not None:
+
+        # Build the filter list
+        if author:
             params.update({"author": author})
-        if search is not None:
+        if dataset_name:
+            params.update({"search": dataset_name})
+
+        for attr in (
+            benchmark,
+            language_creators,
+            language,
+            multilinguality,
+            size_categories,
+            task_categories,
+            task_ids,
+        ):
+            if attr:
+                if not isinstance(attr, (list, tuple)):
+                    attr = [attr]
+                for data in attr:
+                    if not data.startswith(f"{attr}:"):
+                        data = f"{attr}:{data}"
+                    filter_list.append(data)
+
+        if search:
             params.update({"search": search})
         if sort is not None:
             params.update({"sort": "lastModified" if sort == "last_modified" else sort})
@@ -1649,6 +1779,11 @@ class HfApi:
         if full:
             params.update({"full": True})
 
+        filter_value = params.get("filter", [])
+        if filter_value:
+            filter_list.extend([filter_value] if isinstance(filter_value, str) else list(filter_value))
+        params.update({"filter": filter_list})
+
         items = paginate(path, params=params, headers=headers)
         if limit is not None:
             items = islice(items, limit)  # Do not iterate over all pages
@@ -1664,11 +1799,11 @@ class HfApi:
         dataset_str = ""
 
         # Handling author
-        if dataset_filter.author is not None:
+        if dataset_filter.author:
             dataset_str = f"{dataset_filter.author}/"
 
         # Handling dataset_name
-        if dataset_filter.dataset_name is not None:
+        if dataset_filter.dataset_name:
             dataset_str += dataset_filter.dataset_name
 
         filter_list = []
@@ -2307,18 +2442,14 @@ class HfApi:
         Returns:
             True if the repository exists, False otherwise.
 
-        <Tip>
-
         Examples:
             ```py
             >>> from huggingface_hub import repo_exists
-            >>> repo_exists("huggingface/transformers")
+            >>> repo_exists("google/gemma-7b")
             True
-            >>> repo_exists("huggingface/not-a-repo")
+            >>> repo_exists("google/not-a-repo")
             False
             ```
-
-        </Tip>
         """
         try:
             self.repo_info(repo_id=repo_id, repo_type=repo_type, token=token)
@@ -2328,6 +2459,53 @@ class HfApi:
         except RepositoryNotFoundError:
             return False
 
+    @validate_hf_hub_args
+    def revision_exists(
+        self,
+        repo_id: str,
+        revision: str,
+        *,
+        repo_type: Optional[str] = None,
+        token: Optional[str] = None,
+    ) -> bool:
+        """
+        Checks if a specific revision exists on a repo on the Hugging Face Hub.
+
+        Args:
+            repo_id (`str`):
+                A namespace (user or an organization) and a repo name separated
+                by a `/`.
+            revision (`str`):
+                The revision of the repository to check.
+            repo_type (`str`, *optional*):
+                Set to `"dataset"` or `"space"` if getting repository info from a dataset or a space,
+                `None` or `"model"` if getting repository info from a model. Default is `None`.
+            token (`bool` or `str`, *optional*):
+                A valid authentication token (see https://huggingface.co/settings/token).
+                If `None` or `True` and machine is logged in (through `huggingface-cli login`
+                or [`~huggingface_hub.login`]), token will be retrieved from the cache.
+                If `False`, token is not sent in the request header.
+
+        Returns:
+            True if the repository and the revision exists, False otherwise.
+
+        Examples:
+            ```py
+            >>> from huggingface_hub import revision_exists
+            >>> revision_exists("google/gemma-7b", "float16")
+            True
+            >>> revision_exists("google/gemma-7b", "not-a-revision")
+            False
+            ```
+        """
+        try:
+            self.repo_info(repo_id=repo_id, revision=revision, repo_type=repo_type, token=token)
+            return True
+        except RevisionNotFoundError:
+            return False
+        except RepositoryNotFoundError:
+            return False
+
     @validate_hf_hub_args
     def file_exists(
         self,
@@ -2362,8 +2540,6 @@ class HfApi:
         Returns:
             True if the file exists, False otherwise.
 
-        <Tip>
-
         Examples:
             ```py
             >>> from huggingface_hub import file_exists
@@ -2374,8 +2550,6 @@ class HfApi:
             >>> file_exists("bigcode/not-a-repo", "config.json")
             False
             ```
-
-        </Tip>
         """
         url = hf_hub_url(
             repo_id=repo_id, repo_type=repo_type, revision=revision, filename=filename, endpoint=self.endpoint
@@ -3243,6 +3417,9 @@ class HfApi:
                 raise
 
     @validate_hf_hub_args
+    @_deprecate_arguments(
+        version="0.24.0", deprecated_args=("organization", "name"), custom_message="Use `repo_id` instead."
+    )
     def update_repo_visibility(
         self,
         repo_id: str,
@@ -3510,7 +3687,8 @@ class HfApi:
             [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError)
                 If parent commit is not a valid commit OID.
             [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError)
-                If the Hub API returns an HTTP 400 error (bad request)
+                If a README.md file with an invalid metadata section is committed. In this case, the commit will fail
+                early, before trying to upload any file.
             [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError)
                 If `create_pr` is `True` and revision is neither `None` nor `"main"`.
             [`~utils.RepositoryNotFoundError`]:
@@ -3552,6 +3730,32 @@ class HfApi:
             f" {nb_deletions} deletion(s)."
         )
 
+        # If updating a README.md file, make sure the metadata format is valid
+        # It's better to fail early than to fail after all the files have been uploaded.
+        for addition in additions:
+            if addition.path_in_repo == "README.md":
+                with addition.as_file() as file:
+                    response = get_session().post(
+                        f"{ENDPOINT}/api/validate-yaml",
+                        json={"content": file.read().decode(), "repoType": repo_type},
+                        headers=self._build_hf_headers(token=token),
+                    )
+                    # Handle warnings (example: empty metadata)
+                    response_content = response.json()
+                    message = "\n".join(
+                        [f"- {warning.get('message')}" for warning in response_content.get("warnings", [])]
+                    )
+                    if message:
+                        warnings.warn(f"Warnings while validating metadata in README.md:\n{message}")
+
+                    # Raise on errors
+                    try:
+                        hf_raise_for_status(response)
+                    except BadRequestError as e:
+                        errors = response_content.get("errors", [])
+                        message = "\n".join([f"- {error.get('message')}" for error in errors])
+                        raise ValueError(f"Invalid metadata in README.md.\n{message}") from e
+
         # If updating twice the same file or update then delete a file in a single commit
         _warn_on_overwriting_operations(operations)
 
@@ -3565,7 +3769,7 @@ class HfApi:
             num_threads=num_threads,
             free_memory=False,  # do not remove `CommitOperationAdd.path_or_fileobj` on LFS files for "normal" users
         )
-        files_to_copy = _fetch_lfs_files_to_copy(
+        files_to_copy = _fetch_files_to_copy(
             copies=copies,
             repo_type=repo_type,
             repo_id=repo_id,
@@ -5145,7 +5349,7 @@ class HfApi:
         safetensors file, we parse the metadata from this file. If it's a sharded safetensors repo, we parse the
         metadata from the index file and then parse the metadata from each shard.
 
-        To parse metadata from a single safetensors file, use [`get_safetensors_metadata`].
+        To parse metadata from a single safetensors file, use [`parse_safetensors_file_metadata`].
 
         For more details regarding the safetensors format, check out https://huggingface.co/docs/safetensors/index#format.
 
@@ -5688,18 +5892,19 @@ class HfApi:
             raise ValueError(f"Invalid discussion_status, must be one of {DISCUSSION_STATUS}")
 
         headers = self._build_hf_headers(token=token)
-        query_dict: Dict[str, str] = {}
+        path = f"{self.endpoint}/api/{repo_type}s/{repo_id}/discussions"
+
+        params: Dict[str, Union[str, int]] = {}
         if discussion_type is not None:
-            query_dict["type"] = discussion_type
+            params["type"] = discussion_type
         if discussion_status is not None:
-            query_dict["status"] = discussion_status
+            params["status"] = discussion_status
         if author is not None:
-            query_dict["author"] = author
+            params["author"] = author
 
         def _fetch_discussion_page(page_index: int):
-            query_string = urlencode({**query_dict, "page_index": page_index})
-            path = f"{self.endpoint}/api/{repo_type}s/{repo_id}/discussions?{query_string}"
-            resp = get_session().get(path, headers=headers)
+            params["p"] = page_index
+            resp = get_session().get(path, headers=headers, params=params)
             hf_raise_for_status(resp)
             paginated_discussions = resp.json()
             total = paginated_discussions["count"]
@@ -7633,6 +7838,15 @@ class HfApi:
 
         Returns: [`Collection`]
 
+        Raises:
+            `HTTPError`:
+                HTTP 403 if you only have read-only access to the repo. This can be the case if you don't have `write`
+                or `admin` role in the organization the repo belongs to or if you passed a `read` token.
+            `HTTPError`:
+                HTTP 404 if the item you try to add to the collection does not exist on the Hub.
+            `HTTPError`:
+                HTTP 409 if the item you try to add to the collection is already in the collection (and exists_ok=False)
+
         Example:
 
         ```py
@@ -8303,6 +8517,7 @@ list_spaces = api.list_spaces
 space_info = api.space_info
 
 repo_exists = api.repo_exists
+revision_exists = api.revision_exists
 file_exists = api.file_exists
 repo_info = api.repo_info
 list_repo_files = api.list_repo_files