huggingface-hub 0.29.0rc2__py3-none-any.whl → 1.1.3__py3-none-any.whl

huggingface_hub/hub_mixin.py

@@ -3,7 +3,7 @@ import json
 import os
 from dataclasses import Field, asdict, dataclass, is_dataclass
 from pathlib import Path
-from typing import Any, Callable, ClassVar, Dict, List, Optional, Protocol, Tuple, Type, TypeVar, Union
+from typing import Any, Callable, ClassVar, Optional, Protocol, Type, TypeVar, Union
 
 import packaging.version
 
@@ -38,7 +38,7 @@ logger = logging.get_logger(__name__)
 
 # Type alias for dataclass instances, copied from https://github.com/python/typeshed/blob/9f28171658b9ca6c32a7cb93fbb99fc92b17858b/stdlib/_typeshed/__init__.pyi#L349
 class DataclassInstance(Protocol):
-    __dataclass_fields__: ClassVar[Dict[str, Field]]
+    __dataclass_fields__: ClassVar[dict[str, Field]]
 
 
 # Generic variable that is either ModelHubMixin or a subclass thereof
@@ -47,7 +47,7 @@ T = TypeVar("T", bound="ModelHubMixin")
 ARGS_T = TypeVar("ARGS_T")
 ENCODER_T = Callable[[ARGS_T], Any]
 DECODER_T = Callable[[Any], ARGS_T]
-CODER_T = Tuple[ENCODER_T, DECODER_T]
+CODER_T = tuple[ENCODER_T, DECODER_T]
 
 
 DEFAULT_MODEL_CARD = """
@@ -58,7 +58,8 @@ DEFAULT_MODEL_CARD = """
 ---
 
 This model has been pushed to the Hub using the [PytorchModelHubMixin](https://huggingface.co/docs/huggingface_hub/package_reference/mixins#huggingface_hub.PyTorchModelHubMixin) integration:
-- Library: {{ repo_url | default("[More Information Needed]", true) }}
+- Code: {{ repo_url | default("[More Information Needed]", true) }}
+- Paper: {{ paper_url | default("[More Information Needed]", true) }}
 - Docs: {{ docs_url | default("[More Information Needed]", true) }}
 """
 
@@ -67,8 +68,9 @@ This model has been pushed to the Hub using the [PytorchModelHubMixin](https://h
 class MixinInfo:
     model_card_template: str
     model_card_data: ModelCardData
-    repo_url: Optional[str] = None
     docs_url: Optional[str] = None
+    paper_url: Optional[str] = None
+    repo_url: Optional[str] = None
 
 
 class ModelHubMixin:
@@ -88,11 +90,13 @@ class ModelHubMixin:
     Args:
         repo_url (`str`, *optional*):
             URL of the library repository. Used to generate model card.
+        paper_url (`str`, *optional*):
+            URL of the library paper. Used to generate model card.
         docs_url (`str`, *optional*):
             URL of the library documentation. Used to generate model card.
         model_card_template (`str`, *optional*):
             Template of the model card. Used to generate model card. Defaults to a generic template.
-        language (`str` or `List[str]`, *optional*):
+        language (`str` or `list[str]`, *optional*):
             Language supported by the library. Used to generate model card.
         library_name (`str`, *optional*):
             Name of the library integrating ModelHubMixin. Used to generate model card.
@@ -109,11 +113,11 @@ class ModelHubMixin:
             E.g: "https://coqui.ai/cpml".
         pipeline_tag (`str`, *optional*):
             Tag of the pipeline. Used to generate model card. E.g. "text-classification".
-        tags (`List[str]`, *optional*):
-            Tags to be added to the model card. Used to generate model card. E.g. ["x-custom-tag", "arxiv:2304.12244"]
-        coders (`Dict[Type, Tuple[Callable, Callable]]`, *optional*):
+        tags (`list[str]`, *optional*):
+            Tags to be added to the model card. Used to generate model card. E.g. ["computer-vision"]
+        coders (`dict[Type, tuple[Callable, Callable]]`, *optional*):
             Dictionary of custom types and their encoders/decoders. Used to encode/decode arguments that are not
-            jsonable by default. E.g dataclasses, argparse.Namespace, OmegaConf, etc.
+            jsonable by default. E.g. dataclasses, argparse.Namespace, OmegaConf, etc.
 
     Example:
 
@@ -124,8 +128,9 @@ class ModelHubMixin:
     >>> class MyCustomModel(
     ...         ModelHubMixin,
     ...         library_name="my-library",
-    ...         tags=["x-custom-tag", "arxiv:2304.12244"],
+    ...         tags=["computer-vision"],
     ...         repo_url="https://github.com/huggingface/my-cool-library",
+    ...         paper_url="https://arxiv.org/abs/2304.12244",
     ...         docs_url="https://huggingface.co/docs/my-cool-library",
     ...         # ^ optional metadata to generate model card
     ...     ):
@@ -140,12 +145,10 @@ class ModelHubMixin:
     ...
     ...     @classmethod
     ...     def from_pretrained(
-    ...         cls: Type[T],
+    ...         cls: type[T],
     ...         pretrained_model_name_or_path: Union[str, Path],
     ...         *,
     ...         force_download: bool = False,
-    ...         resume_download: Optional[bool] = None,
-    ...         proxies: Optional[Dict] = None,
     ...         token: Optional[Union[str, bool]] = None,
     ...         cache_dir: Optional[Union[str, Path]] = None,
     ...         local_files_only: bool = False,
@@ -183,10 +186,10 @@ class ModelHubMixin:
     _hub_mixin_info: MixinInfo
     # ^ information about the library integrating ModelHubMixin (used to generate model card)
     _hub_mixin_inject_config: bool  # whether `_from_pretrained` expects `config` or not
-    _hub_mixin_init_parameters: Dict[str, inspect.Parameter]  # __init__ parameters
-    _hub_mixin_jsonable_default_values: Dict[str, Any]  # default values for __init__ parameters
-    _hub_mixin_jsonable_custom_types: Tuple[Type, ...]  # custom types that can be encoded/decoded
-    _hub_mixin_coders: Dict[Type, CODER_T]  # encoders/decoders for custom types
+    _hub_mixin_init_parameters: dict[str, inspect.Parameter]  # __init__ parameters
+    _hub_mixin_jsonable_default_values: dict[str, Any]  # default values for __init__ parameters
+    _hub_mixin_jsonable_custom_types: tuple[Type, ...]  # custom types that can be encoded/decoded
+    _hub_mixin_coders: dict[Type, CODER_T]  # encoders/decoders for custom types
     # ^ internal values to handle config
 
     def __init_subclass__(
@@ -194,20 +197,21 @@ class ModelHubMixin:
         *,
         # Generic info for model card
         repo_url: Optional[str] = None,
+        paper_url: Optional[str] = None,
         docs_url: Optional[str] = None,
         # Model card template
         model_card_template: str = DEFAULT_MODEL_CARD,
         # Model card metadata
-        language: Optional[List[str]] = None,
+        language: Optional[list[str]] = None,
         library_name: Optional[str] = None,
         license: Optional[str] = None,
         license_name: Optional[str] = None,
         license_link: Optional[str] = None,
         pipeline_tag: Optional[str] = None,
-        tags: Optional[List[str]] = None,
+        tags: Optional[list[str]] = None,
         # How to encode/decode arguments with custom type into a JSON config?
         coders: Optional[
-            Dict[Type, CODER_T]
+            dict[Type, CODER_T]
             # Key is a type.
             # Value is a tuple (encoder, decoder).
             # Example: {MyCustomType: (lambda x: x.value, lambda data: MyCustomType(data))}
@@ -234,6 +238,7 @@ class ModelHubMixin:
 
             # Inherit other info
             info.docs_url = cls._hub_mixin_info.docs_url
+            info.paper_url = cls._hub_mixin_info.paper_url
             info.repo_url = cls._hub_mixin_info.repo_url
         cls._hub_mixin_info = info
 
@@ -242,6 +247,8 @@ class ModelHubMixin:
             info.model_card_template = model_card_template
         if repo_url is not None:
             info.repo_url = repo_url
+        if paper_url is not None:
+            info.paper_url = paper_url
         if docs_url is not None:
             info.docs_url = docs_url
         if language is not None:
@@ -257,12 +264,14 @@ class ModelHubMixin:
         if pipeline_tag is not None:
             info.model_card_data.pipeline_tag = pipeline_tag
         if tags is not None:
+            normalized_tags = list(tags)
             if info.model_card_data.tags is not None:
-                info.model_card_data.tags.extend(tags)
+                info.model_card_data.tags.extend(normalized_tags)
             else:
-                info.model_card_data.tags = tags
+                info.model_card_data.tags = normalized_tags
 
-        info.model_card_data.tags = sorted(set(info.model_card_data.tags))
+        if info.model_card_data.tags is not None:
+            info.model_card_data.tags = sorted(set(info.model_card_data.tags))
 
         # Handle encoders/decoders for args
         cls._hub_mixin_coders = coders or {}
@@ -277,7 +286,7 @@ class ModelHubMixin:
         }
         cls._hub_mixin_inject_config = "config" in inspect.signature(cls._from_pretrained).parameters
 
-    def __new__(cls: Type[T], *args, **kwargs) -> T:
+    def __new__(cls: type[T], *args, **kwargs) -> T:
         """Create a new instance of the class and handle config.
 
         3 cases:
@@ -334,6 +343,8 @@ class ModelHubMixin:
     @classmethod
     def _is_jsonable(cls, value: Any) -> bool:
         """Check if a value is JSON serializable."""
+        if is_dataclass(value):
+            return True
         if isinstance(value, cls._hub_mixin_jsonable_custom_types):
             return True
         return is_jsonable(value)
@@ -341,6 +352,8 @@ class ModelHubMixin:
     @classmethod
     def _encode_arg(cls, arg: Any) -> Any:
         """Encode an argument into a JSON serializable format."""
+        if is_dataclass(arg):
+            return asdict(arg)  # type: ignore[arg-type]
         for type_, (encoder, _) in cls._hub_mixin_coders.items():
             if isinstance(arg, type_):
                 if arg is None:
@@ -349,7 +362,7 @@ class ModelHubMixin:
         return arg
 
     @classmethod
-    def _decode_arg(cls, expected_type: Type[ARGS_T], value: Any) -> Optional[ARGS_T]:
+    def _decode_arg(cls, expected_type: type[ARGS_T], value: Any) -> Optional[ARGS_T]:
         """Decode a JSON serializable value into an argument."""
         if is_simple_optional_type(expected_type):
             if value is None:
@@ -372,7 +385,7 @@ class ModelHubMixin:
         config: Optional[Union[dict, DataclassInstance]] = None,
         repo_id: Optional[str] = None,
         push_to_hub: bool = False,
-        model_card_kwargs: Optional[Dict[str, Any]] = None,
+        model_card_kwargs: Optional[dict[str, Any]] = None,
         **push_to_hub_kwargs,
     ) -> Optional[str]:
         """
@@ -388,7 +401,7 @@ class ModelHubMixin:
             repo_id (`str`, *optional*):
                 ID of your repository on the Hub. Used only if `push_to_hub=True`. Will default to the folder name if
                 not provided.
-            model_card_kwargs (`Dict[str, Any]`, *optional*):
+            model_card_kwargs (`dict[str, Any]`, *optional*):
                 Additional arguments passed to the model card template to customize the model card.
             push_to_hub_kwargs:
                 Additional key word arguments passed along to the [`~ModelHubMixin.push_to_hub`] method.
@@ -447,12 +460,10 @@ class ModelHubMixin:
     @classmethod
     @validate_hf_hub_args
     def from_pretrained(
-        cls: Type[T],
+        cls: type[T],
         pretrained_model_name_or_path: Union[str, Path],
         *,
         force_download: bool = False,
-        resume_download: Optional[bool] = None,
-        proxies: Optional[Dict] = None,
         token: Optional[Union[str, bool]] = None,
         cache_dir: Optional[Union[str, Path]] = None,
         local_files_only: bool = False,
@@ -473,17 +484,14 @@ class ModelHubMixin:
             force_download (`bool`, *optional*, defaults to `False`):
                 Whether to force (re-)downloading the model weights and configuration files from the Hub, overriding
                 the existing cache.
-            proxies (`Dict[str, str]`, *optional*):
-                A dictionary of proxy servers to use by protocol or endpoint, e.g., `{'http': 'foo.bar:3128',
-                'http://hostname': 'foo.bar:4012'}`. The proxies are used on every request.
             token (`str` or `bool`, *optional*):
                 The token to use as HTTP bearer authorization for remote files. By default, it will use the token
-                cached when running `huggingface-cli login`.
+                cached when running `hf auth login`.
             cache_dir (`str`, `Path`, *optional*):
                 Path to the folder where cached files are stored.
             local_files_only (`bool`, *optional*, defaults to `False`):
                 If `True`, avoid downloading the file and return the path to the local cached file if it exists.
-            model_kwargs (`Dict`, *optional*):
+            model_kwargs (`dict`, *optional*):
                 Additional kwargs to pass to the model during initialization.
         """
         model_id = str(pretrained_model_name_or_path)
@@ -501,8 +509,6 @@ class ModelHubMixin:
                     revision=revision,
                     cache_dir=cache_dir,
                     force_download=force_download,
-                    proxies=proxies,
-                    resume_download=resume_download,
                     token=token,
                     local_files_only=local_files_only,
                 )
@@ -542,7 +548,7 @@ class ModelHubMixin:
                     if key not in model_kwargs and key in config:
                         model_kwargs[key] = config[key]
             elif any(param.kind == inspect.Parameter.VAR_KEYWORD for param in cls._hub_mixin_init_parameters.values()):
-                for key, value in config.items():
+                for key, value in config.items():  # type: ignore[union-attr]
                     if key not in model_kwargs:
                         model_kwargs[key] = value
 
@@ -555,8 +561,6 @@ class ModelHubMixin:
             revision=revision,
             cache_dir=cache_dir,
             force_download=force_download,
-            proxies=proxies,
-            resume_download=resume_download,
             local_files_only=local_files_only,
             token=token,
             **model_kwargs,
@@ -571,14 +575,12 @@ class ModelHubMixin:
 
     @classmethod
     def _from_pretrained(
-        cls: Type[T],
+        cls: type[T],
         *,
         model_id: str,
         revision: Optional[str],
         cache_dir: Optional[Union[str, Path]],
         force_download: bool,
-        proxies: Optional[Dict],
-        resume_download: Optional[bool],
         local_files_only: bool,
         token: Optional[Union[str, bool]],
         **model_kwargs,
@@ -601,12 +603,9 @@ class ModelHubMixin:
             force_download (`bool`, *optional*, defaults to `False`):
                 Whether to force (re-)downloading the model weights and configuration files from the Hub, overriding
                 the existing cache.
-            proxies (`Dict[str, str]`, *optional*):
-                A dictionary of proxy servers to use by protocol or endpoint (e.g., `{'http': 'foo.bar:3128',
-                'http://hostname': 'foo.bar:4012'}`).
             token (`str` or `bool`, *optional*):
                 The token to use as HTTP bearer authorization for remote files. By default, it will use the token
-                cached when running `huggingface-cli login`.
+                cached when running `hf auth login`.
             cache_dir (`str`, `Path`, *optional*):
                 Path to the folder where cached files are stored.
             local_files_only (`bool`, *optional*, defaults to `False`):
@@ -627,10 +626,10 @@ class ModelHubMixin:
         token: Optional[str] = None,
         branch: Optional[str] = None,
         create_pr: Optional[bool] = None,
-        allow_patterns: Optional[Union[List[str], str]] = None,
-        ignore_patterns: Optional[Union[List[str], str]] = None,
-        delete_patterns: Optional[Union[List[str], str]] = None,
-        model_card_kwargs: Optional[Dict[str, Any]] = None,
+        allow_patterns: Optional[Union[list[str], str]] = None,
+        ignore_patterns: Optional[Union[list[str], str]] = None,
+        delete_patterns: Optional[Union[list[str], str]] = None,
+        model_card_kwargs: Optional[dict[str, Any]] = None,
     ) -> str:
         """
         Upload model checkpoint to the Hub.
@@ -651,18 +650,18 @@ class ModelHubMixin:
                 If `None` (default), the repo will be public unless the organization's default is private.
             token (`str`, *optional*):
                 The token to use as HTTP bearer authorization for remote files. By default, it will use the token
-                cached when running `huggingface-cli login`.
+                cached when running `hf auth login`.
             branch (`str`, *optional*):
                 The git branch on which to push the model. This defaults to `"main"`.
             create_pr (`boolean`, *optional*):
                 Whether or not to create a Pull Request from `branch` with that commit. Defaults to `False`.
-            allow_patterns (`List[str]` or `str`, *optional*):
+            allow_patterns (`list[str]` or `str`, *optional*):
                 If provided, only files matching at least one pattern are pushed.
-            ignore_patterns (`List[str]` or `str`, *optional*):
+            ignore_patterns (`list[str]` or `str`, *optional*):
                 If provided, files matching any of the patterns are not pushed.
-            delete_patterns (`List[str]` or `str`, *optional*):
+            delete_patterns (`list[str]` or `str`, *optional*):
                 If provided, remote files matching any of the patterns will be deleted from the repo.
-            model_card_kwargs (`Dict[str, Any]`, *optional*):
+            model_card_kwargs (`dict[str, Any]`, *optional*):
                 Additional arguments passed to the model card template to customize the model card.
 
         Returns:
@@ -692,6 +691,7 @@ class ModelHubMixin:
             card_data=self._hub_mixin_info.model_card_data,
             template_str=self._hub_mixin_info.model_card_template,
             repo_url=self._hub_mixin_info.repo_url,
+            paper_url=self._hub_mixin_info.paper_url,
             docs_url=self._hub_mixin_info.docs_url,
             **kwargs,
         )
@@ -718,6 +718,7 @@ class PyTorchModelHubMixin(ModelHubMixin):
     ...         PyTorchModelHubMixin,
     ...         library_name="keras-nlp",
     ...         repo_url="https://github.com/keras-team/keras-nlp",
+    ...         paper_url="https://arxiv.org/abs/2304.12244",
     ...         docs_url="https://keras.io/keras_nlp/",
     ...         # ^ optional metadata to generate model card
     ...     ):
@@ -743,7 +744,7 @@ class PyTorchModelHubMixin(ModelHubMixin):
     ```
     """
 
-    def __init_subclass__(cls, *args, tags: Optional[List[str]] = None, **kwargs) -> None:
+    def __init_subclass__(cls, *args, tags: Optional[list[str]] = None, **kwargs) -> None:
         tags = tags or []
         tags.append("pytorch_model_hub_mixin")
         kwargs["tags"] = tags
@@ -752,7 +753,7 @@ class PyTorchModelHubMixin(ModelHubMixin):
     def _save_pretrained(self, save_directory: Path) -> None:
         """Save weights from a Pytorch model to a local directory."""
         model_to_save = self.module if hasattr(self, "module") else self  # type: ignore
-        save_model_as_safetensor(model_to_save, str(save_directory / constants.SAFETENSORS_SINGLE_FILE))
+        save_model_as_safetensor(model_to_save, str(save_directory / constants.SAFETENSORS_SINGLE_FILE))  # type: ignore [arg-type]
 
     @classmethod
     def _from_pretrained(
@@ -762,8 +763,6 @@ class PyTorchModelHubMixin(ModelHubMixin):
         revision: Optional[str],
         cache_dir: Optional[Union[str, Path]],
         force_download: bool,
-        proxies: Optional[Dict],
-        resume_download: Optional[bool],
         local_files_only: bool,
         token: Union[str, bool, None],
         map_location: str = "cpu",
@@ -784,8 +783,6 @@ class PyTorchModelHubMixin(ModelHubMixin):
                     revision=revision,
                     cache_dir=cache_dir,
                     force_download=force_download,
-                    proxies=proxies,
-                    resume_download=resume_download,
                     token=token,
                     local_files_only=local_files_only,
                 )
@@ -797,8 +794,6 @@ class PyTorchModelHubMixin(ModelHubMixin):
                     revision=revision,
                     cache_dir=cache_dir,
                     force_download=force_download,
-                    proxies=proxies,
-                    resume_download=resume_download,
                     token=token,
                     local_files_only=local_files_only,
                 )
@@ -828,7 +823,7 @@ class PyTorchModelHubMixin(ModelHubMixin):
         return model
 
 
-def _load_dataclass(datacls: Type[DataclassInstance], data: dict) -> DataclassInstance:
+def _load_dataclass(datacls: type[DataclassInstance], data: dict) -> DataclassInstance:
     """Load a dataclass instance from a dictionary.
 
     Fields not expected by the dataclass are ignored.