transformers 5.0.0__py3-none-any.whl → 5.0.0rc0__py3-none-any.whl

transformers/models/metaclip_2/modular_metaclip_2.py

@@ -1,3 +1,5 @@
+from typing import Optional
+
 import torch
 from torch import nn
 
@@ -223,13 +225,11 @@ class MetaClip2PreTrainedModel(CLIPPreTrainedModel):
         if isinstance(module, MetaClip2TextEmbeddings):
             init.normal_(module.token_embedding.weight, mean=0.0, std=factor * 0.02)
             init.normal_(module.position_embedding.weight, mean=0.0, std=factor * 0.02)
-            init.copy_(module.position_ids, torch.arange(module.position_ids.shape[-1]).expand((1, -1)))
         elif isinstance(module, MetaClip2VisionEmbeddings):
             factor = self.config.initializer_factor
             init.normal_(module.class_embedding, mean=0.0, std=module.embed_dim**-0.5 * factor)
             init.normal_(module.patch_embedding.weight, std=module.config.initializer_range * factor)
             init.normal_(module.position_embedding.weight, std=module.config.initializer_range * factor)
-            init.copy_(module.position_ids, torch.arange(module.position_ids.shape[-1]).expand((1, -1)))
         elif isinstance(module, MetaClip2Attention):
             factor = self.config.initializer_factor
             in_proj_std = (module.embed_dim**-0.5) * ((2 * module.config.num_hidden_layers) ** -0.5) * factor
@@ -281,8 +281,8 @@ class MetaClip2TextTransformer(CLIPTextTransformer):
     def forward(
         self,
         input_ids,
-        attention_mask: torch.Tensor | None = None,
-        position_ids: torch.Tensor | None = None,
+        attention_mask: Optional[torch.Tensor] = None,
+        position_ids: Optional[torch.Tensor] = None,
         **kwargs: Unpack[TransformersKwargs],
     ) -> BaseModelOutputWithPooling:
         input_shape = input_ids.size()
@@ -358,9 +358,9 @@ class MetaClip2TextModel(CLIPTextModel):
     @auto_docstring
     def forward(
         self,
-        input_ids: torch.Tensor | None = None,
-        attention_mask: torch.Tensor | None = None,
-        position_ids: torch.Tensor | None = None,
+        input_ids: Optional[torch.Tensor] = None,
+        attention_mask: Optional[torch.Tensor] = None,
+        position_ids: Optional[torch.Tensor] = None,
         **kwargs: Unpack[TransformersKwargs],
     ):
         r"""
@@ -421,9 +421,9 @@ class MetaClip2TextModelWithProjection(CLIPTextModelWithProjection):
     @auto_docstring
     def forward(
         self,
-        input_ids: torch.Tensor | None = None,
-        attention_mask: torch.Tensor | None = None,
-        position_ids: torch.Tensor | None = None,
+        input_ids: Optional[torch.Tensor] = None,
+        attention_mask: Optional[torch.Tensor] = None,
+        position_ids: Optional[torch.Tensor] = None,
         **kwargs: Unpack[TransformersKwargs],
     ):
         r"""
@@ -467,16 +467,14 @@ class MetaClip2Model(CLIPModel):
 
     ```python
     >>> from PIL import Image
-    >>> import httpx
-        >>> from io import BytesIO
+    >>> import requests
     >>> from transformers import AutoProcessor, MetaClip2Model
 
     >>> model = MetaClip2Model.from_pretrained("facebook/metaclip-2-worldwide-huge-quickgelu")
     >>> processor = AutoProcessor.from_pretrained("facebook/metaclip-2-worldwide-huge-quickgelu")
 
     >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
-    >>> with httpx.stream("GET", url) as response:
-        ...     image = Image.open(BytesIO(response.read()))
+    >>> image = Image.open(requests.get(url, stream=True).raw)
 
     >>> inputs = processor(
     ...     text=["a photo of a cat", "a photo of a dog"], images=image, return_tensors="pt", padding=True
@@ -514,11 +512,11 @@ class MetaClip2Model(CLIPModel):
     @auto_docstring
     def forward(
         self,
-        input_ids: torch.LongTensor | None = None,
-        pixel_values: torch.FloatTensor | None = None,
-        attention_mask: torch.Tensor | None = None,
-        position_ids: torch.LongTensor | None = None,
-        return_loss: bool | None = None,
+        input_ids: Optional[torch.LongTensor] = None,
+        pixel_values: Optional[torch.FloatTensor] = None,
+        attention_mask: Optional[torch.Tensor] = None,
+        position_ids: Optional[torch.LongTensor] = None,
+        return_loss: Optional[bool] = None,
         interpolate_pos_encoding: bool = False,
         **kwargs: Unpack[TransformersKwargs],
     ):
@@ -530,16 +528,14 @@ class MetaClip2Model(CLIPModel):
 
         ```python
         >>> from PIL import Image
-        >>> import httpx
-        >>> from io import BytesIO
+        >>> import requests
         >>> from transformers import AutoProcessor, MetaClip2Model
 
         >>> model = MetaClip2Model.from_pretrained("facebook/metaclip-2-worldwide-huge-quickgelu")
         >>> processor = AutoProcessor.from_pretrained("facebook/metaclip-2-worldwide-huge-quickgelu")
 
         >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
-        >>> with httpx.stream("GET", url) as response:
-        ...     image = Image.open(BytesIO(response.read()))
+        >>> image = Image.open(requests.get(url, stream=True).raw)
 
         >>> inputs = processor(
         ...     text=["a photo of a cat", "a photo of a dog"], images=image, return_tensors="pt", padding=True
@@ -559,16 +555,17 @@ class MetaClip2Model(CLIPModel):
             **kwargs,
         )
 
-    @can_return_tuple
-    @auto_docstring
     def get_text_features(
         self,
-        input_ids: torch.Tensor,
-        attention_mask: torch.Tensor | None = None,
-        position_ids: torch.Tensor | None = None,
-        **kwargs: Unpack[TransformersKwargs],
-    ) -> tuple | BaseModelOutputWithPooling:
+        input_ids: Optional[torch.Tensor] = None,
+        attention_mask: Optional[torch.Tensor] = None,
+        position_ids: Optional[torch.Tensor] = None,
+    ):
         r"""
+        Returns:
+            text_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The text embeddings obtained by
+            applying the projection layer to the pooled output of [`MetaClip2TextModel`].
+
         Examples:
 
         ```python
@@ -584,33 +581,30 @@ class MetaClip2Model(CLIPModel):
             input_ids=input_ids,
             attention_mask=attention_mask,
             position_ids=position_ids,
-            return_dict=True,
-            **kwargs,
         )
 
-    @can_return_tuple
-    @auto_docstring
     def get_image_features(
         self,
-        pixel_values: torch.FloatTensor | None = None,
+        pixel_values: Optional[torch.FloatTensor] = None,
         interpolate_pos_encoding: bool = False,
-        **kwargs: Unpack[TransformersKwargs],
-    ) -> tuple | BaseModelOutputWithPooling:
+    ):
         r"""
+        Returns:
+            image_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The image embeddings obtained by
+            applying the projection layer to the pooled output of [`MetaClip2VisionModel`].
+
         Examples:
 
         ```python
         >>> from PIL import Image
-        >>> import httpx
-        >>> from io import BytesIO
+        >>> import requests
         >>> from transformers import AutoProcessor, MetaClip2Model
 
         >>> model = MetaClip2Model.from_pretrained("facebook/metaclip-2-worldwide-huge-quickgelu")
         >>> processor = AutoProcessor.from_pretrained("facebook/metaclip-2-worldwide-huge-quickgelu")
 
         >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
-        >>> with httpx.stream("GET", url) as response:
-        ...     image = Image.open(BytesIO(response.read()))
+        >>> image = Image.open(requests.get(url, stream=True).raw)
 
         >>> inputs = processor(images=image, return_tensors="pt")
 
@@ -619,8 +613,6 @@ class MetaClip2Model(CLIPModel):
         return super().get_image_features(
             pixel_values=pixel_values,
             interpolate_pos_encoding=interpolate_pos_encoding,
-            return_dict=True,
-            **kwargs,
         )
 
 
@@ -645,16 +637,14 @@ class MetaClip2VisionModel(CLIPVisionModel):
 
     ```python
     >>> from PIL import Image
-    >>> import httpx
-    >>> from io import BytesIO
+    >>> import requests
     >>> from transformers import AutoProcessor, MetaClip2VisionModel
 
     >>> model = MetaClip2VisionModel.from_pretrained("facebook/metaclip-2-worldwide-huge-quickgelu")
     >>> processor = AutoProcessor.from_pretrained("facebook/metaclip-2-worldwide-huge-quickgelu")
 
     >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
-    >>> with httpx.stream("GET", url) as response:
-    ...     image = Image.open(BytesIO(response.read()))
+    >>> image = Image.open(requests.get(url, stream=True).raw)
 
     >>> inputs = processor(images=image, return_tensors="pt")
 
@@ -667,7 +657,7 @@ class MetaClip2VisionModel(CLIPVisionModel):
     @auto_docstring
     def forward(
         self,
-        pixel_values: torch.FloatTensor | None = None,
+        pixel_values: Optional[torch.FloatTensor] = None,
         interpolate_pos_encoding: bool = False,
         **kwargs: Unpack[TransformersKwargs],
     ):
@@ -676,16 +666,14 @@ class MetaClip2VisionModel(CLIPVisionModel):
 
         ```python
         >>> from PIL import Image
-        >>> import httpx
-        >>> from io import BytesIO
+        >>> import requests
         >>> from transformers import AutoProcessor, MetaClip2VisionModel
 
         >>> model = MetaClip2VisionModel.from_pretrained("facebook/metaclip-2-worldwide-huge-quickgelu")
         >>> processor = AutoProcessor.from_pretrained("facebook/metaclip-2-worldwide-huge-quickgelu")
 
         >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
-        >>> with httpx.stream("GET", url) as response:
-        ...     image = Image.open(BytesIO(response.read()))
+        >>> image = Image.open(requests.get(url, stream=True).raw)
 
         >>> inputs = processor(images=image, return_tensors="pt")
 
@@ -721,16 +709,14 @@ class MetaClip2VisionModelWithProjection(CLIPVisionModelWithProjection):
 
     ```python
     >>> from PIL import Image
-    >>> import httpx
-    >>> from io import BytesIO
+    >>> import requests
     >>> from transformers import AutoProcessor, MetaClip2VisionModelWithProjection
 
     >>> model = MetaClip2VisionModelWithProjection.from_pretrained("facebook/metaclip-2-worldwide-huge-quickgelu")
     >>> processor = AutoProcessor.from_pretrained("facebook/metaclip-2-worldwide-huge-quickgelu")
 
     >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
-    >>> with httpx.stream("GET", url) as response:
-    ...     image = Image.open(BytesIO(response.read()))
+    >>> image = Image.open(requests.get(url, stream=True).raw)
 
     >>> inputs = processor(images=image, return_tensors="pt")
 
@@ -742,7 +728,7 @@ class MetaClip2VisionModelWithProjection(CLIPVisionModelWithProjection):
     @auto_docstring
     def forward(
         self,
-        pixel_values: torch.FloatTensor | None = None,
+        pixel_values: Optional[torch.FloatTensor] = None,
         interpolate_pos_encoding: bool = False,
         **kwargs: Unpack[TransformersKwargs],
     ):
@@ -751,16 +737,14 @@ class MetaClip2VisionModelWithProjection(CLIPVisionModelWithProjection):
 
         ```python
         >>> from PIL import Image
-        >>> import httpx
-        >>> from io import BytesIO
+        >>> import requests
         >>> from transformers import AutoProcessor, MetaClip2VisionModelWithProjection
 
         >>> model = MetaClip2VisionModelWithProjection.from_pretrained("facebook/metaclip-2-worldwide-huge-quickgelu")
         >>> processor = AutoProcessor.from_pretrained("facebook/metaclip-2-worldwide-huge-quickgelu")
 
         >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
-        >>> with httpx.stream("GET", url) as response:
-        ...     image = Image.open(BytesIO(response.read()))
+        >>> image = Image.open(requests.get(url, stream=True).raw)
 
         >>> inputs = processor(images=image, return_tensors="pt")
 

transformers/models/mgp_str/configuration_mgp_str.py

@@ -1,3 +1,4 @@
+# coding=utf-8
 # Copyright 2023 The HuggingFace Inc. team. All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

transformers/models/mgp_str/modeling_mgp_str.py

@@ -1,3 +1,4 @@
+# coding=utf-8
 # Copyright 2023 Alibaba Research and The HuggingFace Inc. team. All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -15,6 +16,7 @@
 
 import collections.abc
 from dataclasses import dataclass
+from typing import Optional, Union
 
 import torch
 import torch.nn.functional as F
@@ -50,7 +52,7 @@ def drop_path(input: torch.Tensor, drop_prob: float = 0.0, training: bool = Fals
 class MgpstrDropPath(nn.Module):
     """Drop paths (Stochastic Depth) per sample (when applied in main path of residual blocks)."""
 
-    def __init__(self, drop_prob: float | None = None) -> None:
+    def __init__(self, drop_prob: Optional[float] = None) -> None:
         super().__init__()
         self.drop_prob = drop_prob
 
@@ -84,10 +86,10 @@ class MgpstrModelOutput(ModelOutput):
         heads.
     """
 
-    logits: tuple[torch.FloatTensor] | None = None
-    hidden_states: tuple[torch.FloatTensor] | None = None
-    attentions: tuple[torch.FloatTensor] | None = None
-    a3_attentions: tuple[torch.FloatTensor] | None = None
+    logits: Optional[tuple[torch.FloatTensor]] = None
+    hidden_states: Optional[tuple[torch.FloatTensor]] = None
+    attentions: Optional[tuple[torch.FloatTensor]] = None
+    a3_attentions: Optional[tuple[torch.FloatTensor]] = None
 
 
 class MgpstrEmbeddings(nn.Module):
@@ -317,11 +319,10 @@ class MgpstrModel(MgpstrPreTrainedModel):
     def forward(
         self,
         pixel_values: torch.FloatTensor,
-        output_attentions: bool | None = None,
-        output_hidden_states: bool | None = None,
-        return_dict: bool | None = None,
-        **kwargs,
-    ) -> tuple[torch.FloatTensor] | BaseModelOutput:
+        output_attentions: Optional[bool] = None,
+        output_hidden_states: Optional[bool] = None,
+        return_dict: Optional[bool] = None,
+    ) -> Union[tuple[torch.FloatTensor], BaseModelOutput]:
         output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
         output_hidden_states = (
             output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
@@ -380,12 +381,11 @@ class MgpstrForSceneTextRecognition(MgpstrPreTrainedModel):
     def forward(
         self,
         pixel_values: torch.FloatTensor,
-        output_attentions: bool | None = None,
-        output_a3_attentions: bool | None = None,
-        output_hidden_states: bool | None = None,
-        return_dict: bool | None = None,
-        **kwargs,
-    ) -> tuple[torch.FloatTensor] | MgpstrModelOutput:
+        output_attentions: Optional[bool] = None,
+        output_a3_attentions: Optional[bool] = None,
+        output_hidden_states: Optional[bool] = None,
+        return_dict: Optional[bool] = None,
+    ) -> Union[tuple[torch.FloatTensor], MgpstrModelOutput]:
         r"""
         output_a3_attentions (`bool`, *optional*):
             Whether or not to return the attentions tensors of a3 modules. See `a3_attentions` under returned tensors
@@ -398,14 +398,12 @@ class MgpstrForSceneTextRecognition(MgpstrPreTrainedModel):
         ...     MgpstrProcessor,
         ...     MgpstrForSceneTextRecognition,
         ... )
-        >>> import httpx
-        >>> from io import BytesIO
+        >>> import requests
         >>> from PIL import Image
 
         >>> # load image from the IIIT-5k dataset
         >>> url = "https://i.postimg.cc/ZKwLg2Gw/367-14.png"
-        >>> with httpx.stream("GET", url) as response:
-        ...     image = Image.open(BytesIO(response.read())).convert("RGB")
+        >>> image = Image.open(requests.get(url, stream=True).raw).convert("RGB")
 
         >>> processor = MgpstrProcessor.from_pretrained("alibaba-damo/mgp-str-base")
         >>> pixel_values = processor(images=image, return_tensors="pt").pixel_values

transformers/models/mgp_str/processing_mgp_str.py

@@ -1,3 +1,4 @@
+# coding=utf-8
 # Copyright 2023 The HuggingFace Inc. team. All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -18,7 +19,6 @@ from transformers.utils import is_torch_available
 from transformers.utils.generic import ExplicitEnum
 
 from ...processing_utils import ProcessorMixin
-from ...utils import auto_docstring
 from ...utils.import_utils import requires
 
 
@@ -36,8 +36,20 @@ SUPPORTED_ANNOTATION_FORMATS = (DecodeType.CHARACTER, DecodeType.BPE, DecodeType
 
 
 @requires(backends=("sentencepiece",))
-@auto_docstring
 class MgpstrProcessor(ProcessorMixin):
+    r"""
+    Constructs a MGP-STR processor which wraps an image processor and MGP-STR tokenizers into a single
+
+    [`MgpstrProcessor`] offers all the functionalities of `ViTImageProcessor`] and [`MgpstrTokenizer`]. See the
+    [`~MgpstrProcessor.__call__`] and [`~MgpstrProcessor.batch_decode`] for more information.
+
+    Args:
+        image_processor (`ViTImageProcessor`, *optional*):
+            An instance of `ViTImageProcessor`. The image processor is a required input.
+        tokenizer ([`MgpstrTokenizer`], *optional*):
+            The tokenizer is a required input.
+    """
+
     def __init__(self, image_processor=None, tokenizer=None, **kwargs):
         self.char_tokenizer = tokenizer
         self.bpe_tokenizer = AutoTokenizer.from_pretrained("openai-community/gpt2")
@@ -45,8 +57,13 @@ class MgpstrProcessor(ProcessorMixin):
 
         super().__init__(image_processor, tokenizer)
 
-    @auto_docstring
     def __call__(self, text=None, images=None, return_tensors=None, **kwargs):
+        """
+        When used in normal mode, this method forwards all its arguments to ViTImageProcessor's
+        [`~ViTImageProcessor.__call__`] and returns its output. This method also forwards the `text` and `kwargs`
+        arguments to MgpstrTokenizer's [`~MgpstrTokenizer.__call__`] if `text` is not `None` to encode the text. Please
+        refer to the docstring of the above methods for more information.
+        """
         if images is None and text is None:
             raise ValueError("You need to specify either an `images` or `text` input to process.")
 

transformers/models/mgp_str/tokenization_mgp_str.py

@@ -1,3 +1,4 @@
+# coding=utf-8
 # Copyright 2023 The HuggingFace Inc. team.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -15,6 +16,7 @@
 
 import json
 import os
+from typing import Optional
 
 from ...tokenization_python import PreTrainedTokenizer
 from ...utils import logging
@@ -86,7 +88,7 @@ class MgpstrTokenizer(PreTrainedTokenizer):
         """Converts an index (integer) in a token (str) using the vocab."""
         return self.decoder.get(index)
 
-    def save_vocabulary(self, save_directory: str, filename_prefix: str | None = None) -> tuple[str]:
+    def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> tuple[str]:
         if not os.path.isdir(save_directory):
             logger.error(f"Vocabulary path ({save_directory}) should be a directory")
             return

transformers/models/mimi/configuration_mimi.py

@@ -1,3 +1,4 @@
+# coding=utf-8
 # Copyright 2024 Meta Platforms, Inc. and affiliates, and the HuggingFace Inc. team. All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -14,6 +15,7 @@
 """Mimi model configuration"""
 
 import math
+from typing import Optional
 
 import numpy as np
 
@@ -125,8 +127,6 @@ class MimiConfig(PreTrainedConfig):
             Initial scale of the residual rescaling operation done in the Transformer models.
         attention_bias (`bool`, defaults to `False`, *optional*, defaults to `False`):
             Whether to use a bias in the query, key, value and output projection layers during self-attention.
-        tie_word_embeddings (`bool`, *optional*, defaults to `True`):
-            Whether to tie weight embeddings
     Example:
 
     ```python
@@ -146,45 +146,44 @@ class MimiConfig(PreTrainedConfig):
 
     def __init__(
         self,
-        sampling_rate: int | None = 24_000,
-        frame_rate: int | None = None,
-        audio_channels: int | None = 1,
-        hidden_size: int | None = 512,
-        num_filters: int | None = 64,
-        num_residual_layers: int | None = 1,
-        upsampling_ratios: list[int] | None = None,
-        kernel_size: int | None = 7,
-        last_kernel_size: int | None = 3,
-        residual_kernel_size: int | None = 3,
-        dilation_growth_rate: int | None = 2,
-        use_causal_conv: bool | None = True,
-        pad_mode: str | None = "constant",
-        compress: int | None = 2,
-        trim_right_ratio: float | None = 1.0,
-        codebook_size: int | None = 2048,
-        codebook_dim: int | None = 256,
-        num_quantizers: int | None = 32,
-        use_conv_shortcut: bool | None = False,
-        vector_quantization_hidden_dimension: int | None = 256,
-        num_semantic_quantizers: int | None = 1,
-        upsample_groups: int | None = 512,
-        num_hidden_layers: int | None = 8,
-        intermediate_size: int | None = 2048,
-        num_attention_heads: int | None = 8,
-        num_key_value_heads: int | None = 8,
-        head_dim: int | None = None,
-        hidden_act: str | None = "gelu",
-        max_position_embeddings: int | None = 8000,
-        initializer_range: float | None = 0.02,
-        norm_eps: int | None = 1e-5,
-        use_cache: bool | None = False,
-        use_streaming: bool | None = False,
-        rope_parameters: RopeParameters | dict[str, RopeParameters] | None = None,
-        sliding_window: int | None = 250,
-        attention_dropout: float | None = 0.0,
-        layer_scale_initial_scale: float | None = 0.01,
-        attention_bias: bool | None = False,
-        tie_word_embeddings: bool | None = True,
+        sampling_rate: Optional[int] = 24_000,
+        frame_rate: Optional[int] = None,
+        audio_channels: Optional[int] = 1,
+        hidden_size: Optional[int] = 512,
+        num_filters: Optional[int] = 64,
+        num_residual_layers: Optional[int] = 1,
+        upsampling_ratios: Optional[list[int]] = None,
+        kernel_size: Optional[int] = 7,
+        last_kernel_size: Optional[int] = 3,
+        residual_kernel_size: Optional[int] = 3,
+        dilation_growth_rate: Optional[int] = 2,
+        use_causal_conv: Optional[bool] = True,
+        pad_mode: Optional[str] = "constant",
+        compress: Optional[int] = 2,
+        trim_right_ratio: Optional[float] = 1.0,
+        codebook_size: Optional[int] = 2048,
+        codebook_dim: Optional[int] = 256,
+        num_quantizers: Optional[int] = 32,
+        use_conv_shortcut: Optional[bool] = False,
+        vector_quantization_hidden_dimension: Optional[int] = 256,
+        num_semantic_quantizers: Optional[int] = 1,
+        upsample_groups: Optional[int] = 512,
+        num_hidden_layers: Optional[int] = 8,
+        intermediate_size: Optional[int] = 2048,
+        num_attention_heads: Optional[int] = 8,
+        num_key_value_heads: Optional[int] = 8,
+        head_dim: Optional[int] = None,
+        hidden_act: Optional[str] = "gelu",
+        max_position_embeddings: Optional[int] = 8000,
+        initializer_range: Optional[float] = 0.02,
+        norm_eps: Optional[int] = 1e-5,
+        use_cache: Optional[bool] = False,
+        use_streaming: Optional[bool] = False,
+        rope_parameters: Optional[RopeParameters | dict[str, RopeParameters]] = None,
+        sliding_window: Optional[int] = 250,
+        attention_dropout: Optional[float] = 0.0,
+        layer_scale_initial_scale: Optional[float] = 0.01,
+        attention_bias: Optional[bool] = False,
         **kwargs,
     ):
         self.sampling_rate = sampling_rate
@@ -222,7 +221,6 @@ class MimiConfig(PreTrainedConfig):
         self.head_dim = head_dim or hidden_size // num_attention_heads
         self.layer_scale_initial_scale = layer_scale_initial_scale
         self.attention_bias = attention_bias
-        self.tie_word_embeddings = tie_word_embeddings
         self.rope_parameters = rope_parameters
 
         # Handle backward compatibility for frame_rate: