transformers 5.0.0rc0__py3-none-any.whl → 5.0.0rc1__py3-none-any.whl

transformers/models/sam3/modeling_sam3.py

@@ -14,9 +14,8 @@
 # limitations under the License.
 
 
-import collections.abc
 import math
-from collections.abc import Callable
+from collections.abc import Callable, Iterable
 from dataclasses import dataclass
 from typing import Optional, Union
 
@@ -40,7 +39,7 @@ from ...modeling_outputs import (
 from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel
 from ...processing_utils import Unpack
 from ...pytorch_utils import compile_compatible_method_lru_cache
-from ...utils import auto_docstring
+from ...utils import auto_docstring, logging
 from ...utils.generic import TransformersKwargs, check_model_inputs
 from ..auto import AutoModel
 from .configuration_sam3 import (
@@ -54,6 +53,9 @@ from .configuration_sam3 import (
 )
 
 
+logger = logging.get_logger(__name__)
+
+
 @dataclass
 @auto_docstring
 class Sam3VisionEncoderOutput(ModelOutput):
@@ -123,8 +125,8 @@ class Sam3DETRDecoderOutput(ModelOutput):
         Decoder hidden states from all layers.
     reference_boxes (`torch.FloatTensor` of shape `(num_layers, batch_size, num_queries, 4)`):
         Predicted reference boxes from all decoder layers in (cx, cy, w, h) format.
-    presence_logits (`torch.FloatTensor` of shape `(num_layers, batch_size)`, *optional*):
-        Presence logits from all decoder layers (None if using instance queries).
+    presence_logits (`torch.FloatTensor` of shape `(num_layers, batch_size, 1)`):
+        Presence logits from all decoder layers indicating object presence confidence.
     hidden_states (`tuple[torch.FloatTensor]`, *optional*):
         Tuple of hidden states from all decoder layers.
     attentions (`tuple[torch.FloatTensor]`, *optional*):
@@ -133,7 +135,7 @@ class Sam3DETRDecoderOutput(ModelOutput):
 
     intermediate_hidden_states: torch.FloatTensor = None
     reference_boxes: torch.FloatTensor = None
-    presence_logits: Optional[torch.FloatTensor] = None
+    presence_logits: torch.FloatTensor = None
     hidden_states: Optional[tuple[torch.FloatTensor]] = None
     attentions: Optional[tuple[torch.FloatTensor]] = None
 
@@ -372,6 +374,19 @@ class Sam3Attention(nn.Module):
         if self.config._attn_implementation != "eager":
             attention_interface = ALL_ATTENTION_FUNCTIONS[self.config._attn_implementation]
 
+        if (
+            "flash" in self.config._attn_implementation
+            and attention_mask is not None
+            and attention_mask.dtype != torch.bool
+        ):
+            # Relative position bias tensors are represented as float masks and are incompatible with Flash Attention
+            # Fallback to SDPA for this call only so the rest of the model can still benefit from FA
+            attention_interface = ALL_ATTENTION_FUNCTIONS["sdpa"]
+            logger.warning_once(
+                "Sam3Attention: falling back to SDPA for relative-position cross-attention because "
+                "Flash Attention does not support additive bias masks."
+            )
+
         attn_output, attn_weights = attention_interface(
             self,
             query,
@@ -531,8 +546,8 @@ class Sam3ViTPatchEmbeddings(nn.Module):
         image_size, patch_size = config.pretrain_image_size, config.patch_size
         num_channels, hidden_size = config.num_channels, config.hidden_size
 
-        image_size = image_size if isinstance(image_size, collections.abc.Iterable) else (image_size, image_size)
-        patch_size = patch_size if isinstance(patch_size, collections.abc.Iterable) else (patch_size, patch_size)
+        image_size = image_size if isinstance(image_size, Iterable) else (image_size, image_size)
+        patch_size = patch_size if isinstance(patch_size, Iterable) else (patch_size, patch_size)
         num_patches = (image_size[1] // patch_size[1]) * (image_size[0] // patch_size[0])
         self.image_size = image_size
         self.patch_size = patch_size
@@ -542,7 +557,7 @@ class Sam3ViTPatchEmbeddings(nn.Module):
         self.projection = nn.Conv2d(num_channels, hidden_size, kernel_size=patch_size, stride=patch_size, bias=False)
 
     def forward(self, pixel_values: torch.Tensor) -> torch.Tensor:
-        embeddings = self.projection(pixel_values).flatten(2).transpose(1, 2)
+        embeddings = self.projection(pixel_values.to(self.projection.weight.dtype)).flatten(2).transpose(1, 2)
         return embeddings
 
 
@@ -938,6 +953,7 @@ class Sam3FPNLayer(nn.Module):
         self.proj2 = nn.Conv2d(in_channels=fpn_dim, out_channels=fpn_dim, kernel_size=3, padding=1)
 
     def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
+        hidden_states = hidden_states.to(self.proj1.weight.dtype)
         for layer in self.scale_layers:
             hidden_states = layer(hidden_states)
 
@@ -1253,7 +1269,7 @@ class Sam3DetrEncoderLayer(nn.Module):
         vision_feats: Tensor,
         prompt_feats: Tensor,
         vision_pos_encoding: Tensor,
-        prompt_mask: Tensor,
+        prompt_cross_attn_mask: Optional[Tensor] = None,
         **kwargs: Unpack[TransformersKwargs],
     ):
         """
@@ -1263,7 +1279,7 @@ class Sam3DetrEncoderLayer(nn.Module):
             vision_feats: Vision features [batch_size, vision_len, hidden_size] (main hidden states)
             prompt_feats: Text prompt features [batch_size, text_len, hidden_size]
             vision_pos_encoding: Position encoding for vision [batch_size, vision_len, hidden_size]
-            prompt_mask: Padding mask for prompts [batch_size, text_len] where True=valid, False=padding
+            prompt_cross_attn_mask: Cross-attention mask for prompt features
 
         Returns:
             Updated vision features [batch_size, vision_len, hidden_size]
@@ -1284,15 +1300,6 @@ class Sam3DetrEncoderLayer(nn.Module):
         residual = hidden_states
         hidden_states = self.layer_norm2(hidden_states)
 
-        prompt_cross_attn_mask = None
-        if prompt_mask is not None:
-            prompt_cross_attn_mask = create_bidirectional_mask(
-                config=self.config,
-                input_embeds=hidden_states,
-                attention_mask=prompt_mask,
-                encoder_hidden_states=prompt_feats,
-            )
-
         hidden_states, _ = self.cross_attn(
             query=hidden_states,
             key=prompt_feats,
@@ -1412,13 +1419,22 @@ class Sam3DetrEncoder(Sam3PreTrainedModel):
             spatial_shapes,
         ) = self._prepare_multilevel_features(vision_features, vision_pos_embeds)
 
+        prompt_cross_attn_mask = None
+        if text_mask is not None:
+            prompt_cross_attn_mask = create_bidirectional_mask(
+                config=self.config,
+                input_embeds=features_flattened,
+                attention_mask=text_mask,
+                encoder_hidden_states=text_features,
+            )
+
         hidden_states = features_flattened
         for layer in self.layers:
             hidden_states = layer(
                 hidden_states,
                 prompt_feats=text_features,
                 vision_pos_encoding=pos_embeds_flattened,
-                prompt_mask=text_mask,
+                prompt_cross_attn_mask=prompt_cross_attn_mask,
                 **kwargs,
             )
         return Sam3DETREncoderOutput(
@@ -1484,31 +1500,27 @@ class Sam3DetrDecoderLayer(nn.Module):
         text_features: torch.Tensor,
         vision_features: torch.Tensor,
         vision_pos_encoding: torch.Tensor,
-        text_mask: Optional[torch.Tensor] = None,
+        text_cross_attn_mask: Optional[torch.Tensor] = None,
         vision_cross_attn_mask: Optional[torch.Tensor] = None,
-        presence_token: Optional[torch.Tensor] = None,
         **kwargs: Unpack[TransformersKwargs],
-    ) -> tuple[torch.Tensor, Optional[torch.Tensor]]:
+    ) -> torch.Tensor:
         """
         Forward pass for decoder layer.
 
         Args:
-            hidden_states: Query features [batch_size, num_queries, hidden_size]
+            hidden_states: Query features [batch_size, num_queries + 1, hidden_size] (includes presence token at position 0)
             query_pos: Query position embeddings [batch_size, num_queries, hidden_size]
             text_features: Text features [batch_size, seq_len, hidden_size]
             vision_features: Vision features [batch_size, height*width, hidden_size]
             vision_pos_encoding: Vision position encoding [batch_size, height*width, hidden_size]
-            text_mask: Text padding mask [batch_size, seq_len] where True=valid, False=padding
-            vision_cross_attn_mask: Vision cross-attention mask [batch_size, num_heads, num_queries, height*width]
-            presence_token: Optional presence token [batch_size, 1, hidden_size]
+            text_cross_attn_mask: Text cross-attention mask
+            vision_cross_attn_mask: Vision cross-attention mask, already expanded for presence token
 
         Returns:
-            Tuple of (updated hidden states, updated presence token)
+            Updated hidden states (including presence token at position 0)
         """
-        # Concatenate presence token if provided
-        if presence_token is not None:
-            hidden_states = torch.cat([presence_token, hidden_states], dim=1)
-            query_pos = torch.cat([torch.zeros_like(presence_token), query_pos], dim=1)
+        # Prepend zeros to query_pos for presence token
+        query_pos = F.pad(query_pos, (0, 0, 1, 0), mode="constant", value=0)
 
         # Self-attention with query position encoding
         residual = hidden_states
@@ -1527,15 +1539,6 @@ class Sam3DetrDecoderLayer(nn.Module):
         residual = hidden_states
         query_with_pos = hidden_states + query_pos
 
-        text_cross_attn_mask = None
-        if text_mask is not None:
-            text_cross_attn_mask = create_bidirectional_mask(
-                config=self.config,
-                input_embeds=hidden_states,
-                attention_mask=text_mask,
-                encoder_hidden_states=text_features,
-            )
-
         attn_output, _ = self.text_cross_attn(
             query=query_with_pos,
             key=text_features,
@@ -1546,20 +1549,6 @@ class Sam3DetrDecoderLayer(nn.Module):
         hidden_states = residual + self.text_cross_attn_dropout(attn_output)
         hidden_states = self.text_cross_attn_layer_norm(hidden_states)
 
-        # Expand vision cross-attention mask for presence token if needed
-        combined_vision_mask = vision_cross_attn_mask
-        if presence_token is not None and combined_vision_mask is not None:
-            batch_size, num_heads = combined_vision_mask.shape[:2]
-            presence_mask = torch.zeros(
-                batch_size,
-                num_heads,
-                1,
-                combined_vision_mask.shape[-1],
-                device=combined_vision_mask.device,
-                dtype=combined_vision_mask.dtype,
-            )
-            combined_vision_mask = torch.cat([presence_mask, combined_vision_mask], dim=2)
-
         # Vision cross-attention: queries attend to vision features (with RPB)
         residual = hidden_states
         query_with_pos = hidden_states + query_pos
@@ -1568,7 +1557,7 @@ class Sam3DetrDecoderLayer(nn.Module):
             query=query_with_pos,
             key=key_with_pos,
             value=vision_features,
-            attention_mask=combined_vision_mask,
+            attention_mask=vision_cross_attn_mask,
             **kwargs,
         )
         hidden_states = residual + self.vision_cross_attn_dropout(attn_output)
@@ -1580,13 +1569,7 @@ class Sam3DetrDecoderLayer(nn.Module):
         hidden_states = residual + self.mlp_dropout(hidden_states)
         hidden_states = self.mlp_layer_norm(hidden_states)
 
-        # Extract presence token if it was added
-        presence_token_out = None
-        if presence_token is not None:
-            presence_token_out = hidden_states[:, :1]
-            hidden_states = hidden_states[:, 1:]
-
-        return hidden_states, presence_token_out
+        return hidden_states
 
 
 class Sam3DetrDecoder(Sam3PreTrainedModel):
@@ -1715,11 +1698,23 @@ class Sam3DetrDecoder(Sam3PreTrainedModel):
         """
         batch_size = vision_features.shape[0]
 
-        hidden_states = self.query_embed.weight.unsqueeze(0).expand(batch_size, -1, -1)
+        query_embeds = self.query_embed.weight.unsqueeze(0).expand(batch_size, -1, -1)
         reference_boxes = self.reference_points.weight.unsqueeze(0).expand(batch_size, -1, -1)
         reference_boxes = reference_boxes.sigmoid()
         presence_token = self.presence_token.weight.unsqueeze(0).expand(batch_size, -1, -1)
 
+        # Concatenate presence token with query embeddings
+        hidden_states = torch.cat([presence_token, query_embeds], dim=1)
+
+        text_cross_attn_mask = None
+        if text_mask is not None:
+            text_cross_attn_mask = create_bidirectional_mask(
+                config=self.config,
+                input_embeds=hidden_states,
+                attention_mask=text_mask,
+                encoder_hidden_states=text_features,
+            )
+
         intermediate_outputs = []
         intermediate_boxes = [reference_boxes]
         intermediate_presence_logits = []
@@ -1734,43 +1729,45 @@ class Sam3DetrDecoder(Sam3PreTrainedModel):
             vision_cross_attn_mask = None
             if spatial_shapes is not None and spatial_shapes.shape[0] == 1:
                 spatial_shape = (spatial_shapes[0, 0], spatial_shapes[0, 1])
-                vision_cross_attn_mask = self._get_rpb_matrix(reference_boxes, spatial_shape)
+                rpb_matrix = self._get_rpb_matrix(reference_boxes, spatial_shape)
+                # Prepend zeros row for presence token (it attends to all vision tokens equally)
+                vision_cross_attn_mask = F.pad(rpb_matrix, (0, 0, 1, 0), mode="constant", value=0)
 
-            hidden_states, presence_token = layer(
+            hidden_states = layer(
                 hidden_states,
                 query_pos=query_pos,
                 text_features=text_features,
                 vision_features=vision_features,
                 vision_pos_encoding=vision_pos_encoding,
-                text_mask=text_mask,
+                text_cross_attn_mask=text_cross_attn_mask,
                 vision_cross_attn_mask=vision_cross_attn_mask,
-                presence_token=presence_token,
                 **kwargs,
             )
 
+            # Extract query hidden states (without presence token) for box refinement
+            query_hidden_states = hidden_states[:, 1:]
+
             # Box refinement: predict delta and update reference boxes
             reference_boxes_before_sigmoid = inverse_sigmoid(reference_boxes)
-            delta_boxes = self.box_head(self.output_layer_norm(hidden_states))
+            delta_boxes = self.box_head(self.output_layer_norm(query_hidden_states))
             new_reference_boxes = (delta_boxes + reference_boxes_before_sigmoid).sigmoid()
             reference_boxes = new_reference_boxes.detach()
 
-            intermediate_outputs.append(self.output_layer_norm(hidden_states))
+            intermediate_outputs.append(self.output_layer_norm(query_hidden_states))
             intermediate_boxes.append(new_reference_boxes)
 
             # Process presence token
-            if presence_token is not None:
-                presence_logits = self.presence_head(self.presence_layer_norm(presence_token)).squeeze(-1)
-                presence_logits = presence_logits.clamp(
-                    min=-self.clamp_presence_logit_max_val, max=self.clamp_presence_logit_max_val
-                )
-                intermediate_presence_logits.append(presence_logits)
+            presence_hidden = hidden_states[:, :1]
+            presence_logits = self.presence_head(self.presence_layer_norm(presence_hidden)).squeeze(-1)
+            presence_logits = presence_logits.clamp(
+                min=-self.clamp_presence_logit_max_val, max=self.clamp_presence_logit_max_val
+            )
+            intermediate_presence_logits.append(presence_logits)
 
         # Stack outputs from all layers
         intermediate_outputs = torch.stack(intermediate_outputs)
         intermediate_boxes = torch.stack(intermediate_boxes[:-1])
-        intermediate_presence_logits = (
-            torch.stack(intermediate_presence_logits) if intermediate_presence_logits else None
-        )
+        intermediate_presence_logits = torch.stack(intermediate_presence_logits)
 
         return Sam3DETRDecoderOutput(
             intermediate_hidden_states=intermediate_outputs,

transformers/models/sam3_tracker/modeling_sam3_tracker.py

@@ -107,7 +107,12 @@ class Sam3TrackerFeedForward(nn.Module):
         return hidden_states
 
 
-@auto_docstring
+@auto_docstring(
+    custom_intro="""
+    Segment Anything Model 3 (SAM 3) for generating segmentation masks, given an input image and
+    input points and labels, boxes, or masks.
+    """
+)
 class Sam3TrackerPreTrainedModel(PreTrainedModel):
     config_class = Sam3TrackerConfig
     base_model_prefix = "sam3_tracker"

transformers/models/sam3_tracker/modular_sam3_tracker.py

@@ -136,7 +136,12 @@ class Sam3TrackerFeedForward(Sam2FeedForward):
     pass
 
 
-@auto_docstring
+@auto_docstring(
+    custom_intro="""
+    Segment Anything Model 3 (SAM 3) for generating segmentation masks, given an input image and
+    input points and labels, boxes, or masks.
+    """
+)
 class Sam3TrackerPreTrainedModel(Sam2PreTrainedModel):
     @torch.no_grad()
     def _init_weights(self, module):

transformers/models/sam3_tracker_video/modeling_sam3_tracker_video.py

@@ -1719,6 +1719,7 @@ class Sam3TrackerVideoModel(Sam3TrackerVideoPreTrainedModel):
         frame: Optional[torch.Tensor] = None,
         reverse: bool = False,
         run_mem_encoder: bool = True,
+        **kwargs,
     ) -> Sam3TrackerVideoSegmentationOutput:
         r"""
         inference_session (`Sam3TrackerVideoInferenceSession`):

transformers/models/sam3_video/modeling_sam3_video.py

@@ -1697,6 +1697,7 @@ class Sam3VideoModel(Sam3VideoPreTrainedModel):
         frame_idx: Optional[int] = None,
         frame: Optional[torch.Tensor] = None,
         reverse: bool = False,
+        **kwargs,
     ):
         r"""
         inference_session (`Sam3VideoInferenceSession`):

transformers/models/seamless_m4t/modeling_seamless_m4t.py

@@ -1770,6 +1770,7 @@ class SeamlessM4TDecoder(SeamlessM4TPreTrainedModel):
         output_hidden_states: Optional[bool] = None,
         return_dict: Optional[bool] = None,
         cache_position: Optional[torch.Tensor] = None,
+        **kwargs,
     ) -> Union[tuple, BaseModelOutputWithPastAndCrossAttentions]:
         output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
         output_hidden_states = (
@@ -1914,6 +1915,7 @@ class SeamlessM4TTextToUnitModel(SeamlessM4TPreTrainedModel):
         output_hidden_states: Optional[bool] = None,
         return_dict: Optional[bool] = None,
         cache_position: Optional[torch.Tensor] = None,
+        **kwargs,
     ) -> Union[tuple[torch.Tensor], Seq2SeqModelOutput]:
         output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
         output_hidden_states = (
@@ -2035,6 +2037,7 @@ class SeamlessM4TTextToUnitForConditionalGeneration(SeamlessM4TPreTrainedModel,
         output_hidden_states: Optional[bool] = None,
         return_dict: Optional[bool] = None,
         cache_position: Optional[torch.Tensor] = None,
+        **kwargs,
     ) -> Union[Seq2SeqLMOutput, tuple[torch.FloatTensor]]:
         r"""
         labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
@@ -2354,7 +2357,7 @@ class SeamlessM4TCodeHifiGan(PreTrainedModel):
         return input_lengths
 
     def forward(
-        self, input_ids: torch.LongTensor, spkr_id: torch.Tensor, lang_id: torch.Tensor
+        self, input_ids: torch.LongTensor, spkr_id: torch.Tensor, lang_id: torch.Tensor, **kwargs
     ) -> tuple[torch.Tensor]:
         """
         Args:
@@ -2996,6 +2999,7 @@ class SeamlessM4TForTextToSpeech(SeamlessM4TPreTrainedModel, GenerationMixin):
         output_hidden_states: Optional[bool] = None,
         return_dict: Optional[bool] = None,
         cache_position: Optional[torch.Tensor] = None,
+        **kwargs,
     ) -> Union[Seq2SeqLMOutput, tuple[torch.FloatTensor]]:
         r"""
         labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):

transformers/models/seamless_m4t/tokenization_seamless_m4t.py

@@ -60,7 +60,7 @@ class SeamlessM4TTokenizer(TokenizersBackend):
     Args:
         vocab (`list` or `dict`, *optional*):
             List of (token, score) tuples or dict mapping tokens to indices. If not provided, uses default vocab.
-        merges (`list`, *optional*):
+        merges (`str` or `list`, *optional*):
             List of merge rules for BPE model. If not provided, uses empty list.
         bos_token (`str`, *optional*, defaults to `"<s>"`):
             The beginning of sequence token that was used during pretraining. Can be used a sequence classifier token.
@@ -104,15 +104,15 @@ class SeamlessM4TTokenizer(TokenizersBackend):
 
     vocab_files_names = VOCAB_FILES_NAMES
     model_input_names = ["input_ids", "attention_mask"]
-    slow_tokenizer_class = None
+    model = BPE
 
-    prefix_tokens: list[int] = []
-    suffix_tokens: list[int] = []
+    prefix_tokens: list[int] = None
+    suffix_tokens: list[int] = None
 
     def __init__(
         self,
-        vocab: Optional[list] = None,
-        merges: Optional[list] = None,
+        vocab: Optional[Union[str, dict[str, int]]] = None,
+        merges: Optional[Union[str, list[str]]] = None,
         bos_token="<s>",
         eos_token="</s>",
         sep_token="</s>",
@@ -126,59 +126,14 @@ class SeamlessM4TTokenizer(TokenizersBackend):
         vocab_file=None,
         **kwargs,
     ):
-        if vocab is None:
-            vocab = {
-                str(pad_token): 0,
-                str(unk_token): 1,
-                str(bos_token): 2,
-                str(eos_token): 3,
-            }
-
-        # Process vocab - SeamlessM4T uses fairseq vocab alignment: <pad>=0, <unk>=1, <s>=2, </s>=3, then SPM pieces[3:]
-        if isinstance(vocab, list):
-            # Convert list of (token, score) tuples to dict {token: idx}
-            # Check if vocab is already in SeamlessM4T order (pad, unk, s, /s) or tokenizer.json order (unk, s, /s, ...)
-            first_tokens = [str(item[0]) if isinstance(item, (list, tuple)) else str(item) for item in vocab[:4]]
-            is_seamless_order = (
-                len(first_tokens) >= 4
-                and first_tokens[0] == str(pad_token)
-                and first_tokens[1] == str(unk_token)
-                and first_tokens[2] == str(bos_token)
-                and first_tokens[3] == str(eos_token)
-            )
-
-            if is_seamless_order:
-                # Already in correct order, use list index directly as token ID
-                vocab_dict = {}
-                for idx, item in enumerate(vocab):
-                    token = str(item[0]) if isinstance(item, (list, tuple)) else str(item)
-                    vocab_dict[token] = idx
-                self._vocab = vocab_dict
-            else:
-                # Reorder to fairseq: <pad>, <unk>, <s>, </s>, ... (rest of vocab)
-                vocab_dict = {}
-                vocab_dict[str(pad_token)] = 0
-                vocab_dict[str(unk_token)] = 1
-                vocab_dict[str(bos_token)] = 2
-                vocab_dict[str(eos_token)] = 3
-
-                # Add rest of vocab starting from index 4, skipping tokens we already added
-                idx = 4
-                for item in vocab:
-                    token = str(item[0]) if isinstance(item, (list, tuple)) else str(item)
-                    if token not in vocab_dict:
-                        vocab_dict[token] = idx
-                        idx += 1
-
-                self._vocab = vocab_dict
-        else:
-            self._vocab = vocab
-
-        if merges is None:
-            self._merges = []
-        else:
-            self._merges = [tuple(merge) if isinstance(merge, list) else merge for merge in merges]
+        self._vocab = vocab or {
+            str(pad_token): 0,
+            str(unk_token): 1,
+            str(bos_token): 2,
+            str(eos_token): 3,
+        }
 
+        self._merges = merges or []
         self._tokenizer = Tokenizer(
             BPE(
                 vocab=self._vocab,
@@ -216,7 +171,6 @@ class SeamlessM4TTokenizer(TokenizersBackend):
             kwargs.setdefault("additional_special_tokens", additional_special_tokens)
 
         super().__init__(
-            tokenizer_object=self._tokenizer,
             bos_token=bos_token,
             eos_token=eos_token,
             sep_token=sep_token,
@@ -245,6 +199,20 @@ class SeamlessM4TTokenizer(TokenizersBackend):
 
         self.set_tgt_lang_special_tokens(self._tgt_lang)
 
+    @classmethod
+    def convert_from_spm_model(cls, vocab, **kwargs):
+        """When converting from spm, offset is needed to account for special tokens."""
+        _vocab = {
+            "<pad>": 0,
+            "<unk>": 1,
+            "<s>": 2,
+            "</s>": 3,
+        }
+        for i, token in enumerate(list(vocab.keys())):
+            _vocab[token] = i + 1  # offset by 1 to account for special tokens
+        kwargs["vocab"] = _vocab
+        return kwargs
+
     @property
     def src_lang(self) -> str:
         return self._src_lang

transformers/models/seamless_m4t_v2/modeling_seamless_m4t_v2.py

@@ -1812,6 +1812,7 @@ class SeamlessM4Tv2Decoder(SeamlessM4Tv2PreTrainedModel):
         output_hidden_states: Optional[bool] = None,
         return_dict: Optional[bool] = None,
         cache_position: Optional[torch.Tensor] = None,
+        **kwargs,
     ) -> Union[tuple, BaseModelOutputWithPastAndCrossAttentions]:
         output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
         output_hidden_states = (
@@ -1995,6 +1996,7 @@ class SeamlessM4Tv2TextToUnitDecoder(SeamlessM4Tv2PreTrainedModel):
         output_attentions: Optional[bool] = None,
         output_hidden_states: Optional[bool] = None,
         return_dict: Optional[bool] = None,
+        **kwargs,
     ) -> Union[tuple, SeamlessM4Tv2TextToUnitDecoderOutput]:
         r"""
         Args:
@@ -2122,6 +2124,7 @@ class SeamlessM4Tv2TextToUnitModel(SeamlessM4Tv2PreTrainedModel):
         output_attentions: Optional[bool] = None,
         output_hidden_states: Optional[bool] = None,
         return_dict: Optional[bool] = None,
+        **kwargs,
     ) -> Union[tuple[torch.Tensor], Seq2SeqModelOutput]:
         output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
         output_hidden_states = (
@@ -2556,7 +2559,7 @@ class SeamlessM4Tv2CodeHifiGan(PreTrainedModel):
 
     # Copied from transformers.models.seamless_m4t.modeling_seamless_m4t.SeamlessM4TCodeHifiGan.forward with SeamlessM4T->SeamlessM4Tv2, spkr_id->speaker_id
     def forward(
-        self, input_ids: torch.LongTensor, speaker_id: torch.Tensor, lang_id: torch.Tensor
+        self, input_ids: torch.LongTensor, speaker_id: torch.Tensor, lang_id: torch.Tensor, **kwargs
     ) -> tuple[torch.Tensor]:
         """
         Args:
@@ -3214,6 +3217,7 @@ class SeamlessM4Tv2ForTextToSpeech(SeamlessM4Tv2PreTrainedModel, GenerationMixin
         output_hidden_states: Optional[bool] = None,
         return_dict: Optional[bool] = None,
         cache_position: Optional[torch.Tensor] = None,
+        **kwargs,
     ) -> Union[Seq2SeqLMOutput, tuple[torch.FloatTensor]]:
         r"""
         labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):

transformers/models/seed_oss/modeling_seed_oss.py

@@ -40,7 +40,7 @@ from ...modeling_rope_utils import ROPE_INIT_FUNCTIONS, dynamic_rope_update
 from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel
 from ...processing_utils import Unpack
 from ...utils import TransformersKwargs, auto_docstring, can_return_tuple
-from ...utils.generic import check_model_inputs
+from ...utils.generic import check_model_inputs, maybe_autocast
 from .configuration_seed_oss import SeedOssConfig
 
 
@@ -350,7 +350,7 @@ class SeedOssRotaryEmbedding(nn.Module):
         position_ids_expanded = position_ids[:, None, :].float()
 
         device_type = x.device.type if isinstance(x.device.type, str) and x.device.type != "mps" else "cpu"
-        with torch.autocast(device_type=device_type, enabled=False):  # Force float32
+        with maybe_autocast(device_type=device_type, enabled=False):  # Force float32
             freqs = (inv_freq_expanded.float() @ position_ids_expanded.float()).transpose(1, 2)
             emb = torch.cat((freqs, freqs), dim=-1)
             cos = emb.cos() * self.attention_scaling

transformers/models/segformer/modeling_segformer.py

@@ -434,6 +434,7 @@ class SegformerModel(SegformerPreTrainedModel):
         output_attentions: Optional[bool] = None,
         output_hidden_states: Optional[bool] = None,
         return_dict: Optional[bool] = None,
+        **kwargs,
     ) -> Union[tuple, BaseModelOutput]:
         output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
         output_hidden_states = (
@@ -486,6 +487,7 @@ class SegformerForImageClassification(SegformerPreTrainedModel):
         output_attentions: Optional[bool] = None,
         output_hidden_states: Optional[bool] = None,
         return_dict: Optional[bool] = None,
+        **kwargs,
     ) -> Union[tuple, SegFormerImageClassifierOutput]:
         r"""
         labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
@@ -572,7 +574,7 @@ class SegformerDecodeHead(SegformerPreTrainedModel):
 
         self.config = config
 
-    def forward(self, encoder_hidden_states: torch.FloatTensor) -> torch.Tensor:
+    def forward(self, encoder_hidden_states: torch.FloatTensor, **kwargs) -> torch.Tensor:
         batch_size = encoder_hidden_states[-1].shape[0]
 
         all_hidden_states = ()
@@ -627,6 +629,7 @@ class SegformerForSemanticSegmentation(SegformerPreTrainedModel):
         output_attentions: Optional[bool] = None,
         output_hidden_states: Optional[bool] = None,
         return_dict: Optional[bool] = None,
+        **kwargs,
     ) -> Union[tuple, SemanticSegmenterOutput]:
         r"""
         labels (`torch.LongTensor` of shape `(batch_size, height, width)`, *optional*):

transformers/models/seggpt/modeling_seggpt.py

@@ -647,6 +647,7 @@ class SegGptModel(SegGptPreTrainedModel):
         output_attentions: Optional[bool] = None,
         output_hidden_states: Optional[bool] = None,
         return_dict: Optional[bool] = None,
+        **kwargs,
     ) -> Union[tuple, SegGptEncoderOutput]:
         r"""
         prompt_pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`):
@@ -843,6 +844,7 @@ class SegGptForImageSegmentation(SegGptPreTrainedModel):
         output_attentions: Optional[bool] = None,
         output_hidden_states: Optional[bool] = None,
         return_dict: Optional[bool] = None,
+        **kwargs,
     ) -> Union[tuple, SegGptImageSegmentationOutput]:
         r"""
         prompt_pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`):