transformers 5.0.0__py3-none-any.whl → 5.0.0rc0__py3-none-any.whl

transformers/models/mm_grounding_dino/modeling_mm_grounding_dino.py

@@ -4,6 +4,7 @@
 #             the file from the modular. If any change should be done, please apply the change to the
 #                          modular_mm_grounding_dino.py file directly. One of our CI enforces this.
 #                🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨
+# coding=utf-8
 # Copyright 2025 The HuggingFace Inc. team.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -20,6 +21,7 @@
 import math
 import warnings
 from dataclasses import dataclass
+from typing import Optional, Union
 
 import torch
 import torch.nn.functional as F
@@ -31,7 +33,7 @@ from ...file_utils import ModelOutput, is_timm_available, requires_backends
 from ...integrations import use_kernel_forward_from_hub
 from ...modeling_utils import PreTrainedModel
 from ...pytorch_utils import meshgrid
-from ...utils import auto_docstring, torch_compilable_check
+from ...utils import auto_docstring
 from ...utils.backbone_utils import load_backbone
 from ..auto.modeling_auto import AutoModel
 from .configuration_mm_grounding_dino import MMGroundingDinoConfig
@@ -182,16 +184,16 @@ class MMGroundingDinoMultiscaleDeformableAttention(nn.Module):
 
         self.disable_custom_kernels = config.disable_custom_kernels
 
-    def with_pos_embed(self, tensor: torch.Tensor, position_embeddings: Tensor | None):
+    def with_pos_embed(self, tensor: torch.Tensor, position_embeddings: Optional[Tensor]):
         return tensor if position_embeddings is None else tensor + position_embeddings
 
     def forward(
         self,
         hidden_states: torch.Tensor,
-        attention_mask: torch.Tensor | None = None,
+        attention_mask: Optional[torch.Tensor] = None,
         encoder_hidden_states=None,
         encoder_attention_mask=None,
-        position_embeddings: torch.Tensor | None = None,
+        position_embeddings: Optional[torch.Tensor] = None,
         reference_points=None,
         spatial_shapes=None,
         spatial_shapes_list=None,
@@ -205,10 +207,10 @@ class MMGroundingDinoMultiscaleDeformableAttention(nn.Module):
         batch_size, num_queries, _ = hidden_states.shape
         batch_size, sequence_length, _ = encoder_hidden_states.shape
         # Ignore copy
-        torch_compilable_check(
-            (spatial_shapes[:, 0] * spatial_shapes[:, 1]).sum() == sequence_length,
-            "Make sure to align the spatial shapes with the sequence length of the encoder hidden states",
-        )
+        if (spatial_shapes[:, 0] * spatial_shapes[:, 1]).sum() != sequence_length:
+            raise ValueError(
+                "Make sure to align the spatial shapes with the sequence length of the encoder hidden states"
+            )
 
         value = self.value_proj(encoder_hidden_states)
         if attention_mask is not None:
@@ -292,8 +294,8 @@ class MMGroundingDinoBiMultiHeadAttention(nn.Module):
         self,
         vision_features: torch.FloatTensor,
         text_features: torch.FloatTensor,
-        vision_attention_mask: torch.BoolTensor | None = None,
-        text_attention_mask: torch.BoolTensor | None = None,
+        vision_attention_mask: Optional[torch.BoolTensor] = None,
+        text_attention_mask: Optional[torch.BoolTensor] = None,
     ) -> tuple[tuple[torch.FloatTensor, torch.FloatTensor], tuple[torch.FloatTensor, torch.FloatTensor]]:
         """Image-to-text and text-to-image cross-attention
 
@@ -424,7 +426,7 @@ def drop_path(input: torch.Tensor, drop_prob: float = 0.0, training: bool = Fals
 class MMGroundingDinoDropPath(nn.Module):
     """Drop paths (Stochastic Depth) per sample (when applied in main path of residual blocks)."""
 
-    def __init__(self, drop_prob: float | None = None) -> None:
+    def __init__(self, drop_prob: Optional[float] = None) -> None:
         super().__init__()
         self.drop_prob = drop_prob
 
@@ -455,8 +457,8 @@ class MMGroundingDinoFusionLayer(nn.Module):
         self,
         vision_features: torch.FloatTensor,
         text_features: torch.FloatTensor,
-        attention_mask_vision: torch.BoolTensor | None = None,
-        attention_mask_text: torch.BoolTensor | None = None,
+        attention_mask_vision: Optional[torch.BoolTensor] = None,
+        attention_mask_text: Optional[torch.BoolTensor] = None,
     ) -> tuple[tuple[torch.FloatTensor, torch.FloatTensor], tuple[torch.FloatTensor, torch.FloatTensor]]:
         """Image and text features fusion
 
@@ -550,7 +552,7 @@ class MMGroundingDinoPreTrainedModel(PreTrainedModel):
         elif isinstance(module, MMGroundingDinoFusionLayer):
             init.constant_(module.vision_param, 1e-4)
             init.constant_(module.text_param, 1e-4)
-        elif isinstance(module, (nn.Linear, nn.Conv2d)):
+        elif isinstance(module, (nn.Linear, nn.Conv2d, nn.BatchNorm2d)):
             init.normal_(module.weight, mean=0.0, std=std)
             if module.bias is not None:
                 init.zeros_(module.bias)
@@ -748,11 +750,11 @@ class MMGroundingDinoEncoderOutput(ModelOutput):
         each layer plus the initial embedding outputs.
     """
 
-    last_hidden_state_vision: torch.FloatTensor | None = None
-    last_hidden_state_text: torch.FloatTensor | None = None
-    vision_hidden_states: tuple[torch.FloatTensor] | None = None
-    text_hidden_states: tuple[torch.FloatTensor] | None = None
-    attentions: tuple[tuple[torch.FloatTensor]] | None = None
+    last_hidden_state_vision: Optional[torch.FloatTensor] = None
+    last_hidden_state_text: Optional[torch.FloatTensor] = None
+    vision_hidden_states: Optional[tuple[torch.FloatTensor]] = None
+    text_hidden_states: Optional[tuple[torch.FloatTensor]] = None
+    attentions: Optional[tuple[tuple[torch.FloatTensor]]] = None
 
 
 class MMGroundingDinoMultiheadAttention(nn.Module):
@@ -783,8 +785,8 @@ class MMGroundingDinoMultiheadAttention(nn.Module):
         queries: torch.Tensor,
         keys: torch.Tensor,
         values: torch.Tensor,
-        attention_mask: torch.FloatTensor | None = None,
-        output_attentions: bool | None = False,
+        attention_mask: Optional[torch.FloatTensor] = None,
+        output_attentions: Optional[bool] = False,
     ) -> tuple[torch.Tensor]:
         batch_size, seq_length, _ = queries.shape
         query_layer = (
@@ -847,14 +849,14 @@ class MMGroundingDinoTextEnhancerLayer(nn.Module):
         self.num_heads = config.encoder_attention_heads // 2
         self.dropout = config.text_enhancer_dropout
 
-    def with_pos_embed(self, hidden_state: Tensor, position_embeddings: Tensor | None):
+    def with_pos_embed(self, hidden_state: Tensor, position_embeddings: Optional[Tensor]):
         return hidden_state if position_embeddings is None else hidden_state + position_embeddings
 
     def forward(
         self,
         hidden_states: torch.FloatTensor,
-        attention_masks: torch.BoolTensor | None = None,
-        position_embeddings: torch.FloatTensor | None = None,
+        attention_masks: Optional[torch.BoolTensor] = None,
+        position_embeddings: Optional[torch.FloatTensor] = None,
     ) -> tuple[torch.FloatTensor, torch.FloatTensor]:
         """Text self-attention to enhance projection of text features generated by
         the text encoder (AutoModel based on text_config) within MMGroundingDinoEncoderLayer
@@ -928,7 +930,7 @@ class MMGroundingDinoDeformableLayer(nn.Module):
         self,
         hidden_states: torch.Tensor,
         attention_mask: torch.Tensor,
-        position_embeddings: torch.Tensor | None = None,
+        position_embeddings: Optional[torch.Tensor] = None,
         reference_points=None,
         spatial_shapes=None,
         spatial_shapes_list=None,
@@ -1043,8 +1045,8 @@ class MMGroundingDinoEncoderLayer(nn.Module):
     def get_text_position_embeddings(
         self,
         text_features: Tensor,
-        text_position_embedding: torch.Tensor | None,
-        text_position_ids: torch.Tensor | None,
+        text_position_embedding: Optional[torch.Tensor],
+        text_position_ids: Optional[torch.Tensor],
     ) -> Tensor:
         batch_size, seq_length, _ = text_features.shape
         if text_position_embedding is None and text_position_ids is None:
@@ -1071,11 +1073,11 @@ class MMGroundingDinoEncoderLayer(nn.Module):
         level_start_index: Tensor,
         key_padding_mask: Tensor,
         reference_points: Tensor,
-        text_features: Tensor | None = None,
-        text_attention_mask: Tensor | None = None,
-        text_position_embedding: Tensor | None = None,
-        text_self_attention_masks: Tensor | None = None,
-        text_position_ids: Tensor | None = None,
+        text_features: Optional[Tensor] = None,
+        text_attention_mask: Optional[Tensor] = None,
+        text_position_embedding: Optional[Tensor] = None,
+        text_self_attention_masks: Optional[Tensor] = None,
+        text_position_ids: Optional[Tensor] = None,
     ):
         text_position_embedding = self.get_text_position_embeddings(
             text_features, text_position_embedding, text_position_ids
@@ -1131,12 +1133,12 @@ class MMGroundingDinoEncoder(MMGroundingDinoPreTrainedModel):
         self.post_init()
 
     @staticmethod
-    def get_reference_points(spatial_shapes_list, valid_ratios, device):
+    def get_reference_points(spatial_shapes, valid_ratios, device):
         """
         Get reference points for each feature map.
 
         Args:
-            spatial_shapes_list (`list[tuple[int, int]]`):
+            spatial_shapes (`torch.LongTensor` of shape `(num_feature_levels, 2)`):
                 Spatial shapes of each feature map.
             valid_ratios (`torch.FloatTensor` of shape `(batch_size, num_feature_levels, 2)`):
                 Valid ratios of each feature map.
@@ -1146,7 +1148,7 @@ class MMGroundingDinoEncoder(MMGroundingDinoPreTrainedModel):
             `torch.FloatTensor` of shape `(batch_size, num_queries, num_feature_levels, 2)`
         """
         reference_points_list = []
-        for level, (height, width) in enumerate(spatial_shapes_list):
+        for level, (height, width) in enumerate(spatial_shapes):
             ref_y, ref_x = meshgrid(
                 torch.linspace(0.5, height - 0.5, height, dtype=torch.float32, device=device),
                 torch.linspace(0.5, width - 0.5, width, dtype=torch.float32, device=device),
@@ -1170,16 +1172,15 @@ class MMGroundingDinoEncoder(MMGroundingDinoPreTrainedModel):
         spatial_shapes_list: list[tuple[int, int]],
         level_start_index: Tensor,
         valid_ratios=None,
-        text_features: Tensor | None = None,
-        text_attention_mask: Tensor | None = None,
-        text_position_embedding: Tensor | None = None,
-        text_self_attention_masks: Tensor | None = None,
-        text_position_ids: Tensor | None = None,
+        text_features: Optional[Tensor] = None,
+        text_attention_mask: Optional[Tensor] = None,
+        text_position_embedding: Optional[Tensor] = None,
+        text_self_attention_masks: Optional[Tensor] = None,
+        text_position_ids: Optional[Tensor] = None,
         output_attentions=None,
         output_hidden_states=None,
         return_dict=None,
-        **kwargs,
-    ) -> tuple | MMGroundingDinoEncoderOutput:
+    ):
         r"""
         Args:
             vision_features (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`):
@@ -1229,7 +1230,7 @@ class MMGroundingDinoEncoder(MMGroundingDinoPreTrainedModel):
         )
         return_dict = return_dict if return_dict is not None else self.config.use_return_dict
 
-        reference_points = self.get_reference_points(spatial_shapes_list, valid_ratios, device=vision_features.device)
+        reference_points = self.get_reference_points(spatial_shapes, valid_ratios, device=vision_features.device)
 
         encoder_vision_states = () if output_hidden_states else None
         encoder_text_states = () if output_hidden_states else None
@@ -1300,11 +1301,11 @@ class MMGroundingDinoDecoderOutput(ModelOutput):
         Stacked intermediate reference points (reference points of each layer of the decoder).
     """
 
-    last_hidden_state: torch.FloatTensor | None = None
-    intermediate_hidden_states: torch.FloatTensor | None = None
-    intermediate_reference_points: torch.FloatTensor | None = None
-    hidden_states: tuple[torch.FloatTensor] | None = None
-    attentions: tuple[tuple[torch.FloatTensor]] | None = None
+    last_hidden_state: Optional[torch.FloatTensor] = None
+    intermediate_hidden_states: Optional[torch.FloatTensor] = None
+    intermediate_reference_points: Optional[torch.FloatTensor] = None
+    hidden_states: Optional[tuple[torch.FloatTensor]] = None
+    attentions: Optional[tuple[tuple[torch.FloatTensor]]] = None
 
 
 class MMGroundingDinoDecoderLayer(nn.Module):
@@ -1337,23 +1338,23 @@ class MMGroundingDinoDecoderLayer(nn.Module):
         self.fc2 = nn.Linear(config.decoder_ffn_dim, self.embed_dim)
         self.final_layer_norm = nn.LayerNorm(self.embed_dim, config.layer_norm_eps)
 
-    def with_pos_embed(self, tensor: torch.Tensor, position_embeddings: Tensor | None):
+    def with_pos_embed(self, tensor: torch.Tensor, position_embeddings: Optional[Tensor]):
         return tensor if position_embeddings is None else tensor + position_embeddings
 
     def forward(
         self,
         hidden_states: torch.Tensor,
-        position_embeddings: torch.Tensor | None = None,
+        position_embeddings: Optional[torch.Tensor] = None,
         reference_points=None,
         spatial_shapes=None,
         spatial_shapes_list=None,
         level_start_index=None,
-        vision_encoder_hidden_states: torch.Tensor | None = None,
-        vision_encoder_attention_mask: torch.Tensor | None = None,
-        text_encoder_hidden_states: torch.Tensor | None = None,
-        text_encoder_attention_mask: torch.Tensor | None = None,
-        self_attn_mask: torch.Tensor | None = None,
-        output_attentions: bool | None = False,
+        vision_encoder_hidden_states: Optional[torch.Tensor] = None,
+        vision_encoder_attention_mask: Optional[torch.Tensor] = None,
+        text_encoder_hidden_states: Optional[torch.Tensor] = None,
+        text_encoder_attention_mask: Optional[torch.Tensor] = None,
+        self_attn_mask: Optional[torch.Tensor] = None,
+        output_attentions: Optional[bool] = False,
     ):
         residual = hidden_states
 
@@ -1475,8 +1476,7 @@ class MMGroundingDinoDecoder(MMGroundingDinoPreTrainedModel):
         output_attentions=None,
         output_hidden_states=None,
         return_dict=None,
-        **kwargs,
-    ) -> tuple | MMGroundingDinoDecoderOutput:
+    ):
         r"""
         Args:
             inputs_embeds (`torch.FloatTensor` of shape `(batch_size, num_queries, hidden_size)`):
@@ -1711,21 +1711,21 @@ class MMGroundingDinoModelOutput(ModelOutput):
         Coordinates of top `config.num_queries` scoring bounding boxes in the first stage.
     """
 
-    last_hidden_state: torch.FloatTensor | None = None
-    init_reference_points: torch.FloatTensor | None = None
-    intermediate_hidden_states: torch.FloatTensor | None = None
-    intermediate_reference_points: torch.FloatTensor | None = None
-    decoder_hidden_states: tuple[torch.FloatTensor] | None = None
-    decoder_attentions: tuple[tuple[torch.FloatTensor]] | None = None
-    encoder_last_hidden_state_vision: torch.FloatTensor | None = None
-    encoder_last_hidden_state_text: torch.FloatTensor | None = None
-    encoder_vision_hidden_states: tuple[torch.FloatTensor] | None = None
-    encoder_text_hidden_states: tuple[torch.FloatTensor] | None = None
-    encoder_attentions: tuple[tuple[torch.FloatTensor]] | None = None
-    enc_outputs_class: torch.FloatTensor | None = None
-    enc_outputs_coord_logits: torch.FloatTensor | None = None
-    encoder_logits: torch.FloatTensor | None = None
-    encoder_pred_boxes: torch.FloatTensor | None = None
+    last_hidden_state: Optional[torch.FloatTensor] = None
+    init_reference_points: Optional[torch.FloatTensor] = None
+    intermediate_hidden_states: Optional[torch.FloatTensor] = None
+    intermediate_reference_points: Optional[torch.FloatTensor] = None
+    decoder_hidden_states: Optional[tuple[torch.FloatTensor]] = None
+    decoder_attentions: Optional[tuple[tuple[torch.FloatTensor]]] = None
+    encoder_last_hidden_state_vision: Optional[torch.FloatTensor] = None
+    encoder_last_hidden_state_text: Optional[torch.FloatTensor] = None
+    encoder_vision_hidden_states: Optional[tuple[torch.FloatTensor]] = None
+    encoder_text_hidden_states: Optional[tuple[torch.FloatTensor]] = None
+    encoder_attentions: Optional[tuple[tuple[torch.FloatTensor]]] = None
+    enc_outputs_class: Optional[torch.FloatTensor] = None
+    enc_outputs_coord_logits: Optional[torch.FloatTensor] = None
+    encoder_logits: Optional[torch.FloatTensor] = None
+    encoder_pred_boxes: Optional[torch.FloatTensor] = None
 
 
 class MMGroundingDinoSinePositionEmbedding(nn.Module):
@@ -1783,42 +1783,33 @@ def generate_masks_with_special_tokens_and_transfer_map(input_ids: torch.LongTen
         - **attention_mask** (`torch.BoolTensor` of shape `(batch_size, sequence_length, sequence_length)`)
         - **position_ids** (`torch.LongTensor` of shape `(batch_size, sequence_length)`)
     """
-    batch_size, seq_len = input_ids.shape
-    device = input_ids.device
-
-    # Identify special token positions
-    special_mask = torch.isin(input_ids, torch.tensor(SPECIAL_TOKENS, device=device))
-
-    # For each position, find the previous and next special token indices
-    indices = torch.arange(seq_len, device=device).unsqueeze(0).expand(batch_size, -1)
-
-    # Previous special token: cummax of special token indices
-    prev_special = torch.where(special_mask, indices, torch.tensor(-1, device=device))
-    prev_special = torch.cummax(prev_special, dim=1)[0]
-
-    # Next special token: flip, cummin, flip back
-    next_special = torch.where(special_mask, indices, torch.tensor(seq_len, device=device))
-    next_special = torch.flip(torch.cummin(torch.flip(next_special, dims=[1]), dim=1)[0], dims=[1])
-
-    # Tokens with the same next_special belong to the same block
-    # Exclude blocks whose closing delimiter is at position 0 or seq_len-1
-    valid_block = (next_special != 0) & (next_special != seq_len - 1) & (next_special != seq_len)
-
-    # Build attention mask: tokens attend to each other if they share the same next_special
-    next_i = next_special.unsqueeze(2)  # (B, N, 1)
-    next_j = next_special.unsqueeze(1)  # (B, 1, N)
-    attention_mask = (next_i == next_j) & valid_block.unsqueeze(1)
-
-    # Always allow self-attention
-    identity = torch.eye(seq_len, device=device, dtype=torch.bool).unsqueeze(0).expand(batch_size, -1, -1)
-    attention_mask = identity | attention_mask
+    batch_size, num_token = input_ids.shape
+    # special_tokens_mask: batch_size, num_token. 1 for special tokens. 0 for normal tokens
+    special_tokens_mask = torch.zeros((batch_size, num_token), device=input_ids.device).bool()
+    for special_token in SPECIAL_TOKENS:
+        special_tokens_mask = torch.logical_or(special_tokens_mask, input_ids == special_token)
+
+    # idxs: each row is a list of indices of special tokens
+    idxs = torch.nonzero(special_tokens_mask)
+
+    # generate attention mask and positional ids
+    attention_mask = torch.eye(num_token, device=input_ids.device).bool().unsqueeze(0).repeat(batch_size, 1, 1)
+    position_ids = torch.zeros((batch_size, num_token), device=input_ids.device)
+    previous_col = 0
+    for i in range(idxs.shape[0]):
+        row, col = idxs[i]
+        if (col == 0) or (col == num_token - 1):
+            attention_mask[row, col, col] = True
+            position_ids[row, col] = 0
+        else:
+            attention_mask[row, previous_col + 1 : col + 1, previous_col + 1 : col + 1] = True
+            position_ids[row, previous_col + 1 : col + 1] = torch.arange(
+                0, col - previous_col, device=input_ids.device
+            )
 
-    # Position IDs: distance from previous special token
-    position_ids = indices - prev_special - 1
-    position_ids = torch.where(valid_block, position_ids, torch.zeros_like(position_ids))
-    position_ids = torch.clamp(position_ids, min=0).to(torch.long)
+        previous_col = col
 
-    return attention_mask, position_ids
+    return attention_mask, position_ids.to(torch.long)
 
 
 @auto_docstring(
@@ -1897,13 +1888,13 @@ class MMGroundingDinoModel(MMGroundingDinoPreTrainedModel):
         valid_ratio = torch.stack([valid_ratio_width, valid_ratio_height], -1)
         return valid_ratio
 
-    def generate_encoder_output_proposals(self, enc_output, padding_mask, spatial_shapes_list):
+    def generate_encoder_output_proposals(self, enc_output, padding_mask, spatial_shapes):
         """Generate the encoder output proposals from encoded enc_output.
 
         Args:
             enc_output (`torch.Tensor[batch_size, sequence_length, hidden_size]`): Output of the encoder.
             padding_mask (`torch.Tensor[batch_size, sequence_length]`): Padding mask for `enc_output`.
-            spatial_shapes_list (`list[tuple[int, int]]`): Spatial shapes of each feature map.
+            spatial_shapes (`torch.Tensor[num_feature_levels, 2]`): Spatial shapes of the feature maps.
 
         Returns:
             `tuple(torch.FloatTensor)`: A tuple of feature map and bbox prediction.
@@ -1915,7 +1906,7 @@ class MMGroundingDinoModel(MMGroundingDinoPreTrainedModel):
         batch_size = enc_output.shape[0]
         proposals = []
         current_position = 0
-        for level, (height, width) in enumerate(spatial_shapes_list):
+        for level, (height, width) in enumerate(spatial_shapes):
             mask_flatten_ = padding_mask[:, current_position : (current_position + height * width)]
             mask_flatten_ = mask_flatten_.view(batch_size, height, width, 1)
             valid_height = torch.sum(~mask_flatten_[:, :, 0, 0], 1)
@@ -1953,15 +1944,14 @@ class MMGroundingDinoModel(MMGroundingDinoPreTrainedModel):
         self,
         pixel_values: Tensor,
         input_ids: Tensor,
-        token_type_ids: Tensor | None = None,
-        attention_mask: Tensor | None = None,
-        pixel_mask: Tensor | None = None,
+        token_type_ids: Optional[Tensor] = None,
+        attention_mask: Optional[Tensor] = None,
+        pixel_mask: Optional[Tensor] = None,
         encoder_outputs=None,
         output_attentions=None,
         output_hidden_states=None,
         return_dict=None,
-        **kwargs,
-    ) -> tuple | MMGroundingDinoModelOutput:
+    ):
         r"""
         input_ids (`torch.LongTensor` of shape `(batch_size, text_sequence_length)`):
             Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide
@@ -1979,12 +1969,10 @@ class MMGroundingDinoModel(MMGroundingDinoPreTrainedModel):
         ```python
         >>> from transformers import AutoProcessor, AutoModel
         >>> from PIL import Image
-        >>> import httpx
-        >>> from io import BytesIO
+        >>> import requests
 
         >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
-        >>> with httpx.stream("GET", url) as response:
-        ...     image = Image.open(BytesIO(response.read()))
+        >>> image = Image.open(requests.get(url, stream=True).raw)
         >>> text = "a cat."
 
         >>> processor = AutoProcessor.from_pretrained("IDEA-Research/grounding-dino-tiny")
@@ -2132,7 +2120,7 @@ class MMGroundingDinoModel(MMGroundingDinoPreTrainedModel):
         encoder_pred_boxes = None
         if self.config.two_stage:
             object_query_embedding, output_proposals = self.generate_encoder_output_proposals(
-                encoder_outputs[0], ~mask_flatten, spatial_shapes_list
+                encoder_outputs[0], ~mask_flatten, spatial_shapes
             )
 
             # hack implementation as in two-stage Deformable DETR
@@ -2304,27 +2292,27 @@ class MMGroundingDinoObjectDetectionOutput(ModelOutput):
         Encoded candidate labels sequence. Used in processor to post process object detection result.
     """
 
-    loss: torch.FloatTensor | None = None
-    loss_dict: dict | None = None
-    logits: torch.FloatTensor | None = None
-    pred_boxes: torch.FloatTensor | None = None
-    auxiliary_outputs: list[dict] | None = None
-    last_hidden_state: torch.FloatTensor | None = None
-    init_reference_points: torch.FloatTensor | None = None
-    intermediate_hidden_states: torch.FloatTensor | None = None
-    intermediate_reference_points: torch.FloatTensor | None = None
-    decoder_hidden_states: tuple[torch.FloatTensor] | None = None
-    decoder_attentions: tuple[tuple[torch.FloatTensor]] | None = None
-    encoder_last_hidden_state_vision: torch.FloatTensor | None = None
-    encoder_last_hidden_state_text: torch.FloatTensor | None = None
-    encoder_vision_hidden_states: tuple[torch.FloatTensor] | None = None
-    encoder_text_hidden_states: tuple[torch.FloatTensor] | None = None
-    encoder_attentions: tuple[tuple[torch.FloatTensor]] | None = None
-    enc_outputs_class: torch.FloatTensor | None = None
-    enc_outputs_coord_logits: torch.FloatTensor | None = None
-    encoder_logits: torch.FloatTensor | None = None
-    encoder_pred_boxes: torch.FloatTensor | None = None
-    input_ids: torch.LongTensor | None = None
+    loss: Optional[torch.FloatTensor] = None
+    loss_dict: Optional[dict] = None
+    logits: Optional[torch.FloatTensor] = None
+    pred_boxes: Optional[torch.FloatTensor] = None
+    auxiliary_outputs: Optional[list[dict]] = None
+    last_hidden_state: Optional[torch.FloatTensor] = None
+    init_reference_points: Optional[torch.FloatTensor] = None
+    intermediate_hidden_states: Optional[torch.FloatTensor] = None
+    intermediate_reference_points: Optional[torch.FloatTensor] = None
+    decoder_hidden_states: Optional[tuple[torch.FloatTensor]] = None
+    decoder_attentions: Optional[tuple[tuple[torch.FloatTensor]]] = None
+    encoder_last_hidden_state_vision: Optional[torch.FloatTensor] = None
+    encoder_last_hidden_state_text: Optional[torch.FloatTensor] = None
+    encoder_vision_hidden_states: Optional[tuple[torch.FloatTensor]] = None
+    encoder_text_hidden_states: Optional[tuple[torch.FloatTensor]] = None
+    encoder_attentions: Optional[tuple[tuple[torch.FloatTensor]]] = None
+    enc_outputs_class: Optional[torch.FloatTensor] = None
+    enc_outputs_coord_logits: Optional[torch.FloatTensor] = None
+    encoder_logits: Optional[torch.FloatTensor] = None
+    encoder_pred_boxes: Optional[torch.FloatTensor] = None
+    input_ids: Optional[torch.LongTensor] = None
 
 
 def build_label_maps(logits: torch.FloatTensor, input_ids: torch.LongTensor) -> tuple[torch.FloatTensor]:
@@ -2435,15 +2423,14 @@ class MMGroundingDinoForObjectDetection(MMGroundingDinoPreTrainedModel):
         self,
         pixel_values: torch.FloatTensor,
         input_ids: torch.LongTensor,
-        token_type_ids: torch.LongTensor | None = None,
-        attention_mask: torch.LongTensor | None = None,
-        pixel_mask: torch.BoolTensor | None = None,
-        encoder_outputs: MMGroundingDinoEncoderOutput | tuple | None = None,
-        output_attentions: bool | None = None,
-        output_hidden_states: bool | None = None,
-        return_dict: bool | None = None,
-        labels: list[dict[str, torch.LongTensor | torch.FloatTensor]] | None = None,
-        **kwargs,
+        token_type_ids: Optional[torch.LongTensor] = None,
+        attention_mask: Optional[torch.LongTensor] = None,
+        pixel_mask: Optional[torch.BoolTensor] = None,
+        encoder_outputs: Optional[Union[MMGroundingDinoEncoderOutput, tuple]] = None,
+        output_attentions: Optional[bool] = None,
+        output_hidden_states: Optional[bool] = None,
+        return_dict: Optional[bool] = None,
+        labels: Optional[list[dict[str, Union[torch.LongTensor, torch.FloatTensor]]]] = None,
     ):
         r"""
         input_ids (`torch.LongTensor` of shape `(batch_size, text_sequence_length)`):
@@ -2465,8 +2452,7 @@ class MMGroundingDinoForObjectDetection(MMGroundingDinoPreTrainedModel):
         Examples:
 
         ```python
-        >>> import httpx
-        >>> from io import BytesIO
+        >>> import requests
 
         >>> import torch
         >>> from PIL import Image
@@ -2478,9 +2464,8 @@ class MMGroundingDinoForObjectDetection(MMGroundingDinoPreTrainedModel):
         >>> processor = AutoProcessor.from_pretrained(model_id)
         >>> model = AutoModelForZeroShotObjectDetection.from_pretrained(model_id).to(device)
 
-        >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
-        >>> with httpx.stream("GET", url) as response:
-        ...     image = Image.open(BytesIO(response.read()))
+        >>> image_url = "http://images.cocodataset.org/val2017/000000039769.jpg"
+        >>> image = Image.open(requests.get(image_url, stream=True).raw)
         >>> # Check for cats and remote controls
         >>> text_labels = [["a cat", "a remote control"]]
 

transformers/models/mm_grounding_dino/modular_mm_grounding_dino.py

@@ -1,3 +1,4 @@
+# coding=utf-8
 # Copyright 2025 The HuggingFace Inc. team.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -50,7 +51,7 @@ class MMGroundingDinoConfig(PreTrainedConfig):
     documentation from [`PreTrainedConfig`] for more information.
 
     Args:
-        backbone_config (`Union[dict, "PreTrainedConfig"]`, *optional*, defaults to `SwinConfig()`):
+        backbone_config (`PreTrainedConfig` or `dict`, *optional*, defaults to `ResNetConfig()`):
             The configuration of the backbone model.
         backbone (`str`, *optional*):
             Name of backbone to use when `backbone_config` is `None`. If `use_pretrained_backbone` is `True`, this
@@ -140,8 +141,6 @@ class MMGroundingDinoConfig(PreTrainedConfig):
             The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
         layer_norm_eps (`float`, *optional*, defaults to 1e-05):
             The epsilon used by the layer normalization layers.
-        tie_word_embeddings (`bool`, *optional*, defaults to `True`):
-            Whether to tie weight embeddings
 
     Examples:
 
@@ -209,7 +208,6 @@ class MMGroundingDinoConfig(PreTrainedConfig):
         positional_embedding_temperature=20,
         init_std=0.02,
         layer_norm_eps=1e-5,
-        tie_word_embeddings=True,
         **kwargs,
     ):
         if backbone_config is None and backbone is None:
@@ -293,9 +291,9 @@ class MMGroundingDinoConfig(PreTrainedConfig):
         self.positional_embedding_temperature = positional_embedding_temperature
         self.init_std = init_std
         self.layer_norm_eps = layer_norm_eps
-        self.tie_word_embeddings = tie_word_embeddings
 
         super().__init__(is_encoder_decoder=is_encoder_decoder, **kwargs)
+        self.tie_encoder_decoder = True
 
 
 class MMGroundingDinoContrastiveEmbedding(GroundingDinoContrastiveEmbedding):

transformers/models/mobilebert/configuration_mobilebert.py

@@ -1,3 +1,4 @@
+# coding=utf-8
 # Copyright 2020 The HuggingFace Team. All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -124,12 +125,9 @@ class MobileBertConfig(PreTrainedConfig):
         normalization_type="no_norm",
         classifier_activation=True,
         classifier_dropout=None,
-        tie_word_embeddings=True,
         **kwargs,
     ):
-        super().__init__(**kwargs)
-        self.pad_token_id = pad_token_id
-        self.tie_word_embeddings = tie_word_embeddings
+        super().__init__(pad_token_id=pad_token_id, **kwargs)
 
         self.vocab_size = vocab_size
         self.hidden_size = hidden_size