transformers 5.0.0rc0__py3-none-any.whl → 5.0.0rc2__py3-none-any.whl

transformers/models/pe_video/modular_pe_video.py

@@ -0,0 +1,219 @@
+# coding=utf-8
+# Copyright 2025 the HuggingFace Inc. team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from dataclasses import dataclass
+from typing import Any, Optional
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+from ...modeling_attn_mask_utils import _prepare_4d_attention_mask
+from ...modeling_outputs import BaseModelOutputWithPooling, MaskedLMOutput
+from ...utils import ModelOutput, auto_docstring, can_return_tuple
+from ...utils.generic import check_model_inputs
+from ..auto import AutoModel, AutoModelForImageClassification
+from ..pe_audio_video.modeling_pe_audio_video import (
+    PeAudioVideoContrastiveHead,
+    PeAudioVideoEncoder,
+    PeAudioVideoEncoderPatchEmbedder,
+    PeAudioVideoPreTrainedModel,
+)
+from .configuration_pe_video import PeVideoConfig, PeVideoEncoderConfig
+
+
+# TODO: not sure about the typing for text_model_output
+@dataclass
+# @auto_docstring
+class PeVideoOutput(ModelOutput):
+    loss: Optional[torch.FloatTensor] = None
+    logits_video_text: Optional[torch.FloatTensor] = None
+    text_video_embeds: Optional[torch.FloatTensor] = None
+    video_embeds: Optional[torch.FloatTensor] = None
+    text_outputs: BaseModelOutputWithPooling = None
+    video_outputs: BaseModelOutputWithPooling = None
+
+    def to_tuple(self) -> tuple[Any]:
+        return tuple(
+            self[k] if k not in ["text_outputs", "video_outputs"] else getattr(self, k).to_tuple() for k in self.keys()
+        )
+
+
+class PeVideoContrastiveHead(PeAudioVideoContrastiveHead): ...
+
+
+class PeVideoEncoderPatchEmbedder(PeAudioVideoEncoderPatchEmbedder): ...
+
+
+class PeVideoEncoderEmbedder(nn.Module):
+    def __init__(self, config: PeVideoEncoderConfig):
+        super().__init__()
+        self.vision_model = AutoModelForImageClassification.from_config(config.vision_config)
+        self.proj = nn.Linear(config.vision_config.num_labels, config.hidden_size, bias=False)
+        self.data_proj = nn.Linear(config.hidden_size, config.hidden_size)
+
+    def forward(
+        self,
+        pixel_values_videos: torch.Tensor,
+        padding_mask: Optional[torch.Tensor] = None,
+    ) -> tuple[torch.Tensor, Optional[torch.Tensor]]:
+        input_shape = pixel_values_videos.shape
+
+        pixel_values_videos = pixel_values_videos.view(-1, *input_shape[2:])
+        vision_encoder_outputs = self.vision_model(pixel_values_videos)
+
+        logits = vision_encoder_outputs.logits.view(*input_shape[:2], -1)
+        logits = F.normalize(logits, dim=-1)
+
+        vision_features = self.proj(logits)
+        inputs_embeds = self.data_proj(vision_features)
+
+        return inputs_embeds, padding_mask
+
+
+class PeVideoPreTrainedModel(PeAudioVideoPreTrainedModel):
+    base_model_prefix = "video_model"
+    main_input_name = "pixel_values_videos"
+
+
+@auto_docstring(
+    custom_intro="""
+    The PeVideo Encoder model.
+    """
+)
+class PeVideoEncoder(PeAudioVideoEncoder):
+    base_model_prefix = "video_model.video_encoder"
+    main_input_name = "pixel_values_videos"
+
+    def __init__(self, config: PeVideoEncoderConfig):
+        super().__init__(config)
+        self.embedder = PeVideoEncoderEmbedder(config)
+
+    @can_return_tuple
+    @check_model_inputs
+    def forward(
+        self,
+        pixel_values_videos: torch.Tensor,
+        padding_mask_videos: Optional[torch.Tensor] = None,
+        **kwargs,
+    ) -> BaseModelOutputWithPooling:
+        inputs_embeds, padding_mask = self.embedder(pixel_values_videos, padding_mask=padding_mask_videos)
+        inputs_embeds, attention_mask = self.patch_embedder(inputs_embeds, padding_mask=padding_mask)
+
+        if attention_mask is not None:
+            attention_mask = _prepare_4d_attention_mask(attention_mask, inputs_embeds.dtype)
+
+        position_ids = torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device).unsqueeze(0)
+        position_embeddings = self.rotary_emb(inputs_embeds, position_ids)
+
+        hidden_states = inputs_embeds
+        for encoder_layer in self.layers[: self.config.num_hidden_layers]:
+            hidden_states = encoder_layer(
+                hidden_states,
+                attention_mask=attention_mask,
+                position_embeddings=position_embeddings,
+                **kwargs,
+            )
+
+        hidden_states = self.norm(hidden_states)
+        hidden_states = self.output(hidden_states)
+
+        return BaseModelOutputWithPooling(
+            last_hidden_state=hidden_states[:, 1:],
+            pooler_output=hidden_states[:, 0],
+        )
+
+
+class PeVideoModel(PeVideoPreTrainedModel):
+    main_input_name = "input_ids"
+
+    def __init__(self, config: PeVideoConfig):
+        super().__init__(config)
+        self.text_model = AutoModel.from_config(config.text_config)
+        self.video_encoder = PeVideoEncoder(config.video_config)
+
+        self.text_video_head = PeVideoContrastiveHead(config.text_config.hidden_size, config.text_config.hidden_size)
+        self.video_head = PeVideoContrastiveHead(config.video_config.hidden_size, config.text_config.hidden_size)
+
+        self.text_video_logit_scale = nn.Parameter(torch.zeros(1))
+        self.text_video_logit_bias = nn.Parameter(torch.zeros(1))
+
+        self.post_init()
+
+    def get_text_features(self, input_ids, attention_mask=None):
+        # TODO: should it be named feature or embeds
+        text_outputs: MaskedLMOutput = self.text_model(
+            input_ids=input_ids,
+            attention_mask=attention_mask,
+            return_dict=True,
+        )
+
+        text_features = text_outputs.last_hidden_state
+        text_features = self.text_video_head(text_features)
+        return text_features
+
+    def get_video_features(self, pixel_values_videos, padding_mask_videos=None):
+        # TODO: should it be named feature or embeds
+        video_outputs: BaseModelOutputWithPooling = self.video_encoder(
+            pixel_values_videos=pixel_values_videos,
+            padding_mask_videos=padding_mask_videos,
+            return_dict=True,
+        )
+        video_features = self.video_head(video_outputs.pooler_output)
+        return video_features
+
+    @can_return_tuple
+    def forward(
+        self,
+        input_ids: torch.Tensor,
+        pixel_values_videos: torch.Tensor,
+        attention_mask: Optional[torch.Tensor] = None,
+        padding_mask_videos: Optional[torch.Tensor] = None,
+        return_loss: Optional[bool] = None,
+        **kwargs,
+    ) -> PeVideoOutput:
+        video_outputs: BaseModelOutputWithPooling = self.video_encoder(
+            pixel_values_videos=pixel_values_videos, padding_mask_videos=padding_mask_videos, **kwargs
+        )
+        kwargs["output_hidden_states"] = True
+        text_outputs: MaskedLMOutput = self.text_model(input_ids=input_ids, attention_mask=attention_mask, **kwargs)
+
+        video_embeds = video_outputs.pooler_output
+        video_embeds = self.video_head(video_embeds)
+
+        text_video_embeds = text_outputs.hidden_states[-1][:, 0]
+        text_video_embeds = self.text_video_head(text_video_embeds)
+
+        logits_video_text = video_embeds @ text_video_embeds.T
+        logits_video_text = logits_video_text * self.text_video_logit_scale + self.text_video_logit_bias
+
+        loss = None
+        if return_loss:
+            labels = torch.eye(logits_video_text.shape[0], device=logits_video_text.device)
+            loss = -F.logsigmoid(labels * logits_video_text).sum() / logits_video_text.shape[0]
+
+        return PeVideoOutput(
+            logits_video_text=logits_video_text,
+            text_video_embeds=text_video_embeds,
+            video_embeds=video_embeds,
+            text_outputs=text_outputs,
+            video_outputs=video_outputs,
+            loss=loss,
+        )
+
+
+__all__ = [
+    "PeVideoEncoder",
+    "PeVideoModel",
+]

transformers/models/pe_video/processing_pe_video.py

@@ -0,0 +1,10 @@
+from ...processing_utils import ProcessorMixin
+
+
+class PeVideoProcessor(ProcessorMixin):
+    attributes = ["video_processor", "tokenizer"]
+    video_processor_class = "PeVideoVideoProcessor"
+    tokenizer_class = "AutoTokenizer"
+
+
+__all__ = ["PeVideoProcessor"]

transformers/models/pe_video/video_processing_pe_video.py

@@ -0,0 +1,66 @@
+# coding=utf-8
+# Copyright 2025 the HuggingFace Inc. team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from typing import Optional, Union
+
+import torch
+
+from ...image_processing_utils import BatchFeature
+from ...image_utils import PILImageResampling
+from ...processing_utils import Unpack, VideosKwargs
+from ...video_processing_utils import BaseVideoProcessor, VideoMetadata
+from ...video_utils import VideoInput
+
+
+class PeVideoVideoProcessor(BaseVideoProcessor):
+    resample = PILImageResampling.BILINEAR
+
+    def sample_frames(
+        self,
+        metadata: VideoMetadata,
+        num_frames: Optional[int] = None,
+        fps: Optional[Union[int, float]] = None,
+        **kwargs,
+    ):
+        if num_frames:
+            total_frames = metadata.total_num_frames
+            num_frames = num_frames if num_frames is not None else self.num_frames
+            assert num_frames is not None, "`num_frames` must be specified if `fixed_len_video == True`"
+            frame_idxs = [int(i * (total_frames - 1) / (num_frames - 1)) for i in range(num_frames)]
+            return torch.tensor(frame_idxs)
+        else:
+            return super().sample_frames(metadata, num_frames, fps, **kwargs)
+
+    def _preprocess(
+        self,
+        videos: VideoInput,
+        **kwargs: Unpack[VideosKwargs],
+    ) -> BatchFeature:
+        # Always set `return_tensors` to `None` since it won't pad variable length videos
+        # We'll handle this after we call the parent' method
+        return_tensors = kwargs.pop("return_tensors", None)
+        result = super()._preprocess(videos, **kwargs)
+        pixels = result.pixel_values_videos
+        data = {"pixel_values_videos": pixels}
+        if return_tensors:
+            lengths = torch.tensor([video.size(0) for video in pixels])
+            pixels = torch.nn.utils.rnn.pad_sequence(pixels, batch_first=True, padding_value=0.0)
+            data["pixel_values_videos"] = pixels
+            if lengths.unique().size(0) > 1:
+                mask = torch.arange(lengths.max())[None] < lengths[:, None]
+                data["padding_mask_videos"] = mask
+        return BatchFeature(data=data, tensor_type=return_tensors)
+
+
+__all__ = ["PeVideoVideoProcessor"]

transformers/models/pegasus/configuration_pegasus.py

@@ -143,6 +143,7 @@ class PegasusConfig(PreTrainedConfig):
         self.use_cache = use_cache
         self.num_hidden_layers = encoder_layers
         self.scale_embedding = scale_embedding  # scale factor will be sqrt(d_model) if True
+
         super().__init__(
             pad_token_id=pad_token_id,
             eos_token_id=eos_token_id,

transformers/models/pegasus/modeling_pegasus.py

@@ -443,6 +443,8 @@ class PegasusPreTrainedModel(PreTrainedModel):
         super()._init_weights(module)
         if isinstance(module, PegasusSinusoidalPositionalEmbedding):
             init.copy_(module.weight, module.create_weight())
+        elif isinstance(module, PegasusForConditionalGeneration):
+            init.zeros_(module.final_logits_bias)
 
 
 class PegasusEncoder(PegasusPreTrainedModel):
@@ -518,6 +520,7 @@ class PegasusEncoder(PegasusPreTrainedModel):
         output_attentions=None,
         output_hidden_states=None,
         return_dict=None,
+        **kwargs,
     ):
         r"""
         Args:
@@ -695,6 +698,7 @@ class PegasusDecoder(PegasusPreTrainedModel):
         output_hidden_states=None,
         return_dict=None,
         cache_position=None,
+        **kwargs,
     ):
         r"""
         Args:
@@ -946,6 +950,7 @@ class PegasusModel(PegasusPreTrainedModel):
         output_hidden_states: Optional[bool] = None,
         return_dict: Optional[bool] = None,
         cache_position: Optional[torch.Tensor] = None,
+        **kwargs,
     ) -> Union[tuple, Seq2SeqModelOutput]:
         r"""
         decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
@@ -1111,6 +1116,7 @@ class PegasusForConditionalGeneration(PegasusPreTrainedModel, GenerationMixin):
         output_hidden_states: Optional[bool] = None,
         return_dict: Optional[bool] = None,
         cache_position: Optional[torch.Tensor] = None,
+        **kwargs,
     ) -> Union[tuple, Seq2SeqLMOutput]:
         r"""
         decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
@@ -1216,6 +1222,7 @@ class PegasusDecoderWrapper(PegasusPreTrainedModel):
     def __init__(self, config):
         super().__init__(config)
         self.decoder = PegasusDecoder(config)
+        self.post_init()
 
     def forward(self, *args, **kwargs):
         return self.decoder(*args, **kwargs)
@@ -1283,6 +1290,7 @@ class PegasusForCausalLM(PegasusPreTrainedModel, GenerationMixin):
         return_dict: Optional[bool] = None,
         cache_position: Optional[torch.LongTensor] = None,
         logits_to_keep: Union[int, torch.Tensor] = 0,
+        **kwargs,
     ) -> Union[tuple, CausalLMOutputWithCrossAttentions]:
         r"""
         labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):

transformers/models/pegasus/tokenization_pegasus.py

@@ -14,6 +14,8 @@
 # limitations under the License.
 """Tokenization class for model PEGASUS."""
 
+from typing import Optional, Union
+
 from tokenizers import Regex, Tokenizer, decoders, normalizers, pre_tokenizers, processors
 from tokenizers.models import Unigram
 
@@ -70,15 +72,17 @@ class PegasusTokenizer(TokenizersBackend):
             that uses the tokens 2 - 104 only for pretraining
         offset (`int`, *optional*, defaults to 103):
             Offset for additional special tokens.
-        vocab (`dict`, *optional*):
-            Custom vocabulary dictionary. If not provided, a blank vocabulary is initialized.
+        vocab (`str` or `list[tuple[str, float]]`, *optional*):
+            Custom vocabulary with `(token, score)` tuples. If not provided, a blank vocabulary is initialized.
     """
 
     vocab_files_names = VOCAB_FILES_NAMES
     model_input_names = ["input_ids", "attention_mask"]
+    model = Unigram
 
     def __init__(
         self,
+        vocab: Optional[Union[str, list[tuple[str, float]]]] = None,
         pad_token="<pad>",
         eos_token="</s>",
         unk_token="<unk>",
@@ -86,60 +90,27 @@ class PegasusTokenizer(TokenizersBackend):
         mask_token_sent="<mask_1>",
         additional_special_tokens=None,
         offset=103,
-        vocab=None,
-        vocab_file=None,
         **kwargs,
     ):
         self.offset = offset
-        self.vocab_file = vocab_file
 
         if additional_special_tokens is None:
             additional_special_tokens = [mask_token_sent] if mask_token_sent is not None else []
             additional_special_tokens += [f"<unk_{i}>" for i in range(2, self.offset)]
 
-        if vocab is not None:
-            # For Pegasus, insert special tokens at the beginning
-            special_tokens_set = {pad_token, eos_token, mask_token_sent, mask_token, unk_token}
-            special_tokens_set.update(additional_special_tokens)
-
-            # Build special tokens in correct order
-            _vocab_list = [
-                (str(pad_token), 0.0),
-                (str(eos_token), 0.0),
-            ]
-            if mask_token_sent:
-                _vocab_list.append((str(mask_token_sent), 0.0))
-            for token in additional_special_tokens:
-                if token not in [pad_token, eos_token, mask_token_sent]:
-                    _vocab_list.append((str(token), 0.0))
-            if mask_token not in [t for t, _ in _vocab_list]:
-                _vocab_list.append((str(mask_token), 0.0))
-            _vocab_list.append((str(unk_token), 0.0))
-
-            # Filter out special tokens from main vocab and combine
-            filtered_vocab = [(t, s) for t, s in vocab if t not in special_tokens_set]
-            _vocab_list = _vocab_list + filtered_vocab
-        else:
-            _vocab_list = [(str(unk_token), 0.0)]
-
-        self._vocab = {token: idx for idx, (token, _) in enumerate(_vocab_list)}
-
-        self._tokenizer = Tokenizer(Unigram(vocab=_vocab_list, unk_id=self._vocab.get(str(unk_token), 0)))
+        if vocab is None:
+            vocab = [(str(unk_token), 0.0), (str(pad_token), 0.0), (str(eos_token), 0.0), (str(mask_token), 0.0)]
 
+        self._vocab = vocab
+        self._tokenizer = Tokenizer(Unigram(vocab=vocab, unk_id=self._vocab.index((str(unk_token), 0.0), 1)))
         self._tokenizer.normalizer = normalizers.Sequence(
             [normalizers.Replace(Regex(r"\n"), " "), normalizers.Replace(Regex(r" {2,}"), " ")]
         )
 
-        self._tokenizer.post_processor = processors.TemplateProcessing(
-            single=f"$A {eos_token}",
-            pair=f"$A $B {eos_token}",
-            special_tokens=[(str(eos_token), self._vocab.get(str(eos_token), 1))],
-        )
-
-        tokenizer_object = self._tokenizer
+        self._tokenizer.pre_tokenizer = pre_tokenizers.Metaspace(replacement="▁", prepend_scheme="always", split=True)
+        self._tokenizer.decoder = decoders.Metaspace(replacement="▁", prepend_scheme="always", split=True)
 
         super().__init__(
-            tokenizer_object=tokenizer_object,
             pad_token=pad_token,
             eos_token=eos_token,
             unk_token=unk_token,
@@ -149,9 +120,11 @@ class PegasusTokenizer(TokenizersBackend):
             additional_special_tokens=additional_special_tokens,
             **kwargs,
         )
-
-        self._tokenizer.pre_tokenizer = pre_tokenizers.Metaspace(replacement="▁", prepend_scheme="always", split=True)
-        self._tokenizer.decoder = decoders.Metaspace(replacement="▁", prepend_scheme="always", split=True)
+        self._tokenizer.post_processor = processors.TemplateProcessing(
+            single=f"$A {eos_token}",
+            pair=f"$A $B {eos_token}",
+            special_tokens=[(str(eos_token), self.convert_tokens_to_ids(str(eos_token)))],
+        )
 
 
 __all__ = ["PegasusTokenizer"]

transformers/models/pegasus_x/modeling_pegasus_x.py

@@ -821,6 +821,7 @@ class PegasusXEncoder(PegasusXPreTrainedModel):
         output_attentions=None,
         output_hidden_states=None,
         return_dict=None,
+        **kwargs,
     ):
         r"""
         Args:
@@ -989,6 +990,7 @@ class PegasusXDecoder(PegasusXPreTrainedModel):
         output_hidden_states=None,
         return_dict=None,
         cache_position=None,
+        **kwargs,
     ):
         r"""
         Args:
@@ -1241,6 +1243,7 @@ class PegasusXModel(PegasusXPreTrainedModel):
         output_hidden_states: Optional[bool] = None,
         return_dict: Optional[bool] = None,
         cache_position: Optional[torch.Tensor] = None,
+        **kwargs,
     ) -> Union[tuple, Seq2SeqModelOutput]:
         r"""
         decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
@@ -1388,6 +1391,7 @@ class PegasusXForConditionalGeneration(PegasusXPreTrainedModel, GenerationMixin)
         output_hidden_states: Optional[bool] = None,
         return_dict: Optional[bool] = None,
         cache_position: Optional[torch.Tensor] = None,
+        **kwargs,
     ) -> Union[tuple, Seq2SeqLMOutput]:
         r"""
         decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
@@ -1472,6 +1476,7 @@ class PegasusXDecoderWrapper(PegasusXPreTrainedModel):
     def __init__(self, config):
         super().__init__(config)
         self.decoder = PegasusXDecoder(config)
+        self.post_init()
 
     def forward(self, *args, **kwargs):
         return self.decoder(*args, **kwargs)

transformers/models/perceiver/image_processing_perceiver_fast.py

@@ -113,7 +113,6 @@ class PerceiverImageProcessorFast(BaseImageProcessorFast):
             processed_images_grouped[shape] = stacked_images
 
         processed_images = reorder_images(processed_images_grouped, grouped_images_index)
-        processed_images = torch.stack(processed_images, dim=0) if return_tensors else processed_images
 
         return BatchFeature(data={"pixel_values": processed_images}, tensor_type=return_tensors)
 

transformers/models/perceiver/modeling_perceiver.py

@@ -551,9 +551,13 @@ class PerceiverPreTrainedModel(PreTrainedModel):
             # Here we need the check explicitly, as we slice the weight in the `zeros_` call, so it looses the flag
             if module.padding_idx is not None and not getattr(module.weight, "_is_hf_initialized", False):
                 init.zeros_(module.weight[module.padding_idx])
-        elif isinstance(module, nn.LayerNorm):
+        elif isinstance(module, (nn.LayerNorm, nn.BatchNorm2d)):
             init.zeros_(module.bias)
             init.ones_(module.weight)
+            if getattr(module, "running_mean", None) is not None:
+                init.zeros_(module.running_mean)
+                init.ones_(module.running_var)
+                init.zeros_(module.num_batches_tracked)
 
 
 @auto_docstring(
@@ -615,6 +619,7 @@ class PerceiverModel(PerceiverPreTrainedModel):
         output_hidden_states: Optional[bool] = None,
         interpolate_pos_encoding: bool = False,
         return_dict: Optional[bool] = None,
+        **kwargs,
     ) -> Union[tuple, PerceiverModelOutput]:
         r"""
         inputs (`torch.FloatTensor`):
@@ -850,6 +855,7 @@ class PerceiverForMaskedLM(PerceiverPreTrainedModel):
         labels: Optional[torch.Tensor] = None,
         return_dict: Optional[bool] = None,
         input_ids: Optional[torch.Tensor] = None,
+        **kwargs,
     ) -> Union[tuple, PerceiverMaskedLMOutput]:
         r"""
         inputs (`torch.FloatTensor`):
@@ -975,6 +981,7 @@ class PerceiverForSequenceClassification(PerceiverPreTrainedModel):
         labels: Optional[torch.Tensor] = None,
         return_dict: Optional[bool] = None,
         input_ids: Optional[torch.Tensor] = None,
+        **kwargs,
     ) -> Union[tuple, PerceiverClassifierOutput]:
         r"""
         inputs (`torch.FloatTensor`):
@@ -1107,6 +1114,7 @@ class PerceiverForImageClassificationLearned(PerceiverPreTrainedModel):
         interpolate_pos_encoding: bool = False,
         return_dict: Optional[bool] = None,
         pixel_values: Optional[torch.Tensor] = None,
+        **kwargs,
     ) -> Union[tuple, PerceiverClassifierOutput]:
         r"""
         inputs (`torch.FloatTensor`):
@@ -1229,6 +1237,7 @@ class PerceiverForImageClassificationFourier(PerceiverPreTrainedModel):
         labels: Optional[torch.Tensor] = None,
         return_dict: Optional[bool] = None,
         pixel_values: Optional[torch.Tensor] = None,
+        **kwargs,
     ) -> Union[tuple, PerceiverClassifierOutput]:
         r"""
         inputs (`torch.FloatTensor`):
@@ -1350,6 +1359,7 @@ class PerceiverForImageClassificationConvProcessing(PerceiverPreTrainedModel):
         labels: Optional[torch.Tensor] = None,
         return_dict: Optional[bool] = None,
         pixel_values: Optional[torch.Tensor] = None,
+        **kwargs,
     ) -> Union[tuple, PerceiverClassifierOutput]:
         r"""
         inputs (`torch.FloatTensor`):
@@ -1487,6 +1497,7 @@ class PerceiverForOpticalFlow(PerceiverPreTrainedModel):
         output_hidden_states: Optional[bool] = None,
         labels: Optional[torch.Tensor] = None,
         return_dict: Optional[bool] = None,
+        **kwargs,
     ) -> Union[tuple, PerceiverClassifierOutput]:
         r"""
         inputs (`torch.FloatTensor`):
@@ -1695,6 +1706,7 @@ class PerceiverForMultimodalAutoencoding(PerceiverPreTrainedModel):
         output_hidden_states: Optional[bool] = None,
         labels: Optional[torch.Tensor] = None,
         return_dict: Optional[bool] = None,
+        **kwargs,
     ) -> Union[tuple, PerceiverClassifierOutput]:
         r"""
         inputs (`torch.FloatTensor`):

transformers/models/perception_lm/image_processing_perception_lm_fast.py

@@ -307,7 +307,6 @@ class PerceptionLMImageProcessorFast(BaseImageProcessorFast):
             processed_images_grouped[shape] = stacked_images
         processed_images = reorder_images(processed_images_grouped, grouped_images_index)
         processed_images = [p[None] if p.ndim == 3 else p for p in processed_images]  # add tiles dimension if needed
-        processed_images = torch.stack(processed_images, dim=0) if return_tensors else processed_images
         return BatchFeature(data={"pixel_values": processed_images}, tensor_type=return_tensors)
 
 

transformers/models/perception_lm/modeling_perception_lm.py

@@ -451,6 +451,7 @@ class PerceptionLMForConditionalGeneration(PerceptionLMPreTrainedModel, Generati
         attention_mask=None,
         cache_position=None,
         logits_to_keep=None,
+        is_first_iteration=False,
         **kwargs,
     ):
         # Overwritten -- in specific circumstances we don't want to forward image inputs to the model
@@ -462,12 +463,15 @@ class PerceptionLMForConditionalGeneration(PerceptionLMPreTrainedModel, Generati
             attention_mask=attention_mask,
             cache_position=cache_position,
             logits_to_keep=logits_to_keep,
+            is_first_iteration=is_first_iteration,
             **kwargs,
         )
 
-        if cache_position[0] == 0:
-            # If we're in cached decoding stage, pixel values should be None because input ids do not contain special image token anymore
-            # Otherwise we need pixel values to be passed to model
+        if is_first_iteration or not kwargs.get("use_cache", True):
+            # Pixel values are used only in the first iteration if available
+            # In subsquent iterations, they are already merged with text and cached
+            # NOTE: first iteration doesn't have to be prefill, it can be the first
+            # iteration with a question and cached system prompt (continue generate from cache)
             model_inputs["pixel_values"] = pixel_values
             model_inputs["pixel_values_videos"] = pixel_values_videos
         return model_inputs

transformers/models/perception_lm/modular_perception_lm.py

@@ -293,6 +293,7 @@ class PerceptionLMForConditionalGeneration(LlavaForConditionalGeneration):
         attention_mask=None,
         cache_position=None,
         logits_to_keep=None,
+        is_first_iteration=False,
         **kwargs,
     ):
         # Overwritten -- in specific circumstances we don't want to forward image inputs to the model
@@ -304,12 +305,15 @@ class PerceptionLMForConditionalGeneration(LlavaForConditionalGeneration):
             attention_mask=attention_mask,
             cache_position=cache_position,
             logits_to_keep=logits_to_keep,
+            is_first_iteration=is_first_iteration,
             **kwargs,
         )
 
-        if cache_position[0] == 0:
-            # If we're in cached decoding stage, pixel values should be None because input ids do not contain special image token anymore
-            # Otherwise we need pixel values to be passed to model
+        if is_first_iteration or not kwargs.get("use_cache", True):
+            # Pixel values are used only in the first iteration if available
+            # In subsquent iterations, they are already merged with text and cached
+            # NOTE: first iteration doesn't have to be prefill, it can be the first
+            # iteration with a question and cached system prompt (continue generate from cache)
             model_inputs["pixel_values"] = pixel_values
             model_inputs["pixel_values_videos"] = pixel_values_videos
         return model_inputs