transformers 5.0.0__py3-none-any.whl → 5.0.0rc0__py3-none-any.whl

transformers/models/pegasus/tokenization_pegasus.py

@@ -1,3 +1,4 @@
+# coding=utf-8
 # Copyright 2020 Google and The HuggingFace Inc. team.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -69,17 +70,15 @@ class PegasusTokenizer(TokenizersBackend):
             that uses the tokens 2 - 104 only for pretraining
         offset (`int`, *optional*, defaults to 103):
             Offset for additional special tokens.
-        vocab (`str` or `list[tuple[str, float]]`, *optional*):
-            Custom vocabulary with `(token, score)` tuples. If not provided, a blank vocabulary is initialized.
+        vocab (`dict`, *optional*):
+            Custom vocabulary dictionary. If not provided, a blank vocabulary is initialized.
     """
 
     vocab_files_names = VOCAB_FILES_NAMES
     model_input_names = ["input_ids", "attention_mask"]
-    model = Unigram
 
     def __init__(
         self,
-        vocab: str | list[tuple[str, float]] | None = None,
         pad_token="<pad>",
         eos_token="</s>",
         unk_token="<unk>",
@@ -87,27 +86,60 @@ class PegasusTokenizer(TokenizersBackend):
         mask_token_sent="<mask_1>",
         additional_special_tokens=None,
         offset=103,
+        vocab=None,
+        vocab_file=None,
         **kwargs,
     ):
         self.offset = offset
+        self.vocab_file = vocab_file
 
         if additional_special_tokens is None:
             additional_special_tokens = [mask_token_sent] if mask_token_sent is not None else []
             additional_special_tokens += [f"<unk_{i}>" for i in range(2, self.offset)]
 
-        if vocab is None:
-            vocab = [(str(unk_token), 0.0), (str(pad_token), 0.0), (str(eos_token), 0.0), (str(mask_token), 0.0)]
+        if vocab is not None:
+            # For Pegasus, insert special tokens at the beginning
+            special_tokens_set = {pad_token, eos_token, mask_token_sent, mask_token, unk_token}
+            special_tokens_set.update(additional_special_tokens)
+
+            # Build special tokens in correct order
+            _vocab_list = [
+                (str(pad_token), 0.0),
+                (str(eos_token), 0.0),
+            ]
+            if mask_token_sent:
+                _vocab_list.append((str(mask_token_sent), 0.0))
+            for token in additional_special_tokens:
+                if token not in [pad_token, eos_token, mask_token_sent]:
+                    _vocab_list.append((str(token), 0.0))
+            if mask_token not in [t for t, _ in _vocab_list]:
+                _vocab_list.append((str(mask_token), 0.0))
+            _vocab_list.append((str(unk_token), 0.0))
+
+            # Filter out special tokens from main vocab and combine
+            filtered_vocab = [(t, s) for t, s in vocab if t not in special_tokens_set]
+            _vocab_list = _vocab_list + filtered_vocab
+        else:
+            _vocab_list = [(str(unk_token), 0.0)]
+
+        self._vocab = {token: idx for idx, (token, _) in enumerate(_vocab_list)}
+
+        self._tokenizer = Tokenizer(Unigram(vocab=_vocab_list, unk_id=self._vocab.get(str(unk_token), 0)))
 
-        self._vocab = vocab
-        self._tokenizer = Tokenizer(Unigram(vocab=vocab, unk_id=self._vocab.index((str(unk_token), 0.0), 1)))
         self._tokenizer.normalizer = normalizers.Sequence(
             [normalizers.Replace(Regex(r"\n"), " "), normalizers.Replace(Regex(r" {2,}"), " ")]
         )
 
-        self._tokenizer.pre_tokenizer = pre_tokenizers.Metaspace(replacement="▁", prepend_scheme="always", split=True)
-        self._tokenizer.decoder = decoders.Metaspace(replacement="▁", prepend_scheme="always", split=True)
+        self._tokenizer.post_processor = processors.TemplateProcessing(
+            single=f"$A {eos_token}",
+            pair=f"$A $B {eos_token}",
+            special_tokens=[(str(eos_token), self._vocab.get(str(eos_token), 1))],
+        )
+
+        tokenizer_object = self._tokenizer
 
         super().__init__(
+            tokenizer_object=tokenizer_object,
             pad_token=pad_token,
             eos_token=eos_token,
             unk_token=unk_token,
@@ -117,11 +149,9 @@ class PegasusTokenizer(TokenizersBackend):
             additional_special_tokens=additional_special_tokens,
             **kwargs,
         )
-        self._tokenizer.post_processor = processors.TemplateProcessing(
-            single=f"$A {eos_token}",
-            pair=f"$A $B {eos_token}",
-            special_tokens=[(str(eos_token), self.convert_tokens_to_ids(str(eos_token)))],
-        )
+
+        self._tokenizer.pre_tokenizer = pre_tokenizers.Metaspace(replacement="▁", prepend_scheme="always", split=True)
+        self._tokenizer.decoder = decoders.Metaspace(replacement="▁", prepend_scheme="always", split=True)
 
 
 __all__ = ["PegasusTokenizer"]

transformers/models/pegasus_x/configuration_pegasus_x.py

@@ -1,3 +1,4 @@
+# coding=utf-8
 # Copyright 2022, Google and The HuggingFace Inc. team. All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -129,7 +130,6 @@ class PegasusXConfig(PreTrainedConfig):
         num_global_tokens=32,
         block_size=512,
         stagger_local_blocks=True,
-        tie_word_embeddings=True,
         **kwargs,
     ):
         self.vocab_size = vocab_size
@@ -155,13 +155,12 @@ class PegasusXConfig(PreTrainedConfig):
         self.num_global_tokens = num_global_tokens
         self.block_size = block_size
         self.stagger_local_blocks = stagger_local_blocks
-        self.pad_token_id = pad_token_id
-        self.eos_token_id = eos_token_id
-        self.decoder_start_token_id = decoder_start_token_id
-        self.tie_word_embeddings = tie_word_embeddings
 
         super().__init__(
+            pad_token_id=pad_token_id,
+            eos_token_id=eos_token_id,
             is_encoder_decoder=is_encoder_decoder,
+            decoder_start_token_id=decoder_start_token_id,
             forced_eos_token_id=forced_eos_token_id,
             **kwargs,
         )

transformers/models/pegasus_x/modeling_pegasus_x.py

@@ -1,3 +1,4 @@
+# coding=utf-8
 # Copyright 2022, Google and The HuggingFace Inc. team. All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -16,6 +17,7 @@
 import math
 from collections.abc import Callable
 from dataclasses import dataclass
+from typing import Optional, Union
 
 import numpy as np
 import torch
@@ -81,7 +83,7 @@ class PegasusXScaledWordEmbedding(nn.Embedding):
     This module overrides nn.Embeddings' forward by multiplying with embeddings scale.
     """
 
-    def __init__(self, num_embeddings: int, embedding_dim: int, padding_idx: int, embed_scale: float | None = 1.0):
+    def __init__(self, num_embeddings: int, embedding_dim: int, padding_idx: int, embed_scale: Optional[float] = 1.0):
         super().__init__(num_embeddings, embedding_dim, padding_idx)
         self.embed_scale = embed_scale
 
@@ -99,7 +101,7 @@ class PegasusXSinusoidalPositionalEmbedding(nn.Module):
 
     @torch.no_grad()
     def forward(
-        self, input_embeds: torch.Tensor, past_key_values_length: int = 0, position_ids: torch.Tensor | None = None
+        self, input_embeds: torch.Tensor, past_key_values_length: int = 0, position_ids: Optional[torch.Tensor] = None
     ) -> torch.Tensor:
         """`input_ids_shape` is expected to be [bsz x seqlen]."""
         batch_size, seq_len = input_embeds.shape[:2]
@@ -125,8 +127,8 @@ def eager_attention_forward(
     query: torch.Tensor,
     key: torch.Tensor,
     value: torch.Tensor,
-    attention_mask: torch.Tensor | None,
-    scaling: float | None = None,
+    attention_mask: Optional[torch.Tensor],
+    scaling: Optional[float] = None,
     dropout: float = 0.0,
     **kwargs: Unpack[TransformersKwargs],
 ):
@@ -161,8 +163,8 @@ class PegasusXAttention(nn.Module):
         is_decoder: bool = False,
         bias: bool = True,
         is_causal: bool = False,
-        config: PegasusXConfig | None = None,
-        layer_idx: int | None = None,
+        config: Optional[PegasusXConfig] = None,
+        layer_idx: Optional[int] = None,
     ):
         super().__init__()
         self.embed_dim = embed_dim
@@ -195,15 +197,15 @@ class PegasusXAttention(nn.Module):
     def forward(
         self,
         hidden_states: torch.Tensor,
-        key_value_states: torch.Tensor | None = None,
-        past_key_values: Cache | None = None,
-        attention_mask: torch.Tensor | None = None,
+        key_value_states: Optional[torch.Tensor] = None,
+        past_key_values: Optional[Cache] = None,
+        attention_mask: Optional[torch.Tensor] = None,
         output_attentions: bool = False,
-        cache_position: torch.Tensor | None = None,
+        cache_position: Optional[torch.Tensor] = None,
         # TODO: we need a refactor so that the different attention modules can get their specific kwargs
         # ATM, we have mixed things encoder, decoder, and encoder-decoder attn
         **kwargs: Unpack[FlashAttentionKwargs],
-    ) -> tuple[torch.Tensor, torch.Tensor | None, tuple[torch.Tensor] | None]:
+    ) -> tuple[torch.Tensor, Optional[torch.Tensor], Optional[tuple[torch.Tensor]]]:
         """Input shape: Batch x Time x Channel"""
 
         # if key_value_states are provided this layer is used as a cross-attention layer
@@ -313,9 +315,9 @@ class PegasusXGlobalLocalAttention(nn.Module):
         self,
         token_hidden_states: torch.Tensor,
         global_hidden_states: torch.Tensor,
-        attention_mask: torch.Tensor | None = None,
+        attention_mask: Optional[torch.Tensor] = None,
         output_attentions: bool = False,
-    ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor | None]:
+    ) -> tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]:
         """Input shape: Batch x Time x Channel"""
         dim = DimensionInfo(
             batch_size=token_hidden_states.shape[0],
@@ -625,7 +627,7 @@ class PegasusXEncoderLayer(GradientCheckpointingLayer):
 
 
 class PegasusXDecoderLayer(GradientCheckpointingLayer):
-    def __init__(self, config: PegasusXConfig, layer_idx: int | None = None):
+    def __init__(self, config: PegasusXConfig, layer_idx: Optional[int] = None):
         super().__init__()
         self.embed_dim = config.d_model
 
@@ -660,13 +662,13 @@ class PegasusXDecoderLayer(GradientCheckpointingLayer):
     def forward(
         self,
         hidden_states: torch.Tensor,
-        attention_mask: torch.Tensor | None = None,
-        encoder_hidden_states: torch.Tensor | None = None,
-        encoder_attention_mask: torch.Tensor | None = None,
-        past_key_values: Cache | None = None,
-        output_attentions: bool | None = False,
-        use_cache: bool | None = True,
-        cache_position: torch.Tensor | None = None,
+        attention_mask: Optional[torch.Tensor] = None,
+        encoder_hidden_states: Optional[torch.Tensor] = None,
+        encoder_attention_mask: Optional[torch.Tensor] = None,
+        past_key_values: Optional[Cache] = None,
+        output_attentions: Optional[bool] = False,
+        use_cache: Optional[bool] = True,
+        cache_position: Optional[torch.Tensor] = None,
     ) -> torch.Tensor:
         """
         Args:
@@ -819,7 +821,6 @@ class PegasusXEncoder(PegasusXPreTrainedModel):
         output_attentions=None,
         output_hidden_states=None,
         return_dict=None,
-        **kwargs,
     ):
         r"""
         Args:
@@ -988,7 +989,6 @@ class PegasusXDecoder(PegasusXPreTrainedModel):
         output_hidden_states=None,
         return_dict=None,
         cache_position=None,
-        **kwargs,
     ):
         r"""
         Args:
@@ -1228,21 +1228,20 @@ class PegasusXModel(PegasusXPreTrainedModel):
     @auto_docstring
     def forward(
         self,
-        input_ids: torch.Tensor | None = None,
-        attention_mask: torch.Tensor | None = None,
-        decoder_input_ids: torch.Tensor | None = None,
-        decoder_attention_mask: torch.Tensor | None = None,
-        encoder_outputs: tuple[torch.FloatTensor] | None = None,
-        past_key_values: Cache | None = None,
-        inputs_embeds: torch.Tensor | None = None,
-        decoder_inputs_embeds: torch.Tensor | None = None,
-        use_cache: bool | None = None,
-        output_attentions: bool | None = None,
-        output_hidden_states: bool | None = None,
-        return_dict: bool | None = None,
-        cache_position: torch.Tensor | None = None,
-        **kwargs,
-    ) -> tuple | Seq2SeqModelOutput:
+        input_ids: Optional[torch.Tensor] = None,
+        attention_mask: Optional[torch.Tensor] = None,
+        decoder_input_ids: Optional[torch.Tensor] = None,
+        decoder_attention_mask: Optional[torch.Tensor] = None,
+        encoder_outputs: Optional[tuple[torch.FloatTensor]] = None,
+        past_key_values: Optional[Cache] = None,
+        inputs_embeds: Optional[torch.Tensor] = None,
+        decoder_inputs_embeds: Optional[torch.Tensor] = None,
+        use_cache: Optional[bool] = None,
+        output_attentions: Optional[bool] = None,
+        output_hidden_states: Optional[bool] = None,
+        return_dict: Optional[bool] = None,
+        cache_position: Optional[torch.Tensor] = None,
+    ) -> Union[tuple, Seq2SeqModelOutput]:
         r"""
         decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
             Indices of decoder input sequence tokens in the vocabulary.
@@ -1375,22 +1374,21 @@ class PegasusXForConditionalGeneration(PegasusXPreTrainedModel, GenerationMixin)
     @auto_docstring
     def forward(
         self,
-        input_ids: torch.Tensor | None = None,
-        attention_mask: torch.Tensor | None = None,
-        decoder_input_ids: torch.Tensor | None = None,
-        decoder_attention_mask: torch.Tensor | None = None,
-        encoder_outputs: tuple[torch.FloatTensor] | None = None,
-        past_key_values: Cache | None = None,
-        inputs_embeds: torch.Tensor | None = None,
-        decoder_inputs_embeds: torch.Tensor | None = None,
-        labels: torch.Tensor | None = None,
-        use_cache: bool | None = None,
-        output_attentions: bool | None = None,
-        output_hidden_states: bool | None = None,
-        return_dict: bool | None = None,
-        cache_position: torch.Tensor | None = None,
-        **kwargs,
-    ) -> tuple | Seq2SeqLMOutput:
+        input_ids: Optional[torch.Tensor] = None,
+        attention_mask: Optional[torch.Tensor] = None,
+        decoder_input_ids: Optional[torch.Tensor] = None,
+        decoder_attention_mask: Optional[torch.Tensor] = None,
+        encoder_outputs: Optional[tuple[torch.FloatTensor]] = None,
+        past_key_values: Optional[Cache] = None,
+        inputs_embeds: Optional[torch.Tensor] = None,
+        decoder_inputs_embeds: Optional[torch.Tensor] = None,
+        labels: Optional[torch.Tensor] = None,
+        use_cache: Optional[bool] = None,
+        output_attentions: Optional[bool] = None,
+        output_hidden_states: Optional[bool] = None,
+        return_dict: Optional[bool] = None,
+        cache_position: Optional[torch.Tensor] = None,
+    ) -> Union[tuple, Seq2SeqLMOutput]:
         r"""
         decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
             Indices of decoder input sequence tokens in the vocabulary.
@@ -1474,7 +1472,6 @@ class PegasusXDecoderWrapper(PegasusXPreTrainedModel):
     def __init__(self, config):
         super().__init__(config)
         self.decoder = PegasusXDecoder(config)
-        self.post_init()
 
     def forward(self, *args, **kwargs):
         return self.decoder(*args, **kwargs)

transformers/models/perceiver/configuration_perceiver.py

@@ -1,3 +1,4 @@
+# coding=utf-8
 # Copyright Deepmind and The HuggingFace Inc. team. All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

transformers/models/perceiver/image_processing_perceiver.py

@@ -1,3 +1,4 @@
+# coding=utf-8
 # Copyright 2022 The HuggingFace Inc. team. All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -13,6 +14,8 @@
 # limitations under the License.
 """Image processor class for Perceiver."""
 
+from typing import Optional, Union
+
 import numpy as np
 
 from ...image_processing_utils import BaseImageProcessor, BatchFeature, get_size_dict
@@ -85,15 +88,15 @@ class PerceiverImageProcessor(BaseImageProcessor):
     def __init__(
         self,
         do_center_crop: bool = True,
-        crop_size: dict[str, int] | None = None,
+        crop_size: Optional[dict[str, int]] = None,
         do_resize: bool = True,
-        size: dict[str, int] | None = None,
+        size: Optional[dict[str, int]] = None,
         resample: PILImageResampling = PILImageResampling.BICUBIC,
         do_rescale: bool = True,
-        rescale_factor: int | float = 1 / 255,
+        rescale_factor: Union[int, float] = 1 / 255,
         do_normalize: bool = True,
-        image_mean: float | list[float] | None = None,
-        image_std: float | list[float] | None = None,
+        image_mean: Optional[Union[float, list[float]]] = None,
+        image_std: Optional[Union[float, list[float]]] = None,
         **kwargs,
     ) -> None:
         super().__init__(**kwargs)
@@ -117,9 +120,9 @@ class PerceiverImageProcessor(BaseImageProcessor):
         self,
         image: np.ndarray,
         crop_size: dict[str, int],
-        size: int | None = None,
-        data_format: str | ChannelDimension | None = None,
-        input_data_format: str | ChannelDimension | None = None,
+        size: Optional[int] = None,
+        data_format: Optional[Union[str, ChannelDimension]] = None,
+        input_data_format: Optional[Union[str, ChannelDimension]] = None,
         **kwargs,
     ) -> np.ndarray:
         """
@@ -163,8 +166,8 @@ class PerceiverImageProcessor(BaseImageProcessor):
         image: np.ndarray,
         size: dict[str, int],
         resample: PILImageResampling = PILImageResampling.BICUBIC,
-        data_format: str | ChannelDimension | None = None,
-        input_data_format: str | ChannelDimension | None = None,
+        data_format: Optional[Union[str, ChannelDimension]] = None,
+        input_data_format: Optional[Union[str, ChannelDimension]] = None,
         **kwargs,
     ) -> np.ndarray:
         """
@@ -210,19 +213,19 @@ class PerceiverImageProcessor(BaseImageProcessor):
     def preprocess(
         self,
         images: ImageInput,
-        do_center_crop: bool | None = None,
-        crop_size: dict[str, int] | None = None,
-        do_resize: bool | None = None,
-        size: dict[str, int] | None = None,
-        resample: PILImageResampling | None = None,
-        do_rescale: bool | None = None,
-        rescale_factor: float | None = None,
-        do_normalize: bool | None = None,
-        image_mean: float | list[float] | None = None,
-        image_std: float | list[float] | None = None,
-        return_tensors: str | TensorType | None = None,
+        do_center_crop: Optional[bool] = None,
+        crop_size: Optional[dict[str, int]] = None,
+        do_resize: Optional[bool] = None,
+        size: Optional[dict[str, int]] = None,
+        resample: Optional[PILImageResampling] = None,
+        do_rescale: Optional[bool] = None,
+        rescale_factor: Optional[float] = None,
+        do_normalize: Optional[bool] = None,
+        image_mean: Optional[Union[float, list[float]]] = None,
+        image_std: Optional[Union[float, list[float]]] = None,
+        return_tensors: Optional[Union[str, TensorType]] = None,
         data_format: ChannelDimension = ChannelDimension.FIRST,
-        input_data_format: str | ChannelDimension | None = None,
+        input_data_format: Optional[Union[str, ChannelDimension]] = None,
     ) -> PIL.Image.Image:
         """
         Preprocess an image or batch of images.

transformers/models/perceiver/image_processing_perceiver_fast.py

@@ -1,3 +1,4 @@
+# coding=utf-8
 # Copyright 2025 The HuggingFace Inc. team. All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -13,10 +14,10 @@
 # limitations under the License.
 """Fast Image processor class for Perceiver."""
 
-from typing import Optional
+from typing import Optional, Union
 
 import torch
-import torchvision.transforms.v2.functional as tvF
+from torchvision.transforms.v2 import functional as F
 
 from ...image_processing_utils_fast import BaseImageProcessorFast, BatchFeature
 from ...image_transforms import group_images_by_shape, reorder_images
@@ -77,16 +78,16 @@ class PerceiverImageProcessorFast(BaseImageProcessorFast):
         images: list["torch.Tensor"],
         do_resize: bool,
         size: SizeDict,
-        interpolation: Optional["tvF.InterpolationMode"],
+        interpolation: Optional["F.InterpolationMode"],
         do_center_crop: bool,
         crop_size: SizeDict,
         do_rescale: bool,
         rescale_factor: float,
         do_normalize: bool,
-        image_mean: float | list[float] | None,
-        image_std: float | list[float] | None,
-        disable_grouping: bool | None,
-        return_tensors: str | TensorType | None,
+        image_mean: Optional[Union[float, list[float]]],
+        image_std: Optional[Union[float, list[float]]],
+        disable_grouping: Optional[bool],
+        return_tensors: Optional[Union[str, TensorType]],
         **kwargs,
     ) -> BatchFeature:
         # Group images by size for batched resizing
@@ -112,6 +113,7 @@ class PerceiverImageProcessorFast(BaseImageProcessorFast):
             processed_images_grouped[shape] = stacked_images
 
         processed_images = reorder_images(processed_images_grouped, grouped_images_index)
+        processed_images = torch.stack(processed_images, dim=0) if return_tensors else processed_images
 
         return BatchFeature(data={"pixel_values": processed_images}, tensor_type=return_tensors)