transformers 5.0.0rc0__py3-none-any.whl → 5.0.0rc1__py3-none-any.whl

transformers/models/xglm/tokenization_xglm.py

@@ -14,7 +14,7 @@
 # limitations under the License.
 """Tokenization classes for XGLM."""
 
-from typing import Optional
+from typing import Optional, Union
 
 from tokenizers import Regex, Tokenizer, decoders, normalizers, pre_tokenizers, processors
 from tokenizers.models import Unigram
@@ -50,7 +50,7 @@ class XGLMTokenizer(TokenizersBackend):
             The unknown token.
         pad_token (`str`, *optional*, defaults to `"<pad>"`):
             The token used for padding.
-        vocab (`dict`, *optional*):
+        vocab (`str`, `dict` or `list`, *optional*):
             Custom vocabulary dictionary. If not provided, a minimal vocabulary is created.
         merges (`list[tuple[str, str]]`, *optional*):
             Custom merge rules for BPE. If not provided, merges are generated from the vocabulary.
@@ -60,18 +60,17 @@ class XGLMTokenizer(TokenizersBackend):
 
     vocab_files_names = VOCAB_FILES_NAMES
     model_input_names = ["input_ids", "attention_mask"]
-    slow_tokenizer_class = None
+    model = Unigram
 
     def __init__(
         self,
+        vocab: Optional[Union[str, list[tuple[str, float]]]] = None,
         bos_token: str = "<s>",
         eos_token: str = "</s>",
         sep_token: str = "</s>",
         cls_token: str = "<s>",
         unk_token: str = "<unk>",
         pad_token: str = "<pad>",
-        vocab: Optional[dict] = None,
-        merges: Optional[list[tuple[str, str]]] = None,
         add_prefix_space: bool = True,
         **kwargs,
     ):
@@ -106,11 +105,7 @@ class XGLMTokenizer(TokenizersBackend):
         prepend_scheme = "always" if add_prefix_space else "never"
         self._tokenizer.pre_tokenizer = pre_tokenizers.Metaspace(replacement="▁", prepend_scheme=prepend_scheme)
         self._tokenizer.decoder = decoders.Metaspace(replacement="▁", prepend_scheme=prepend_scheme)
-
-        tokenizer_object = self._tokenizer
-
         super().__init__(
-            tokenizer_object=tokenizer_object,
             bos_token=bos_token,
             eos_token=eos_token,
             sep_token=sep_token,

transformers/models/xlm/modeling_xlm.py

@@ -1082,6 +1082,7 @@ class XLMForSequenceClassification(XLMPreTrainedModel):
         output_attentions: Optional[bool] = None,
         output_hidden_states: Optional[bool] = None,
         return_dict: Optional[bool] = None,
+        **kwargs,
     ) -> Union[tuple, SequenceClassifierOutput]:
         r"""
         langs (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
@@ -1190,6 +1191,7 @@ class XLMForQuestionAnsweringSimple(XLMPreTrainedModel):
         output_attentions: Optional[bool] = None,
         output_hidden_states: Optional[bool] = None,
         return_dict: Optional[bool] = None,
+        **kwargs,
     ) -> Union[tuple, QuestionAnsweringModelOutput]:
         r"""
         langs (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
@@ -1291,6 +1293,7 @@ class XLMForQuestionAnswering(XLMPreTrainedModel):
         output_attentions: Optional[bool] = None,
         output_hidden_states: Optional[bool] = None,
         return_dict: Optional[bool] = None,
+        **kwargs,
     ) -> Union[tuple, XLMForQuestionAnsweringOutput]:
         r"""
         langs (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
@@ -1406,6 +1409,7 @@ class XLMForTokenClassification(XLMPreTrainedModel):
         output_attentions: Optional[bool] = None,
         output_hidden_states: Optional[bool] = None,
         return_dict: Optional[bool] = None,
+        **kwargs,
     ) -> Union[tuple, TokenClassifierOutput]:
         r"""
         langs (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
@@ -1491,6 +1495,7 @@ class XLMForMultipleChoice(XLMPreTrainedModel):
         output_attentions: Optional[bool] = None,
         output_hidden_states: Optional[bool] = None,
         return_dict: Optional[bool] = None,
+        **kwargs,
     ) -> Union[tuple, MultipleChoiceModelOutput]:
         r"""
         input_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`):

transformers/models/xlm_roberta/tokenization_xlm_roberta.py

@@ -14,7 +14,7 @@
 # limitations under the License
 """Tokenization classes for XLM-RoBERTa model (Tokenizers backend)."""
 
-from typing import Optional
+from typing import Optional, Union
 
 from tokenizers import Tokenizer, decoders, normalizers, pre_tokenizers, processors
 from tokenizers.models import Unigram
@@ -47,16 +47,17 @@ class XLMRobertaTokenizer(TokenizersBackend):
         pad_token (`str`, optional, defaults to `"<pad>"`): The padding token.
         mask_token (`str`, optional, defaults to `"<mask>"`): The mask token.
         add_prefix_space (`bool`, optional, defaults to `True`): Whether to add an initial space.
-        vocab (`dict`, optional): Custom vocabulary dictionary.
-        merges (`list`, optional): Custom merges list.
+        vocab (`str`, `dict` or `list`, optional): Custom vocabulary dictionary.
     """
 
     vocab_files_names = VOCAB_FILES_NAMES
     model_input_names = ["input_ids", "attention_mask"]
-    slow_tokenizer_class = None
+    model = Unigram
 
     def __init__(
         self,
+        vocab: Optional[Union[str, list[tuple[str, float]]]] = None,
+        add_prefix_space: bool = True,
         bos_token: str = "<s>",
         eos_token: str = "</s>",
         sep_token: str = "</s>",
@@ -64,9 +65,6 @@ class XLMRobertaTokenizer(TokenizersBackend):
         unk_token: str = "<unk>",
         pad_token: str = "<pad>",
         mask_token: str = "<mask>",
-        add_prefix_space: bool = True,
-        vocab: Optional[dict] = None,
-        vocab_file: Optional[str] = None,
         **kwargs,
     ):
         self.add_prefix_space = add_prefix_space
@@ -99,11 +97,7 @@ class XLMRobertaTokenizer(TokenizersBackend):
             ]
         )
         self._tokenizer.decoder = decoders.Metaspace(replacement="▁", prepend_scheme=prepend_scheme)
-
-        tokenizer_object = self._tokenizer
-
         super().__init__(
-            tokenizer_object=tokenizer_object,
             bos_token=bos_token,
             eos_token=eos_token,
             sep_token=sep_token,
@@ -116,14 +110,13 @@ class XLMRobertaTokenizer(TokenizersBackend):
         )
 
         self._tokenizer.post_processor = processors.TemplateProcessing(
-            single=["$A", "</s>"],
-            pair=["$A", "</s>", "$B", "</s>"],
+            single=[str(bos_token), "$A", str(eos_token)],
+            pair=[str(bos_token), "$A", str(eos_token), "$B", str(eos_token)],
             special_tokens=[
-                ("</s>", self.eos_token_id),
+                (str(bos_token), self.bos_token_id),
+                (str(eos_token), self.eos_token_id),
             ],
         )
 
-        self.vocab_file = vocab_file
-
 
 __all__ = ["XLMRobertaTokenizer"]

transformers/models/xlnet/tokenization_xlnet.py

@@ -14,7 +14,7 @@
 # limitations under the License.
 """Tokenization classes for XLNet model."""
 
-from typing import Optional
+from typing import Optional, Union
 
 from tokenizers import AddedToken, Regex, Tokenizer, decoders, normalizers, pre_tokenizers, processors
 from tokenizers.models import Unigram
@@ -98,10 +98,11 @@ class XLNetTokenizer(TokenizersBackend):
 
     vocab_files_names = VOCAB_FILES_NAMES
     padding_side = "left"
+    model = Unigram
 
     def __init__(
         self,
-        vocab: Optional[list] = None,
+        vocab: Optional[Union[str, list[tuple[str, float]]]] = None,
         unk_id: int = 0,
         do_lower_case=False,
         remove_space=True,
@@ -159,13 +160,8 @@ class XLNetTokenizer(TokenizersBackend):
         self.do_lower_case = do_lower_case
         self.remove_space = remove_space
         self.keep_accents = keep_accents
-
         mask_token = AddedToken(mask_token, lstrip=True, rstrip=False) if isinstance(mask_token, str) else mask_token
-
-        tokenizer_object = self._tokenizer
-
         super().__init__(
-            tokenizer_object=tokenizer_object,
             unk_id=unk_id,
             do_lower_case=do_lower_case,
             remove_space=remove_space,

transformers/models/yoso/modeling_yoso.py

@@ -642,6 +642,7 @@ class YosoModel(YosoPreTrainedModel):
         output_attentions: Optional[bool] = None,
         output_hidden_states: Optional[bool] = None,
         return_dict: Optional[bool] = None,
+        **kwargs,
     ) -> Union[tuple, BaseModelOutputWithCrossAttentions]:
         output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
         output_hidden_states = (
@@ -734,6 +735,7 @@ class YosoForMaskedLM(YosoPreTrainedModel):
         output_attentions: Optional[bool] = None,
         output_hidden_states: Optional[bool] = None,
         return_dict: Optional[bool] = None,
+        **kwargs,
     ) -> Union[tuple, MaskedLMOutput]:
         r"""
         labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
@@ -823,6 +825,7 @@ class YosoForSequenceClassification(YosoPreTrainedModel):
         output_attentions: Optional[bool] = None,
         output_hidden_states: Optional[bool] = None,
         return_dict: Optional[bool] = None,
+        **kwargs,
     ) -> Union[tuple, SequenceClassifierOutput]:
         r"""
         labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
@@ -904,6 +907,7 @@ class YosoForMultipleChoice(YosoPreTrainedModel):
         output_attentions: Optional[bool] = None,
         output_hidden_states: Optional[bool] = None,
         return_dict: Optional[bool] = None,
+        **kwargs,
     ) -> Union[tuple, MultipleChoiceModelOutput]:
         r"""
         input_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`):
@@ -1009,6 +1013,7 @@ class YosoForTokenClassification(YosoPreTrainedModel):
         output_attentions: Optional[bool] = None,
         output_hidden_states: Optional[bool] = None,
         return_dict: Optional[bool] = None,
+        **kwargs,
     ) -> Union[tuple, TokenClassifierOutput]:
         r"""
         labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
@@ -1085,6 +1090,7 @@ class YosoForQuestionAnswering(YosoPreTrainedModel):
         output_attentions: Optional[bool] = None,
         output_hidden_states: Optional[bool] = None,
         return_dict: Optional[bool] = None,
+        **kwargs,
     ) -> Union[tuple, QuestionAnsweringModelOutput]:
         return_dict = return_dict if return_dict is not None else self.config.use_return_dict
 

transformers/models/zamba/modeling_zamba.py

@@ -870,6 +870,7 @@ class ZambaModel(ZambaPreTrainedModel):
         output_hidden_states: Optional[bool] = None,
         return_dict: Optional[bool] = None,
         cache_position: Optional[torch.LongTensor] = None,
+        **kwargs,
     ) -> Union[tuple, BaseModelOutputWithPast]:
         output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
         output_hidden_states = (
@@ -1192,6 +1193,7 @@ class ZambaForSequenceClassification(ZambaPreTrainedModel):
         output_attentions: Optional[bool] = None,
         output_hidden_states: Optional[bool] = None,
         return_dict: Optional[bool] = None,
+        **kwargs,
     ) -> Union[tuple, SequenceClassifierOutputWithPast]:
         r"""
         labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):

transformers/models/zamba2/modeling_zamba2.py

@@ -41,6 +41,7 @@ from ...modeling_rope_utils import ROPE_INIT_FUNCTIONS, dynamic_rope_update
 from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel
 from ...processing_utils import Unpack
 from ...utils import auto_docstring, logging
+from ...utils.generic import maybe_autocast
 from ...utils.import_utils import is_causal_conv1d_available, is_mamba_ssm_available
 from .configuration_zamba2 import Zamba2Config
 
@@ -263,7 +264,7 @@ class Zamba2RotaryEmbedding(nn.Module):
         position_ids_expanded = position_ids[:, None, :].float()
 
         device_type = x.device.type if isinstance(x.device.type, str) and x.device.type != "mps" else "cpu"
-        with torch.autocast(device_type=device_type, enabled=False):  # Force float32
+        with maybe_autocast(device_type=device_type, enabled=False):  # Force float32
             freqs = (inv_freq_expanded.float() @ position_ids_expanded.float()).transpose(1, 2)
             emb = torch.cat((freqs, freqs), dim=-1)
             cos = emb.cos() * self.attention_scaling
@@ -424,7 +425,6 @@ class Zamba2Attention(nn.Module):
         attention_mask: Optional[torch.Tensor] = None,
         past_key_values: Optional[Zamba2HybridDynamicCache] = None,
         position_embeddings: Optional[tuple[torch.Tensor, torch.Tensor]] = None,
-        position_ids: Optional[torch.Tensor] = None,
         **kwargs: Unpack[FlashAttentionKwargs],
     ) -> tuple[torch.Tensor, Optional[torch.Tensor], Optional[tuple[torch.Tensor]]]:
         input_shape = hidden_states.shape[:-1]
@@ -1294,6 +1294,7 @@ class Zamba2Model(Zamba2PreTrainedModel):
         output_hidden_states: Optional[bool] = None,
         return_dict: Optional[bool] = None,
         cache_position: Optional[torch.LongTensor] = None,
+        **kwargs,
     ) -> Union[tuple, BaseModelOutputWithPast]:
         output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
         output_hidden_states = (
@@ -1638,6 +1639,7 @@ class Zamba2ForSequenceClassification(Zamba2PreTrainedModel):
         output_attentions: Optional[bool] = None,
         output_hidden_states: Optional[bool] = None,
         return_dict: Optional[bool] = None,
+        **kwargs,
     ) -> Union[tuple, SequenceClassifierOutputWithPast]:
         r"""
         labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):

transformers/models/zamba2/modular_zamba2.py

@@ -232,7 +232,6 @@ class Zamba2Attention(ZambaAttention):
         attention_mask: Optional[torch.Tensor] = None,
         past_key_values: Optional[Zamba2HybridDynamicCache] = None,
         position_embeddings: Optional[tuple[torch.Tensor, torch.Tensor]] = None,
-        position_ids: Optional[torch.Tensor] = None,
         **kwargs: Unpack[FlashAttentionKwargs],
     ) -> tuple[torch.Tensor, Optional[torch.Tensor], Optional[tuple[torch.Tensor]]]:
         input_shape = hidden_states.shape[:-1]
@@ -993,6 +992,7 @@ class Zamba2Model(ZambaModel, Zamba2PreTrainedModel):
         output_hidden_states: Optional[bool] = None,
         return_dict: Optional[bool] = None,
         cache_position: Optional[torch.LongTensor] = None,
+        **kwargs,
     ) -> Union[tuple, BaseModelOutputWithPast]:
         output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
         output_hidden_states = (

transformers/models/zoedepth/modeling_zoedepth.py

@@ -1251,6 +1251,7 @@ class ZoeDepthForDepthEstimation(ZoeDepthPreTrainedModel):
         output_attentions: Optional[bool] = None,
         output_hidden_states: Optional[bool] = None,
         return_dict: Optional[bool] = None,
+        **kwargs,
     ) -> Union[tuple[torch.Tensor], DepthEstimatorOutput]:
         r"""
         labels (`torch.LongTensor` of shape `(batch_size, height, width)`, *optional*):

transformers/pipelines/__init__.py

@@ -18,7 +18,7 @@ import warnings
 from pathlib import Path
 from typing import TYPE_CHECKING, Any, Optional, Union
 
-from huggingface_hub import model_info
+from huggingface_hub import is_offline_mode, model_info
 
 from ..configuration_utils import PreTrainedConfig
 from ..dynamic_module_utils import get_class_from_dynamic_module
@@ -38,7 +38,6 @@ from ..utils import (
     extract_commit_hash,
     find_adapter_config_file,
     is_kenlm_available,
-    is_offline_mode,
     is_peft_available,
     is_pyctcdecode_available,
     is_torch_available,
@@ -278,7 +277,7 @@ SUPPORTED_TASKS = {
     "image-to-text": {
         "impl": ImageToTextPipeline,
         "pt": (AutoModelForImageTextToText,) if is_torch_available() else (),
-        "default": {"model": ("ydshieh/vit-gpt2-coco-en", "5bebf1e")},
+        "default": {"model": ("ydshieh/vit-gpt2-coco-en", "e460201")},
         "type": "multimodal",
     },
     "image-text-to-text": {

transformers/pipelines/base.py

@@ -950,20 +950,13 @@ class Pipeline(_ScikitCompat, PushToHubMixin):
             pipe_information["output_modalities"] = self.model.output_modalities
         return f"{self.__class__.__name__}: {pipe_information}"
 
-    def save_pretrained(
-        self,
-        save_directory: str | os.PathLike,
-        safe_serialization: bool = True,
-        **kwargs: Any,
-    ):
+    def save_pretrained(self, save_directory: str | os.PathLike, **kwargs: Any):
         """
         Save the pipeline's model and tokenizer.
 
         Args:
             save_directory (`str` or `os.PathLike`):
                 A path to the directory where to saved. It will be created if it doesn't exist.
-            safe_serialization (`str`):
-                Whether to save the model using `safetensors` or PyTorch serialization.
             kwargs (`dict[str, Any]`, *optional*):
                 Additional key word arguments passed along to the [`~utils.PushToHubMixin.push_to_hub`] method.
         """
@@ -992,7 +985,6 @@ class Pipeline(_ScikitCompat, PushToHubMixin):
             # Save the pipeline custom code
             custom_object_save(self, save_directory)
 
-        kwargs["safe_serialization"] = safe_serialization
         self.model.save_pretrained(save_directory, **kwargs)
 
         if self.tokenizer is not None:

transformers/pipelines/document_question_answering.py

@@ -146,7 +146,9 @@ class DocumentQuestionAnsweringPipeline(ChunkPipeline):
 
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
-        if self.tokenizer is not None and not self.tokenizer.__class__.__name__.endswith("Fast"):
+        if self.tokenizer is not None and not (
+            self.tokenizer.__class__.__name__.endswith("Fast") or self.tokenizer.backend == "tokenizers"
+        ):
             raise ValueError(
                 "`DocumentQuestionAnsweringPipeline` requires a fast tokenizer, but a slow tokenizer "
                 f"(`{self.tokenizer.__class__.__name__}`) is provided."

transformers/pipelines/text_generation.py

@@ -486,7 +486,7 @@ class TextGenerationPipeline(Pipeline):
                             ]
                         else:
                             # When we're not starting from a prefill, the output is a new assistant message
-                            if self.tokenizer.response_schema:
+                            if getattr(self.tokenizer, "response_schema", False):
                                 assistant_message = self.tokenizer.parse_response(all_text)
                             else:
                                 # If there's no schema, then we have to assume it's all content

transformers/processing_utils.py

@@ -28,7 +28,7 @@ from typing import Annotated, Any, Literal, Optional, TypedDict, TypeVar, Union
 
 import numpy as np
 import typing_extensions
-from huggingface_hub import create_repo
+from huggingface_hub import create_repo, is_offline_mode
 from huggingface_hub.dataclasses import validate_typed_dict
 from huggingface_hub.errors import EntryNotFoundError
 
@@ -54,7 +54,6 @@ from .utils import (
     cached_file,
     copy_func,
     direct_transformers_import,
-    is_offline_mode,
     is_torch_available,
     list_repo_templates,
     logging,
@@ -696,14 +695,10 @@ class ProcessorMixin(PushToHubMixin):
         # extra attributes to be kept
         attrs_to_save += ["auto_map"]
 
-        if "tokenizer" in output:
-            del output["tokenizer"]
-        if "qformer_tokenizer" in output:
-            del output["qformer_tokenizer"]
-        if "protein_tokenizer" in output:
-            del output["protein_tokenizer"]
-        if "char_tokenizer" in output:
-            del output["char_tokenizer"]
+        for attribute in self.__class__.get_attributes():
+            if "tokenizer" in attribute and attribute in output:
+                del output[attribute]
+
         if "chat_template" in output:
             del output["chat_template"]
 
@@ -1465,7 +1460,24 @@ class ProcessorMixin(PushToHubMixin):
         # get args from processor init signature
         sub_processors = cls.get_attributes()
         for sub_processor_type in sub_processors:
-            if sub_processor_type in MODALITY_TO_AUTOPROCESSOR_MAPPING:
+            if "FuyuProcessor" in cls.__name__ and "tokenizer" in sub_processor_type:
+                from .tokenization_utils_tokenizers import TokenizersBackend
+
+                tokenizer = TokenizersBackend.from_pretrained(pretrained_model_name_or_path, **kwargs)
+                if "token_type_ids" in tokenizer.model_input_names:
+                    tokenizer.model_input_names.remove("token_type_ids")
+                args.append(tokenizer)
+            elif "PixtralProcessor" in cls.__name__ and "tokenizer" in sub_processor_type:
+                from tokenizers import pre_tokenizers
+
+                from .models.llama import LlamaTokenizer
+
+                tokenizer = LlamaTokenizer.from_pretrained(pretrained_model_name_or_path, **kwargs)
+                tokenizer._tokenizer.pre_tokenizer = pre_tokenizers.Sequence(
+                    [pre_tokenizers.ByteLevel(False), tokenizer._tokenizer.pre_tokenizer]
+                )
+                args.append(tokenizer)
+            elif sub_processor_type in MODALITY_TO_AUTOPROCESSOR_MAPPING:
                 auto_processor_class = MODALITY_TO_AUTOPROCESSOR_MAPPING[sub_processor_type]
                 sub_processor = auto_processor_class.from_pretrained(pretrained_model_name_or_path, **kwargs)
                 args.append(sub_processor)