transformers 5.0.0rc2__py3-none-any.whl → 5.0.0rc3__py3-none-any.whl

transformers/tokenization_mistral_common.py

@@ -14,39 +14,39 @@
 import os
 import re
 import shutil
-import warnings
-from collections.abc import Callable, Mapping, Sized
+from collections.abc import Callable, Sequence
 from enum import Enum
 from pathlib import Path
-from typing import Any, Union, overload
+from typing import Any, Literal, Union, overload
 
 import numpy as np
 from huggingface_hub import create_repo
 
 from transformers.audio_utils import load_audio_as
 from transformers.tokenization_utils_base import (
-    LARGE_INTEGER,
     VERY_LARGE_INTEGER,
+    AddedToken,
     BatchEncoding,
     EncodedInput,
     PreTokenizedInput,
+    PreTrainedTokenizerBase,
     TextInput,
     TruncationStrategy,
 )
 from transformers.utils import PaddingStrategy, TensorType, add_end_docstrings, logging, to_py_obj
-from transformers.utils.generic import is_torch_tensor
-from transformers.utils.hub import PushToHubMixin
 from transformers.utils.import_utils import is_mistral_common_available, is_torch_available, requires
 
 
 if is_mistral_common_available():
     from mistral_common.protocol.instruct.request import ChatCompletionRequest
     from mistral_common.protocol.instruct.validator import ValidationMode
-    from mistral_common.tokens.tokenizers.base import SpecialTokenPolicy, TokenizerVersion
-    from mistral_common.tokens.tokenizers.image import MultiModalVersion
+    from mistral_common.tokens.tokenizers.base import SpecialTokenPolicy, SpecialTokens
     from mistral_common.tokens.tokenizers.mistral import MistralTokenizer
     from mistral_common.tokens.tokenizers.tekken import Tekkenizer
-    from mistral_common.tokens.tokenizers.utils import download_tokenizer_from_hf_hub
+    from mistral_common.tokens.tokenizers.utils import (
+        download_tokenizer_from_hf_hub,
+        get_one_valid_tokenizer_file,
+    )
 
 
 if is_torch_available():
@@ -103,6 +103,10 @@ ENCODE_KWARGS_DOCSTRING = r"""
 """
 
 ENCODE_PLUS_ADDITIONAL_KWARGS_DOCSTRING = r"""
+            return_token_type_ids (`bool`, *optional*):
+                Whether to return token type IDs. For `MistralCommonBackend` it returns a list of zeros of the sequence length as only one sequence is supported.
+
+                [What are token type IDs?](../glossary#token-type-ids)
             return_attention_mask (`bool`, *optional*):
                 Whether to return the attention mask. If left to the default, will return the attention mask according
                 to the specific tokenizer's default, defined by the `return_outputs` attribute.
@@ -118,6 +122,8 @@ ENCODE_PLUS_ADDITIONAL_KWARGS_DOCSTRING = r"""
                 Whether or not to return the lengths of the encoded inputs.
             verbose (`bool`, *optional*, defaults to `True`):
                 Whether or not to print more information and warnings.
+            return_offsets_mapping (`Literal[False]`, *optional*): False, kept to match Transformers' signature.
+            split_special_tokens (`Literal[False]`, *optional*): False, kept to match Transformers' signature.
             **kwargs: passed to the `self.tokenize()` method
 
         Return:
@@ -149,8 +155,35 @@ class MistralTokenizerType(str, Enum):
     tekken = "tekken"
 
 
+@overload
+def _maybe_remove_lang(text: str, skip_special_tokens: bool) -> str: ...
+@overload
+def _maybe_remove_lang(text: list[str], skip_special_tokens: bool) -> list[str]: ...
+def _maybe_remove_lang(text: str | list[str], skip_special_tokens: bool) -> str | list[str]:
+    # in the specific case of Voxtral, the added f"lang:xx" (always a two char language code since it follows ISO 639-1 alpha-2 format)
+    # is not considered as a special token by mistral-common and is encoded/ decoded as normal text.
+    # Nevertheless we should remove it to ease users life.
+    if not skip_special_tokens:
+        return text
+
+    if isinstance(text, str):
+        return re.sub(r"^lang:[a-z]{2}", "", text)
+
+    return [re.sub(r"^lang:[a-z]{2}", "", string) for string in text]
+
+
+_MAP_SPECIAL_TOKENS = {
+    "bos_token": SpecialTokens.bos.value,
+    "eos_token": SpecialTokens.eos.value,
+    "pad_token": SpecialTokens.pad.value,
+    "unk_token": SpecialTokens.unk.value,
+}
+
+_VALID_INIT_KWARGS = {"_from_auto", "backend", "files_loaded"}
+
+
 @requires(backends=("mistral-common",))
-class MistralCommonBackend(PushToHubMixin):
+class MistralCommonBackend(PreTrainedTokenizerBase):
     """
     Class to wrap `mistral-common` tokenizers.
 
@@ -165,34 +198,13 @@ class MistralCommonBackend(PushToHubMixin):
     For more info on `mistral-common`, see [mistral-common](https://github.com/mistralai/mistral-common).
 
     This class is a wrapper around a `mistral_common.tokens.tokenizers.mistral.MistralTokenizer`.
-    It provides a Hugging Face compatible interface to tokenize using the official mistral-common tokenizer.
-
-    Supports the following methods from the `PreTrainedTokenizerBase` class:
-
-    - [`~MistralCommonBackend.get_vocab`]: Returns the vocabulary as a dictionary of token to index.
-        This is a lossy conversion for Tekkenizer as some decoding errors are collapsed into the same token.
-    - [`~MistralCommonBackend.encode`]: Encode a string to a list of integers.
-    - [`~MistralCommonBackend.decode`]: Decode a list of integers to a string.
-    - [`~MistralCommonBackend.batch_decode`]: Decode a batch of list of integers to a list of strings.
-    - [`~MistralCommonBackend.convert_tokens_to_ids`]: Convert a list of tokens to a list of integers.
-    - [`~MistralCommonBackend.convert_ids_to_tokens`]: Convert a list of integers to a list of tokens.
-    - [`~MistralCommonBackend.tokenize`]: Tokenize a string.
-    - [`~MistralCommonBackend.get_special_tokens_mask`]: Get the special tokens mask for a list of tokens.
-    - [`~MistralCommonBackend.prepare_for_model`]: Prepare a list of inputs for the model.
-    - [`~MistralCommonBackend.pad`]: Pad a list of inputs to the same length.
-    - [`~MistralCommonBackend.truncate_sequences`]: Truncate a list of sequences to the same length.
-    - [`~MistralCommonBackend.apply_chat_template`]: Apply a chat template to a list of messages.
-    - [`~MistralCommonBackend.__call__`]: Tokenize a string or a list of strings.
-    - [`~MistralCommonBackend.from_pretrained`]: Download and cache a pretrained tokenizer from the Hugging Face model hub or local directory.
-    - [`~MistralCommonBackend.save_pretrained`]: Save a tokenizer to a directory, so it can be reloaded using the `from_pretrained` class method.
-    - [`~MistralCommonBackend.push_to_hub`]: Upload tokenizer to the Hugging Face model hub.
-
-    Here are the key differences with the `PreTrainedTokenizerBase` class:
-
-    - Pair of sequences are not supported. The signature have been kept for compatibility but all arguments related to pair of sequences are ignored. The return values of pairs are returned as `None`.
+    It provides a Hugging Face compatible interface to tokenize using the official mistral-common tokenizer and inherits from the `PreTrainedTokenizerBase` class.
+
+    Here are the key behavior differences with the `PythonBackend` class:
+
+    - Pair of sequences are not supported. The signature has been kept for compatibility but all arguments related to pair of sequences are ignored. The return values for pairs are returned as `None`.
     - The `is_split_into_words` argument is not supported.
-    - The `return_token_type_ids` argument is not supported.
-    - It is not possible to add new tokens to the tokenizer. Also the special tokens are handled differently from Transformers. In `mistral-common`, special tokens are never encoded directly. This means that: `tokenizer.encode("<s>")` will not return the ID of the `<s>` token. Instead, it will return a list of IDs corresponding to the tokenization of the string `"<s>"`. For more information, see the [mistral-common documentation](https://mistralai.github.io/mistral-common/usage/tokenizers/#special-tokens).
+    - It is not possible to add new tokens to the tokenizer. Special tokens are handled differently from Transformers. In `mistral-common`, special tokens are never encoded directly. This means that: `tokenizer.encode("<s>")` will not return the ID of the `<s>` token. Instead, it will return a list of IDs corresponding to the tokenization of the string `"<s>"`. For more information, see the [mistral-common documentation](https://mistralai.github.io/mistral-common/usage/tokenizers/#special-tokens).
 
     If you have suggestions to improve this class, please open an issue on the [mistral-common GitHub repository](https://github.com/mistralai/mistral-common/issues) if it is related to the tokenizer or on the [Transformers GitHub repository](https://github.com/huggingface/transformers/issues) if it is related to the Hugging Face interface.
     """
@@ -200,6 +212,12 @@ class MistralCommonBackend(PushToHubMixin):
     model_input_names: list[str] = ["input_ids", "attention_mask"]
     padding_side: str = "left"
     truncation_side: str = "right"
+    SPECIAL_TOKENS_ATTRIBUTES = [
+        "bos_token",
+        "eos_token",
+        "unk_token",
+        "pad_token",
+    ]
 
     def __init__(
         self,
@@ -226,7 +244,7 @@ class MistralCommonBackend(PushToHubMixin):
                 Path to the tokenizer file to load the `MistralTokenizer`.
             mode (`Union[str, ValidationMode]`, *optional*, defaults to `ValidationMode.test`):
                 The mode to use for the tokenizer. This will be passed to the `MistralTokenizer` constructor. Possible values are:
-                - `"finetuning"` or `ValidationMode.finetuning`: The finetuning mode.
+                - `"finetuning"` or `ValidationMode.finetuning`: The fine-tuning mode.
                 - `"test"` or `ValidationMode.test`: The test mode.
                 It changes how the tokenizer validates the input and prepares the request to the model.
             model_max_length (`int`, *optional*):
@@ -240,60 +258,40 @@ class MistralCommonBackend(PushToHubMixin):
             truncation_side (`str`, *optional*):
                 The side on which the model should have truncation applied. Should be selected between ['right', 'left'].
                 Default value is picked from the class attribute of the same name.
-            model_input_names (`List[string]`, *optional*):
+            model_input_names (`List[str]`, *optional*):
                 The list of inputs accepted by the forward pass of the model (like `"token_type_ids"` or
                 `"attention_mask"`). Default value is picked from the class attribute of the same name.
             clean_up_tokenization_spaces (`bool`, *optional*, defaults to `False`):
-                Whether or not the model should cleanup the spaces that were added when splitting the input text during the
+                Whether or not the model should clean up the spaces that were added when splitting the input text during the
                 tokenization process.
         """
-        if kwargs:
+        if kwargs and not set(kwargs.keys()).issubset(_VALID_INIT_KWARGS):
             raise ValueError(f"Kwargs {list(kwargs.keys())} are not supported to init `MistralCommonBackend`.")
 
         self._tokenizer_path = Path(tokenizer_path)
         self._mode = self._get_validation_mode(mode)
+
         self.tokenizer: MistralTokenizer = MistralTokenizer.from_file(str(self._tokenizer_path), mode=self._mode)
         self._tokenizer_type = (
             MistralTokenizerType.tekken
             if isinstance(self.tokenizer.instruct_tokenizer.tokenizer, Tekkenizer)
             else MistralTokenizerType.spm
         )
-        self.truncation_side = truncation_side
-        self.padding_side = padding_side
-        self.model_max_length = model_max_length
-        self.cleanup_tokenization_spaces = clean_up_tokenization_spaces
-        self.deprecation_warnings = {}  # Use to store when we have already noticed a deprecation warning (avoid overlogging).
-        self._all_special_tokens_ids = self._get_all_special_ids()
-
-        if model_input_names is not None:
-            if (
-                not isinstance(model_input_names, (list, tuple))
-                and len(model_input_names) == 0
-                and not all(isinstance(i, str) for i in model_input_names)
-            ):
-                raise ValueError(
-                    "`model_input_names` should be a non-empty list or tuple of str but got an empty value."
-                )
-            self.model_input_names = model_input_names
-
         self._cache_get_vocab: dict[str, int] | None = None
 
-    @staticmethod
-    def clean_up_tokenization(text: str) -> str:
-        """
-        Clean up a list of simple English tokenization artifacts like spaces before punctuation.
-        """
-        return (
-            text.replace(" .", ".")
-            .replace(" ?", "?")
-            .replace(" !", "!")
-            .replace(" ,", ",")
-            .replace(" ' ", "'")
-            .replace(" n't", "n't")
-            .replace(" 'm", "'m")
-            .replace(" 's", "'s")
-            .replace(" 've", "'ve")
-            .replace(" 're", "'re")
+        self._all_special_ids = self._get_all_special_ids()
+        self._all_special_tokens = self.convert_ids_to_tokens(self.all_special_ids)
+
+        super().__init__(
+            truncation_side=truncation_side,
+            padding_side=padding_side,
+            model_max_length=model_max_length,
+            clean_up_tokenization_spaces=clean_up_tokenization_spaces,
+            extra_special_tokens=None,  # Not used by this backend.
+            model_specific_special_tokens=None,  # Not used by this backend.
+            model_input_names=model_input_names or self.model_input_names,
+            **_MAP_SPECIAL_TOKENS,
+            **kwargs,
         )
 
     @property
@@ -306,75 +304,19 @@ class MistralCommonBackend(PushToHubMixin):
         """
         return self._mode
 
-    @property
-    def bos_token_id(self) -> int:
-        """
-        Id of the beginning of sentence token in the vocabulary.
-        """
-        return self.tokenizer.instruct_tokenizer.tokenizer.bos_id
-
-    @property
-    def eos_token_id(self) -> int:
-        """
-        Id of the end of sentence token in the vocabulary.
-        """
-        return self.tokenizer.instruct_tokenizer.tokenizer.eos_id
-
-    @property
-    def unk_token_id(self) -> int:
-        """
-        Id of the unknown token in the vocabulary.
-        """
-        return self.tokenizer.instruct_tokenizer.tokenizer.unk_id
-
-    @property
-    def pad_token_id(self) -> int:
-        """
-        Id of the padding token in the vocabulary.
-        """
-        return self.tokenizer.instruct_tokenizer.tokenizer.pad_id
-
-    @property
-    def bos_token(self) -> str:
-        """
-        String associated to the beginning of sentence token in the vocabulary.
-        """
-        return self.convert_ids_to_tokens(self.bos_token_id)
-
-    @property
-    def eos_token(self) -> str:
-        """
-        String associated to the end of sentence token in the vocabulary.
-        """
-        return self.convert_ids_to_tokens(self.eos_token_id)
-
-    @property
-    def unk_token(self) -> str:
-        """
-        String associated to the unknown token in the vocabulary.
-        """
-        return self.convert_ids_to_tokens(self.unk_token_id)
-
-    @property
-    def pad_token(self) -> str:
-        """
-        String associated to the padding token in the vocabulary.
-        """
-        return self.convert_ids_to_tokens(self.pad_token_id)
-
     @property
     def all_special_ids(self) -> list[int]:
         """
         `list[int]`: List the ids of the special tokens(`'<unk>'`, `'<cls>'`, etc.).
         """
-        return sorted(self._all_special_tokens_ids)
+        return sorted(self._all_special_ids)
 
     @property
     def all_special_tokens(self) -> list[str]:
         """
         `list[str]`: A list of all unique special tokens.
         """
-        return self.convert_ids_to_tokens(self.all_special_ids)
+        return self._all_special_tokens
 
     @property
     def vocab_size(self) -> int:
@@ -435,6 +377,8 @@ class MistralCommonBackend(PushToHubMixin):
         padding_side: str | None = None,
         return_tensors: str | TensorType | None = None,
         verbose: bool = True,
+        return_offsets_mapping: Literal[False] = False,
+        split_special_tokens: Literal[False] = False,
         **kwargs,
     ) -> list[int]:
         """
@@ -446,37 +390,81 @@ class MistralCommonBackend(PushToHubMixin):
             text_pair (`None`, *optional*):
                 Not supported by `MistralCommonBackend.encode`. Kept to match `PreTrainedTokenizerBase.encode` signature.
         """
+        if return_offsets_mapping or split_special_tokens:
+            raise ValueError(
+                "`MistralCommonBackend` does not support `return_offsets_mapping` and `split_special_tokens`."
+            )
+
+        if truncation in [TruncationStrategy.ONLY_FIRST, TruncationStrategy.ONLY_SECOND, "only_first", "only_second"]:
+            raise ValueError(
+                "Truncation strategy `only_first` and `only_second` are not supported by `MistralCommonBackend`."
+            )
+
         if kwargs:
             raise ValueError(f"Kwargs {list(kwargs.keys())} are not supported by `MistralCommonBackend.encode`.")
+
         if text_pair:
             raise ValueError("`MistralCommonBackend.encode` does not support `text_pair`.")
 
-        padding_strategy, truncation_strategy, max_length, _ = self._get_padding_truncation_strategies(
+        return super().encode(
+            text=text,
+            text_pair=text_pair,
+            add_special_tokens=add_special_tokens,
             padding=padding,
             truncation=truncation,
             max_length=max_length,
-            pad_to_multiple_of=pad_to_multiple_of,
-            verbose=verbose,
-        )
-
-        encoded_inputs = self._encode_plus(
-            text,
-            add_special_tokens=add_special_tokens,
-            padding_strategy=padding_strategy,
-            truncation_strategy=truncation_strategy,
-            max_length=max_length,
             stride=stride,
+            return_tensors=return_tensors,
             pad_to_multiple_of=pad_to_multiple_of,
             padding_side=padding_side,
-            return_tensors=return_tensors,
-            return_attention_mask=False,
-            return_overflowing_tokens=False,
-            return_special_tokens_mask=False,
-            return_length=False,
             verbose=verbose,
         )
 
-        return encoded_inputs["input_ids"]
+    def _decode(
+        self,
+        token_ids: int | list[int],
+        skip_special_tokens: bool = False,
+        clean_up_tokenization_spaces: bool | None = None,
+        **kwargs,
+    ) -> str:
+        if kwargs:
+            raise ValueError(f"Kwargs {list(kwargs.keys())} are not supported by `MistralCommonBackend.decode`.")
+
+        token_ids = to_py_obj(token_ids)
+
+        if isinstance(token_ids, int):
+            token_ids = [token_ids]
+
+        special_token_policy = SpecialTokenPolicy.IGNORE if skip_special_tokens else SpecialTokenPolicy.KEEP
+
+        text = self.tokenizer.decode(token_ids, special_token_policy=special_token_policy)
+
+        # Apply tokenizer-specific cleanup if available and requested
+        clean_up_tokenization_spaces = (
+            clean_up_tokenization_spaces
+            if clean_up_tokenization_spaces is not None
+            else self.clean_up_tokenization_spaces
+        )
+        if clean_up_tokenization_spaces:
+            # Call custom cleanup method if it exists (e.g., for CLVP's [SPACE] token replacement)
+            if hasattr(self, "clean_up_tokenization") and callable(self.clean_up_tokenization):
+                text = self.clean_up_tokenization(text)
+            else:
+                # Otherwise apply standard cleanup
+                text = (
+                    text.replace(" .", ".")
+                    .replace(" ?", "?")
+                    .replace(" !", "!")
+                    .replace(" ,", ",")
+                    .replace(" ' ", "'")
+                    .replace(" n't", "n't")
+                    .replace(" 'm", "'m")
+                    .replace(" 's", "'s")
+                    .replace(" 've", "'ve")
+                    .replace(" 're", "'re")
+                )
+
+        return _maybe_remove_lang(text=text, skip_special_tokens=skip_special_tokens)
 
     def decode(
         self,
@@ -484,7 +472,7 @@ class MistralCommonBackend(PushToHubMixin):
         skip_special_tokens: bool = False,
         clean_up_tokenization_spaces: bool | None = None,
         **kwargs,
-    ) -> Union[str, list[str]]:
+    ) -> str | list[str]:
         """
         Converts a sequence of ids in a string, using the tokenizer and vocabulary with options to remove special
         tokens and clean up tokenization spaces.
@@ -509,16 +497,7 @@ class MistralCommonBackend(PushToHubMixin):
         if kwargs:
             raise ValueError(f"Kwargs {list(kwargs.keys())} are not supported by `MistralCommonBackend.decode`.")
 
-        token_ids = to_py_obj(token_ids)
-
-        if isinstance(token_ids, (list, tuple)) and len(token_ids) > 0 and isinstance(token_ids[0], (list, tuple)):
-            return self._batch_decode(
-                sequences=token_ids,
-                skip_special_tokens=skip_special_tokens,
-                clean_up_tokenization_spaces=clean_up_tokenization_spaces,
-            )
-
-        return self._decode(
+        return super().decode(
             token_ids=token_ids,
             skip_special_tokens=skip_special_tokens,
             clean_up_tokenization_spaces=clean_up_tokenization_spaces,
@@ -555,63 +534,12 @@ class MistralCommonBackend(PushToHubMixin):
         if kwargs:
             raise ValueError(f"Kwargs {list(kwargs.keys())} are not supported by `MistralCommonBackend.batch_decode`.")
 
-        return self._batch_decode(
+        return super().batch_decode(
             sequences=sequences,
             skip_special_tokens=skip_special_tokens,
             clean_up_tokenization_spaces=clean_up_tokenization_spaces,
         )
 
-    def _decode(
-        self,
-        token_ids: Union[int, list[int], list[list[int]], np.ndarray, "torch.Tensor"],
-        skip_special_tokens: bool = False,
-        clean_up_tokenization_spaces: bool | None = None,
-    ) -> str:
-        clean_up_tokenization_spaces = clean_up_tokenization_spaces or self.cleanup_tokenization_spaces
-
-        # Convert inputs to python lists
-        if isinstance(token_ids, int):
-            token_ids = [token_ids]
-
-        token_ids = to_py_obj(token_ids)
-
-        special_token_policy = SpecialTokenPolicy.IGNORE if skip_special_tokens else SpecialTokenPolicy.KEEP
-
-        decoded_string = self.tokenizer.decode(token_ids, special_token_policy=special_token_policy)
-        if clean_up_tokenization_spaces:
-            decoded_string = self.clean_up_tokenization(decoded_string)
-
-        # in the specific case of Voxtral, the added f"lang:xx" (always a two char language code since it follows ISO 639-1 alpha-2 format)
-        # is not considered as a special token by mistral-common and is encoded/ decoded as normal text.
-        # Nevertheless we should remove it to ease users life.
-        if skip_special_tokens:
-            decoded_string = re.sub(r"^lang:[a-z]{2}", "", decoded_string)
-
-        return decoded_string
-
-    def _batch_decode(
-        self,
-        sequences: Union[list[int], list[list[int]], np.ndarray, "torch.Tensor"],
-        skip_special_tokens: bool = False,
-        clean_up_tokenization_spaces: bool | None = None,
-    ) -> list[str]:
-        return [
-            self._decode(
-                seq,
-                skip_special_tokens=skip_special_tokens,
-                clean_up_tokenization_spaces=clean_up_tokenization_spaces,
-            )
-            for seq in sequences
-        ]
-
-    def _is_control_token(self, token_id: int) -> bool:
-        if self._tokenizer_type == MistralTokenizerType.spm:
-            return token_id in self.tokenizer.instruct_tokenizer.tokenizer._control_tokens
-        elif self._tokenizer_type == MistralTokenizerType.tekken:
-            return token_id < self.tokenizer.instruct_tokenizer.tokenizer.num_special_tokens
-        else:
-            raise ValueError(f"Unknown tokenizer type: {self._tokenizer_type}")
-
     @overload
     def convert_ids_to_tokens(self, ids: int, skip_special_tokens: bool = False) -> str: ...
     @overload
@@ -632,22 +560,22 @@ class MistralCommonBackend(PushToHubMixin):
         """
 
         if isinstance(ids, int):
-            one_token = True
+            return_int = True
             ids = [ids]
         else:
-            one_token = False
+            return_int = False
 
         tokens: list[str] = []
         for token_id in ids:
-            if self._is_control_token(token_id) and skip_special_tokens:
+            if self.tokenizer.instruct_tokenizer.tokenizer.is_special(token_id) and skip_special_tokens:
                 continue
             tokens.append(self.tokenizer.instruct_tokenizer.tokenizer.id_to_piece(token_id))
 
-        if one_token:
-            if tokens == []:
-                raise ValueError(f"Invalid token id {ids}.")
-
+        if return_int and tokens == []:
+            raise ValueError(f"Invalid token id {ids[0]}.")
+        elif return_int:
             return tokens[0]
+
         return tokens
 
     def _tekken_piece_to_id(self, piece: str, warn: bool) -> int:
@@ -708,7 +636,13 @@ class MistralCommonBackend(PushToHubMixin):
         tokens_ids = self.tokenizer.instruct_tokenizer.tokenizer.encode(text, bos=add_special_tokens, eos=add_eos)
         return tokens_ids
 
-    def tokenize(self, text: TextInput, **kwargs) -> list[str]:
+    def tokenize(
+        self,
+        text: TextInput,
+        return_offsets_mapping: Literal[False] = False,
+        split_special_tokens: Literal[False] = False,
+        **kwargs,
+    ) -> list[str]:
         """
         Converts a string into a sequence of tokens, using the tokenizer.
 
@@ -717,6 +651,8 @@ class MistralCommonBackend(PushToHubMixin):
         Args:
             text (`str`):
                 The sequence to be encoded.
+            return_offsets_mapping (`Literal[False]`, *optional*): False, kept to match Transformers' signature.
+            split_special_tokens (`Literal[False]`, *optional*): False, kept to match Transformers' signature.
             **kwargs (additional keyword arguments):
                 Not supported by `MistralCommonBackend.tokenize`.
                 Will raise an error if used.
@@ -724,40 +660,164 @@ class MistralCommonBackend(PushToHubMixin):
         Returns:
             `list[str]`: The list of tokens.
         """
+        if return_offsets_mapping or split_special_tokens:
+            raise ValueError(
+                "`MistralCommonBackend` does not support `return_offsets_mapping` and `split_special_tokens`."
+            )
+
         if kwargs:
             raise ValueError(f"Kwargs {list(kwargs.keys())} are not supported by `MistralCommonBackend.tokenize`.")
 
         return self.convert_ids_to_tokens(self._text_to_ids(text, add_special_tokens=False), skip_special_tokens=False)
 
-    def _encode_plus(
+    def _get_all_special_ids(self) -> set[int]:
+        if self._tokenizer_type == MistralTokenizerType.tekken:
+            return self.tokenizer.instruct_tokenizer.tokenizer._special_token_ids
+        elif self._tokenizer_type == MistralTokenizerType.spm:
+            return {
+                token_id
+                for token_id in range(self.tokenizer.instruct_tokenizer.tokenizer.n_words)
+                if self.tokenizer.instruct_tokenizer.tokenizer.is_special(token_id)
+            }
+        else:
+            raise ValueError(f"Unknown tokenizer type: {self._tokenizer_type}")
+
+    def get_special_tokens_mask(
+        self, token_ids_0: list[int], token_ids_1: None = None, already_has_special_tokens: bool = False
+    ) -> list[int]:
+        """
+        Retrieves sequence ids from a token list that has no special tokens added. This method is called when adding
+        special tokens using the tokenizer `prepare_for_model` or `encode_plus` methods.
+
+        Args:
+            token_ids_0 (`list[int]`): List of ids of the sequence.
+            token_ids_1 (`None`, *optional*): None, kept to match Transformers' implementation.
+            already_has_special_tokens (`bool`, *optional*, defaults to `False`):
+                Whether or not the token list is already formatted with special tokens for the model.
+
+        Returns:
+            A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.
+        """
+        if token_ids_1 is not None:
+            raise ValueError(
+                "`token_ids_1` is not supported by `MistralCommonBackend` and should be `None`, kept for compatibility."
+            )
+
+        if already_has_special_tokens:
+            return [1 if int(token_id) in self._all_special_ids else 0 for token_id in token_ids_0]
+
+        if self.mode == ValidationMode.test:
+            # [BOS] seq0
+            return [1] + ([0] * len(token_ids_0))
+        else:
+            # [BOS] seq0 [EOS]
+            return [1] + ([0] * len(token_ids_0)) + [1]
+
+    def _encode_plus(  # type: ignore[override]
         self,
-        text: TextInput | EncodedInput,
+        text: TextInput | PreTokenizedInput | EncodedInput,
+        text_pair: None = None,
         add_special_tokens: bool = True,
         padding_strategy: PaddingStrategy = PaddingStrategy.DO_NOT_PAD,
         truncation_strategy: TruncationStrategy = TruncationStrategy.DO_NOT_TRUNCATE,
         max_length: int | None = None,
         stride: int = 0,
+        is_split_into_words: bool = False,
         pad_to_multiple_of: int | None = None,
         padding_side: str | None = None,
         return_tensors: str | TensorType | None = None,
+        return_token_type_ids: bool | None = None,
         return_attention_mask: bool | None = None,
         return_overflowing_tokens: bool = False,
         return_special_tokens_mask: bool = False,
         return_length: bool = False,
         verbose: bool = True,
+        return_offsets_mapping: Literal[False] = False,
+        split_special_tokens: Literal[False] = False,
+        **kwargs,
     ) -> BatchEncoding:
+        # Detect batched inputs (list of sequences)
+        if text_pair is not None:
+            raise ValueError("`MistralCommonBackend` does not support `text_pair != None` for `_encode_plus`.")
+
+        if return_offsets_mapping or split_special_tokens:
+            raise ValueError(
+                "`MistralCommonBackend` does not support `return_offsets_mapping` and `split_special_tokens`."
+            )
+
+        if kwargs:
+            raise ValueError(f"Kwargs {list(kwargs.keys())} are not supported by `MistralCommonBackend._encode_plus`.")
+
+        is_batched = isinstance(text, (list, tuple)) and (
+            (not text and not is_split_into_words)
+            or (text and is_split_into_words and isinstance(text[0], (list, tuple)))
+            or (text and not is_split_into_words and isinstance(text[0], (str, list, tuple)))
+        )
+
+        if is_batched:
+            batch_outputs = {}
+            one_overflowed = False
+            for current_text in text:
+                current_output = self._encode_plus(
+                    text=current_text,
+                    text_pair=None,
+                    add_special_tokens=add_special_tokens,
+                    padding_strategy=PaddingStrategy.DO_NOT_PAD,  # we pad in batch afterward
+                    truncation_strategy=truncation_strategy,
+                    max_length=max_length,
+                    stride=stride,
+                    is_split_into_words=is_split_into_words,
+                    pad_to_multiple_of=None,  # we pad in batch afterward
+                    padding_side=None,  # we pad in batch afterward
+                    return_tensors=None,  # We convert the whole batch to tensors at the end
+                    return_token_type_ids=return_token_type_ids,
+                    return_attention_mask=False,  # we pad in batch afterward
+                    return_overflowing_tokens=return_overflowing_tokens,
+                    return_special_tokens_mask=return_special_tokens_mask,
+                    return_length=return_length,
+                    verbose=verbose,
+                )
+                for key, value in current_output.items():
+                    batch_outputs.setdefault(key, []).append(value)
+
+                # To ensure the list is built for each sample, we need to add this.
+                if return_overflowing_tokens and not return_tensors:
+                    if "overflowing_tokens" not in current_output:
+                        batch_outputs.setdefault("overflowing_tokens", []).append([0])
+                        batch_outputs.setdefault("num_truncated_tokens", []).append([0])
+                    else:
+                        one_overflowed = True
+
+            # Remove overflow-related keys before tensor conversion if return_tensors is set
+            # Slow tokenizers don't support returning these as tensors
+            if return_overflowing_tokens and (return_tensors or not one_overflowed):
+                batch_outputs.pop("overflowing_tokens", None)
+                batch_outputs.pop("num_truncated_tokens", None)
+
+            batch_outputs = self.pad(
+                batch_outputs,
+                padding=padding_strategy.value,
+                max_length=max_length,
+                pad_to_multiple_of=pad_to_multiple_of,
+                padding_side=padding_side,
+                return_attention_mask=return_attention_mask,
+            )
+
+            return BatchEncoding(batch_outputs, tensor_type=return_tensors)
+
         def get_input_ids(text):
             if isinstance(text, str):
-                return self._text_to_ids(text, add_special_tokens)
+                return self._text_to_ids(text, False)
             elif isinstance(text, (list, tuple)) and len(text) > 0 and isinstance(text[0], int):
                 return text
             else:
                 raise ValueError(f"Input {text} is not valid. Should be a string, or a list/tuple of integers.")
 
-        ids = get_input_ids(text)
+        first_ids = get_input_ids(text)
 
         return self.prepare_for_model(
-            ids,
+            first_ids,
+            pair_ids=None,
             add_special_tokens=add_special_tokens,
             padding=padding_strategy.value,
             truncation=truncation_strategy.value,
@@ -768,202 +828,62 @@ class MistralCommonBackend(PushToHubMixin):
             return_tensors=return_tensors,
             prepend_batch_axis=True,
             return_attention_mask=return_attention_mask,
+            return_token_type_ids=return_token_type_ids,
             return_overflowing_tokens=return_overflowing_tokens,
             return_special_tokens_mask=return_special_tokens_mask,
             return_length=return_length,
             verbose=verbose,
         )
 
-    def _batch_encode_plus(
+    @add_end_docstrings(ENCODE_KWARGS_DOCSTRING, ENCODE_PLUS_ADDITIONAL_KWARGS_DOCSTRING)
+    def prepare_for_model(
         self,
-        batch_text: list[TextInput] | list[EncodedInput],
+        ids: list[int],
+        pair_ids: None = None,
         add_special_tokens: bool = True,
-        padding_strategy: PaddingStrategy = PaddingStrategy.DO_NOT_PAD,
-        truncation_strategy: TruncationStrategy = TruncationStrategy.DO_NOT_TRUNCATE,
+        padding: bool | str | PaddingStrategy = False,
+        truncation: bool | str | TruncationStrategy | None = None,
         max_length: int | None = None,
         stride: int = 0,
         pad_to_multiple_of: int | None = None,
         padding_side: str | None = None,
         return_tensors: str | TensorType | None = None,
+        return_token_type_ids: bool | None = None,
         return_attention_mask: bool | None = None,
         return_overflowing_tokens: bool = False,
         return_special_tokens_mask: bool = False,
         return_length: bool = False,
         verbose: bool = True,
+        prepend_batch_axis: bool = False,
+        return_offsets_mapping: Literal[False] = False,
+        split_special_tokens: Literal[False] = False,
+        **kwargs,
     ) -> BatchEncoding:
-        def get_input_ids(text):
-            if isinstance(text, str):
-                return self._text_to_ids(text, add_special_tokens)
-            elif isinstance(text, (list, tuple)) and len(text) > 0 and isinstance(text[0], int):
-                return text
-            else:
-                raise ValueError("Input is not valid. Should be a string or a list/tuple of integers.")
-
-        input_ids = []
-        for ids in batch_text:
-            input_ids.append(get_input_ids(ids))
-
-        batch_outputs = self._batch_prepare_for_model(
-            input_ids,
-            add_special_tokens=add_special_tokens,
-            padding_strategy=padding_strategy,
-            truncation_strategy=truncation_strategy,
-            max_length=max_length,
-            stride=stride,
-            pad_to_multiple_of=pad_to_multiple_of,
-            padding_side=padding_side,
-            return_attention_mask=return_attention_mask,
-            return_overflowing_tokens=return_overflowing_tokens,
-            return_special_tokens_mask=return_special_tokens_mask,
-            return_length=return_length,
-            return_tensors=return_tensors,
-            verbose=verbose,
-        )
-
-        return BatchEncoding(batch_outputs)
-
-    def _get_all_special_ids(self) -> set[int]:
-        if self._tokenizer_type == MistralTokenizerType.tekken:
-            return {t["rank"] for t in self.tokenizer.instruct_tokenizer.tokenizer._all_special_tokens}
-        elif self._tokenizer_type == MistralTokenizerType.spm:
-            return self.tokenizer.instruct_tokenizer.tokenizer._control_tokens
-        else:
-            raise ValueError(f"Unknown tokenizer type: {self._tokenizer_type}")
-
-    def get_special_tokens_mask(
-        self, token_ids_0: list, token_ids_1: None = None, already_has_special_tokens: bool = False
-    ) -> list[int]:
         """
-        Retrieves sequence ids from a token list that has no special tokens added. This method is called when adding
-        special tokens using the tokenizer `prepare_for_model` or `encode_plus` methods.
+        Prepares a sequence of input id so that it can be used by the model. It
+        adds special tokens, truncates sequences if overflowing while taking into account the special tokens and
+        manages a moving window (with user defined stride) for overflowing tokens.
 
         Args:
-            token_ids_0 (`list[int]`):
-                List of ids of the sequence.
-            token_ids_1 (`list[int]`, *optional*):
+            ids (`list[int]`):
+                Tokenized input ids of the first sequence.
+            pair_ids (`None`, *optional*):
                 Not supported by `MistralCommonBackend`. Kept to match the interface of `PreTrainedTokenizerBase`.
-            already_has_special_tokens (`bool`, *optional*, defaults to `False`):
-                Whether or not the token list is already formatted with special tokens for the model.
-
-        Returns:
-            A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.
         """
-        if token_ids_1 is not None:
+        if return_offsets_mapping or split_special_tokens:
             raise ValueError(
-                "`token_ids_1` is not supported by `MistralCommonBackend` and should be `None`, kept for compatibility."
+                "`MistralCommonBackend` does not support `return_offsets_mapping` and `split_special_tokens`."
             )
-        if already_has_special_tokens:
+
+        if pair_ids is not None:
             raise ValueError(
-                "`already_has_special_tokens` is not supported by `MistralCommonBackend` and should be `False`."
+                "`pair_ids` is not supported by `MistralCommonBackend` and should be `None`, kept for compatibility."
             )
 
-        special_tokens_mask = [1 if token in self._all_special_tokens_ids else 0 for token in token_ids_0]
-        return special_tokens_mask
-
-    def _batch_prepare_for_model(
-        self,
-        batch_ids: list[PreTokenizedInput | list[int]],
-        add_special_tokens: bool = True,
-        padding_strategy: PaddingStrategy = PaddingStrategy.DO_NOT_PAD,
-        truncation_strategy: TruncationStrategy = TruncationStrategy.DO_NOT_TRUNCATE,
-        max_length: int | None = None,
-        stride: int = 0,
-        pad_to_multiple_of: int | None = None,
-        padding_side: str | None = None,
-        return_tensors: str | None = None,
-        return_attention_mask: bool | None = None,
-        return_overflowing_tokens: bool = False,
-        return_special_tokens_mask: bool = False,
-        return_length: bool = False,
-        verbose: bool = True,
-    ) -> BatchEncoding:
-        """
-        Prepares a sequence of input id so that it can be used by the model. It
-        adds special tokens, truncates sequences if overflowing while taking into account the special tokens and
-        manages a moving window (with user defined stride) for overflowing tokens.
-
-        Args:
-            batch_ids: list of tokenized input ids
-        """
-
-        batch_outputs = {}
-        for ids in batch_ids:
-            outputs = self.prepare_for_model(
-                ids,
-                add_special_tokens=add_special_tokens,
-                padding=PaddingStrategy.DO_NOT_PAD.value,  # we pad in batch afterward
-                truncation=truncation_strategy.value,
-                max_length=max_length,
-                stride=stride,
-                pad_to_multiple_of=None,  # we pad in batch afterward
-                padding_side=None,  # we pad in batch afterward
-                return_attention_mask=False,  # we pad in batch afterward
-                return_overflowing_tokens=return_overflowing_tokens,
-                return_special_tokens_mask=return_special_tokens_mask,
-                return_length=return_length,
-                return_tensors=None,  # We convert the whole batch to tensors at the end
-                prepend_batch_axis=False,
-                verbose=verbose,
-            )
-
-            for key, value in outputs.items():
-                if key not in batch_outputs:
-                    batch_outputs[key] = []
-                batch_outputs[key].append(value)
-
-        batch_outputs = self.pad(
-            batch_outputs,
-            padding=padding_strategy.value,
-            max_length=max_length,
-            pad_to_multiple_of=pad_to_multiple_of,
-            padding_side=padding_side,
-            return_attention_mask=return_attention_mask,
-        )
-
-        batch_outputs = BatchEncoding(batch_outputs, tensor_type=return_tensors)
-
-        return batch_outputs
-
-    @add_end_docstrings(ENCODE_KWARGS_DOCSTRING, ENCODE_PLUS_ADDITIONAL_KWARGS_DOCSTRING)
-    def prepare_for_model(
-        self,
-        ids: list[int],
-        pair_ids: None = None,
-        add_special_tokens: bool = True,
-        padding: bool | str | PaddingStrategy = False,
-        truncation: bool | str | TruncationStrategy | None = None,
-        max_length: int | None = None,
-        stride: int = 0,
-        pad_to_multiple_of: int | None = None,
-        padding_side: str | None = None,
-        return_tensors: str | TensorType | None = None,
-        return_attention_mask: bool | None = None,
-        return_overflowing_tokens: bool = False,
-        return_special_tokens_mask: bool = False,
-        return_length: bool = False,
-        verbose: bool = True,
-        prepend_batch_axis: bool = False,
-        **kwargs,
-    ) -> BatchEncoding:
-        """
-        Prepares a sequence of input id so that it can be used by the model. It
-        adds special tokens, truncates sequences if overflowing while taking into account the special tokens and
-        manages a moving window (with user defined stride) for overflowing tokens.
-
-        Args:
-            ids (`list[int]`):
-                Tokenized input ids of the first sequence.
-            pair_ids (`None`, *optional*):
-                Not supported by `MistralCommonBackend`. Kept to match the interface of `PreTrainedTokenizerBase`.
-        """
-        if pair_ids is not None:
-            raise ValueError(
-                "`pair_ids` is not supported by `MistralCommonBackend` and should be `None`, kept for compatibility."
-            )
-        if kwargs:
-            raise ValueError(
-                f"Kwargs {list(kwargs.keys())} are not supported by `MistralCommonBackend.prepare_for_model`."
-            )
+        if kwargs:
+            raise ValueError(
+                f"Kwargs {list(kwargs.keys())} are not supported by `MistralCommonBackend.prepare_for_model`."
+            )
 
         padding_strategy, truncation_strategy, max_length, _ = self._get_padding_truncation_strategies(
             padding=padding,
@@ -971,39 +891,65 @@ class MistralCommonBackend(PushToHubMixin):
             max_length=max_length,
             pad_to_multiple_of=pad_to_multiple_of,
             verbose=verbose,
+            **kwargs,
         )
 
-        len_ids = len(ids)
+        # Validation
+        if (
+            return_overflowing_tokens
+            and truncation_strategy == TruncationStrategy.LONGEST_FIRST
+            and pair_ids is not None
+        ):
+            raise ValueError(
+                "Not possible to return overflowing tokens for pair of sequences with the "
+                "`longest_first`. Please select another truncation strategy than `longest_first`, "
+                "for instance `only_second` or `only_first`."
+            )
 
-        # Load from model defaults
+        # Defaults
+        if return_token_type_ids is None:
+            return_token_type_ids = "token_type_ids" in self.model_input_names
         if return_attention_mask is None:
             return_attention_mask = "attention_mask" in self.model_input_names
 
-        encoded_inputs = {}
+        # Truncation
+        num_special = self.num_special_tokens_to_add(pair=False) if add_special_tokens else 0
+        total_len = len(ids) + len(pair_ids or []) + num_special
 
-        # Truncation: Handle max sequence length
         overflowing_tokens = []
-        if truncation_strategy != TruncationStrategy.DO_NOT_TRUNCATE and max_length and len_ids > max_length:
+        if truncation_strategy != TruncationStrategy.DO_NOT_TRUNCATE and max_length and total_len > max_length:
             ids, _, overflowing_tokens = self.truncate_sequences(
                 ids,
-                num_tokens_to_remove=len_ids - max_length,
+                pair_ids=None,
+                num_tokens_to_remove=total_len - max_length,
                 truncation_strategy=truncation_strategy,
                 stride=stride,
             )
 
-        if return_overflowing_tokens:
-            encoded_inputs["overflowing_tokens"] = overflowing_tokens
-            encoded_inputs["num_truncated_tokens"] = len_ids - max_length
+        # Add special tokens
+        if add_special_tokens:
+            sequence = self.build_inputs_with_special_tokens(ids, None)
+            token_type_ids = self.create_token_type_ids_from_sequences(ids, None)
+        else:
+            sequence = ids
+            token_type_ids = [0] * len(sequence)
 
-        # Build output dictionary
-        encoded_inputs[self.model_input_names[0]] = ids
+        # Build output
+        encoded_inputs = {"input_ids": sequence}
+        if return_token_type_ids:
+            encoded_inputs["token_type_ids"] = token_type_ids
         if return_special_tokens_mask:
-            if add_special_tokens:
-                encoded_inputs["special_tokens_mask"] = self.get_special_tokens_mask(ids, None)
-            else:
-                encoded_inputs["special_tokens_mask"] = [0] * len(ids)
+            encoded_inputs["special_tokens_mask"] = (
+                self.get_special_tokens_mask(ids, None) if add_special_tokens else [0] * len(sequence)
+            )
+        if return_overflowing_tokens and not return_tensors and overflowing_tokens:
+            encoded_inputs["overflowing_tokens"] = overflowing_tokens
+            encoded_inputs["num_truncated_tokens"] = total_len - max_length if max_length else 0
+
+        # Check sequence length and warn if needed
+        self._eventual_warn_about_too_long_sequence(encoded_inputs["input_ids"], max_length, verbose)
 
-        # Padding
+        # Pad
         if padding_strategy != PaddingStrategy.DO_NOT_PAD or return_attention_mask:
             encoded_inputs = self.pad(
                 encoded_inputs,
@@ -1017,362 +963,9 @@ class MistralCommonBackend(PushToHubMixin):
         if return_length:
             encoded_inputs["length"] = len(encoded_inputs["input_ids"])
 
-        batch_outputs = BatchEncoding(
-            encoded_inputs, tensor_type=return_tensors, prepend_batch_axis=prepend_batch_axis
-        )
-
-        return batch_outputs
-
-    def _get_padding_truncation_strategies(
-        self,
-        padding: str | PaddingStrategy | bool = False,
-        truncation: str | TruncationStrategy | bool | None = None,
-        max_length: int | None = None,
-        pad_to_multiple_of: int | None = None,
-        verbose: bool = True,
-        **kwargs,
-    ):
-        """
-        Find the correct padding/truncation strategy.
-        """
-
-        # Backward compatibility for previous behavior, maybe we should deprecate it:
-        # If you only set max_length, it activates truncation for max_length
-        if max_length is not None and padding is False and truncation is None:
-            if verbose:
-                if not self.deprecation_warnings.get("Truncation-not-explicitly-activated", False):
-                    logger.warning(
-                        "Truncation was not explicitly activated but `max_length` is provided a specific value, please"
-                        " use `truncation=True` to explicitly truncate examples to max length. Defaulting to"
-                        " 'longest_first' truncation strategy."
-                    )
-                self.deprecation_warnings["Truncation-not-explicitly-activated"] = True
-            truncation = "longest_first"
-
-        # Get padding strategy
-        if padding is not False:
-            if padding is True:
-                if verbose:
-                    if max_length is not None and (
-                        truncation is None or truncation is False or truncation == "do_not_truncate"
-                    ):
-                        warnings.warn(
-                            "`max_length` is ignored when `padding`=`True` and there is no truncation strategy. "
-                            "To pad to max length, use `padding='max_length'`."
-                        )
-                padding_strategy = PaddingStrategy.LONGEST  # Default to pad to the longest sequence in the batch
-            elif not isinstance(padding, PaddingStrategy):
-                padding_strategy = PaddingStrategy(padding)
-            elif isinstance(padding, PaddingStrategy):
-                padding_strategy = padding
-        else:
-            padding_strategy = PaddingStrategy.DO_NOT_PAD
-
-        # Get truncation strategy
-        if truncation is not False and truncation is not None:
-            if truncation is True:
-                truncation_strategy = (
-                    TruncationStrategy.LONGEST_FIRST
-                )  # Default to truncate the longest sequences in pairs of inputs
-            elif not isinstance(truncation, TruncationStrategy):
-                truncation_strategy = TruncationStrategy(truncation)
-            elif isinstance(truncation, TruncationStrategy):
-                truncation_strategy = truncation
-            if truncation in [TruncationStrategy.ONLY_FIRST, TruncationStrategy.ONLY_SECOND]:
-                raise ValueError(
-                    "Truncation strategy `only_first` and `only_second` are not supported by `MistralCommonBackend`."
-                )
-        else:
-            truncation_strategy = TruncationStrategy.DO_NOT_TRUNCATE
-
-        # Set max length if needed
-        if max_length is None:
-            if padding_strategy == PaddingStrategy.MAX_LENGTH:
-                if self.model_max_length > LARGE_INTEGER:
-                    if verbose:
-                        if not self.deprecation_warnings.get("Asking-to-pad-to-max_length", False):
-                            logger.warning(
-                                "Asking to pad to max_length but no maximum length is provided and the model has no"
-                                " predefined maximum length. Default to no padding."
-                            )
-                        self.deprecation_warnings["Asking-to-pad-to-max_length"] = True
-                    padding_strategy = PaddingStrategy.DO_NOT_PAD
-                else:
-                    max_length = self.model_max_length
-
-            if truncation_strategy != TruncationStrategy.DO_NOT_TRUNCATE:
-                if self.model_max_length > LARGE_INTEGER:
-                    if verbose:
-                        if not self.deprecation_warnings.get("Asking-to-truncate-to-max_length", False):
-                            logger.warning(
-                                "Asking to truncate to max_length but no maximum length is provided and the model has"
-                                " no predefined maximum length. Default to no truncation."
-                            )
-                        self.deprecation_warnings["Asking-to-truncate-to-max_length"] = True
-                    truncation_strategy = TruncationStrategy.DO_NOT_TRUNCATE
-                else:
-                    max_length = self.model_max_length
-
-        # Test if we have a padding token
-        if padding_strategy != PaddingStrategy.DO_NOT_PAD and (self.pad_token_id is None or self.pad_token_id < 0):
-            raise ValueError(
-                "Asking to pad but the tokenizer does not have a padding token. "
-                "Please select a token to use as `pad_token` `(tokenizer.pad_token = tokenizer.eos_token e.g.)` "
-                "or add a new pad token via `tokenizer.add_special_tokens({'pad_token': '[PAD]'})`."
-            )
-
-        # Check that we will truncate to a multiple of pad_to_multiple_of if both are provided
-        if (
-            truncation_strategy != TruncationStrategy.DO_NOT_TRUNCATE
-            and padding_strategy != PaddingStrategy.DO_NOT_PAD
-            and pad_to_multiple_of is not None
-            and max_length is not None
-            and (max_length % pad_to_multiple_of != 0)
-        ):
-            raise ValueError(
-                "Truncation and padding are both activated but "
-                f"truncation length ({max_length}) is not a multiple of pad_to_multiple_of ({pad_to_multiple_of})."
-            )
-
-        return padding_strategy, truncation_strategy, max_length, kwargs
-
-    def _pad(
-        self,
-        encoded_inputs: dict[str, EncodedInput] | BatchEncoding,
-        max_length: int | None = None,
-        padding_strategy: PaddingStrategy = PaddingStrategy.DO_NOT_PAD,
-        pad_to_multiple_of: int | None = None,
-        padding_side: str | None = None,
-        return_attention_mask: bool | None = None,
-    ) -> dict:
-        """
-        Pad encoded inputs (on left/right and up to predefined length or max length in the batch)
-
-        Args:
-            encoded_inputs:
-                Dictionary of tokenized inputs (`list[int]`) or batch of tokenized inputs (`list[list[int]]`).
-            max_length: maximum length of the returned list and optionally padding length (see below).
-                Will truncate by taking into account the special tokens.
-            padding_strategy: PaddingStrategy to use for padding.
-
-                - PaddingStrategy.LONGEST Pad to the longest sequence in the batch
-                - PaddingStrategy.MAX_LENGTH: Pad to the max length (default)
-                - PaddingStrategy.DO_NOT_PAD: Do not pad
-                The tokenizer padding sides are defined in `padding_side` argument:
-
-                    - 'left': pads on the left of the sequences
-                    - 'right': pads on the right of the sequences
-            pad_to_multiple_of: (optional) Integer if set will pad the sequence to a multiple of the provided value.
-                This is especially useful to enable the use of Tensor Core on NVIDIA hardware with compute capability
-                `>= 7.5` (Volta).
-            padding_side:
-                The side on which the model should have padding applied. Should be selected between ['right', 'left'].
-                Default value is picked from the class attribute of the same name.
-            return_attention_mask:
-                (optional) Set to False to avoid returning attention mask (default: set to model specifics)
-        """
-        # Load from model defaults
-        if return_attention_mask is None:
-            return_attention_mask = "attention_mask" in self.model_input_names
-
-        required_input = encoded_inputs[self.model_input_names[0]]
-
-        if padding_strategy == PaddingStrategy.LONGEST:
-            max_length = len(required_input)
-
-        if max_length is not None and pad_to_multiple_of is not None and (max_length % pad_to_multiple_of != 0):
-            max_length = ((max_length // pad_to_multiple_of) + 1) * pad_to_multiple_of
-
-        needs_to_be_padded = padding_strategy != PaddingStrategy.DO_NOT_PAD and len(required_input) != max_length
-
-        # Initialize attention mask if not present.
-        if return_attention_mask and "attention_mask" not in encoded_inputs:
-            encoded_inputs["attention_mask"] = [1] * len(required_input)
-
-        if needs_to_be_padded:
-            difference = max_length - len(required_input)
-            padding_side = padding_side if padding_side is not None else self.padding_side
-
-            if padding_side == "right":
-                if return_attention_mask:
-                    encoded_inputs["attention_mask"] = encoded_inputs["attention_mask"] + [0] * difference
-                if "special_tokens_mask" in encoded_inputs:
-                    encoded_inputs["special_tokens_mask"] = encoded_inputs["special_tokens_mask"] + [1] * difference
-                encoded_inputs[self.model_input_names[0]] = required_input + [self.pad_token_id] * difference
-            elif padding_side == "left":
-                if return_attention_mask:
-                    encoded_inputs["attention_mask"] = [0] * difference + encoded_inputs["attention_mask"]
-                if "special_tokens_mask" in encoded_inputs:
-                    encoded_inputs["special_tokens_mask"] = [1] * difference + encoded_inputs["special_tokens_mask"]
-                encoded_inputs[self.model_input_names[0]] = [self.pad_token_id] * difference + required_input
-            else:
-                raise ValueError(f"Invalid padding strategy:{padding_side}")
-
-        return encoded_inputs
+        return BatchEncoding(encoded_inputs, tensor_type=return_tensors, prepend_batch_axis=prepend_batch_axis)
 
-    def pad(
-        self,
-        encoded_inputs: BatchEncoding
-        | list[BatchEncoding]
-        | dict[str, EncodedInput]
-        | dict[str, list[EncodedInput]]
-        | list[dict[str, EncodedInput]],
-        padding: bool | str | PaddingStrategy = True,
-        max_length: int | None = None,
-        pad_to_multiple_of: int | None = None,
-        padding_side: str | None = None,
-        return_attention_mask: bool | None = None,
-        return_tensors: str | TensorType | None = None,
-        verbose: bool = True,
-    ) -> BatchEncoding:
-        """
-        Pad a single encoded input or a batch of encoded inputs up to predefined length or to the max sequence length
-        in the batch.
-
-        Padding side (left/right) padding token ids are defined at the tokenizer level (with `self.padding_side`,
-        `self.pad_token_id`).
-        <Tip>
-
-        If the `encoded_inputs` passed are dictionary of numpy arrays, PyTorch tensors, the
-        result will use the same type unless you provide a different tensor type with `return_tensors`. In the case of
-        PyTorch tensors, you will lose the specific device of your tensors however.
-
-        </Tip>
-
-        Args:
-            encoded_inputs ([`BatchEncoding`], list of [`BatchEncoding`], `Dict[str, list[int]]`, `Dict[str, list[list[int]]` or `List[Dict[str, list[int]]]`):
-                Tokenized inputs. Can represent one input ([`BatchEncoding`] or `Dict[str, list[int]]`) or a batch of
-                tokenized inputs (list of [`BatchEncoding`], *Dict[str, list[list[int]]]* or *List[Dict[str,
-                list[int]]]*) so you can use this method during preprocessing as well as in a PyTorch Dataloader
-                collate function.
-
-                Instead of `list[int]` you can have tensors (numpy arrays, PyTorch tensors), see
-                the note above for the return type.
-            padding (`bool`, `str` or [`~utils.PaddingStrategy`], *optional*, defaults to `True`):
-                 Select a strategy to pad the returned sequences (according to the model's padding side and padding
-                 index) among:
-
-                - `True` or `'longest'` (default): Pad to the longest sequence in the batch (or no padding if only a single
-                  sequence if provided).
-                - `'max_length'`: Pad to a maximum length specified with the argument `max_length` or to the maximum
-                  acceptable input length for the model if that argument is not provided.
-                - `False` or `'do_not_pad'`: No padding (i.e., can output a batch with sequences of different
-                  lengths).
-            max_length (`int`, *optional*):
-                Maximum length of the returned list and optionally padding length (see above).
-            pad_to_multiple_of (`int`, *optional*):
-                If set will pad the sequence to a multiple of the provided value.
-
-                This is especially useful to enable the use of Tensor Cores on NVIDIA hardware with compute capability
-                `>= 7.5` (Volta).
-            padding_side (`str`, *optional*):
-                The side on which the model should have padding applied. Should be selected between ['right', 'left'].
-                Default value is picked from the class attribute of the same name.
-            return_attention_mask (`bool`, *optional*):
-                Whether to return the attention mask. If left to the default, will return the attention mask according
-                to the specific tokenizer's default, defined by the `return_outputs` attribute.
-
-                [What are attention masks?](../glossary#attention-mask)
-            return_tensors (`str` or [`~utils.TensorType`], *optional*):
-                If set, will return tensors instead of list of python integers. Acceptable values are:
-
-                - `'pt'`: Return PyTorch `torch.Tensor` objects.
-                - `'np'`: Return Numpy `np.ndarray` objects.
-            verbose (`bool`, *optional*, defaults to `True`):
-                Whether or not to print more information and warnings.
-        """
-        # If we have a list of dicts, let's convert it in a dict of lists
-        # We do this to allow using this method as a collate_fn function in PyTorch Dataloader
-        if isinstance(encoded_inputs, (list, tuple)) and isinstance(encoded_inputs[0], Mapping):
-            # Call .keys() explicitly for compatibility with TensorDict and other Mapping subclasses
-            encoded_inputs = {key: [example[key] for example in encoded_inputs] for key in encoded_inputs[0].keys()}
-
-        # The model's main input name, usually `input_ids`, has been passed for padding
-        if self.model_input_names[0] not in encoded_inputs:
-            raise ValueError(
-                "You should supply an encoding or a list of encodings to this method "
-                f"that includes {self.model_input_names[0]}, but you provided {list(encoded_inputs.keys())}"
-            )
-
-        required_input = encoded_inputs[self.model_input_names[0]]
-
-        if required_input is None or (isinstance(required_input, Sized) and len(required_input) == 0):
-            if return_attention_mask:
-                encoded_inputs["attention_mask"] = []
-            return encoded_inputs
-
-        # If we have PyTorch/NumPy tensors/arrays as inputs, we cast them as python objects
-        # and rebuild them afterwards if no return_tensors is specified
-        # Note that we lose the specific device the tensor may be on for PyTorch
-
-        first_element = required_input[0]
-        if isinstance(first_element, (list, tuple)):
-            # first_element might be an empty list/tuple in some edge cases so we grab the first non empty element.
-            for item in required_input:
-                if len(item) != 0:
-                    first_element = item[0]
-                    break
-        # At this state, if `first_element` is still a list/tuple, it's an empty one so there is nothing to do.
-        if not isinstance(first_element, (int, list, tuple)):
-            if is_torch_tensor(first_element):
-                return_tensors = "pt" if return_tensors is None else return_tensors
-            elif isinstance(first_element, np.ndarray):
-                return_tensors = "np" if return_tensors is None else return_tensors
-            else:
-                raise ValueError(
-                    f"type of {first_element} unknown: {type(first_element)}. "
-                    "Should be one of a python, numpy, or pytorch object."
-                )
-
-            for key, value in encoded_inputs.items():
-                encoded_inputs[key] = to_py_obj(value)
-
-        # Convert padding_strategy in PaddingStrategy
-        padding_strategy, _, max_length, _ = self._get_padding_truncation_strategies(
-            padding=padding, max_length=max_length, verbose=verbose
-        )
-
-        required_input = encoded_inputs[self.model_input_names[0]]
-        if required_input and not isinstance(required_input[0], (list, tuple)):
-            encoded_inputs = self._pad(
-                encoded_inputs,
-                max_length=max_length,
-                padding_strategy=padding_strategy,
-                pad_to_multiple_of=pad_to_multiple_of,
-                padding_side=padding_side,
-                return_attention_mask=return_attention_mask,
-            )
-            return BatchEncoding(encoded_inputs, tensor_type=return_tensors)
-
-        batch_size = len(required_input)
-        assert all(len(v) == batch_size for v in encoded_inputs.values()), (
-            "Some items in the output dictionary have a different batch size than others."
-        )
-
-        if padding_strategy == PaddingStrategy.LONGEST:
-            max_length = max(len(inputs) for inputs in required_input)
-            padding_strategy = PaddingStrategy.MAX_LENGTH
-
-        batch_outputs = {}
-        for i in range(batch_size):
-            inputs = {k: v[i] for k, v in encoded_inputs.items()}
-            outputs = self._pad(
-                inputs,
-                max_length=max_length,
-                padding_strategy=padding_strategy,
-                pad_to_multiple_of=pad_to_multiple_of,
-                padding_side=padding_side,
-                return_attention_mask=return_attention_mask,
-            )
-
-            for key, value in outputs.items():
-                if key not in batch_outputs:
-                    batch_outputs[key] = []
-                batch_outputs[key].append(value)
-
-        return BatchEncoding(batch_outputs, tensor_type=return_tensors)
-
-    def truncate_sequences(
+    def truncate_sequences(  # type: ignore[override]
         self,
         ids: list[int],
         pair_ids: None = None,
@@ -1407,47 +1000,36 @@ class MistralCommonBackend(PushToHubMixin):
             `Tuple[list[int], None, list[int]]`: The truncated `ids` and the list of
             overflowing tokens. `None` is returned to match Transformers signature.
         """
-        if kwargs:
-            raise ValueError(
-                f"Kwargs {list(kwargs.keys())} are not supported by `MistralCommonBackend.truncate_sequences`."
-            )
+
         if pair_ids:
             raise ValueError("`pair_ids` is not supported by `MistralCommonBackend.truncate_sequences`.")
 
-        if num_tokens_to_remove <= 0:
-            return (ids, None, [])
-
         if not isinstance(truncation_strategy, TruncationStrategy):
             truncation_strategy = TruncationStrategy(truncation_strategy)
 
-        if truncation_strategy in [TruncationStrategy.ONLY_FIRST, TruncationStrategy.ONLY_SECOND]:
-            raise ValueError(
-                f"Only {TruncationStrategy.LONGEST_FIRST} and {TruncationStrategy.DO_NOT_TRUNCATE} are supported."
-            )
+        if truncation_strategy in [
+            TruncationStrategy.ONLY_FIRST,
+            TruncationStrategy.ONLY_SECOND,
+        ]:
+            raise ValueError(f"{truncation_strategy=} is not supported by `MistralCommonBackend`.")
+
+        if num_tokens_to_remove <= 0:
+            return ids, None, []
 
         overflowing_tokens = []
-        if truncation_strategy == TruncationStrategy.LONGEST_FIRST:
-            if len(ids) > num_tokens_to_remove:
-                window_len = min(len(ids), stride + num_tokens_to_remove)
-                if self.truncation_side == "left":
-                    overflowing_tokens = ids[:window_len]
-                    ids = ids[num_tokens_to_remove:]
-                elif self.truncation_side == "right":
-                    overflowing_tokens = ids[-window_len:]
-                    ids = ids[:-num_tokens_to_remove]
-                else:
-                    raise ValueError(f"invalid truncation strategy: {self.truncation_side}, use 'left' or 'right'.")
 
+        if truncation_strategy == TruncationStrategy.LONGEST_FIRST:
+            window_len = min(len(ids), stride + num_tokens_to_remove)
+            if self.truncation_side == "left":
+                overflowing_tokens = ids[:window_len]
+                ids = ids[num_tokens_to_remove:]
             else:
-                error_msg = (
-                    f"We need to remove {num_tokens_to_remove} to truncate the input "
-                    f"but the first sequence has a length {len(ids)}. "
-                )
-                logger.error(error_msg)
+                overflowing_tokens = ids[-window_len:]
+                ids = ids[:-num_tokens_to_remove]
 
-        return (ids, None, overflowing_tokens)
+        return ids, None, overflowing_tokens
 
-    def apply_chat_template(
+    def apply_chat_template(  # type: ignore[override]
         self,
         conversation: list[dict[str, str]] | list[list[dict[str, str]]],
         tools: list[dict | Callable] | None = None,
@@ -1475,8 +1057,8 @@ class MistralCommonBackend(PushToHubMixin):
                 [chat templating guide](https://huggingface.co/docs/transformers/main/en/chat_templating#automated-function-conversion-for-tool-use)
                 for more information.
             add_generation_prompt (`bool`, *optional*):
-                This argument is a no-op for `MistralCommonBackend`. However it cannot be used at the same time as `continue_final_message` to keep the API consistent and
-                if any conversation ends with an assistant message, it will raise an error. In such case, use `continue_final_message` instead.
+                This argument is a no-op for `MistralCommonBackend`. However, it cannot be used at the same time as `continue_final_message` to keep the API consistent.
+                If any conversation ends with an assistant message, it will raise an error. In such cases, use `continue_final_message` instead.
             continue_final_message (bool, *optional*):
                 If this is set, the chat will be formatted so that the final
                 message in the chat is open-ended, without any EOS tokens. The model will continue this message
@@ -1511,8 +1093,7 @@ class MistralCommonBackend(PushToHubMixin):
                 Will raise an error if used.
 
         Returns:
-            `Union[str, list[int], list[str], list[list[int]], BatchEncoding]`: A list of token ids representing the tokenized chat so far, including control
-            tokens. This output is ready to pass to the model, either directly or via methods like `generate()`.
+            `Union[str, list[int], list[str], list[list[int]], BatchEncoding]`: The tokenized chat so far, including control tokens. This output is ready to pass to the model, either directly or via methods like `generate()`.
         """
         if kwargs:
             raise ValueError(
@@ -1659,6 +1240,83 @@ class MistralCommonBackend(PushToHubMixin):
             )
             return outputs
 
+    def build_inputs_with_special_tokens(self, token_ids_0: list[int], token_ids_1: None = None) -> list[int]:
+        """
+        Build model inputs from a sequence by adding special tokens.
+
+        This method dynamically builds inputs based on the tokenizer's `mode`:
+        - `"test"`: seq0 [EOS]
+        - `"finetuning"`: [BOS] seq0
+
+        Args:
+            token_ids_0 (`list[int]`):
+                List of IDs to which the special tokens will be added.
+            token_ids_1 (`None`, *optional*): None, kept to match Transformers' signature.
+
+        Returns:
+            `list[int]`: List of input IDs with the appropriate special tokens.
+        """
+        if token_ids_1 is not None:
+            raise ValueError(
+                "`MistralCommonBackend` does not implement `token_ids_1 != None` for `build_inputs_with_special_tokens`."
+            )
+
+        if self.mode == ValidationMode.test:
+            # [BOS] seq0
+            return [self.bos_token_id] + token_ids_0
+
+        else:
+            # [BOS] seq0 [EOS]
+            return [self.bos_token_id] + token_ids_0 + [self.eos_token_id]
+
+    def create_token_type_ids_from_sequences(self, token_ids_0: list[int], token_ids_1: None = None) -> list[int]:
+        """
+        Create a mask of zeroes from the token ids with special tokens added.
+
+        Kept to match Transformers' implementation.
+
+        Args:
+            token_ids_0 (`list[int]`):
+                List of IDs.
+            token_ids_1 (`None`, *optional*): None, kept to match Transformers' signature.
+
+
+        Returns:
+            `list[int]`: Token type IDs according to the configured pattern.
+        """
+        if token_ids_1 is not None:
+            raise ValueError(
+                "`MistralCommonBackend` does not implement `token_ids_1 != None` for `create_token_type_ids_from_sequences`."
+            )
+
+        sequence = self.build_inputs_with_special_tokens(token_ids_0)
+
+        return [0] * len(sequence)
+
+    def num_special_tokens_to_add(self, pair: Literal[False] = False) -> int:
+        """
+        Returns the number of added tokens when encoding a sequence with special tokens.
+
+        <Tip>
+
+        This encodes a dummy input and checks the number of added tokens, and is therefore not efficient. Do not put
+        this inside your training loop.
+
+        </Tip>
+
+        Args:
+            pair (`Literal[False]`, *optional*): False, kept to match Transformer's signature.
+
+        Returns:
+            `int`: Number of special tokens added to sequences.
+        """
+        if pair:
+            raise ValueError(
+                "`MistralCommonBackend` does not implement `pair = True` for `num_special_tokens_to_add`."
+            )
+
+        return len(self.build_inputs_with_special_tokens([], None))
+
     @add_end_docstrings(ENCODE_KWARGS_DOCSTRING, ENCODE_PLUS_ADDITIONAL_KWARGS_DOCSTRING)
     def __call__(
         self,
@@ -1679,6 +1337,8 @@ class MistralCommonBackend(PushToHubMixin):
         return_special_tokens_mask: bool = False,
         return_length: bool = False,
         verbose: bool = True,
+        return_offsets_mapping: Literal[False] = False,
+        split_special_tokens: Literal[False] = False,
         **kwargs,
     ) -> BatchEncoding:
         """
@@ -1696,92 +1356,49 @@ class MistralCommonBackend(PushToHubMixin):
             text_pair_target (`None`, *optional*):
                 Not supported by `MistralCommonBackend`. Kept to match the signature of `PreTrainedTokenizerBase.__call__`.
         """
-        if kwargs:
-            raise ValueError(f"Kwargs {list(kwargs.keys())} are not supported by `MistralCommonBackend.__call__`.")
+        if return_offsets_mapping or split_special_tokens:
+            raise ValueError(
+                "`MistralCommonBackend` does not support `return_offsets_mapping` and `split_special_tokens`."
+            )
 
-        if text_pair or text_target or text_pair_target:
+        if truncation in [TruncationStrategy.ONLY_FIRST, TruncationStrategy.ONLY_SECOND, "only_first", "only_second"]:
             raise ValueError(
-                "`text_pair`, `text_target` and `text_pair_target` are not supported by `MistralCommonBackend`."
+                "Truncation strategy `only_first` and `only_second` are not supported by `MistralCommonBackend`."
             )
 
-        def _is_valid_text_input(t):
-            if isinstance(t, str):
-                # Strings are fine
-                return True
-            elif isinstance(t, (list, tuple)):
-                # List are fine as long as they are...
-                if len(t) == 0:
-                    # ... empty
-                    return True
-                elif isinstance(t[0], (str, int)):
-                    # ... list of strings or int
-                    return True
-                elif isinstance(t[0], (list, tuple)):
-                    # ... list with an empty list or with a list of strings or with a list of ints
-                    return len(t[0]) == 0 or isinstance(t[0][0], (str, int))
-                else:
-                    return False
-            else:
-                return False
+        if kwargs:
+            raise ValueError(f"Kwargs {list(kwargs.keys())} are not supported by `MistralCommonBackend.__call__`.")
 
-        if not _is_valid_text_input(text):
+        if text_pair or text_target or text_pair_target:
             raise ValueError(
-                "text input must be of type `str` (single example), `list[str]` (batch or single encoded example) "
-                "or `list[list[int]]` (batch of encoded examples)."
+                "`text_pair`, `text_target` and `text_pair_target` are not supported by `MistralCommonBackend`."
             )
 
-        is_batched = isinstance(text, (list, tuple)) and isinstance(text[0], (str, list, tuple))
-
-        padding_strategy, truncation_strategy, max_length, kwargs = self._get_padding_truncation_strategies(
+        return super().__call__(
+            text=text,
+            text_pair=text_pair,
+            text_target=text_target,
+            add_special_tokens=add_special_tokens,
             padding=padding,
             truncation=truncation,
             max_length=max_length,
+            stride=stride,
             pad_to_multiple_of=pad_to_multiple_of,
+            padding_side=padding_side,
+            return_tensors=return_tensors,
+            return_attention_mask=return_attention_mask,
+            return_overflowing_tokens=return_overflowing_tokens,
+            return_special_tokens_mask=return_special_tokens_mask,
+            return_length=return_length,
             verbose=verbose,
-            **kwargs,
         )
 
-        if is_batched:
-            return self._batch_encode_plus(
-                batch_text=text,
-                add_special_tokens=add_special_tokens,
-                padding_strategy=padding_strategy,
-                truncation_strategy=truncation_strategy,
-                max_length=max_length,
-                stride=stride,
-                pad_to_multiple_of=pad_to_multiple_of,
-                padding_side=padding_side,
-                return_tensors=return_tensors,
-                return_attention_mask=return_attention_mask,
-                return_overflowing_tokens=return_overflowing_tokens,
-                return_special_tokens_mask=return_special_tokens_mask,
-                return_length=return_length,
-                verbose=verbose,
-            )
-        else:
-            return self._encode_plus(
-                text=text,
-                add_special_tokens=add_special_tokens,
-                padding_strategy=padding_strategy,
-                truncation_strategy=truncation_strategy,
-                max_length=max_length,
-                stride=stride,
-                pad_to_multiple_of=pad_to_multiple_of,
-                padding_side=padding_side,
-                return_tensors=return_tensors,
-                return_attention_mask=return_attention_mask,
-                return_overflowing_tokens=return_overflowing_tokens,
-                return_special_tokens_mask=return_special_tokens_mask,
-                return_length=return_length,
-                verbose=verbose,
-            )
-
     @classmethod
     def from_pretrained(
         cls,
         pretrained_model_name_or_path: str | os.PathLike,
         *init_inputs,
-        mode: Union[str, ValidationMode] = ValidationMode.test,
+        mode: str | ValidationMode = ValidationMode.test,
         cache_dir: str | os.PathLike | None = None,
         force_download: bool = False,
         local_files_only: bool = False,
@@ -1808,9 +1425,9 @@ class MistralCommonBackend(PushToHubMixin):
                   `./my_model_directory/`.
             mode (`Union[str, ValidationMode]`, *optional*, defaults to `ValidationMode.test`):
                 Validation mode for the `MistralTokenizer` tokenizer. Possible values are:
-                - `"finetuning"` or `ValidationMode.finetuning`: The finetuning mode.
+                - `"finetuning"` or `ValidationMode.finetuning`: The fine-tuning mode.
                 - `"test"` or `ValidationMode.test`: The test mode.
-                It changes how the tokenizer validates the input and prepare the request to the model.
+                It changes how the tokenizer validates the input and prepares the request to the model.
             cache_dir (`str` or `os.PathLike`, *optional*):
                 Path to a directory in which a downloaded predefined tokenizer vocabulary files should be cached if the
                 standard cache should not be used.
@@ -1837,11 +1454,11 @@ class MistralCommonBackend(PushToHubMixin):
                 Default value is picked from the class attribute of the same name.
             truncation_side (`str`, *optional*, defaults to `"right"`):
                 The side on which the model should have truncation applied. Should be selected between ['right', 'left'].
-            model_input_names (`List[string]`, *optional*):
+            model_input_names (`List[str]`, *optional*):
                 The list of inputs accepted by the forward pass of the model (like `"token_type_ids"` or
                 `"attention_mask"`). Default value is picked from the class attribute of the same name.
             clean_up_tokenization_spaces (`bool`, *optional*, defaults to `False`):
-                Whether or not the model should cleanup the spaces that were added when splitting the input text during the
+                Whether or not the model should clean up the spaces that were added when splitting the input text during the
                 tokenization process.
             kwargs (additional keyword arguments, *optional*):
                 Not supported by `MistralCommonBackend.from_pretrained`.
@@ -1851,11 +1468,13 @@ class MistralCommonBackend(PushToHubMixin):
             raise ValueError("`init_inputs` are not supported by `MistralCommonBackend.from_pretrained`.")
 
         # Handle kwargs and AutoTokenizer/AutoProcessor case
-        # These kwargs are passed by AutoTokenizer/AutoProcessor but are not used by MistralCommonBackend
-        if kwargs and not set(kwargs.keys()).issubset(
-            {"trust_remote_code", "_from_pipeline", "_commit_hash", "dtype", "_from_auto", "subfolder"}
-        ):
-            raise ValueError(f"Some kwargs in {kwargs} are not supported by `MistralCommonBackend.from_pretrained`.")
+        valid_kwargs = _VALID_INIT_KWARGS.union(
+            {"trust_remote_code", "_from_pipeline", "_commit_hash", "dtype", "subfolder"}
+        )
+        if kwargs and not set(kwargs.keys()).issubset(valid_kwargs):
+            raise ValueError(
+                f"Some kwargs in {list(kwargs.keys())} are not supported by `MistralCommonBackend.from_pretrained`."
+            )
 
         mode = cls._get_validation_mode(mode)
 
@@ -1869,35 +1488,8 @@ class MistralCommonBackend(PushToHubMixin):
                 local_files_only=local_files_only,
             )
         else:
-            valid_tokenizer_files = []
-            tokenizer_file: str
-
-            instruct_versions = list(TokenizerVersion.__members__)
-            mm_versions = list(MultiModalVersion.__members__) + [""]  # allow no mm version
-            sentencepiece_suffixes = [f".model.{v}{m}" for v in instruct_versions for m in mm_versions] + [".model"]
-
-            for path in os.listdir(pretrained_model_name_or_path):
-                pathlib_repo_file = Path(path)
-                file_name = pathlib_repo_file.name
-                suffix = "".join(pathlib_repo_file.suffixes)
-                if file_name == "tekken.json" or suffix in sentencepiece_suffixes:
-                    valid_tokenizer_files.append(file_name)
-
-            if len(valid_tokenizer_files) == 0:
-                raise ValueError(f"No tokenizer file found in directory: {pretrained_model_name_or_path}")
-            # If there are multiple tokenizer files, we use tekken.json if it exists, otherwise the versioned one.
-            if len(valid_tokenizer_files) > 1:
-                if "tekken.json" in valid_tokenizer_files:
-                    tokenizer_file = "tekken.json"
-                else:
-                    tokenizer_file = max(valid_tokenizer_files)
-                logger.warning(
-                    f"Multiple tokenizer files found in directory: {pretrained_model_name_or_path}. Using {tokenizer_file}."
-                )
-            else:
-                tokenizer_file = valid_tokenizer_files[0]
-
-            tokenizer_path = os.path.join(pretrained_model_name_or_path, tokenizer_file)
+            candidate_files = os.listdir(pretrained_model_name_or_path)
+            tokenizer_path = os.path.join(pretrained_model_name_or_path, get_one_valid_tokenizer_file(candidate_files))
 
         return cls(
             tokenizer_path=tokenizer_path,
@@ -1909,7 +1501,7 @@ class MistralCommonBackend(PushToHubMixin):
             clean_up_tokenization_spaces=clean_up_tokenization_spaces,
         )
 
-    def save_pretrained(
+    def save_pretrained(  # type: ignore[override]
         self,
         save_directory: str | os.PathLike | Path,
         push_to_hub: bool = False,
@@ -1971,7 +1563,7 @@ class MistralCommonBackend(PushToHubMixin):
         return (str(save_directory / self._tokenizer_path.name),)
 
     @staticmethod
-    def _get_validation_mode(mode: Union[str, ValidationMode]) -> ValidationMode:
+    def _get_validation_mode(mode: str | ValidationMode) -> ValidationMode:
         """Get the validation mode from a string or a ValidationMode."""
         _invalid_mode_msg = (
             f"Invalid `mistral-common` tokenizer mode: {mode}. Possible values are 'finetuning' or 'test'."
@@ -1988,6 +1580,65 @@ class MistralCommonBackend(PushToHubMixin):
             raise ValueError(_invalid_mode_msg)
         return mode
 
+    def add_special_tokens(
+        self,
+        special_tokens_dict: dict[str, str | AddedToken | Sequence[str | AddedToken]],
+        replace_extra_special_tokens: bool = True,
+    ):
+        r"""`MistralCommonBackend` does not implement `add_special_tokens` by design.
+
+        If you would like this behaviour to be implemented, please open an issue in the `Transformers` or `mistral-common` repositories to request it.
+        """
+
+        raise NotImplementedError("`MistralCommonBackend` does not implement `add_special_tokens`.")
+
+    def add_tokens(  # type: ignore[override]
+        self,
+        special_tokens_dict: dict[str, str | AddedToken | Sequence[str | AddedToken]],
+        replace_extra_special_tokens: bool = True,
+    ):
+        """
+        `MistralCommonBackend` does not implement `add_special_tokens` by design.
+
+        If you would like this behaviour to be implemented, please open an issue in the `Transformers` or `mistral-common` repositories to request it.
+        """
+
+        raise NotImplementedError("`MistralCommonBackend` does not implement `add_tokens`.")
+
+    def convert_added_tokens(cls, obj: AddedToken | Any, save: bool = False, add_type_field: bool = True):  # type: ignore[override]
+        """
+        `MistralCommonBackend` does not implement `convert_added_tokens` by design.
+
+        If you would like this behaviour to be implemented, please open an issue in the `Transformers` or `mistral-common` repositories to request it.
+        """
+
+        raise NotImplementedError("`MistralCommonBackend` does not implement `convert_added_tokens`.")
+
+    def get_chat_template(self, chat_template: str | None = None, tools: list[dict] | None = None) -> str:
+        """`MistralCommonBackend` does not implement `get_chat_template` by design as `mistral-common` does not use chat templates."""
+
+        raise NotImplementedError("`MistralCommonBackend` does not implement `get_chat_template`.")
+
+    def save_chat_templates(
+        self,
+        save_directory: str | os.PathLike,
+        tokenizer_config: dict,
+        filename_prefix: str | None,
+        save_jinja_files: bool,
+    ):
+        """`MistralCommonBackend` does not implement `save_chat_templates` by design as `mistral-common` does not use chat templates."""
+
+        raise NotImplementedError("`MistralCommonBackend` does not implement `save_chat_templates`.")
+
+    def save_vocabulary(self, save_directory: str, filename_prefix: str | None = None) -> tuple[str, ...]:
+        """
+        `MistralCommonBackend` does not implement `save_vocabulary` by design.
+
+        This is because `mistral-common` is configured by one tokenizer file. If you'd like to save the vocabulary, please consider using the `save_pretrained` method instead.
+        """
+
+        raise NotImplementedError("`MistralCommonBackend` does not implement `save_vocabulary`.")
+
 
 # Backward compatibility alias for codebases still importing the legacy name.
 MistralCommonTokenizer = MistralCommonBackend