transformers 5.0.0__py3-none-any.whl → 5.0.0rc0__py3-none-any.whl

transformers/tokenization_utils_base.py

@@ -1,4 +1,5 @@
 # base
+# coding=utf-8
 # Copyright 2020 The HuggingFace Inc. team.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -18,21 +19,19 @@ fronting encoding methods) Special token mixing (host the special tokens logic)
 of output with special method for the Fast tokenizers)
 """
 
-from __future__ import annotations
-
 import copy
 import json
 import os
 import re
 import warnings
 from collections import OrderedDict, UserDict
-from collections.abc import Callable, Collection, Mapping, Sequence, Sized
+from collections.abc import Callable, Mapping, Sequence, Sized
 from dataclasses import dataclass
 from pathlib import Path
-from typing import TYPE_CHECKING, Any, NamedTuple, Union
+from typing import TYPE_CHECKING, Any, NamedTuple, Optional, Union
 
 import numpy as np
-from huggingface_hub import create_repo, is_offline_mode, list_repo_files
+from huggingface_hub import create_repo, list_repo_files
 from packaging import version
 
 from . import __version__
@@ -50,6 +49,7 @@ from .utils import (
     extract_commit_hash,
     is_mlx_available,
     is_numpy_array,
+    is_offline_mode,
     is_protobuf_available,
     is_tokenizers_available,
     is_torch_available,
@@ -60,7 +60,6 @@ from .utils import (
     requires_backends,
     to_py_obj,
 )
-from .utils.chat_parsing_utils import recursive_parse
 from .utils.chat_template_utils import render_jinja_template
 from .utils.import_utils import PROTOBUF_IMPORT_ERROR
 
@@ -218,11 +217,11 @@ class BatchEncoding(UserDict):
 
     def __init__(
         self,
-        data: dict[str, Any] | None = None,
-        encoding: EncodingFast | Sequence[EncodingFast] | None = None,
-        tensor_type: None | str | TensorType = None,
+        data: Optional[dict[str, Any]] = None,
+        encoding: Optional[Union[EncodingFast, Sequence[EncodingFast]]] = None,
+        tensor_type: Union[None, str, TensorType] = None,
         prepend_batch_axis: bool = False,
-        n_sequences: int | None = None,
+        n_sequences: Optional[int] = None,
     ):
         super().__init__(data)
 
@@ -240,7 +239,7 @@ class BatchEncoding(UserDict):
         self.convert_to_tensors(tensor_type=tensor_type, prepend_batch_axis=prepend_batch_axis)
 
     @property
-    def n_sequences(self) -> int | None:
+    def n_sequences(self) -> Optional[int]:
         """
         `Optional[int]`: The number of sequences used to generate each sample from the batch encoded in this
         [`BatchEncoding`]. Currently can be one of `None` (unknown), `1` (a single sentence) or `2` (a pair of
@@ -248,7 +247,7 @@ class BatchEncoding(UserDict):
         """
         return self._n_sequences
 
-    def __getitem__(self, item: int | str) -> Any | EncodingFast:
+    def __getitem__(self, item: Union[int, str]) -> Union[Any, EncodingFast]:
         """
         If the key is a string, returns the value of the dict associated to `key` ('input_ids', 'attention_mask',
         etc.).
@@ -298,7 +297,7 @@ class BatchEncoding(UserDict):
         return self._encodings is not None
 
     @property
-    def encodings(self) -> list[EncodingFast] | None:
+    def encodings(self) -> Optional[list[EncodingFast]]:
         """
         `Optional[list[tokenizers.Encoding]]`: The list all encodings from the tokenization process. Returns `None` if
         the input was tokenized through Python (i.e., not a fast) tokenizer.
@@ -323,7 +322,7 @@ class BatchEncoding(UserDict):
             )
         return self._encodings[batch_index].tokens
 
-    def sequence_ids(self, batch_index: int = 0) -> list[int | None]:
+    def sequence_ids(self, batch_index: int = 0) -> list[Optional[int]]:
         """
         Return a list mapping the tokens to the id of their original sentences:
 
@@ -347,7 +346,7 @@ class BatchEncoding(UserDict):
             )
         return self._encodings[batch_index].sequence_ids
 
-    def word_ids(self, batch_index: int = 0) -> list[int | None]:
+    def word_ids(self, batch_index: int = 0) -> list[Optional[int]]:
         """
         Return a list mapping the tokens to their actual word in the initial sentence for a fast tokenizer.
 
@@ -366,7 +365,7 @@ class BatchEncoding(UserDict):
             )
         return self._encodings[batch_index].word_ids
 
-    def token_to_sequence(self, batch_or_token_index: int, token_index: int | None = None) -> int:
+    def token_to_sequence(self, batch_or_token_index: int, token_index: Optional[int] = None) -> int:
         """
         Get the index of the sequence represented by the given token. In the general use case, this method returns `0`
         for a single sequence or the first sequence of a pair, and `1` for the second sequence of a pair
@@ -405,7 +404,7 @@ class BatchEncoding(UserDict):
             token_index = self._seq_len + token_index
         return self._encodings[batch_index].token_to_sequence(token_index)
 
-    def token_to_word(self, batch_or_token_index: int, token_index: int | None = None) -> int:
+    def token_to_word(self, batch_or_token_index: int, token_index: Optional[int] = None) -> int:
         """
         Get the index of the word corresponding (i.e. comprising) to an encoded token in a sequence of the batch.
 
@@ -444,8 +443,8 @@ class BatchEncoding(UserDict):
         return self._encodings[batch_index].token_to_word(token_index)
 
     def word_to_tokens(
-        self, batch_or_word_index: int, word_index: int | None = None, sequence_index: int = 0
-    ) -> TokenSpan | None:
+        self, batch_or_word_index: int, word_index: Optional[int] = None, sequence_index: int = 0
+    ) -> Optional[TokenSpan]:
         """
         Get the encoded token span corresponding to a word in a sequence of the batch.
 
@@ -496,7 +495,7 @@ class BatchEncoding(UserDict):
         span = self._encodings[batch_index].word_to_tokens(word_index, sequence_index)
         return TokenSpan(*span) if span is not None else None
 
-    def token_to_chars(self, batch_or_token_index: int, token_index: int | None = None) -> CharSpan | None:
+    def token_to_chars(self, batch_or_token_index: int, token_index: Optional[int] = None) -> Optional[CharSpan]:
         """
         Get the character span corresponding to an encoded token in a sequence of the batch.
 
@@ -535,7 +534,9 @@ class BatchEncoding(UserDict):
 
         return CharSpan(*span_indices) if span_indices is not None else None
 
-    def char_to_token(self, batch_or_char_index: int, char_index: int | None = None, sequence_index: int = 0) -> int:
+    def char_to_token(
+        self, batch_or_char_index: int, char_index: Optional[int] = None, sequence_index: int = 0
+    ) -> int:
         """
         Get the index of the token in the encoded output comprising a character in the original string for a sequence
         of the batch.
@@ -576,7 +577,7 @@ class BatchEncoding(UserDict):
         return self._encodings[batch_index].char_to_token(char_index, sequence_index)
 
     def word_to_chars(
-        self, batch_or_word_index: int, word_index: int | None = None, sequence_index: int = 0
+        self, batch_or_word_index: int, word_index: Optional[int] = None, sequence_index: int = 0
     ) -> CharSpan:
         """
         Get the character span in the original string corresponding to given word in a sequence of the batch.
@@ -620,7 +621,7 @@ class BatchEncoding(UserDict):
             word_index = batch_or_word_index
         return CharSpan(*(self._encodings[batch_index].word_to_chars(word_index, sequence_index)))
 
-    def char_to_word(self, batch_or_char_index: int, char_index: int | None = None, sequence_index: int = 0) -> int:
+    def char_to_word(self, batch_or_char_index: int, char_index: Optional[int] = None, sequence_index: int = 0) -> int:
         """
         Get the word in the original string corresponding to a character in the original string of a sequence of the
         batch.
@@ -659,7 +660,9 @@ class BatchEncoding(UserDict):
             char_index = batch_or_char_index
         return self._encodings[batch_index].char_to_word(char_index, sequence_index)
 
-    def convert_to_tensors(self, tensor_type: str | TensorType | None = None, prepend_batch_axis: bool = False):
+    def convert_to_tensors(
+        self, tensor_type: Optional[Union[str, TensorType]] = None, prepend_batch_axis: bool = False
+    ):
         """
         Convert the inner content to tensors.
 
@@ -753,7 +756,7 @@ class BatchEncoding(UserDict):
 
         return self
 
-    def to(self, device: str | torch.device, *, non_blocking: bool = False) -> BatchEncoding:
+    def to(self, device: Union[str, "torch.device"], *, non_blocking: bool = False) -> "BatchEncoding":
         """
         Send all values to device by calling `v.to(device, non_blocking=non_blocking)` (PyTorch only).
 
@@ -963,11 +966,11 @@ class PreTrainedTokenizerBase(PushToHubMixin):
 
     vocab_files_names: dict[str, str] = {}
     pretrained_vocab_files_map: dict[str, dict[str, str]] = {}
-    _auto_class: str | None = None
+    _auto_class: Optional[str] = None
 
     # first name has to correspond to main model input name
     # to make sure `tokenizer.pad(...)` works correctly
-    model_input_names: list[str] = ["input_ids", "attention_mask"]
+    model_input_names: list[str] = ["input_ids", "token_type_ids", "attention_mask"]
     padding_side: str = "right"
     truncation_side: str = "right"
     slow_tokenizer_class = None
@@ -1099,7 +1102,7 @@ class PreTrainedTokenizerBase(PushToHubMixin):
     # ---- Special tokens API (moved from SpecialTokensMixin) ----
     def add_special_tokens(
         self,
-        special_tokens_dict: dict[str, str | AddedToken | Sequence[str | AddedToken]],
+        special_tokens_dict: dict[str, Union[str, AddedToken, Sequence[Union[str, AddedToken]]]],
         replace_extra_special_tokens=True,
     ) -> int:
         """
@@ -1203,7 +1206,7 @@ class PreTrainedTokenizerBase(PushToHubMixin):
         return self.add_tokens(tokens_to_add, special_tokens=True)
 
     def add_tokens(
-        self, new_tokens: str | AddedToken | Sequence[str | AddedToken], special_tokens: bool = False
+        self, new_tokens: Union[str, AddedToken, Sequence[Union[str, AddedToken]]], special_tokens: bool = False
     ) -> int:
         """
         #TODO remove this from here! PreTrainedTOkeniuzerBase should be agnostic of AddedToken.
@@ -1243,7 +1246,7 @@ class PreTrainedTokenizerBase(PushToHubMixin):
             new_tokens = [new_tokens]
         return self._add_tokens(new_tokens, special_tokens=special_tokens)
 
-    def _add_tokens(self, new_tokens: list[str] | list[AddedToken], special_tokens: bool = False) -> int:
+    def _add_tokens(self, new_tokens: Union[list[str], list[AddedToken]], special_tokens: bool = False) -> int:
         raise NotImplementedError
 
     @property
@@ -1328,7 +1331,7 @@ class PreTrainedTokenizerBase(PushToHubMixin):
             return super().__getattr__(key)
 
     def get_special_tokens_mask(
-        self, token_ids_0: list[int], token_ids_1: list[int] | None = None, already_has_special_tokens: bool = False
+        self, token_ids_0: list[int], token_ids_1: Optional[list[int]] = None, already_has_special_tokens: bool = False
     ) -> list[int]:
         """
         Retrieve sequence ids from a token list that has no special tokens added.
@@ -1417,7 +1420,7 @@ class PreTrainedTokenizerBase(PushToHubMixin):
         """
         return self.convert_tokens_to_ids(self.all_special_tokens)
 
-    def _set_model_specific_special_tokens(self, special_tokens: dict[str, str | AddedToken]):
+    def _set_model_specific_special_tokens(self, special_tokens: dict[str, Union[str, AddedToken]]):
         """
         Adds new model-specific special tokens (e.g., for multimodal models).
 
@@ -1470,7 +1473,7 @@ class PreTrainedTokenizerBase(PushToHubMixin):
         """
         raise NotImplementedError()
 
-    def convert_tokens_to_ids(self, tokens: str | list[str]) -> int | list[int]:
+    def convert_tokens_to_ids(self, tokens: Union[str, list[str]]) -> Union[int, list[int]]:
         """
         Converts a token string (or a sequence of tokens) in a single integer id (or a sequence of ids), using the
         vocabulary.
@@ -1486,7 +1489,9 @@ class PreTrainedTokenizerBase(PushToHubMixin):
 
         return [self._convert_token_to_id_with_added_voc(token) for token in tokens]
 
-    def convert_ids_to_tokens(self, ids: int | list[int], skip_special_tokens: bool = False) -> str | list[str]:
+    def convert_ids_to_tokens(
+        self, ids: Union[int, list[int]], skip_special_tokens: bool = False
+    ) -> Union[str, list[str]]:
         """
         Converts a single index or a sequence of indices in a token or a sequence of tokens, using the vocabulary and
         added tokens.
@@ -1505,12 +1510,12 @@ class PreTrainedTokenizerBase(PushToHubMixin):
     @classmethod
     def from_pretrained(
         cls,
-        pretrained_model_name_or_path: str | os.PathLike,
+        pretrained_model_name_or_path: Union[str, os.PathLike],
         *init_inputs,
-        cache_dir: str | os.PathLike | None = None,
+        cache_dir: Optional[Union[str, os.PathLike]] = None,
         force_download: bool = False,
         local_files_only: bool = False,
-        token: str | bool | None = None,
+        token: Optional[Union[str, bool]] = None,
         revision: str = "main",
         trust_remote_code=False,
         **kwargs,
@@ -1607,7 +1612,6 @@ class PreTrainedTokenizerBase(PushToHubMixin):
 
         pretrained_model_name_or_path = str(pretrained_model_name_or_path)
         vocab_files = {}
-        additional_files_names = {}
         init_configuration = {}
 
         is_local = os.path.isdir(pretrained_model_name_or_path)
@@ -1625,9 +1629,11 @@ class PreTrainedTokenizerBase(PushToHubMixin):
                     f"Calling {cls.__name__}.from_pretrained() with the path to a single file or url is not "
                     "supported for this tokenizer. Use a model identifier or the path to a directory instead."
                 )
-            file_id = "vocab_file"
-            if pretrained_model_name_or_path.endswith("tokenizer.json"):
-                file_id = "tokenizer_file"
+            # Use first vocab file that's not tokenizer_file
+            file_id = list(cls.vocab_files_names.keys())[0]
+            if file_id == "tokenizer_file" and vocab_files_count > 1:
+                file_id = [k for k in cls.vocab_files_names.keys() if k != "tokenizer_file"][0]
+
             vocab_files[file_id] = pretrained_model_name_or_path
             single_file_id = file_id
         else:
@@ -1645,10 +1651,10 @@ class PreTrainedTokenizerBase(PushToHubMixin):
                 }
 
             vocab_files = {**cls.vocab_files_names, **additional_files_names}
-
-            # Check for versioned tokenizer files
             if "tokenizer_file" in vocab_files:
+                # Try to get the tokenizer config to see if there are versioned tokenizer files.
                 fast_tokenizer_file = FULL_TOKENIZER_FILE
+
                 try:
                     resolved_config_file = cached_file(
                         pretrained_model_name_or_path,
@@ -1664,33 +1670,43 @@ class PreTrainedTokenizerBase(PushToHubMixin):
                         _raise_exceptions_for_missing_entries=False,
                         _commit_hash=commit_hash,
                     )
-                    if resolved_config_file is not None:
-                        with open(resolved_config_file, encoding="utf-8") as reader:
-                            tokenizer_config = json.load(reader)
-                            if "fast_tokenizer_files" in tokenizer_config:
-                                fast_tokenizer_file = get_fast_tokenizer_file(tokenizer_config["fast_tokenizer_files"])
-                        commit_hash = extract_commit_hash(resolved_config_file, commit_hash)
+                except OSError:
+                    # Re-raise any error raised by cached_file in order to get a helpful error message
+                    raise
                 except Exception:
-                    pass
+                    # For any other exception, we throw a generic error.
+                    raise OSError(
+                        f"Can't load tokenizer for '{pretrained_model_name_or_path}'. If you were trying to load it from "
+                        "'https://huggingface.co/models', make sure you don't have a local directory with the same name. "
+                        f"Otherwise, make sure '{pretrained_model_name_or_path}' is the correct path to a directory "
+                        f"containing all relevant files for a {cls.__name__} tokenizer."
+                    )
+
+                commit_hash = extract_commit_hash(resolved_config_file, commit_hash)
+                if resolved_config_file is not None:
+                    with open(resolved_config_file, encoding="utf-8") as reader:
+                        tokenizer_config = json.load(reader)
+                        if "fast_tokenizer_files" in tokenizer_config:
+                            fast_tokenizer_file = get_fast_tokenizer_file(tokenizer_config["fast_tokenizer_files"])
                 vocab_files["tokenizer_file"] = fast_tokenizer_file
 
-            # This block looks for any extra chat template files
-            if is_local:
-                template_dir = Path(pretrained_model_name_or_path, CHAT_TEMPLATE_DIR)
-                if template_dir.is_dir():
-                    for template_file in template_dir.glob("*.jinja"):
-                        template_name = template_file.name.removesuffix(".jinja")
-                        vocab_files[f"chat_template_{template_name}"] = f"{CHAT_TEMPLATE_DIR}/{template_file.name}"
-            else:
-                for template in list_repo_templates(
-                    pretrained_model_name_or_path,
-                    local_files_only=local_files_only,
-                    revision=revision,
-                    cache_dir=cache_dir,
-                    token=token,
-                ):
-                    template = template.removesuffix(".jinja")
-                    vocab_files[f"chat_template_{template}"] = f"{CHAT_TEMPLATE_DIR}/{template}.jinja"
+                # This block looks for any extra chat template files
+                if is_local:
+                    template_dir = Path(pretrained_model_name_or_path, CHAT_TEMPLATE_DIR)
+                    if template_dir.is_dir():
+                        for template_file in template_dir.glob("*.jinja"):
+                            template_name = template_file.name.removesuffix(".jinja")
+                            vocab_files[f"chat_template_{template_name}"] = f"{CHAT_TEMPLATE_DIR}/{template_file.name}"
+                else:
+                    for template in list_repo_templates(
+                        pretrained_model_name_or_path,
+                        local_files_only=local_files_only,
+                        revision=revision,
+                        cache_dir=cache_dir,
+                        token=token,
+                    ):
+                        template = template.removesuffix(".jinja")
+                        vocab_files[f"chat_template_{template}"] = f"{CHAT_TEMPLATE_DIR}/{template}.jinja"
 
         remote_files = []
         if not is_local and not local_files_only:
@@ -1748,6 +1764,11 @@ class PreTrainedTokenizerBase(PushToHubMixin):
             if file_id not in resolved_vocab_files:
                 continue
 
+            if is_local:
+                logger.info(f"loading file {file_path}")
+            else:
+                logger.info(f"loading file {file_path} from cache at {resolved_vocab_files[file_id]}")
+
         return cls._from_pretrained(
             resolved_vocab_files,
             pretrained_model_name_or_path,
@@ -1777,6 +1798,29 @@ class PreTrainedTokenizerBase(PushToHubMixin):
         trust_remote_code=False,
         **kwargs,
     ):
+        # We instantiate fast tokenizers based on a slow tokenizer if we don't have access to the tokenizer.json
+        # file or if `from_slow` is set to True.
+        from_slow = kwargs.get("from_slow", False)
+        gguf_file = kwargs.get("gguf_file")
+        has_tokenizer_file = resolved_vocab_files.get("tokenizer_file", None) is not None
+
+        # If one passes a GGUF file path to `gguf_file` there is no need for this check as the tokenizer will be
+        # loaded directly from the GGUF file.
+        if (from_slow or not has_tokenizer_file) and cls.slow_tokenizer_class is not None and not gguf_file:
+            slow_tokenizer = (cls.slow_tokenizer_class)._from_pretrained(
+                copy.deepcopy(resolved_vocab_files),
+                pretrained_model_name_or_path,
+                copy.deepcopy(init_configuration),
+                *init_inputs,
+                token=token,
+                cache_dir=cache_dir,
+                local_files_only=local_files_only,
+                _commit_hash=_commit_hash,
+                **(copy.deepcopy(kwargs)),
+            )
+        else:
+            slow_tokenizer = None
+
         # Prepare tokenizer initialization kwargs
         # Did we saved some inputs and kwargs to reload ?
         tokenizer_config_file = resolved_vocab_files.pop("tokenizer_config_file", None)
@@ -1785,16 +1829,14 @@ class PreTrainedTokenizerBase(PushToHubMixin):
                 init_kwargs = json.load(tokenizer_config_handle)
             # used in the past to check if the tokenizer class matches the class in the repo
             init_kwargs.pop("tokenizer_class", None)
+            if not has_tokenizer_file:
+                init_kwargs.get("tokenizer_file", None)
             saved_init_inputs = init_kwargs.pop("init_inputs", ())
             if not init_inputs:
                 init_inputs = saved_init_inputs
         else:
             init_kwargs = init_configuration
 
-        if resolved_vocab_files.get("tokenizer_file", None) is not None:
-            init_kwargs.pop("add_bos_token", None)
-            init_kwargs.pop("add_eos_token", None)
-
         # If independent chat template file(s) exist, they take priority over template entries in the tokenizer config
         chat_templates = {}
         chat_template_file = resolved_vocab_files.pop("chat_template_file", None)
@@ -1875,6 +1917,8 @@ class PreTrainedTokenizerBase(PushToHubMixin):
                 init_kwargs[args_name] = file_path
         tokenizer_file = resolved_vocab_files.get("tokenizer_file", None)
 
+        if slow_tokenizer is not None:
+            init_kwargs["__slow_tokenizer"] = slow_tokenizer
         init_kwargs["name_or_path"] = pretrained_model_name_or_path
         init_kwargs["is_local"] = _is_local
 
@@ -1993,12 +2037,28 @@ class PreTrainedTokenizerBase(PushToHubMixin):
             if key in init_kwargs and added_tokens_map != {} and init_kwargs[key] is not None:
                 init_kwargs[key] = added_tokens_map.get(str(init_kwargs[key]), init_kwargs[key])
 
-        # From pretrained with the legacy fixes
-        # for `tokenizers` based tokenizer, we actually want to have vocab and merges pre-extracted from whatever inputs
-        # for `none` (PythonBackend) based tokenizer, we also want the vocab file / merge files not extracted.
-        # for `sentencepiece` based tokenizer, we pass the sentencepiece model file directly.
-        init_kwargs = cls.convert_to_native_format(**init_kwargs)
+        # Track which files were loaded (if not already set by AutoTokenizer)
+        if "files_loaded" not in init_kwargs:
+            files_loaded = []
+            # Check which files this tokenizer class actually uses based on vocab_files_names
+            tokenizer_needs_files = set(cls.vocab_files_names.keys()) if hasattr(cls, "vocab_files_names") else set()
+
+            # If tokenizer_file is in the class's vocab_files_names and exists, prioritize it (TokenizersBackend)
+            if "tokenizer_file" in tokenizer_needs_files and resolved_vocab_files.get("tokenizer_file"):
+                files_loaded.append(os.path.basename(resolved_vocab_files["tokenizer_file"]))
+            else:
+                # Otherwise, add the actual vocab files that were used by this tokenizer class
+                for file_key, file_path in resolved_vocab_files.items():
+                    if (
+                        file_path
+                        and file_key not in ["tokenizer_config_file", "special_tokens_map_file", "added_tokens_file"]
+                        and file_key in tokenizer_needs_files
+                    ):
+                        # Extract just the filename from the path
+                        files_loaded.append(os.path.basename(file_path))
+            init_kwargs["files_loaded"] = files_loaded
 
+        # Instantiate the tokenizer.
         try:
             tokenizer = cls(*init_inputs, **init_kwargs)
         except import_protobuf_decode_error():
@@ -2019,14 +2079,120 @@ class PreTrainedTokenizerBase(PushToHubMixin):
                 "Unable to load vocabulary from file. "
                 "Please check that the provided vocabulary is accessible and not corrupted."
             )
-        return tokenizer
 
-    @classmethod
-    def convert_to_native_format(cls, **kwargs):
-        return kwargs
+        # If tokenizer_file exists and tokenizer has a TokenizersBackend, replace the blank tokenizer with tokenizer.json
+        if tokenizer_file is not None and hasattr(tokenizer, "_tokenizer"):
+            from tokenizers import Tokenizer as TokenizerFast
+
+            tokenizer._tokenizer = TokenizerFast.from_file(tokenizer_file)
+            # Re-run post-initialization if the tokenizer has it
+            if hasattr(tokenizer, "_post_init"):
+                tokenizer._post_init()
+            # If only SPM exists, try to get vocab and merges and init to load a tokenizers-backend
+        else:
+            spm_filename = find_sentencepiece_model_file(
+                pretrained_model_name_or_path,
+                revision=kwargs.get("revision"),
+                token=kwargs.get("token"),
+                cache_dir=kwargs.get("cache_dir"),
+                local_files_only=kwargs.get("local_files_only", False),
+                subfolder=kwargs.get("subfolder", ""),
+            )
+            if spm_filename is not None:
+                try:
+                    resolved_spm = cached_file(
+                        pretrained_model_name_or_path,
+                        spm_filename,
+                        cache_dir=kwargs.get("cache_dir"),
+                        force_download=kwargs.get("force_download", False),
+                        proxies=kwargs.get("proxies"),
+                        token=kwargs.get("token"),
+                        revision=kwargs.get("revision"),
+                        local_files_only=kwargs.get("local_files_only", False),
+                        subfolder=kwargs.get("subfolder", ""),
+                    )
+                except Exception:
+                    resolved_spm = None
+                if resolved_spm is not None:
+                    try:
+                        # Mirror AutoTokenizer fallback: extract vocab/merges from SentencePiece
+                        import inspect as _inspect
+
+                        from .tokenization_utils_sentencepiece import SentencePieceExtractor
+
+                        class_sig = _inspect.signature(getattr(cls, "__init__", cls))
+                        vocab_ids, vocab_scores, merges = SentencePieceExtractor(resolved_spm).extract()
+                        files_loaded = [spm_filename]
+                        init_kwargs["backend"] = "tokenizers"
+                        init_kwargs["files_loaded"] = files_loaded
+                        # If tokenizer needs merges too (BPE), pass both; unigram models only need vocab
+                        if "merges" in class_sig.parameters:
+                            return cls.from_pretrained(
+                                pretrained_model_name_or_path,
+                                *init_inputs,
+                                vocab=vocab_scores,
+                                merges=merges,
+                                **init_kwargs,
+                            )
+                        elif "vocab" in class_sig.parameters:
+                            return cls.from_pretrained(
+                                pretrained_model_name_or_path,
+                                *init_inputs,
+                                vocab=vocab_scores,
+                                **init_kwargs,
+                            )
+                    except Exception as e:
+                        logger.warning(
+                            f"Could not extract vocab/merges from the SentencePiece model to initialize a Tokenizers backend: {e}. We are falling back so we are falling back to the standard loading method."
+                        )
+                        pass
+            # Fallback to vocab.json + merges.txt (BPE) or just vocab.json (WordLevel/WordPiece)
+            vocab, merges, files_loaded = load_vocab_and_merges(
+                pretrained_model_name_or_path,
+                cache_dir=kwargs.get("cache_dir"),
+                force_download=kwargs.get("force_download", False),
+                proxies=kwargs.get("proxies"),
+                token=kwargs.get("token"),
+                revision=kwargs.get("revision"),
+                local_files_only=kwargs.get("local_files_only", False),
+                subfolder=kwargs.get("subfolder", ""),
+            )
+
+            if vocab is not None:
+                try:
+                    import inspect as _inspect
+
+                    class_sig = _inspect.signature(getattr(cls, "__init__", cls))
+                    init_kwargs["backend"] = "tokenizers"
+                    init_kwargs["files_loaded"] = files_loaded
+
+                    if merges is not None and "merges" in class_sig.parameters:
+                        return cls.from_pretrained(
+                            pretrained_model_name_or_path,
+                            *init_inputs,
+                            vocab=vocab,
+                            merges=merges,
+                            **init_kwargs,
+                        )
+                    elif "vocab" in class_sig.parameters:
+                        return cls.from_pretrained(
+                            pretrained_model_name_or_path,
+                            *init_inputs,
+                            vocab=vocab,
+                            **init_kwargs,
+                        )
+                except Exception:
+                    pass
+        if added_tokens_decoder != {} and max(list(added_tokens_decoder.keys())[-1], 0) > tokenizer.vocab_size:
+            logger.info(
+                "Special tokens have been added in the vocabulary, make sure the associated word embeddings are"
+                " fine-tuned or trained."
+            )
+
+        return tokenizer
 
     @classmethod
-    def convert_added_tokens(cls, obj: AddedToken | Any, save=False, add_type_field=True):
+    def convert_added_tokens(cls, obj: Union[AddedToken, Any], save=False, add_type_field=True):
         if isinstance(obj, dict) and "__type" in obj and obj["__type"] == "AddedToken":
             obj.pop("__type")
             return AddedToken(**obj)
@@ -2046,9 +2212,9 @@ class PreTrainedTokenizerBase(PushToHubMixin):
 
     def save_pretrained(
         self,
-        save_directory: str | os.PathLike,
-        legacy_format: bool | None = None,
-        filename_prefix: str | None = None,
+        save_directory: Union[str, os.PathLike],
+        legacy_format: Optional[bool] = None,
+        filename_prefix: Optional[str] = None,
         push_to_hub: bool = False,
         **kwargs,
     ) -> tuple[str, ...]:
@@ -2105,13 +2271,9 @@ class PreTrainedTokenizerBase(PushToHubMixin):
         )
 
         tokenizer_config = copy.deepcopy(self.init_kwargs)
-        tokenizer_config.pop("add_bos_token", None)
-        tokenizer_config.pop("add_eos_token", None)
 
         # Let's save the init kwargs
         target_keys = set(self.init_kwargs.keys())
-        target_keys.discard("add_bos_token")
-        target_keys.discard("add_eos_token")
         # Let's save the special tokens map (only the strings)
         target_keys.update(["model_max_length"])
 
@@ -2146,10 +2308,9 @@ class PreTrainedTokenizerBase(PushToHubMixin):
         # Add tokenizer class to the tokenizer config to be able to reload it with from_pretrained
         tokenizer_class = self.__class__.__name__
 
-        # tokenizers backend don't need to save added_tokens_decoder and additional_special_tokens
+        # tokenizers backend don't need to save added_tokens_decoder
         if any(base.__name__ == "TokenizersBackend" for base in self.__class__.__mro__):
             tokenizer_config.pop("added_tokens_decoder", None)
-            tokenizer_config.pop("additional_special_tokens", None)
 
         # Remove the Fast at the end if we can save the slow tokenizer
         if tokenizer_class.endswith("Fast") and getattr(self, "can_save_slow_tokenizer", False):
@@ -2204,10 +2365,10 @@ class PreTrainedTokenizerBase(PushToHubMixin):
 
     def _save_pretrained(
         self,
-        save_directory: str | os.PathLike,
+        save_directory: Union[str, os.PathLike],
         file_names: tuple[str, ...],
-        legacy_format: bool | None = None,
-        filename_prefix: str | None = None,
+        legacy_format: Optional[bool] = None,
+        filename_prefix: Optional[str] = None,
     ) -> tuple[str, ...]:
         """
         Save a tokenizer using the slow-tokenizer/legacy format: vocabulary + added tokens.
@@ -2237,7 +2398,7 @@ class PreTrainedTokenizerBase(PushToHubMixin):
 
         return file_names + vocab_files + (added_tokens_file,)
 
-    def save_vocabulary(self, save_directory: str, filename_prefix: str | None = None) -> tuple[str, ...]:
+    def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> tuple[str, ...]:
         """
         Save only the vocabulary of the tokenizer (vocabulary + added tokens).
 
@@ -2255,7 +2416,7 @@ class PreTrainedTokenizerBase(PushToHubMixin):
         """
         raise NotImplementedError
 
-    def tokenize(self, text: str, pair: str | None = None, add_special_tokens: bool = False, **kwargs) -> list[str]:
+    def tokenize(self, text: str, pair: Optional[str] = None, add_special_tokens: bool = False, **kwargs) -> list[str]:
         """
         Converts a string into a sequence of tokens, replacing unknown tokens with the `unk_token`.
 
@@ -2287,15 +2448,15 @@ class PreTrainedTokenizerBase(PushToHubMixin):
     )
     def encode(
         self,
-        text: TextInput | PreTokenizedInput | EncodedInput,
-        text_pair: TextInput | PreTokenizedInput | EncodedInput | None = None,
+        text: Union[TextInput, PreTokenizedInput, EncodedInput],
+        text_pair: Optional[Union[TextInput, PreTokenizedInput, EncodedInput]] = None,
         add_special_tokens: bool = True,
-        padding: bool | str | PaddingStrategy = False,
-        truncation: bool | str | TruncationStrategy | None = None,
-        max_length: int | None = None,
+        padding: Union[bool, str, PaddingStrategy] = False,
+        truncation: Union[bool, str, TruncationStrategy, None] = None,
+        max_length: Optional[int] = None,
         stride: int = 0,
-        padding_side: str | None = None,
-        return_tensors: str | TensorType | None = None,
+        padding_side: Optional[str] = None,
+        return_tensors: Optional[Union[str, TensorType]] = None,
         **kwargs,
     ) -> list[int]:
         """
@@ -2313,15 +2474,15 @@ class PreTrainedTokenizerBase(PushToHubMixin):
                 the `tokenize` method) or a list of integers (tokenized string ids using the `convert_tokens_to_ids`
                 method).
         """
-        padding_strategy, truncation_strategy, max_length, kwargs_updated = self._get_padding_truncation_strategies(
+        padding_strategy, truncation_strategy, max_length, kwargs = self._get_padding_truncation_strategies(
             padding=padding,
             truncation=truncation,
             max_length=max_length,
+            pad_to_multiple_of=kwargs.get("pad_to_multiple_of"),
+            verbose=kwargs.get("verbose", True),
             **kwargs,
         )
 
-        kwargs.update(kwargs_updated)
-
         encoded_inputs = self._encode_plus(
             text,
             text_pair=text_pair,
@@ -2464,27 +2625,29 @@ class PreTrainedTokenizerBase(PushToHubMixin):
     @add_end_docstrings(ENCODE_KWARGS_DOCSTRING, ENCODE_PLUS_ADDITIONAL_KWARGS_DOCSTRING)
     def __call__(
         self,
-        text: TextInput | PreTokenizedInput | list[TextInput] | list[PreTokenizedInput] | None = None,
-        text_pair: TextInput | PreTokenizedInput | list[TextInput] | list[PreTokenizedInput] | None = None,
-        text_target: TextInput | PreTokenizedInput | list[TextInput] | list[PreTokenizedInput] | None = None,
-        text_pair_target: TextInput | PreTokenizedInput | list[TextInput] | list[PreTokenizedInput] | None = None,
+        text: Union[TextInput, PreTokenizedInput, list[TextInput], list[PreTokenizedInput], None] = None,
+        text_pair: Optional[Union[TextInput, PreTokenizedInput, list[TextInput], list[PreTokenizedInput]]] = None,
+        text_target: Union[TextInput, PreTokenizedInput, list[TextInput], list[PreTokenizedInput], None] = None,
+        text_pair_target: Optional[
+            Union[TextInput, PreTokenizedInput, list[TextInput], list[PreTokenizedInput]]
+        ] = None,
         add_special_tokens: bool = True,
-        padding: bool | str | PaddingStrategy = False,
-        truncation: bool | str | TruncationStrategy | None = None,
-        max_length: int | None = None,
+        padding: Union[bool, str, PaddingStrategy] = False,
+        truncation: Union[bool, str, TruncationStrategy, None] = None,
+        max_length: Optional[int] = None,
         stride: int = 0,
         is_split_into_words: bool = False,
-        pad_to_multiple_of: int | None = None,
-        padding_side: str | None = None,
-        return_tensors: str | TensorType | None = None,
-        return_token_type_ids: bool | None = None,
-        return_attention_mask: bool | None = None,
+        pad_to_multiple_of: Optional[int] = None,
+        padding_side: Optional[str] = None,
+        return_tensors: Optional[Union[str, TensorType]] = None,
+        return_token_type_ids: Optional[bool] = None,
+        return_attention_mask: Optional[bool] = None,
         return_overflowing_tokens: bool = False,
         return_special_tokens_mask: bool = False,
         return_offsets_mapping: bool = False,
         return_length: bool = False,
         verbose: bool = True,
-        tokenizer_kwargs: dict[str, Any] | None = None,
+        tokenizer_kwargs: Optional[dict[str, Any]] = None,
         **kwargs,
     ) -> BatchEncoding:
         """
@@ -2589,19 +2752,19 @@ class PreTrainedTokenizerBase(PushToHubMixin):
 
     def _encode_plus(
         self,
-        text: TextInput | PreTokenizedInput | EncodedInput,
-        text_pair: TextInput | PreTokenizedInput | EncodedInput | None = None,
+        text: Union[TextInput, PreTokenizedInput, EncodedInput],
+        text_pair: Optional[Union[TextInput, PreTokenizedInput, EncodedInput]] = None,
         add_special_tokens: bool = True,
         padding_strategy: PaddingStrategy = PaddingStrategy.DO_NOT_PAD,
         truncation_strategy: TruncationStrategy = TruncationStrategy.DO_NOT_TRUNCATE,
-        max_length: int | None = None,
+        max_length: Optional[int] = None,
         stride: int = 0,
         is_split_into_words: bool = False,
-        pad_to_multiple_of: int | None = None,
-        padding_side: str | None = None,
-        return_tensors: str | TensorType | None = None,
-        return_token_type_ids: bool | None = None,
-        return_attention_mask: bool | None = None,
+        pad_to_multiple_of: Optional[int] = None,
+        padding_side: Optional[str] = None,
+        return_tensors: Optional[Union[str, TensorType]] = None,
+        return_token_type_ids: Optional[bool] = None,
+        return_attention_mask: Optional[bool] = None,
         return_overflowing_tokens: bool = False,
         return_special_tokens_mask: bool = False,
         return_offsets_mapping: bool = False,
@@ -2614,17 +2777,19 @@ class PreTrainedTokenizerBase(PushToHubMixin):
 
     def pad(
         self,
-        encoded_inputs: BatchEncoding
-        | list[BatchEncoding]
-        | dict[str, EncodedInput]
-        | dict[str, list[EncodedInput]]
-        | list[dict[str, EncodedInput]],
-        padding: bool | str | PaddingStrategy = True,
-        max_length: int | None = None,
-        pad_to_multiple_of: int | None = None,
-        padding_side: str | None = None,
-        return_attention_mask: bool | None = None,
-        return_tensors: str | TensorType | None = None,
+        encoded_inputs: Union[
+            BatchEncoding,
+            list[BatchEncoding],
+            dict[str, EncodedInput],
+            dict[str, list[EncodedInput]],
+            list[dict[str, EncodedInput]],
+        ],
+        padding: Union[bool, str, PaddingStrategy] = True,
+        max_length: Optional[int] = None,
+        pad_to_multiple_of: Optional[int] = None,
+        padding_side: Optional[str] = None,
+        return_attention_mask: Optional[bool] = None,
+        return_tensors: Optional[Union[str, TensorType]] = None,
         verbose: bool = True,
     ) -> BatchEncoding:
         """
@@ -2785,12 +2950,12 @@ class PreTrainedTokenizerBase(PushToHubMixin):
 
     def _pad(
         self,
-        encoded_inputs: dict[str, EncodedInput] | BatchEncoding,
-        max_length: int | None = None,
+        encoded_inputs: Union[dict[str, EncodedInput], BatchEncoding],
+        max_length: Optional[int] = None,
         padding_strategy: PaddingStrategy = PaddingStrategy.DO_NOT_PAD,
-        pad_to_multiple_of: int | None = None,
-        padding_side: str | None = None,
-        return_attention_mask: bool | None = None,
+        pad_to_multiple_of: Optional[int] = None,
+        padding_side: Optional[str] = None,
+        return_attention_mask: Optional[bool] = None,
     ) -> dict:
         """
         Pad encoded inputs (on left/right and up to predefined length or max length in the batch)
@@ -2880,10 +3045,10 @@ class PreTrainedTokenizerBase(PushToHubMixin):
 
     def decode(
         self,
-        token_ids: int | list[int] | list[list[int]] | np.ndarray | torch.Tensor,
+        token_ids: Union[int, list[int], list[list[int]], np.ndarray, "torch.Tensor"],
         skip_special_tokens: bool = False,
         **kwargs,
-    ) -> str | list[str]:
+    ) -> Union[str, list[str]]:
         """
         Converts a sequence of ids into a string, or a list of sequences into a list of strings,
         using the tokenizer and vocabulary with options to remove special tokens and clean up
@@ -2928,9 +3093,9 @@ class PreTrainedTokenizerBase(PushToHubMixin):
 
     def batch_decode(
         self,
-        sequences: list[int] | list[list[int]] | np.ndarray | torch.Tensor,
+        sequences: Union[list[int], list[list[int]], np.ndarray, "torch.Tensor"],
         skip_special_tokens: bool = False,
-        clean_up_tokenization_spaces: bool | None = None,
+        clean_up_tokenization_spaces: Optional[bool] = None,
         **kwargs,
     ) -> list[str]:
         """
@@ -2967,14 +3132,14 @@ class PreTrainedTokenizerBase(PushToHubMixin):
 
     def _decode(
         self,
-        token_ids: int | list[int],
+        token_ids: Union[int, list[int]],
         skip_special_tokens: bool = False,
-        clean_up_tokenization_spaces: bool | None = None,
+        clean_up_tokenization_spaces: Optional[bool] = None,
         **kwargs,
     ) -> str:
         raise NotImplementedError
 
-    def _eventual_warn_about_too_long_sequence(self, ids: list[int], max_length: int | None, verbose: bool):
+    def _eventual_warn_about_too_long_sequence(self, ids: list[int], max_length: Optional[int], verbose: bool):
         """
         Depending on the input and internal state we might trigger a warning about a sequence that is too long for its
         corresponding model
@@ -3016,22 +3181,22 @@ class PreTrainedTokenizerBase(PushToHubMixin):
 
     def apply_chat_template(
         self,
-        conversation: list[dict[str, str]] | list[list[dict[str, str]]],
-        tools: list[dict | Callable] | None = None,
-        documents: list[dict[str, str]] | None = None,
-        chat_template: str | None = None,
+        conversation: Union[list[dict[str, str]], list[list[dict[str, str]]]],
+        tools: Optional[list[Union[dict, Callable]]] = None,
+        documents: Optional[list[dict[str, str]]] = None,
+        chat_template: Optional[str] = None,
         add_generation_prompt: bool = False,
         continue_final_message: bool = False,
         tokenize: bool = True,
-        padding: bool | str | PaddingStrategy = False,
+        padding: Union[bool, str, PaddingStrategy] = False,
         truncation: bool = False,
-        max_length: int | None = None,
-        return_tensors: str | TensorType | None = None,
-        return_dict: bool = True,
+        max_length: Optional[int] = None,
+        return_tensors: Optional[Union[str, TensorType]] = None,
+        return_dict: bool = False,
         return_assistant_tokens_mask: bool = False,
-        tokenizer_kwargs: dict[str, Any] | None = None,
+        tokenizer_kwargs: Optional[dict[str, Any]] = None,
         **kwargs,
-    ) -> str | list[int] | list[str] | list[list[int]] | BatchEncoding:
+    ) -> Union[str, list[int], list[str], list[list[int]], BatchEncoding]:
         """
         Converts a list of dictionaries with `"role"` and `"content"` keys to a list of token
         ids. This method is intended for use with chat models, and will read the tokenizer's chat_template attribute to
@@ -3085,7 +3250,7 @@ class PreTrainedTokenizerBase(PushToHubMixin):
                 values are:
                 - `'pt'`: Return PyTorch `torch.Tensor` objects.
                 - `'np'`: Return NumPy `np.ndarray` objects.
-            return_dict (`bool`, defaults to `True`):
+            return_dict (`bool`, defaults to `False`):
                 Whether to return a dictionary with named outputs. Has no effect if tokenize is `False`.
             tokenizer_kwargs (`dict[str: Any]`, *optional*): Additional kwargs to pass to the tokenizer.
             return_assistant_tokens_mask (`bool`, defaults to `False`):
@@ -3100,11 +3265,14 @@ class PreTrainedTokenizerBase(PushToHubMixin):
             set, will return a dict of tokenizer outputs instead.
         """
 
-        if not tokenize:
-            return_dict = False  # dicts are only returned by the tokenizer anyway
+        if return_dict and not tokenize:
+            raise ValueError(
+                "`return_dict=True` is incompatible with `tokenize=False`, because there is no dict "
+                "of tokenizer outputs to return."
+            )
 
-        if return_assistant_tokens_mask and not (return_dict and tokenize):
-            raise ValueError("`return_assistant_tokens_mask=True` requires `return_dict=True` and `tokenize=True`")
+        if return_assistant_tokens_mask and not return_dict:
+            raise ValueError("`return_assistant_tokens_mask=True` is incompatible with `return_dict=False`")
 
         if tokenizer_kwargs is None:
             tokenizer_kwargs = {}
@@ -3189,7 +3357,7 @@ class PreTrainedTokenizerBase(PushToHubMixin):
     def encode_message_with_chat_template(
         self,
         message: dict[str, str],
-        conversation_history: list[dict[str, str]] | None = None,
+        conversation_history: Optional[list[dict[str, str]]] = None,
         **kwargs,
     ) -> list[int]:
         """
@@ -3219,17 +3387,13 @@ class PreTrainedTokenizerBase(PushToHubMixin):
             )
 
         if conversation_history is None or len(conversation_history) == 0:
-            return self.apply_chat_template(
-                [message], add_generation_prompt=False, tokenize=True, return_dict=False, **kwargs
-            )
+            return self.apply_chat_template([message], add_generation_prompt=False, tokenize=True, **kwargs)
 
         conversation = conversation_history + [message]
-        tokens = self.apply_chat_template(
-            conversation, add_generation_prompt=False, tokenize=True, return_dict=False, **kwargs
-        )
+        tokens = self.apply_chat_template(conversation, add_generation_prompt=False, tokenize=True, **kwargs)
 
         prefix_tokens = self.apply_chat_template(
-            conversation_history, add_generation_prompt=False, tokenize=True, return_dict=False, **kwargs
+            conversation_history, add_generation_prompt=False, tokenize=True, **kwargs
         )
         # It's possible that the prefix tokens are not a prefix of the full list of tokens.
         # For example, if the prefix is `<s>User: Hi` and the full conversation is `<s>User: Hi</s><s>Assistant: Hello`.
@@ -3246,7 +3410,7 @@ class PreTrainedTokenizerBase(PushToHubMixin):
                 return tokens[i:]
         return tokens[min_len:]
 
-    def get_chat_template(self, chat_template: str | None = None, tools: list[dict] | None = None) -> str:
+    def get_chat_template(self, chat_template: Optional[str] = None, tools: Optional[list[dict]] = None) -> str:
         """
         Retrieve the chat template string used for tokenizing chat messages. This template is used
         internally by the `apply_chat_template` method and can also be used externally to retrieve the model's chat
@@ -3302,9 +3466,9 @@ class PreTrainedTokenizerBase(PushToHubMixin):
 
     def save_chat_templates(
         self,
-        save_directory: str | os.PathLike,
+        save_directory: Union[str, os.PathLike],
         tokenizer_config: dict,
-        filename_prefix: str | None,
+        filename_prefix: Optional[str],
         save_jinja_files: bool,
     ):
         """
@@ -3355,45 +3519,6 @@ class PreTrainedTokenizerBase(PushToHubMixin):
             tokenizer_config["chat_template"] = self.chat_template
         return tokenizer_config, saved_raw_chat_template_files
 
-    def parse_response(
-        self,
-        response: str | list[str | int | list[int]] | np.ndarray | torch.Tensor,
-        schema: list | dict | None = None,
-    ):
-        """
-        Converts an output string created by generating text from a model into a parsed message dictionary.
-        This method is intended for use with chat models, and will read the tokenizer's `response_schema` attribute to
-        control parsing, although this can be overridden by passing a `response_schema` argument directly.
-
-        This method is currently **highly experimental** and the schema specification is likely to change in future!
-        We recommend not building production code on top of it just yet.
-
-        Args:
-            response (`str`):
-                The output string generated by the model. This can be either a decoded string or list of strings,
-                or token IDs as a list/array.
-            schema (`Union[list, dict]`, *optional*):
-                A response schema that indicates the expected output format and how parsing should be performed.
-                If not provided, the tokenizer's `response_schema` attribute will be used.
-        """
-        batched = (
-            (isinstance(response, list) and not isinstance(response[0], int))
-            or getattr(response, "ndim", 0) > 1  # For torch/numpy tensors
-        )
-
-        if schema is None:
-            if getattr(self, "response_schema", None) is None:
-                raise AttributeError("This tokenizer does not have a `response_schema` for parsing chat responses!")
-            schema = self.response_schema
-        if batched:
-            if not (isinstance(response, list) and isinstance(response[0], str)):
-                response = self.batch_decode(response)
-            return [recursive_parse(single_response, schema) for single_response in response]
-        else:
-            if not isinstance(response, str):
-                response = self.decode(response)
-            return recursive_parse(response, schema)
-
 
 def get_fast_tokenizer_file(tokenization_files: list[str]) -> str:
     """
@@ -3603,20 +3728,15 @@ def _get_prepend_scheme(add_prefix_space: bool, original_tokenizer) -> str:
     return prepend_scheme
 
 
-def generate_merges(vocab, vocab_scores: dict[str, float] | None = None, skip_tokens: Collection[str] | None = None):
-    skip_tokens = set(skip_tokens) if skip_tokens is not None else set()
+def generate_merges(vocab, vocab_scores: Optional[dict[str, float]] = None):
     reverse = vocab_scores is not None
     vocab_scores = dict(vocab_scores) if reverse else vocab
 
     merges = []
     for merge, piece_score in vocab_scores.items():
-        if merge in skip_tokens:
-            continue
         local = []
         for index in range(1, len(merge)):
             piece_l, piece_r = merge[:index], merge[index:]
-            if piece_l in skip_tokens or piece_r in skip_tokens:
-                continue
             if piece_l in vocab and piece_r in vocab:
                 local.append((piece_l, piece_r, piece_score))
         local = sorted(local, key=lambda x: (vocab[x[0]], vocab[x[1]]))