transformers 5.0.0rc0__py3-none-any.whl → 5.0.0rc2__py3-none-any.whl

transformers/models/auto/tokenization_auto.py

@@ -15,7 +15,6 @@
 """Auto Tokenizer class."""
 
 import importlib
-import inspect
 import json
 import os
 from collections import OrderedDict
@@ -26,8 +25,7 @@ from transformers.utils.import_utils import is_mistral_common_available
 from ...configuration_utils import PreTrainedConfig
 from ...dynamic_module_utils import get_class_from_dynamic_module, resolve_trust_remote_code
 from ...modeling_gguf_pytorch_utils import load_gguf_checkpoint
-from ...tokenization_python import PreTrainedTokenizer, PythonBackend
-from ...tokenization_utils_base import TOKENIZER_CONFIG_FILE, find_sentencepiece_model_file, load_vocab_and_merges
+from ...tokenization_utils_base import TOKENIZER_CONFIG_FILE
 from ...utils import (
     extract_commit_hash,
     is_g2p_en_available,
@@ -35,7 +33,7 @@ from ...utils import (
     is_tokenizers_available,
     logging,
 )
-from ...utils.hub import cached_file, has_file
+from ...utils.hub import cached_file
 from ..encoder_decoder import EncoderDecoderConfig
 from .auto_factory import _LazyAutoMapping
 from .configuration_auto import (
@@ -65,11 +63,10 @@ REGISTERED_FAST_ALIASES: dict[str, type[Any]] = {}
 
 TOKENIZER_MAPPING_NAMES = OrderedDict[str, Optional[str]](
     [
-        ("aimv2", "CLIPTokenizerFast" if is_tokenizers_available() else None),
+        ("aimv2", "CLIPTokenizer" if is_tokenizers_available() else None),
         ("albert", "AlbertTokenizer" if is_tokenizers_available() else None),
         ("align", "BertTokenizer" if is_tokenizers_available() else None),
-        ("arcee", "LlamaTokenizerFast" if is_tokenizers_available() else None),
-        ("aria", "LlamaTokenizerFast" if is_tokenizers_available() else None),
+        ("audioflamingo3", "Qwen2Tokenizer" if is_tokenizers_available() else None),
         ("aya_vision", "CohereTokenizer" if is_tokenizers_available() else None),
         ("bark", "BertTokenizer" if is_tokenizers_available() else None),
         ("bart", "RobertaTokenizer" if is_tokenizers_available() else None),
@@ -82,19 +79,15 @@ TOKENIZER_MAPPING_NAMES = OrderedDict[str, Optional[str]](
         ("big_bird", "BigBirdTokenizer" if is_tokenizers_available() else None),
         ("bigbird_pegasus", "PegasusTokenizer" if is_tokenizers_available() else None),
         ("biogpt", "BioGptTokenizer"),
-        ("bitnet", "PreTrainedTokenizerFast" if is_tokenizers_available() else None),
         ("blenderbot", "BlenderbotTokenizer" if is_tokenizers_available() else None),
         ("blenderbot-small", "BlenderbotSmallTokenizer"),
         ("blip", "BertTokenizer" if is_tokenizers_available() else None),
         ("blip-2", "GPT2Tokenizer" if is_tokenizers_available() else None),
-        ("bloom", "TokenizersBackend" if is_tokenizers_available() else None),
-        ("blt", "PreTrainedTokenizerFast" if is_tokenizers_available() else None),
         ("bridgetower", "RobertaTokenizer"),
         ("bros", "BertTokenizer" if is_tokenizers_available() else None),
         ("byt5", "ByT5Tokenizer"),
         ("camembert", "CamembertTokenizer" if is_tokenizers_available() else None),
         ("canine", "CanineTokenizer"),
-        ("chameleon", "LlamaTokenizerFast" if is_tokenizers_available() else None),
         ("chinese_clip", "BertTokenizer" if is_tokenizers_available() else None),
         ("clap", "RobertaTokenizer"),
         ("clip", "CLIPTokenizer" if is_tokenizers_available() else None),
@@ -104,265 +97,219 @@ TOKENIZER_MAPPING_NAMES = OrderedDict[str, Optional[str]](
         ("codegen", "GPT2Tokenizer" if is_tokenizers_available() else None),
         ("cohere", "CohereTokenizer" if is_tokenizers_available() else None),
         ("cohere2", "CohereTokenizer" if is_tokenizers_available() else None),
-        ("colpali", "LlamaTokenizerFast" if is_tokenizers_available() else None),
-        ("colqwen2", "Qwen2TokenizerFast" if is_tokenizers_available() else None),
+        ("colqwen2", "Qwen2Tokenizer" if is_tokenizers_available() else None),
         ("convbert", "BertTokenizer" if is_tokenizers_available() else None),
         ("cpm", "CpmTokenizer" if is_tokenizers_available() else None),
         ("cpmant", "CpmAntTokenizer"),
-        ("csm", "PreTrainedTokenizerFast" if is_tokenizers_available() else None),
         ("ctrl", "CTRLTokenizer"),
         ("data2vec-audio", "Wav2Vec2CTCTokenizer"),
         ("data2vec-text", "RobertaTokenizer"),
         ("dbrx", "GPT2Tokenizer" if is_tokenizers_available() else None),
         ("deberta", "DebertaTokenizer" if is_tokenizers_available() else None),
         ("deberta-v2", "DebertaV2Tokenizer" if is_tokenizers_available() else None),
-        ("deepseek_v2", "LlamaTokenizerFast" if is_tokenizers_available() else None),
-        ("deepseek_v3", "LlamaTokenizerFast" if is_tokenizers_available() else None),
-        ("deepseek_vl", "LlamaTokenizerFast" if is_tokenizers_available() else None),
-        ("deepseek_vl_hybrid", "LlamaTokenizerFast" if is_tokenizers_available() else None),
         ("dia", "DiaTokenizer"),
-        ("diffllama", "LlamaTokenizerFast" if is_tokenizers_available() else None),
         ("distilbert", "BertTokenizer" if is_tokenizers_available() else None),
-        ("dpr", "DPRQuestionEncoderTokenizerFast" if is_tokenizers_available() else None),
+        ("dpr", "DPRQuestionEncoderTokenizer" if is_tokenizers_available() else None),
         ("electra", "BertTokenizer" if is_tokenizers_available() else None),
         ("emu3", "GPT2Tokenizer" if is_tokenizers_available() else None),
         ("ernie", "BertTokenizer" if is_tokenizers_available() else None),
-        ("ernie4_5", "LlamaTokenizerFast" if is_tokenizers_available() else None),
-        ("ernie4_5_moe", "LlamaTokenizerFast" if is_tokenizers_available() else None),
         ("esm", "EsmTokenizer"),
         ("exaone4", "GPT2Tokenizer" if is_tokenizers_available() else None),
-        ("falcon", "PreTrainedTokenizerFast" if is_tokenizers_available() else None),
-        ("falcon_mamba", "GPTNeoXTokenizerFast" if is_tokenizers_available() else None),
+        ("falcon_mamba", "GPTNeoXTokenizer" if is_tokenizers_available() else None),
         ("fastspeech2_conformer", "FastSpeech2ConformerTokenizer" if is_g2p_en_available() else None),
         ("flaubert", "FlaubertTokenizer"),
         ("flava", "BertTokenizer" if is_tokenizers_available() else None),
         ("flex_olmo", "GPT2Tokenizer" if is_tokenizers_available() else None),
         ("florence2", "BartTokenizer" if is_tokenizers_available() else None),
-        ("fnet", "FNetTokenizerFast" if is_tokenizers_available() else None),
+        ("fnet", "FNetTokenizer" if is_tokenizers_available() else None),
         ("fsmt", "FSMTTokenizer"),
         ("funnel", "FunnelTokenizer" if is_tokenizers_available() else None),
-        ("gemma", "GemmaTokenizerFast" if is_tokenizers_available() else None),
-        ("gemma2", "GemmaTokenizerFast" if is_tokenizers_available() else None),
-        ("gemma3", "GemmaTokenizerFast" if is_tokenizers_available() else None),
-        ("gemma3_text", "GemmaTokenizerFast" if is_tokenizers_available() else None),
-        ("gemma3n", "GemmaTokenizerFast" if is_tokenizers_available() else None),
-        ("gemma3n_text", "GemmaTokenizerFast" if is_tokenizers_available() else None),
+        ("gemma", "GemmaTokenizer" if is_tokenizers_available() else None),
+        ("gemma2", "GemmaTokenizer" if is_tokenizers_available() else None),
+        ("gemma3", "GemmaTokenizer" if is_tokenizers_available() else None),
+        ("gemma3_text", "GemmaTokenizer" if is_tokenizers_available() else None),
+        ("gemma3n", "GemmaTokenizer" if is_tokenizers_available() else None),
+        ("gemma3n_text", "GemmaTokenizer" if is_tokenizers_available() else None),
         ("git", "BertTokenizer" if is_tokenizers_available() else None),
-        ("glm", "PreTrainedTokenizerFast" if is_tokenizers_available() else None),
-        ("glm4", "PreTrainedTokenizerFast" if is_tokenizers_available() else None),
-        ("glm4_moe", "PreTrainedTokenizerFast" if is_tokenizers_available() else None),
-        ("glm4v", "PreTrainedTokenizerFast" if is_tokenizers_available() else None),
-        ("glm4v_moe", "PreTrainedTokenizerFast" if is_tokenizers_available() else None),
-        ("got_ocr2", "PreTrainedTokenizerFast" if is_tokenizers_available() else None),
         ("gpt-sw3", "GPTSw3Tokenizer" if is_sentencepiece_available() else None),
         ("gpt2", "GPT2Tokenizer" if is_tokenizers_available() else None),
         ("gpt_bigcode", "GPT2Tokenizer" if is_tokenizers_available() else None),
         ("gpt_neo", "GPT2Tokenizer" if is_tokenizers_available() else None),
         ("gpt_neox", "GPTNeoXTokenizer" if is_tokenizers_available() else None),
         ("gpt_neox_japanese", "GPTNeoXJapaneseTokenizer"),
-        ("gpt_oss", "PreTrainedTokenizerFast" if is_tokenizers_available() else None),
         ("gptj", "GPT2Tokenizer" if is_tokenizers_available() else None),
         ("granite", "GPT2Tokenizer"),
         ("granitemoe", "GPT2Tokenizer"),
         ("granitemoehybrid", "GPT2Tokenizer"),
         ("granitemoeshared", "GPT2Tokenizer"),
         ("grounding-dino", "BertTokenizer" if is_tokenizers_available() else None),
-        ("groupvit", "CLIPTokenizerFast" if is_tokenizers_available() else None),
-        ("helium", "PreTrainedTokenizerFast" if is_tokenizers_available() else None),
+        ("groupvit", "CLIPTokenizer" if is_tokenizers_available() else None),
         ("herbert", "HerbertTokenizer" if is_tokenizers_available() else None),
         ("hubert", "Wav2Vec2CTCTokenizer"),
         ("ibert", "RobertaTokenizer"),
-        ("idefics", "LlamaTokenizerFast" if is_tokenizers_available() else None),
-        ("idefics2", "LlamaTokenizerFast" if is_tokenizers_available() else None),
-        ("idefics3", "LlamaTokenizerFast" if is_tokenizers_available() else None),
+        ("idefics", "LlamaTokenizer" if is_tokenizers_available() else None),
+        ("idefics2", "LlamaTokenizer" if is_tokenizers_available() else None),
         ("instructblip", "GPT2Tokenizer" if is_tokenizers_available() else None),
         ("instructblipvideo", "GPT2Tokenizer" if is_tokenizers_available() else None),
-        ("internvl", "Qwen2TokenizerFast" if is_tokenizers_available() else None),
-        ("jamba", "LlamaTokenizerFast" if is_tokenizers_available() else None),
-        ("janus", "LlamaTokenizerFast" if is_tokenizers_available() else None),
-        ("jetmoe", "LlamaTokenizerFast" if is_tokenizers_available() else None),
+        ("internvl", "Qwen2Tokenizer" if is_tokenizers_available() else None),
+        ("jais2", "GPT2Tokenizer" if is_tokenizers_available() else None),
         ("kosmos-2", "XLMRobertaTokenizer" if is_tokenizers_available() else None),
-        ("kosmos-2.5", "PreTrainedTokenizerFast" if is_tokenizers_available() else None),
+        ("lasr_ctc", "ParakeetTokenizer" if is_tokenizers_available() else None),
+        ("lasr_encoder", "ParakeetTokenizer" if is_tokenizers_available() else None),
         ("layoutlm", "BertTokenizer" if is_tokenizers_available() else None),
         ("layoutlmv2", "LayoutLMv2Tokenizer" if is_tokenizers_available() else None),
         ("layoutlmv3", "LayoutLMv3Tokenizer" if is_tokenizers_available() else None),
         ("layoutxlm", "LayoutXLMTokenizer" if is_tokenizers_available() else None),
         ("led", "LEDTokenizer" if is_tokenizers_available() else None),
-        ("lfm2_vl", "PreTrainedTokenizerFast" if is_tokenizers_available() else None),
         ("lilt", "RobertaTokenizer" if is_tokenizers_available() else None),
-        ("llama", "LlamaTokenizer" if is_tokenizers_available() else None),
-        ("llama4", "LlamaTokenizerFast" if is_tokenizers_available() else None),
-        ("llama4_text", "LlamaTokenizerFast" if is_tokenizers_available() else None),
-        ("llava", "LlamaTokenizerFast" if is_tokenizers_available() else None),
-        ("llava_next", "LlamaTokenizerFast" if is_tokenizers_available() else None),
-        ("llava_next_video", "LlamaTokenizerFast" if is_tokenizers_available() else None),
-        ("llava_onevision", "LlamaTokenizerFast" if is_tokenizers_available() else None),
         ("longformer", "RobertaTokenizer" if is_tokenizers_available() else None),
         ("longt5", "T5Tokenizer" if is_tokenizers_available() else None),
         ("luke", "LukeTokenizer"),
         ("lxmert", "LxmertTokenizer" if is_tokenizers_available() else None),
         ("m2m_100", "M2M100Tokenizer" if is_sentencepiece_available() else None),
-        ("mamba", "GPTNeoXTokenizerFast" if is_tokenizers_available() else None),
-        ("mamba2", "GPTNeoXTokenizerFast" if is_tokenizers_available() else None),
+        ("mamba", "GPTNeoXTokenizer" if is_tokenizers_available() else None),
+        ("mamba2", "GPTNeoXTokenizer" if is_tokenizers_available() else None),
         ("marian", "MarianTokenizer" if is_sentencepiece_available() else None),
+        ("markuplm", "MarkupLMTokenizer" if is_tokenizers_available() else None),
         ("mbart", "MBartTokenizer" if is_tokenizers_available() else None),
         ("mbart50", "MBart50Tokenizer" if is_tokenizers_available() else None),
         ("mega", "RobertaTokenizer"),
         ("megatron-bert", "BertTokenizer" if is_tokenizers_available() else None),
-        ("metaclip_2", "XLMRobertaTokenizerFast" if is_tokenizers_available() else None),
+        ("metaclip_2", "XLMRobertaTokenizer" if is_tokenizers_available() else None),
         ("mgp-str", "MgpstrTokenizer"),
-        ("minimax", "GPT2Tokenizer" if is_tokenizers_available() else None),
         (
             "ministral3",
-            (
-                "MistralCommonBackend"
-                if is_mistral_common_available()
-                else ("LlamaTokenizer" if is_sentencepiece_available() else None),
-                "LlamaTokenizerFast" if is_tokenizers_available() and not is_mistral_common_available() else None,
-            ),
+            "MistralCommonBackend"
+            if is_mistral_common_available()
+            else ("TokenizersBackend" if is_tokenizers_available() else None),
         ),
         (
             "mistral",
             "MistralCommonBackend"
             if is_mistral_common_available()
-            else ("LlamaTokenizerFast" if is_tokenizers_available() else None),
+            else ("TokenizersBackend" if is_tokenizers_available() else None),
         ),
         (
             "mistral3",
-            (
-                "MistralCommonBackend"
-                if is_mistral_common_available()
-                else ("LlamaTokenizer" if is_sentencepiece_available() else None),
-                "LlamaTokenizerFast" if is_tokenizers_available() and not is_mistral_common_available() else None,
-            ),
+            "MistralCommonBackend"
+            if is_mistral_common_available()
+            else ("TokenizersBackend" if is_tokenizers_available() else None),
         ),
         (
             "mixtral",
             "MistralCommonBackend"
             if is_mistral_common_available()
-            else ("LlamaTokenizerFast" if is_tokenizers_available() else None),
+            else ("TokenizersBackend" if is_tokenizers_available() else None),
         ),
-        ("mllama", "LlamaTokenizerFast" if is_tokenizers_available() else None),
         ("mluke", "MLukeTokenizer" if is_sentencepiece_available() else None),
         ("mm-grounding-dino", "BertTokenizer" if is_tokenizers_available() else None),
         ("mobilebert", "MobileBertTokenizer" if is_tokenizers_available() else None),
-        ("modernbert", "PreTrainedTokenizerFast" if is_tokenizers_available() else None),
-        ("moonshine", "PreTrainedTokenizerFast" if is_tokenizers_available() else None),
-        ("moshi", "PreTrainedTokenizerFast" if is_tokenizers_available() else None),
         ("mpnet", "MPNetTokenizer" if is_tokenizers_available() else None),
-        ("mpt", "GPTNeoXTokenizerFast" if is_tokenizers_available() else None),
+        ("mpt", "GPTNeoXTokenizer" if is_tokenizers_available() else None),
         ("mra", "RobertaTokenizer"),
         ("mt5", "T5Tokenizer" if is_tokenizers_available() else None),
         ("musicgen", "T5Tokenizer" if is_tokenizers_available() else None),
         ("musicgen_melody", "T5Tokenizer" if is_tokenizers_available() else None),
         ("mvp", "MvpTokenizer" if is_tokenizers_available() else None),
         ("myt5", "MyT5Tokenizer"),
-        ("nemotron", "PreTrainedTokenizerFast" if is_tokenizers_available() else None),
         ("nezha", "BertTokenizer" if is_tokenizers_available() else None),
         ("nllb", "NllbTokenizer" if is_tokenizers_available() else None),
         ("nllb-moe", "NllbTokenizer" if is_tokenizers_available() else None),
         ("nougat", "NougatTokenizer" if is_tokenizers_available() else None),
-        ("nystromformer", "AlbertTokenizerFast" if is_tokenizers_available() else None),
-        ("olmo", "GPTNeoXTokenizerFast" if is_tokenizers_available() else None),
-        ("olmo2", "GPTNeoXTokenizerFast" if is_tokenizers_available() else None),
+        ("nystromformer", "AlbertTokenizer" if is_tokenizers_available() else None),
+        ("olmo", "GPTNeoXTokenizer" if is_tokenizers_available() else None),
+        ("olmo2", "GPTNeoXTokenizer" if is_tokenizers_available() else None),
         ("olmo3", "GPT2Tokenizer" if is_tokenizers_available() else None),
-        ("olmoe", "GPTNeoXTokenizerFast" if is_tokenizers_available() else None),
-        ("omdet-turbo", "CLIPTokenizerFast" if is_tokenizers_available() else None),
-        ("oneformer", "CLIPTokenizerFast" if is_tokenizers_available() else None),
+        ("olmoe", "GPTNeoXTokenizer" if is_tokenizers_available() else None),
+        ("omdet-turbo", "CLIPTokenizer" if is_tokenizers_available() else None),
+        ("oneformer", "CLIPTokenizer" if is_tokenizers_available() else None),
         ("openai-gpt", "OpenAIGPTTokenizer" if is_tokenizers_available() else None),
         ("opt", "GPT2Tokenizer" if is_tokenizers_available() else None),
-        ("ovis2", "Qwen2TokenizerFast" if is_tokenizers_available() else None),
-        ("owlv2", "CLIPTokenizerFast" if is_tokenizers_available() else None),
-        ("owlvit", "CLIPTokenizerFast" if is_tokenizers_available() else None),
-        ("paligemma", "LlamaTokenizerFast" if is_tokenizers_available() else None),
+        ("ovis2", "Qwen2Tokenizer" if is_tokenizers_available() else None),
+        ("owlv2", "CLIPTokenizer" if is_tokenizers_available() else None),
+        ("owlvit", "CLIPTokenizer" if is_tokenizers_available() else None),
         ("pegasus", "PegasusTokenizer" if is_tokenizers_available() else None),
         ("pegasus_x", "PegasusTokenizer" if is_tokenizers_available() else None),
         ("perceiver", "PerceiverTokenizer"),
-        ("persimmon", "LlamaTokenizerFast" if is_tokenizers_available() else None),
         ("phi", "GPT2Tokenizer" if is_tokenizers_available() else None),
-        ("phi3", "LlamaTokenizerFast" if is_tokenizers_available() else None),
-        ("phimoe", "LlamaTokenizerFast" if is_tokenizers_available() else None),
         ("phobert", "PhobertTokenizer"),
         ("pix2struct", "T5Tokenizer" if is_tokenizers_available() else None),
         (
             "pixtral",
             "MistralCommonBackend"
             if is_mistral_common_available()
-            else ("PreTrainedTokenizerFast" if is_tokenizers_available() else None),
+            else ("TokenizersBackend" if is_tokenizers_available() else None),
         ),
         ("plbart", "PLBartTokenizer" if is_tokenizers_available() else None),
         ("prophetnet", "ProphetNetTokenizer"),
         ("qdqbert", "BertTokenizer" if is_tokenizers_available() else None),
-        ("qwen2", "Qwen2TokenizerFast" if is_tokenizers_available() else None),
-        ("qwen2_5_omni", "Qwen2TokenizerFast" if is_tokenizers_available() else None),
-        ("qwen2_5_vl", "Qwen2TokenizerFast" if is_tokenizers_available() else None),
-        ("qwen2_audio", "Qwen2TokenizerFast" if is_tokenizers_available() else None),
-        ("qwen2_moe", "Qwen2TokenizerFast" if is_tokenizers_available() else None),
-        ("qwen2_vl", "Qwen2TokenizerFast" if is_tokenizers_available() else None),
-        ("qwen3", "Qwen2TokenizerFast" if is_tokenizers_available() else None),
-        ("qwen3_moe", "Qwen2TokenizerFast" if is_tokenizers_available() else None),
-        ("qwen3_next", "Qwen2TokenizerFast" if is_tokenizers_available() else None),
-        ("qwen3_omni_moe", "Qwen2TokenizerFast" if is_tokenizers_available() else None),
-        ("qwen3_vl", "Qwen2TokenizerFast" if is_tokenizers_available() else None),
-        ("qwen3_vl_moe", "Qwen2TokenizerFast" if is_tokenizers_available() else None),
+        ("qwen2", "Qwen2Tokenizer" if is_tokenizers_available() else None),
+        ("qwen2_5_omni", "Qwen2Tokenizer" if is_tokenizers_available() else None),
+        ("qwen2_5_vl", "Qwen2Tokenizer" if is_tokenizers_available() else None),
+        ("qwen2_audio", "Qwen2Tokenizer" if is_tokenizers_available() else None),
+        ("qwen2_moe", "Qwen2Tokenizer" if is_tokenizers_available() else None),
+        ("qwen2_vl", "Qwen2Tokenizer" if is_tokenizers_available() else None),
+        ("qwen3", "Qwen2Tokenizer" if is_tokenizers_available() else None),
+        ("qwen3_moe", "Qwen2Tokenizer" if is_tokenizers_available() else None),
+        ("qwen3_next", "Qwen2Tokenizer" if is_tokenizers_available() else None),
+        ("qwen3_omni_moe", "Qwen2Tokenizer" if is_tokenizers_available() else None),
+        ("qwen3_vl", "Qwen2Tokenizer" if is_tokenizers_available() else None),
+        ("qwen3_vl_moe", "Qwen2Tokenizer" if is_tokenizers_available() else None),
         ("rag", "RagTokenizer"),
         ("realm", "BertTokenizer" if is_tokenizers_available() else None),
-        ("recurrent_gemma", "GemmaTokenizerFast" if is_tokenizers_available() else None),
+        ("recurrent_gemma", "GemmaTokenizer" if is_tokenizers_available() else None),
         ("reformer", "ReformerTokenizer" if is_tokenizers_available() else None),
         ("rembert", "RemBertTokenizer" if is_tokenizers_available() else None),
         ("retribert", "BertTokenizer" if is_tokenizers_available() else None),
         ("roberta", "RobertaTokenizer"),
         ("roberta-prelayernorm", "RobertaTokenizer"),
         ("roc_bert", "RoCBertTokenizer"),
-        ("roformer", "RoFormerTokenizerFast" if is_tokenizers_available() else None),
-        ("rwkv", "GPTNeoXTokenizerFast" if is_tokenizers_available() else None),
+        ("roformer", "RoFormerTokenizer" if is_tokenizers_available() else None),
+        ("rwkv", "GPTNeoXTokenizer" if is_tokenizers_available() else None),
         ("seamless_m4t", "SeamlessM4TTokenizer" if is_tokenizers_available() else None),
         ("seamless_m4t_v2", "SeamlessM4TTokenizer" if is_tokenizers_available() else None),
-        ("shieldgemma2", "GemmaTokenizerFast" if is_tokenizers_available() else None),
+        ("shieldgemma2", "GemmaTokenizer" if is_tokenizers_available() else None),
         ("siglip", "SiglipTokenizer" if is_sentencepiece_available() else None),
-        ("siglip2", "GemmaTokenizerFast" if is_tokenizers_available() else None),
-        ("smollm3", "PreTrainedTokenizerFast" if is_tokenizers_available() else None),
+        ("siglip2", "GemmaTokenizer" if is_tokenizers_available() else None),
         ("speech_to_text", "Speech2TextTokenizer" if is_sentencepiece_available() else None),
         ("speecht5", "SpeechT5Tokenizer" if is_sentencepiece_available() else None),
         ("splinter", "SplinterTokenizer"),
         ("squeezebert", "BertTokenizer" if is_tokenizers_available() else None),
-        ("stablelm", "GPTNeoXTokenizerFast" if is_tokenizers_available() else None),
+        ("stablelm", "GPTNeoXTokenizer" if is_tokenizers_available() else None),
         ("starcoder2", "GPT2Tokenizer" if is_tokenizers_available() else None),
         ("switch_transformers", "T5Tokenizer" if is_tokenizers_available() else None),
         ("t5", "T5Tokenizer" if is_tokenizers_available() else None),
-        ("t5gemma", "GemmaTokenizerFast" if is_tokenizers_available() else None),
+        ("t5gemma", "GemmaTokenizer" if is_tokenizers_available() else None),
         ("tapas", "TapasTokenizer"),
         ("trocr", "XLMRobertaTokenizer" if is_tokenizers_available() else None),
         ("tvp", "BertTokenizer" if is_tokenizers_available() else None),
         ("udop", "UdopTokenizer" if is_tokenizers_available() else None),
         ("umt5", "T5Tokenizer" if is_tokenizers_available() else None),
-        ("video_llava", "LlamaTokenizerFast" if is_tokenizers_available() else None),
+        ("unispeech", "Wav2Vec2CTCTokenizer"),
+        ("unispeech-sat", "Wav2Vec2CTCTokenizer"),
         ("vilt", "BertTokenizer" if is_tokenizers_available() else None),
-        ("vipllava", "LlamaTokenizerFast" if is_tokenizers_available() else None),
         ("visual_bert", "BertTokenizer" if is_tokenizers_available() else None),
         ("vits", "VitsTokenizer"),
         (
             "voxtral",
             "MistralCommonBackend"
             if is_mistral_common_available()
-            else ("PreTrainedTokenizerFast" if is_tokenizers_available() else None),
+            else ("TokenizersBackend" if is_tokenizers_available() else None),
         ),
         ("wav2vec2", "Wav2Vec2CTCTokenizer"),
         ("wav2vec2-bert", "Wav2Vec2CTCTokenizer"),
         ("wav2vec2-conformer", "Wav2Vec2CTCTokenizer"),
         ("wav2vec2_phoneme", "Wav2Vec2PhonemeCTCTokenizer"),
         ("whisper", "WhisperTokenizer" if is_tokenizers_available() else None),
-        ("xclip", "CLIPTokenizerFast" if is_tokenizers_available() else None),
+        ("xclip", "CLIPTokenizer" if is_tokenizers_available() else None),
         ("xglm", "XGLMTokenizer" if is_tokenizers_available() else None),
         ("xlm", "XLMTokenizer"),
         ("xlm-roberta", "XLMRobertaTokenizer" if is_tokenizers_available() else None),
         ("xlm-roberta-xl", "XLMRobertaTokenizer" if is_tokenizers_available() else None),
         ("xlnet", "XLNetTokenizer" if is_tokenizers_available() else None),
-        ("xlstm", "GPTNeoXTokenizerFast" if is_tokenizers_available() else None),
-        ("xmod", "XLMRobertaTokenizerFast" if is_tokenizers_available() else None),
+        ("xlstm", "GPTNeoXTokenizer" if is_tokenizers_available() else None),
+        ("xmod", "XLMRobertaTokenizer" if is_tokenizers_available() else None),
         ("yoso", "AlbertTokenizer" if is_tokenizers_available() else None),
-        ("zamba", "LlamaTokenizerFast" if is_tokenizers_available() else None),
-        ("zamba2", "LlamaTokenizerFast" if is_tokenizers_available() else None),
     ]
 )
 
@@ -389,13 +336,17 @@ def load_merges(merges_file):
 
 
 def tokenizer_class_from_name(class_name: str) -> Union[type[Any], None]:
+    # Bloom tokenizer classes were removed but should map to the fast backend for BC
+    if class_name in {"BloomTokenizer", "BloomTokenizerFast"}:
+        return TokenizersBackend
+
     if class_name in REGISTERED_FAST_ALIASES:
         return REGISTERED_FAST_ALIASES[class_name]
 
     if class_name in REGISTERED_TOKENIZER_CLASSES:
         return REGISTERED_TOKENIZER_CLASSES[class_name]
 
-    if class_name == "PreTrainedTokenizerFast":
+    if class_name == "TokenizersBackend":
         return TokenizersBackend
 
     # V5: TOKENIZER_MAPPING_NAMES now maps to single strings, not tuples
@@ -404,7 +355,7 @@ def tokenizer_class_from_name(class_name: str) -> Union[type[Any], None]:
             module_name = model_type_to_module_name(module_name)
             if (
                 module_name in ["mistral", "mistral3", "mixtral", "ministral", "ministral3", "pixtral", "voxtral"]
-                and class_name == "MistralCommonTokenizer"
+                and class_name == "MistralCommonBackend"
             ):
                 module = importlib.import_module(".tokenization_mistral_common", "transformers")
             else:
@@ -428,402 +379,6 @@ def tokenizer_class_from_name(class_name: str) -> Union[type[Any], None]:
     return None
 
 
-def _find_sentencepiece_model_file(pretrained_model_name_or_path, **kwargs):
-    # Delegate to shared helper to avoid duplication
-    return find_sentencepiece_model_file(pretrained_model_name_or_path, **kwargs)
-
-
-def _load_tokenizers_backend(tokenizer_class, pretrained_model_name_or_path, inputs, kwargs):
-    """
-    Load a tokenizer using only the tokenizers backend (no SentencePiece fallback).
-
-    This function attempts to load with the following priority:
-    1. If tokenizer.json exists, load directly
-    2. If any .model file (SPM) exists, try extracting vocab and merges
-    3. If vocab.json and merges.txt exist, load with those
-    4. If vocab.txt exists (WordPiece models), load with that
-
-    Args:
-        tokenizer_class: The tokenizer class to instantiate
-        pretrained_model_name_or_path: Path or model id
-        inputs: Additional positional arguments for tokenizer init
-        kwargs: Additional keyword arguments
-
-    Returns:
-        An instantiated tokenizer object
-
-    Raises:
-        ValueError: If tokenizer could not be loaded with tokenizers backend
-    """
-    files_loaded = []
-
-    # Try tokenizer.json first
-    try:
-        tokenizer_json_exists = has_file(
-            pretrained_model_name_or_path,
-            "tokenizer.json",
-            revision=kwargs.get("revision"),
-            token=kwargs.get("token"),
-            cache_dir=kwargs.get("cache_dir"),
-            local_files_only=kwargs.get("local_files_only", False),
-        )
-    except Exception:
-        tokenizer_json_exists = False
-
-    if tokenizer_json_exists:
-        files_loaded.append("tokenizer.json")
-        kwargs["backend"] = "tokenizers"
-        kwargs["files_loaded"] = files_loaded
-        # Some old models have uploaded a tokenizer.json but haven't updated tokenizer_config.json to point to the correct tokenizer class
-        tokenizer_class = (
-            TokenizersBackend
-            if tokenizer_class.__name__ in ("PythonBackend", "PreTrainedTokenizer")
-            else tokenizer_class
-        )
-        return tokenizer_class.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs)
-
-    # Try tekken.json (Mistral format)
-    try:
-        if has_file(
-            pretrained_model_name_or_path,
-            "tekken.json",
-            revision=kwargs.get("revision"),
-            token=kwargs.get("token"),
-            cache_dir=kwargs.get("cache_dir"),
-            local_files_only=kwargs.get("local_files_only", False),
-        ):
-            from ...integrations.mistral import convert_tekken_tokenizer
-
-            tekken_file = cached_file(
-                pretrained_model_name_or_path,
-                "tekken.json",
-                **{
-                    k: v
-                    for k, v in kwargs.items()
-                    if k
-                    in ["cache_dir", "force_download", "proxies", "token", "revision", "local_files_only", "subfolder"]
-                },
-            )
-            if tekken_file is not None:
-                files_loaded.append("tekken.json")
-                kwargs["backend"] = "tokenizers"
-                kwargs["files_loaded"] = files_loaded
-                return convert_tekken_tokenizer(tekken_file)
-    except (ImportError, Exception):
-        pass
-
-    # Try extracting from SentencePiece model
-    spm_file = _find_sentencepiece_model_file(pretrained_model_name_or_path, **kwargs)
-    if spm_file is not None:
-        try:
-            resolved_spm = cached_file(
-                pretrained_model_name_or_path,
-                spm_file,
-                cache_dir=kwargs.get("cache_dir"),
-                force_download=kwargs.get("force_download", False),
-                proxies=kwargs.get("proxies"),
-                token=kwargs.get("token"),
-                revision=kwargs.get("revision"),
-                local_files_only=kwargs.get("local_files_only", False),
-                subfolder=kwargs.get("subfolder", ""),
-            )
-        except Exception:
-            resolved_spm = None
-
-        if resolved_spm is not None:
-            try:
-                from ...tokenization_utils_sentencepiece import SentencePieceExtractor
-
-                fast_sig = inspect.signature(getattr(tokenizer_class, "__init__", tokenizer_class))
-                if "vocab" in fast_sig.parameters:
-                    try:
-                        vocab_ids, vocab_scores, merges = SentencePieceExtractor(resolved_spm).extract()
-                        files_loaded.append(spm_file)
-                        kwargs["backend"] = "tokenizers"
-                        kwargs["files_loaded"] = files_loaded
-                        # If tokenizer needs both vocab and merges (BPE models)
-                        if "merges" in fast_sig.parameters:
-                            return tokenizer_class.from_pretrained(
-                                pretrained_model_name_or_path, *inputs, vocab=vocab_scores, merges=merges, **kwargs
-                            )
-                        # If tokenizer only needs vocab (Unigram models like NLLB, SeamlessM4T)
-                        else:
-                            return tokenizer_class.from_pretrained(
-                                pretrained_model_name_or_path, *inputs, vocab=vocab_scores, **kwargs
-                            )
-                    except Exception:
-                        pass
-            except ImportError as e:
-                if "sentencepiece" in str(e).lower() or "SentencePiece" in str(e):
-                    raise ImportError(
-                        f"This checkpoint only contains a SentencePiece model file ({spm_file}), but the `sentencepiece` library is not installed. "
-                        f"Please install sentencepiece to load this tokenizer: `pip install sentencepiece`"
-                    ) from e
-                raise
-            except Exception:
-                pass
-
-    vocab, merges, loaded = load_vocab_and_merges(pretrained_model_name_or_path, **kwargs)
-    if vocab is not None:
-        files_loaded.extend(loaded)
-        if issubclass(tokenizer_class, PreTrainedTokenizer):
-            kwargs["backend"] = "python"
-        else:
-            kwargs["backend"] = "tokenizers"
-        kwargs["files_loaded"] = files_loaded
-        if merges is not None:
-            return tokenizer_class.from_pretrained(
-                pretrained_model_name_or_path, *inputs, vocab=vocab, merges=merges, **kwargs
-            )
-        else:
-            return tokenizer_class.from_pretrained(pretrained_model_name_or_path, *inputs, vocab=vocab, **kwargs)
-
-    # Try vocab.txt (WordPiece models like SplinterTokenizer)
-    try:
-        resolved_vocab_txt = cached_file(
-            pretrained_model_name_or_path,
-            "vocab.txt",
-            cache_dir=kwargs.get("cache_dir"),
-            force_download=kwargs.get("force_download", False),
-            proxies=kwargs.get("proxies"),
-            token=kwargs.get("token"),
-            revision=kwargs.get("revision"),
-            local_files_only=kwargs.get("local_files_only", False),
-            subfolder=kwargs.get("subfolder", ""),
-        )
-    except Exception:
-        resolved_vocab_txt = None
-
-    if resolved_vocab_txt is not None:
-        try:
-            fast_sig = inspect.signature(getattr(tokenizer_class, "__init__", tokenizer_class))
-            if "vocab" in fast_sig.parameters:
-                # Load vocab.txt: each line is a token, line number is the ID
-                vocab = OrderedDict()
-                with open(resolved_vocab_txt, "r", encoding="utf-8") as reader:
-                    tokens = reader.readlines()
-                for index, token in enumerate(tokens):
-                    token = token.rstrip("\n")
-                    vocab[token] = index
-                files_loaded.append("vocab.txt")
-                kwargs["backend"] = "tokenizers"
-                kwargs["files_loaded"] = files_loaded
-                return tokenizer_class.from_pretrained(pretrained_model_name_or_path, *inputs, vocab=vocab, **kwargs)
-        except Exception:
-            pass
-
-    # If all methods failed, raise an error
-    raise ValueError(
-        f"Could not load tokenizer from {pretrained_model_name_or_path} using tokenizers backend. "
-        "No tokenizer.json, tekken.json, vocab.json+merges.txt, vocab.txt, or compatible SentencePiece model found."
-    )
-
-
-def _try_load_tokenizer_with_fallbacks(tokenizer_class, pretrained_model_name_or_path, inputs, kwargs):
-    """
-    Try to load a tokenizer with backend selection.
-
-    This function routes to the appropriate backend based on the 'backend' parameter:
-    - "tokenizers" (default): Uses HuggingFace tokenizers library backend
-    - "sentencepiece": Uses SentencePiece backend
-
-    For the tokenizers backend, attempts to load with the following priority:
-    1. If tokenizer.json exists, load directly
-    2. If any .model file (SPM) exists, try extracting vocab and merges
-    3. If vocab.json and merges.txt exist, load with those
-    4. Fallback to SentencePieceBackend if available
-
-    Args:
-        tokenizer_class: The tokenizer class to instantiate (can be None)
-        pretrained_model_name_or_path: Path or model id
-        inputs: Additional positional arguments for tokenizer init
-        kwargs: Additional keyword arguments (may include 'backend' parameter, defaults to "tokenizers")
-
-    Returns:
-        An instantiated tokenizer object
-
-    Raises:
-        ValueError: If no tokenizer could be loaded
-    """
-    # Extract the backend parameter - default to "tokenizers" to prioritize tokenizers backend
-    backend = kwargs.pop("backend", "tokenizers")
-
-    # Validate backend parameter
-    if backend not in ["sentencepiece", "tokenizers"]:
-        logger.warning(
-            f"Invalid backend '{backend}' specified. Valid options are 'tokenizers' or 'sentencepiece'. "
-            "Defaulting to 'tokenizers' backend."
-        )
-        backend = "tokenizers"
-
-    # Route to SentencePiece backend if requested
-    if backend == "sentencepiece":
-        if SentencePieceBackend is None:
-            raise ValueError(
-                "SentencePiece backend was requested but sentencepiece is not installed. "
-                "Please install it with: pip install sentencepiece"
-            )
-        logger.info("Loading tokenizer with SentencePiece backend")
-        # Track files loaded for SentencePiece backend
-        spm_file = _find_sentencepiece_model_file(pretrained_model_name_or_path, **kwargs)
-        files_loaded = [spm_file] if spm_file else []
-        kwargs["backend"] = "sentencepiece"
-        kwargs["files_loaded"] = files_loaded
-        # Resolve the SPM file path and pass it as vocab_file
-        if spm_file is not None:
-            resolved_vocab_file = cached_file(
-                pretrained_model_name_or_path,
-                spm_file,
-                cache_dir=kwargs.get("cache_dir"),
-                force_download=kwargs.get("force_download", False),
-                proxies=kwargs.get("proxies"),
-                token=kwargs.get("token"),
-                revision=kwargs.get("revision"),
-                local_files_only=kwargs.get("local_files_only", False),
-                subfolder=kwargs.get("subfolder", ""),
-            )
-            kwargs["vocab_file"] = resolved_vocab_file
-        if isinstance(tokenizer_class, type) and issubclass(tokenizer_class, SentencePieceBackend):
-            logger.info("Loading tokenizer with SentencePiece backend using tokenizer class")
-            return tokenizer_class.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs)
-        return SentencePieceBackend.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs)
-
-    # Route to tokenizers backend (default)
-    if backend == "tokenizers":
-        if tokenizer_class is not None:
-            # Check if tokenizer_class inherits from PreTrainedTokenizer (but not from TokenizersBackend/SentencePieceBackend)
-            # These are edge cases with custom logic (e.g., BioGptTokenizer with Moses tokenization)
-            from ...tokenization_python import PreTrainedTokenizer
-
-            # Build list of backend classes to check against
-            backend_classes = [TokenizersBackend] if TokenizersBackend else []
-            if SentencePieceBackend:
-                backend_classes.append(SentencePieceBackend)
-
-            # Check if it's a custom PreTrainedTokenizer (not a backend class)
-            is_custom_pre_trained = (
-                isinstance(tokenizer_class, type)
-                and issubclass(tokenizer_class, PreTrainedTokenizer)
-                and not any(issubclass(tokenizer_class, bc) for bc in backend_classes)
-                and tokenizer_class.__name__ not in ("PythonBackend", "PreTrainedTokenizer")
-            )
-
-            # Check if it's a completely custom tokenizer (not PreTrainedTokenizer, not backend class)
-            # e.g., MistralCommonBackend which has its own from_pretrained logic
-            inherits_from_backend = isinstance(tokenizer_class, type) and any(
-                bc and issubclass(tokenizer_class, bc) for bc in backend_classes
-            )
-            is_completely_custom = (
-                isinstance(tokenizer_class, type)
-                and not issubclass(tokenizer_class, PythonBackend)
-                and not inherits_from_backend
-            )
-
-            if is_custom_pre_trained:
-                logger.info("Loading tokenizer with custom PreTrainedTokenizer backend (edge case)")
-                # Track the backend type for custom tokenizers
-                kwargs["backend"] = "custom"
-                kwargs["files_loaded"] = []  # Custom tokenizers may load various files
-                return tokenizer_class.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs)
-
-            if is_completely_custom:
-                # For completely custom tokenizers (like MistralCommonBackend), try calling from_pretrained directly
-                logger.info("Loading tokenizer with custom tokenizer class (non-PreTrainedTokenizer)")
-                # Filter out AutoTokenizer-specific kwargs that custom tokenizers don't accept
-                custom_kwargs = {k: v for k, v in kwargs.items() if k not in ["backend", "files_loaded"]}
-                custom_kwargs["_from_auto"] = True  # Signal that this is called from AutoTokenizer
-                return tokenizer_class.from_pretrained(pretrained_model_name_or_path, *inputs, **custom_kwargs)
-
-            if TokenizersBackend is None:
-                raise ValueError(
-                    "Tokenizers backend is the default but tokenizers library is not installed. "
-                    "Please install it with: pip install tokenizers"
-                )
-            logger.info("Loading tokenizer with tokenizers backend")
-            try:
-                return _load_tokenizers_backend(tokenizer_class, pretrained_model_name_or_path, inputs, kwargs)
-            except ValueError as e:
-                # If tokenizers backend fails, try falling back to SentencePiece backend if available
-                spm_file = _find_sentencepiece_model_file(pretrained_model_name_or_path, **kwargs)
-                if spm_file is not None and SentencePieceBackend is not None:
-                    logger.info(
-                        f"Tokenizers backend failed: {e}. "
-                        f"Falling back to SentencePieceBackend since {spm_file} file was found."
-                    )
-                    files_loaded = [spm_file]
-                    kwargs["backend"] = "sentencepiece"
-                    kwargs["files_loaded"] = files_loaded
-                    # Resolve the SPM file path and pass it as vocab_file
-                    resolved_vocab_file = cached_file(
-                        pretrained_model_name_or_path,
-                        spm_file,
-                        cache_dir=kwargs.get("cache_dir"),
-                        force_download=kwargs.get("force_download", False),
-                        proxies=kwargs.get("proxies"),
-                        token=kwargs.get("token"),
-                        revision=kwargs.get("revision"),
-                        local_files_only=kwargs.get("local_files_only", False),
-                        subfolder=kwargs.get("subfolder", ""),
-                    )
-                    kwargs["vocab_file"] = resolved_vocab_file
-                    if tokenizer_class is not None and issubclass(tokenizer_class, SentencePieceBackend):
-                        logger.info(
-                            "Falling back to SentencePiece backend using tokenizer class that inherits from SentencePieceBackend."
-                        )
-                        return tokenizer_class.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs)
-                    return SentencePieceBackend.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs)
-                # If no fallback available, try calling tokenizer class directly as last resort
-                if hasattr(tokenizer_class, "from_pretrained"):
-                    logger.info(
-                        f"Tokenizers backend failed: {e}. Trying to load tokenizer directly from tokenizer class."
-                    )
-                    # Filter out AutoTokenizer-specific kwargs that custom tokenizers don't accept
-                    custom_kwargs = {k: v for k, v in kwargs.items() if k not in ["backend", "files_loaded"]}
-                    custom_kwargs["_from_auto"] = True  # Signal that this is called from AutoTokenizer
-                    return tokenizer_class.from_pretrained(pretrained_model_name_or_path, *inputs, **custom_kwargs)
-                # Re-raise if no fallback options available
-                raise
-
-        # If no tokenizer class but tokenizers backend requested, fall back to SentencePiece if available
-        spm_file = _find_sentencepiece_model_file(pretrained_model_name_or_path, **kwargs)
-        if spm_file is not None and SentencePieceBackend is not None:
-            logger.info(
-                f"Tokenizers backend was requested but no tokenizer class found. "
-                f"Falling back to SentencePieceBackend since {spm_file} file was found."
-            )
-            files_loaded = [spm_file]
-            kwargs["backend"] = "sentencepiece"
-            kwargs["files_loaded"] = files_loaded
-            # Resolve the SPM file path and pass it as vocab_file
-            resolved_vocab_file = cached_file(
-                pretrained_model_name_or_path,
-                spm_file,
-                cache_dir=kwargs.get("cache_dir"),
-                force_download=kwargs.get("force_download", False),
-                proxies=kwargs.get("proxies"),
-                token=kwargs.get("token"),
-                revision=kwargs.get("revision"),
-                local_files_only=kwargs.get("local_files_only", False),
-                subfolder=kwargs.get("subfolder", ""),
-            )
-            kwargs["vocab_file"] = resolved_vocab_file
-            if (
-                tokenizer_class is not None
-                and SentencePieceBackend is not None
-                and issubclass(tokenizer_class, SentencePieceBackend)
-            ):
-                logger.info(
-                    "Falling back to SentencePiece backend using tokenizer class that inherits from SentencePieceBackend."
-                )
-                return tokenizer_class.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs)
-            return SentencePieceBackend.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs)
-
-        raise ValueError(
-            f"Could not load tokenizer from {pretrained_model_name_or_path}. "
-            "No tokenizer class could be determined and no SentencePiece model found."
-        )
-
-
 def get_tokenizer_config(
     pretrained_model_name_or_path: Union[str, os.PathLike[str]],
     cache_dir: Optional[Union[str, os.PathLike[str]]] = None,
@@ -1054,11 +609,43 @@ class AutoTokenizer:
 
             return tokenizer_class.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs)
 
+        if gguf_file:
+            gguf_path = cached_file(pretrained_model_name_or_path, gguf_file, **kwargs)
+            config_dict = load_gguf_checkpoint(gguf_path, return_tensors=False)["config"]
+            config = AutoConfig.for_model(**config_dict)
+        elif config is None:
+            try:
+                config = AutoConfig.from_pretrained(
+                    pretrained_model_name_or_path, trust_remote_code=trust_remote_code, **kwargs
+                )
+            except Exception:
+                config = PreTrainedConfig.from_pretrained(pretrained_model_name_or_path, **kwargs)
+
+        config_model_type = config.model_type
+
         # Next, let's try to use the tokenizer_config file to get the tokenizer class.
         tokenizer_config = get_tokenizer_config(pretrained_model_name_or_path, **kwargs)
+        tokenizer_config_class = tokenizer_config.get("tokenizer_class", None)
+        # if there is a config, we can check that the tokenizer class != than model class and can thus assume we need to use `TokenizersBackend`
+        if (
+            tokenizer_config_class is not None
+            and config_model_type is not None
+            and config_model_type != ""
+            and TOKENIZER_MAPPING_NAMES.get(config_model_type, "").replace("Fast", "")
+            != tokenizer_config_class.replace("Fast", "")
+        ):
+            # new model, but we ignore it unless the model type is the same
+            try:
+                return TokenizersBackend.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs)
+            except Exception:
+                return tokenizer_class_from_name(tokenizer_config_class).from_pretrained(
+                    pretrained_model_name_or_path, *inputs, **kwargs
+                )
+
         if "_commit_hash" in tokenizer_config:
             kwargs["_commit_hash"] = tokenizer_config["_commit_hash"]
-        config_tokenizer_class = tokenizer_config.get("tokenizer_class")
+
+        # Check for auto_map early to handle dynamic tokenizers properly
         tokenizer_auto_map = None
         if "auto_map" in tokenizer_config:
             if isinstance(tokenizer_config["auto_map"], (tuple, list)):
@@ -1067,34 +654,15 @@ class AutoTokenizer:
             else:
                 tokenizer_auto_map = tokenizer_config["auto_map"].get("AutoTokenizer", None)
 
-        # If that did not work, let's try to use the config.
-        if config_tokenizer_class is None:
-            if not isinstance(config, PreTrainedConfig):
-                if gguf_file:
-                    gguf_path = cached_file(pretrained_model_name_or_path, gguf_file, **kwargs)
-                    config_dict = load_gguf_checkpoint(gguf_path, return_tensors=False)["config"]
-                    config = AutoConfig.for_model(**config_dict)
-                else:
-                    config = AutoConfig.from_pretrained(
-                        pretrained_model_name_or_path, trust_remote_code=trust_remote_code, **kwargs
-                    )
-            config_tokenizer_class = config.tokenizer_class
-            if hasattr(config, "auto_map") and "AutoTokenizer" in config.auto_map:
-                tokenizer_auto_map = config.auto_map["AutoTokenizer"]
-
-        if (
-            config_tokenizer_class is not None
-            and config_tokenizer_class != "PreTrainedTokenizerFast"
-            and "Fast" in config_tokenizer_class
-        ):
-            config_tokenizer_class = config_tokenizer_class[:-4]
+        if tokenizer_config_class:
+            tokenizer_config_class = tokenizer_config_class.replace("Fast", "")
 
         has_remote_code = tokenizer_auto_map is not None
         has_local_code = type(config) in TOKENIZER_MAPPING or (
-            config_tokenizer_class is not None
+            tokenizer_config_class is not None
             and (
-                tokenizer_class_from_name(config_tokenizer_class) is not None
-                or tokenizer_class_from_name(config_tokenizer_class + "Fast") is not None
+                tokenizer_class_from_name(tokenizer_config_class) is not None
+                or tokenizer_class_from_name(tokenizer_config_class + "Fast") is not None
             )
         )
         if has_remote_code:
@@ -1118,17 +686,24 @@ class AutoTokenizer:
             return tokenizer_class.from_pretrained(
                 pretrained_model_name_or_path, *inputs, trust_remote_code=trust_remote_code, **kwargs
             )
-        elif config_tokenizer_class is not None:
-            fast_tokenizer_class = None
-            if fast_tokenizer_class is None:
-                tokenizer_class_candidate = config_tokenizer_class
-                tokenizer_class = tokenizer_class_from_name(tokenizer_class_candidate)
-                if tokenizer_class is None and not tokenizer_class_candidate.endswith("Fast"):
-                    tokenizer_class = tokenizer_class_from_name(tokenizer_class_candidate + "Fast")
-            else:
-                tokenizer_class = fast_tokenizer_class
+        elif tokenizer_config_class is not None:
+            tokenizer_class_candidate = tokenizer_config_class
+            tokenizer_class = tokenizer_class_from_name(tokenizer_class_candidate)
+            if tokenizer_class is None and not tokenizer_class_candidate.endswith("Fast"):
+                tokenizer_class = tokenizer_class_from_name(tokenizer_class_candidate + "Fast")
+            if tokenizer_class is not None and tokenizer_class.__name__ == "PythonBackend":
+                tokenizer_class = TokenizersBackend
+            # Fallback to TokenizersBackend if the class wasn't found
+            if tokenizer_class is None:
+                tokenizer_class = TokenizersBackend
 
-            return _try_load_tokenizer_with_fallbacks(tokenizer_class, pretrained_model_name_or_path, inputs, kwargs)
+            return tokenizer_class.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs)
+        elif getattr(config, "tokenizer_class"):
+            _class = config.tokenizer_class
+            if "PreTrainedTokenizerFast" not in _class:
+                _class = _class.replace("Fast", "")
+            tokenizer_class = tokenizer_class_from_name(_class)
+            return tokenizer_class.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs)
 
         # Otherwise we have to be creative.
         # if model is an encoder decoder, the encoder tokenizer class is used by default
@@ -1142,19 +717,25 @@ class AutoTokenizer:
                 )
             config = config.encoder
 
-        model_type = config_class_to_model_type(type(config).__name__)
+        model_type = config_class_to_model_type(type(config).__name__) or config.get("model_type", None)
         if model_type is not None:
-            tokenizer_class = TOKENIZER_MAPPING[type(config)]
-
+            tokenizer_class = TOKENIZER_MAPPING.get(type(config), TokenizersBackend)
             if tokenizer_class is not None:
-                return _try_load_tokenizer_with_fallbacks(
-                    tokenizer_class, pretrained_model_name_or_path, inputs, kwargs
-                )
-            else:
-                raise ValueError(
-                    "This tokenizer cannot be instantiated. Please make sure you have `sentencepiece` installed "
-                    "in order to use this tokenizer."
-                )
+                return tokenizer_class.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs)
+
+        # Fallback: try tokenizer_class from tokenizer_config.json
+        tokenizer_config_class = tokenizer_config.get("tokenizer_class", None)
+        if tokenizer_config_class is not None:
+            if tokenizer_config_class != "TokenizersBackend" and "Fast" in tokenizer_config_class:
+                tokenizer_config_class = tokenizer_config_class[:-4]
+            tokenizer_class = tokenizer_class_from_name(tokenizer_config_class)
+            if tokenizer_class is None and not tokenizer_config_class.endswith("Fast"):
+                tokenizer_class = tokenizer_class_from_name(tokenizer_config_class + "Fast")
+            if tokenizer_class is not None and tokenizer_class.__name__ == "PythonBackend":
+                tokenizer_class = TokenizersBackend
+            if tokenizer_class is None:
+                tokenizer_class = TokenizersBackend
+            return tokenizer_class.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs)
 
         raise ValueError(
             f"Unrecognized configuration class {config.__class__} to build an AutoTokenizer.\n"