transformers 5.0.0__py3-none-any.whl → 5.0.0rc0__py3-none-any.whl

transformers/conversion_mapping.py

@@ -1,3 +1,4 @@
+# coding=utf-8
 # Copyright (C) 2025 the HuggingFace Inc. team. All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -17,16 +18,7 @@ from __future__ import annotations
 from copy import deepcopy
 from typing import TYPE_CHECKING
 
-from .core_model_loading import (
-    Chunk,
-    Concatenate,
-    ErnieFuseAndSplitTextVisionExperts,
-    Force16BytesAlignment,
-    MergeModulelist,
-    Transpose,
-    WeightConverter,
-    WeightRenaming,
-)
+from .core_model_loading import Concatenate, MergeModulelist, WeightConverter, WeightRenaming
 from .utils import is_torch_available
 
 
@@ -39,51 +31,8 @@ if TYPE_CHECKING:
     from .quantizers import HfQuantizer
 
 
-_MODEL_TO_CONVERSION_PATTERN = {
-    # Mixtral-style MoE
-    "mixtral": "mixtral",
-    "minimax": "mixtral",
-    "minimax_m2": "mixtral",
-    # Qwen2-style MoE
-    "qwen2_moe": "qwen2_moe",
-    "deepseek_v2": "qwen2_moe",
-    "deepseek_v3": "qwen2_moe",
-    "dots1": "qwen2_moe",
-    "ernie4_5_moe": "qwen2_moe",
-    "glm4_moe": "qwen2_moe",
-    "glm4_moe_lite": "qwen2_moe",
-    "glm4v_moe": "qwen2_moe",
-    "longcat_flash": "qwen2_moe",
-    "solar_open": "qwen2_moe",
-    "qwen3_moe": "qwen2_moe",
-    "qwen3_omni_moe": "qwen2_moe",
-    "qwen3_omni_moe_thinker": "qwen2_moe",
-    "qwen3_next": "qwen2_moe",
-    "hunyuan_v1_moe": "qwen2_moe",
-    "flex_olmo": "qwen2_moe",
-    "olmoe": "qwen2_moe",
-}
-
-
 def _build_checkpoint_conversion_mapping():
     mapping = {
-        "gpt_oss": [
-            # NOTE: These converters are only applied if the model is being loaded from pre-dequantized checkpoint.
-            # If you are dequantizing the model on the fly, these converters will be ignored because the tensors
-            # that match these patterns are only created after dequantization.
-            # That's not an issue for now since the dequantization converters already ensure 16 bytes alignment
-            # by enforcing contiguity.
-            WeightConverter(
-                source_patterns="mlp.experts.gate_up_proj$",
-                target_patterns="mlp.experts.gate_up_proj",
-                operations=[Force16BytesAlignment()],
-            ),
-            WeightConverter(
-                source_patterns="mlp.experts.down_proj$",
-                target_patterns="mlp.experts.down_proj",
-                operations=[Force16BytesAlignment()],
-            ),
-        ],
         "mixtral": [
             WeightRenaming(".block_sparse_moe.gate", ".mlp.gate"),
             WeightConverter(
@@ -126,21 +75,6 @@ def _build_checkpoint_conversion_mapping():
                 operations=[MergeModulelist(dim=0)],
             ),
         ],
-        "qwen3_vl_moe": [
-            WeightConverter(
-                source_patterns=[
-                    "mlp.experts.*.gate_proj.weight",
-                    "mlp.experts.*.up_proj.weight",
-                ],
-                target_patterns="mlp.experts.gate_up_proj",
-                operations=[MergeModulelist(dim=0), Concatenate(dim=1), Transpose(1, 2)],
-            ),
-            WeightConverter(
-                source_patterns="mlp.experts.*.down_proj.weight",
-                target_patterns="mlp.experts.down_proj",
-                operations=[MergeModulelist(dim=0), Transpose(1, 2)],
-            ),
-        ],
         "phimoe": [
             WeightConverter(
                 source_patterns=[
@@ -171,57 +105,6 @@ def _build_checkpoint_conversion_mapping():
                 operations=[MergeModulelist(dim=0)],
             ),
         ],
-        "ernie4_5_vl_moe": [
-            # vision
-            WeightRenaming("vision_model", "vision_tower"),
-            # resampler
-            WeightRenaming("spatial_linear.0", "spatial_linear.fc1"),
-            WeightRenaming("spatial_linear.2", "spatial_linear.fc2"),
-            WeightRenaming("spatial_linear.3", "spatial_linear.ln"),
-            WeightRenaming("temporal_linear.0", "temporal_linear.fc1"),
-            WeightRenaming("temporal_linear.2", "temporal_linear.fc2"),
-            WeightRenaming("temporal_linear.3", "temporal_linear.ln"),
-            # language model
-            WeightRenaming(r"(?<!language_model\.)embed_tokens", "language_model.embed_tokens"),
-            WeightRenaming(r"(?<!language_model\.)layers", "language_model.layers"),
-            WeightConverter(
-                source_patterns="mlp.gate.weight_1",
-                target_patterns="mlp.vision_moe.gate.weight",
-                operations=[Transpose(dim0=0, dim1=1)],
-            ),
-            WeightConverter(
-                source_patterns="mlp.gate.weight",
-                target_patterns="mlp.text_moe.gate.weight",
-                operations=[Transpose(dim0=0, dim1=1)],
-            ),
-            WeightConverter(
-                source_patterns=["mlp.moe_statics.e_score_correction_bias"],
-                target_patterns=[
-                    "mlp.text_moe.gate.moe_statics.e_score_correction_bias",
-                    "mlp.vision_moe.gate.moe_statics.e_score_correction_bias",
-                ],
-                operations=[Chunk(dim=0)],
-            ),
-            WeightConverter(
-                source_patterns=["experts.*.down_proj.weight"],
-                target_patterns=[
-                    "text_moe.experts.down_proj",
-                    "vision_moe.experts.down_proj",
-                ],
-                operations=[ErnieFuseAndSplitTextVisionExperts(stack_dim=0, concat_dim=1)],
-            ),
-            WeightConverter(
-                source_patterns=[
-                    "experts.*.gate_proj.weight",
-                    "experts.*.up_proj.weight",
-                ],
-                target_patterns=[
-                    "text_moe.experts.gate_up_proj",
-                    "vision_moe.experts.gate_up_proj",
-                ],
-                operations=[ErnieFuseAndSplitTextVisionExperts(stack_dim=0, concat_dim=1)],
-            ),
-        ],
         "jamba": [
             WeightConverter(
                 source_patterns=[
@@ -259,12 +142,12 @@ def _build_checkpoint_conversion_mapping():
     if hasattr(torch.nn.utils.parametrizations, "weight_norm"):
         mapping["legacy"] += [
             WeightRenaming(
-                source_patterns=".weight_g$",
-                target_patterns=".parametrizations.weight.original0",
+                source_patterns="weight_g",
+                target_patterns="parametrizations.weight.original0",
             ),
             WeightRenaming(
-                source_patterns=".weight_v$",
-                target_patterns=".parametrizations.weight.original1",
+                source_patterns="weight_v",
+                target_patterns="parametrizations.weight.original1",
             ),
         ]
     else:
@@ -279,19 +162,21 @@ def _build_checkpoint_conversion_mapping():
             ),
         ]
 
+    mapping["deepseek_v2"] = mapping["qwen2_moe"].copy()
+    mapping["deepseek_v3"] = mapping["qwen2_moe"].copy()
+    mapping["dots1"] = mapping["qwen2_moe"].copy()
     mapping["ernie4_5_moe"] = mapping["qwen2_moe"].copy()
-    mapping["ernie4_5_moe"] += [
-        WeightRenaming("mlp.moe_statics.e_score_correction_bias", "mlp.gate.moe_statics.e_score_correction_bias")
-    ]
-    mapping["minimax_m2"] = mapping["mixtral"].copy()
-    mapping["minimax_m2"] += [
-        WeightRenaming(".block_sparse_moe.e_score_correction_bias", ".mlp.e_score_correction_bias"),
-    ]
-
-    for model_type, base_pattern in _MODEL_TO_CONVERSION_PATTERN.items():
-        if model_type in mapping:
-            continue
-        mapping[model_type] = mapping[base_pattern].copy()
+    mapping["glm4_moe"] = mapping["qwen2_moe"].copy()
+    mapping["glm4v_moe"] = mapping["qwen2_moe"].copy()
+    mapping["longcat_flash"] = mapping["qwen2_moe"].copy()
+    mapping["qwen3_moe"] = mapping["qwen2_moe"].copy()
+    mapping["qwen3_omni_moe"] = mapping["qwen2_moe"].copy()
+    mapping["qwen3_next"] = mapping["qwen2_moe"].copy()
+    mapping["qwen3_vl_moe"] = mapping["qwen2_moe"].copy()
+    mapping["hunyuan_v1_moe"] = mapping["qwen2_moe"].copy()
+    mapping["minimax"] = mapping["mixtral"].copy()
+    mapping["flex_olmo"] = mapping["qwen2_moe"].copy()
+    mapping["olmoe"] = mapping["qwen2_moe"].copy()
 
     return mapping
 
@@ -301,22 +186,10 @@ _checkpoint_conversion_mapping_cache = None
 
 def get_checkpoint_conversion_mapping(model_type):
     global _checkpoint_conversion_mapping_cache
-    if _checkpoint_conversion_mapping_cache is None:
-        _checkpoint_conversion_mapping_cache = _build_checkpoint_conversion_mapping()
+    _checkpoint_conversion_mapping_cache = _build_checkpoint_conversion_mapping()
     return deepcopy(_checkpoint_conversion_mapping_cache.get(model_type))
 
 
-def register_checkpoint_conversion_mapping(
-    model_type: str, mapping: list[WeightConverter | WeightRenaming], overwrite: bool = False
-) -> None:
-    global _checkpoint_conversion_mapping_cache
-    if _checkpoint_conversion_mapping_cache is None:
-        _checkpoint_conversion_mapping_cache = _build_checkpoint_conversion_mapping()
-    if model_type in _checkpoint_conversion_mapping_cache and not overwrite:
-        raise ValueError(f"Model type {model_type} already exists in the checkpoint conversion mapping.")
-    _checkpoint_conversion_mapping_cache[model_type] = mapping
-
-
 # DO NOT MODIFY, KEPT FOR BC ONLY
 VLMS = [
     "aria",
@@ -340,8 +213,6 @@ VLMS = [
     "sam3",
     "sam3_tracker",
     "sam3_tracker_video",
-    "paddleocrvl",
-    "ernie4_5_vl_moe",
 ]
 
 
@@ -357,7 +228,7 @@ def get_model_conversion_mapping(
     """
     weight_conversions = []
 
-    # Load models with explicit, user-provided key mapping
+    # Load models with key mapping
     if key_mapping is not None:
         weight_conversions = [WeightRenaming(source_patterns=k, target_patterns=v) for k, v in key_mapping.items()]
     elif any(
@@ -382,13 +253,6 @@ def get_model_conversion_mapping(
 
     # Add the ones from the quantizer as well if provided
     if hf_quantizer is not None:
-        # NOTE: Since get_weight_conversions() only serve to dequantize, we would normally want to apply them first.
-        # However, for now it's not possible to cascade converters (i.e., applying model-specific conversions on top
-        # of tensors created by the dequantization conversions)
-        # This means that if a model has model-specific conversions and is being dequantized, the model-specific conversion
-        # that relies on tensors created by dequantization conversions will not be applied.
-        # GptOss example: with Mxfp4Config(dequantize=True), Force16BytesAlignment converters are ignored because the tensors
-        # "mlp.experts.gate_up_proj$" and "mlp.experts.down_proj$" are only created after dequantization conversions are applied.
         weight_conversions.extend(hf_quantizer.get_weight_conversions())
 
     return weight_conversions

transformers/convert_slow_tokenizer.py

@@ -19,8 +19,8 @@ allow to make our dependency on SentencePiece optional.
 """
 
 import warnings
-from collections.abc import Collection
 from functools import lru_cache
+from typing import Optional
 
 from packaging import version
 from tokenizers import AddedToken, Regex, Tokenizer, decoders, normalizers, pre_tokenizers, processors
@@ -33,64 +33,6 @@ from .utils.import_utils import PROTOBUF_IMPORT_ERROR
 
 logger = logging.get_logger(__name__)
 
-MBART_LANGUAGES = [
-    "ar_AR",
-    "cs_CZ",
-    "de_DE",
-    "en_XX",
-    "es_XX",
-    "et_EE",
-    "fi_FI",
-    "fr_XX",
-    "gu_IN",
-    "hi_IN",
-    "it_IT",
-    "ja_XX",
-    "kk_KZ",
-    "ko_KR",
-    "lt_LT",
-    "lv_LV",
-    "my_MM",
-    "ne_NP",
-    "nl_XX",
-    "ro_RO",
-    "ru_RU",
-    "si_LK",
-    "tr_TR",
-    "vi_VN",
-    "zh_CN",
-]
-
-MBART50_LANGUAGES = MBART_LANGUAGES + [
-    "af_ZA",
-    "az_AZ",
-    "bn_IN",
-    "fa_IR",
-    "he_IL",
-    "hr_HR",
-    "id_ID",
-    "ka_GE",
-    "km_KH",
-    "mk_MK",
-    "ml_IN",
-    "mn_MN",
-    "mr_IN",
-    "pl_PL",
-    "ps_AF",
-    "pt_XX",
-    "sv_SE",
-    "sw_KE",
-    "ta_IN",
-    "te_IN",
-    "th_TH",
-    "tl_XX",
-    "uk_UA",
-    "ur_PK",
-    "xh_ZA",
-    "gl_ES",
-    "sl_SI",
-]
-
 
 def import_protobuf(error_message=""):
     if is_sentencepiece_available():
@@ -119,20 +61,15 @@ def _get_prepend_scheme(add_prefix_space: bool, original_tokenizer) -> str:
     return prepend_scheme
 
 
-def generate_merges(vocab, vocab_scores, skip_tokens: Collection[str] | None = None):
-    skip_tokens = set(skip_tokens) if skip_tokens is not None else set()
+def generate_merges(vocab, vocab_scores):
     reverse = vocab_scores is not None
     vocab_scores = dict(vocab_scores) if reverse else vocab
 
     merges = []
     for merge, piece_score in vocab_scores.items():
-        if merge in skip_tokens:
-            continue
         local = []
         for index in range(1, len(merge)):
             piece_l, piece_r = merge[:index], merge[index:]
-            if piece_l in skip_tokens or piece_r in skip_tokens:
-                continue
             if piece_l in vocab and piece_r in vocab:
                 local.append((piece_l, piece_r, piece_score))
         local = sorted(local, key=lambda x: (vocab[x[0]], vocab[x[1]]))
@@ -150,49 +87,22 @@ class SentencePieceExtractor:
 
     def __init__(self, model: str):
         requires_backends(self, "sentencepiece")
-        requires_backends(self, "protobuf")
-
-        # from .utils import sentencepiece_model_pb2 as model_pb2
-        model_pb2 = import_protobuf()
+        from sentencepiece import SentencePieceProcessor
 
-        m = model_pb2.ModelProto()
-        with open(model, "rb") as f:
-            m.ParseFromString(f.read())
-        self.proto = m
+        self.sp = SentencePieceProcessor()
+        self.sp.Load(model)
 
-    def extract(self, model_type, **kwargs) -> tuple[dict[str, int], list[tuple]]:
+    def extract(self, vocab_scores=None) -> tuple[dict[str, int], list[tuple]]:
         """
         By default will return vocab and merges with respect to their order, by sending `vocab_scores` we're going to
         order the merges with respect to the piece scores instead.
         """
-        self.proto.trainer_spec.unk_id
-        if model_type is None:
-            from tokenizers.models import BPE, Unigram
-
-            model_type = Unigram if self.proto.trainer_spec.model_type == 2 else BPE
-        vocab = [(piece.piece, piece.score) for piece in self.proto.pieces]
-
-        if model_type.__name__ != "BPE":
-            kwargs["unk_id"] = self.proto.trainer_spec.unk_id
-            kwargs["vocab"] = vocab
-        else:
-            from .tokenization_utils_base import generate_merges
+        sp = self.sp
+        vocab = {sp.id_to_piece(index): index for index in range(sp.GetPieceSize())}
 
-            vocab = {word: i for i, (word, score) in enumerate(vocab)}
-            merges = generate_merges(vocab)
-            kwargs["vocab"] = vocab
-            kwargs["merges"] = merges
+        merges = generate_merges(vocab, vocab_scores)
 
-        # control tokens are special
-        # user defined symbols are not
-        # both user and control tokens are AddedTokens
-        # Add user defined symbols (type == 4) from sentencepiece (https://github.com/google/sentencepiece/blob/6225e08edb2577757163b3f5dbba4c0b670ef445/src/sentencepiece_model.proto#L299C29-L299C33)
-        spm_added_tokens = [(id, p.piece, p.type == 3) for id, p in enumerate(self.proto.pieces) if p.type in [3, 4]]
-        kwargs["additional_special_tokens"] = [
-            AddedToken(token, normalized=False, special=special)
-            for id, token, special in sorted(spm_added_tokens, key=lambda x: x[0])
-        ]
-        return kwargs
+        return vocab, merges
 
 
 class GemmaSentencePieceExtractor(SentencePieceExtractor):
@@ -419,7 +329,9 @@ class OpenAIGPTConverter(Converter):
 
 
 class GPT2Converter(Converter):
-    def converted(self, vocab: dict[str, int] | None = None, merges: list[tuple[str, str]] | None = None) -> Tokenizer:
+    def converted(
+        self, vocab: Optional[dict[str, int]] = None, merges: Optional[list[tuple[str, str]]] = None
+    ) -> Tokenizer:
         if not vocab:
             vocab = self.original_tokenizer.encoder
         if not merges:
@@ -488,7 +400,9 @@ class HerbertConverter(Converter):
 
 
 class Qwen2Converter(Converter):
-    def converted(self, vocab: dict[str, int] | None = None, merges: list[tuple[str, str]] | None = None) -> Tokenizer:
+    def converted(
+        self, vocab: Optional[dict[str, int]] = None, merges: Optional[list[tuple[str, str]]] = None
+    ) -> Tokenizer:
         if not vocab:
             vocab = self.original_tokenizer.encoder
         if not merges:
@@ -635,16 +549,6 @@ class SpmConverter(Converter):
     SpmExtractor = SentencePieceExtractor
     special_tokens = {}
 
-    @classmethod
-    def convert_from_spm(cls, vocab=None, **kwargs):
-        """
-        Hook used when converting directly from a SentencePiece model without a slow tokenizer instance.
-        By default, return kwargs unchanged.
-        """
-        if vocab is not None:
-            kwargs["vocab"] = vocab
-        return kwargs
-
     def __init__(self, *args):
         requires_backends(self, "protobuf")
 
@@ -850,25 +754,6 @@ class CamembertConverter(SpmConverter):
             ],
         )
 
-    @classmethod
-    def convert_from_spm(cls, vocab=None, **kwargs):
-        pad_token = str(kwargs.get("pad_token", "<pad>"))
-        unk_token = str(kwargs.get("unk_token", "<unk>"))
-        mask_token = str(kwargs.get("mask_token", "<mask>"))
-
-        vocab_list = [
-            ("<s>NOTUSED", 0.0),
-            (pad_token, 0.0),
-            ("</s>NOTUSED", 0.0),
-            (unk_token, 0.0),
-            ("<unk>NOTUSED", -100.0),
-        ]
-        if vocab is not None:
-            vocab_list.extend(list(vocab)[1:])
-        vocab_list.append((mask_token, 0.0))
-        kwargs["vocab"] = vocab_list
-        return kwargs
-
 
 class DebertaV2Converter(SpmConverter):
     def pre_tokenizer(self, replacement, add_prefix_space):
@@ -955,27 +840,6 @@ class MBartConverter(SpmConverter):
             ],
         )
 
-    @classmethod
-    def convert_from_spm(cls, vocab=None, **kwargs):
-        bos_token = str(kwargs.get("bos_token", "<s>"))
-        pad_token = str(kwargs.get("pad_token", "<pad>"))
-        eos_token = str(kwargs.get("eos_token", "</s>"))
-        unk_token = str(kwargs.get("unk_token", "<unk>"))
-        mask_token = str(kwargs.get("mask_token", "<mask>"))
-
-        vocab_list = [
-            (bos_token, 0.0),
-            (pad_token, 0.0),
-            (eos_token, 0.0),
-            (unk_token, 0.0),
-        ]
-        if vocab is not None:
-            vocab_list.extend(list(vocab)[3:])
-        vocab_list.extend((lang_code, 0.0) for lang_code in MBART_LANGUAGES)
-        vocab_list.append((mask_token, 0.0))
-        kwargs["vocab"] = vocab_list
-        return kwargs
-
 
 class MBart50Converter(SpmConverter):
     def vocab(self, proto):
@@ -1003,27 +867,6 @@ class MBart50Converter(SpmConverter):
             ],
         )
 
-    @classmethod
-    def convert_from_spm(cls, vocab=None, **kwargs):
-        cls_token = str(kwargs.get("cls_token", "<s>"))
-        pad_token = str(kwargs.get("pad_token", "<pad>"))
-        eos_token = str(kwargs.get("eos_token", "</s>"))
-        unk_token = str(kwargs.get("unk_token", "<unk>"))
-        mask_token = str(kwargs.get("mask_token", "<mask>"))
-
-        vocab_list = [
-            (cls_token, 0.0),
-            (pad_token, 0.0),
-            (eos_token, 0.0),
-            (unk_token, 0.0),
-        ]
-        if vocab is not None:
-            vocab_list.extend(list(vocab)[3:])
-        vocab_list.extend((lang_code, 0.0) for lang_code in MBART50_LANGUAGES)
-        vocab_list.append((mask_token, 0.0))
-        kwargs["vocab"] = vocab_list
-        return kwargs
-
 
 class NllbConverter(SpmConverter):
     def vocab(self, proto):
@@ -1049,28 +892,6 @@ class NllbConverter(SpmConverter):
             ],
         )
 
-    @classmethod
-    def convert_from_spm(cls, vocab=None, **kwargs):
-        bos_token = str(kwargs.get("bos_token", "<s>"))
-        pad_token = str(kwargs.get("pad_token", "<pad>"))
-        eos_token = str(kwargs.get("eos_token", "</s>"))
-        unk_token = str(kwargs.get("unk_token", "<unk>"))
-
-        reordered_vocab = {
-            bos_token: 0,
-            pad_token: 1,
-            eos_token: 2,
-            unk_token: 3,
-        }
-        if vocab is not None:
-            tokens = vocab.keys() if isinstance(vocab, dict) else [tok for tok, _ in vocab]
-            for token in tokens:
-                if token in reordered_vocab:
-                    continue
-                reordered_vocab[token] = len(reordered_vocab)
-        kwargs["vocab"] = reordered_vocab
-        return kwargs
-
 
 class SeamlessM4TConverter(SpmConverter):
     def vocab(self, proto):
@@ -1123,26 +944,6 @@ class XLMRobertaConverter(SpmConverter):
             ],
         )
 
-    @classmethod
-    def convert_from_spm(cls, vocab=None, **kwargs):
-        bos_token = str(kwargs.get("bos_token", "<s>"))
-        pad_token = str(kwargs.get("pad_token", "<pad>"))
-        eos_token = str(kwargs.get("eos_token", "</s>"))
-        unk_token = str(kwargs.get("unk_token", "<unk>"))
-        mask_token = str(kwargs.get("mask_token", "<mask>"))
-
-        vocab_list = [
-            (bos_token, 0.0),
-            (pad_token, 0.0),
-            (eos_token, 0.0),
-            (unk_token, 0.0),
-        ]
-        if vocab is not None:
-            vocab_list.extend(list(vocab)[3:])
-        vocab_list.append((mask_token, 0.0))
-        kwargs["vocab"] = vocab_list
-        return kwargs
-
 
 class XLNetConverter(SpmConverter):
     def vocab(self, proto):
@@ -1277,17 +1078,6 @@ class T5Converter(SpmConverter):
             ],
         )
 
-    @classmethod
-    def convert_from_spm(cls, vocab=None, **kwargs):
-        extra_ids = kwargs.get("extra_ids", 100)
-        extra_tokens = [f"<extra_id_{i}>" for i in range(extra_ids - 1, -1, -1)]
-        vocab_list = list(vocab) if vocab is not None else []
-        vocab_list.extend((token, 0.0) for token in extra_tokens)
-
-        kwargs.setdefault("additional_special_tokens", extra_tokens)
-        kwargs["vocab"] = vocab_list
-        return kwargs
-
 
 class UdopConverter(SpmConverter):
     def post_processor(self):
@@ -1381,7 +1171,7 @@ class CLIPConverter(Converter):
         )
         tokenizer.decoder = decoders.ByteLevel()
 
-        # Hack to have a ByteLevel and TemplateProcessor
+        # Hack to have a ByteLevel and TemplaceProcessor
         tokenizer.post_processor = processors.RobertaProcessing(
             sep=(self.original_tokenizer.eos_token, self.original_tokenizer.eos_token_id),
             cls=(self.original_tokenizer.bos_token, self.original_tokenizer.bos_token_id),