transformers 5.0.0rc0__py3-none-any.whl → 5.0.0rc2__py3-none-any.whl

transformers/convert_slow_tokenizer.py

@@ -19,6 +19,7 @@ allow to make our dependency on SentencePiece optional.
 """
 
 import warnings
+from collections.abc import Collection
 from functools import lru_cache
 from typing import Optional
 
@@ -33,6 +34,64 @@ from .utils.import_utils import PROTOBUF_IMPORT_ERROR
 
 logger = logging.get_logger(__name__)
 
+MBART_LANGUAGES = [
+    "ar_AR",
+    "cs_CZ",
+    "de_DE",
+    "en_XX",
+    "es_XX",
+    "et_EE",
+    "fi_FI",
+    "fr_XX",
+    "gu_IN",
+    "hi_IN",
+    "it_IT",
+    "ja_XX",
+    "kk_KZ",
+    "ko_KR",
+    "lt_LT",
+    "lv_LV",
+    "my_MM",
+    "ne_NP",
+    "nl_XX",
+    "ro_RO",
+    "ru_RU",
+    "si_LK",
+    "tr_TR",
+    "vi_VN",
+    "zh_CN",
+]
+
+MBART50_LANGUAGES = MBART_LANGUAGES + [
+    "af_ZA",
+    "az_AZ",
+    "bn_IN",
+    "fa_IR",
+    "he_IL",
+    "hr_HR",
+    "id_ID",
+    "ka_GE",
+    "km_KH",
+    "mk_MK",
+    "ml_IN",
+    "mn_MN",
+    "mr_IN",
+    "pl_PL",
+    "ps_AF",
+    "pt_XX",
+    "sv_SE",
+    "sw_KE",
+    "ta_IN",
+    "te_IN",
+    "th_TH",
+    "tl_XX",
+    "uk_UA",
+    "ur_PK",
+    "xh_ZA",
+    "gl_ES",
+    "sl_SI",
+]
+
 
 def import_protobuf(error_message=""):
     if is_sentencepiece_available():
@@ -61,15 +120,20 @@ def _get_prepend_scheme(add_prefix_space: bool, original_tokenizer) -> str:
     return prepend_scheme
 
 
-def generate_merges(vocab, vocab_scores):
+def generate_merges(vocab, vocab_scores, skip_tokens: Optional[Collection[str]] = None):
+    skip_tokens = set(skip_tokens) if skip_tokens is not None else set()
     reverse = vocab_scores is not None
     vocab_scores = dict(vocab_scores) if reverse else vocab
 
     merges = []
     for merge, piece_score in vocab_scores.items():
+        if merge in skip_tokens:
+            continue
         local = []
         for index in range(1, len(merge)):
             piece_l, piece_r = merge[:index], merge[index:]
+            if piece_l in skip_tokens or piece_r in skip_tokens:
+                continue
             if piece_l in vocab and piece_r in vocab:
                 local.append((piece_l, piece_r, piece_score))
         local = sorted(local, key=lambda x: (vocab[x[0]], vocab[x[1]]))
@@ -87,22 +151,49 @@ class SentencePieceExtractor:
 
     def __init__(self, model: str):
         requires_backends(self, "sentencepiece")
-        from sentencepiece import SentencePieceProcessor
+        requires_backends(self, "protobuf")
+
+        # from .utils import sentencepiece_model_pb2 as model_pb2
+        model_pb2 = import_protobuf()
 
-        self.sp = SentencePieceProcessor()
-        self.sp.Load(model)
+        m = model_pb2.ModelProto()
+        with open(model, "rb") as f:
+            m.ParseFromString(f.read())
+        self.proto = m
 
-    def extract(self, vocab_scores=None) -> tuple[dict[str, int], list[tuple]]:
+    def extract(self, model_type, **kwargs) -> tuple[dict[str, int], list[tuple]]:
         """
         By default will return vocab and merges with respect to their order, by sending `vocab_scores` we're going to
         order the merges with respect to the piece scores instead.
         """
-        sp = self.sp
-        vocab = {sp.id_to_piece(index): index for index in range(sp.GetPieceSize())}
+        self.proto.trainer_spec.unk_id
+        if model_type is None:
+            from tokenizers.models import BPE, Unigram
 
-        merges = generate_merges(vocab, vocab_scores)
+            model_type = Unigram if self.proto.trainer_spec.model_type == 2 else BPE
+        vocab = [(piece.piece, piece.score) for piece in self.proto.pieces]
 
-        return vocab, merges
+        if model_type.__name__ != "BPE":
+            kwargs["unk_id"] = self.proto.trainer_spec.unk_id
+            kwargs["vocab"] = vocab
+        else:
+            from .tokenization_utils_base import generate_merges
+
+            vocab = {word: i for i, (word, score) in enumerate(vocab)}
+            merges = generate_merges(vocab)
+            kwargs["vocab"] = vocab
+            kwargs["merges"] = merges
+
+        # control tokens are special
+        # user defined symbols are not
+        # both user and control tokens are AddedTokens
+        # Add user defined symbols (type == 4) from sentencepiece (https://github.com/google/sentencepiece/blob/6225e08edb2577757163b3f5dbba4c0b670ef445/src/sentencepiece_model.proto#L299C29-L299C33)
+        spm_added_tokens = [(id, p.piece, p.type == 3) for id, p in enumerate(self.proto.pieces) if p.type in [3, 4]]
+        kwargs["additional_special_tokens"] = [
+            AddedToken(token, normalized=False, special=special)
+            for id, token, special in sorted(spm_added_tokens, key=lambda x: x[0])
+        ]
+        return kwargs
 
 
 class GemmaSentencePieceExtractor(SentencePieceExtractor):
@@ -549,6 +640,16 @@ class SpmConverter(Converter):
     SpmExtractor = SentencePieceExtractor
     special_tokens = {}
 
+    @classmethod
+    def convert_from_spm(cls, vocab=None, **kwargs):
+        """
+        Hook used when converting directly from a SentencePiece model without a slow tokenizer instance.
+        By default, return kwargs unchanged.
+        """
+        if vocab is not None:
+            kwargs["vocab"] = vocab
+        return kwargs
+
     def __init__(self, *args):
         requires_backends(self, "protobuf")
 
@@ -754,6 +855,25 @@ class CamembertConverter(SpmConverter):
             ],
         )
 
+    @classmethod
+    def convert_from_spm(cls, vocab=None, **kwargs):
+        pad_token = str(kwargs.get("pad_token", "<pad>"))
+        unk_token = str(kwargs.get("unk_token", "<unk>"))
+        mask_token = str(kwargs.get("mask_token", "<mask>"))
+
+        vocab_list = [
+            ("<s>NOTUSED", 0.0),
+            (pad_token, 0.0),
+            ("</s>NOTUSED", 0.0),
+            (unk_token, 0.0),
+            ("<unk>NOTUSED", -100.0),
+        ]
+        if vocab is not None:
+            vocab_list.extend(list(vocab)[1:])
+        vocab_list.append((mask_token, 0.0))
+        kwargs["vocab"] = vocab_list
+        return kwargs
+
 
 class DebertaV2Converter(SpmConverter):
     def pre_tokenizer(self, replacement, add_prefix_space):
@@ -840,6 +960,27 @@ class MBartConverter(SpmConverter):
             ],
         )
 
+    @classmethod
+    def convert_from_spm(cls, vocab=None, **kwargs):
+        bos_token = str(kwargs.get("bos_token", "<s>"))
+        pad_token = str(kwargs.get("pad_token", "<pad>"))
+        eos_token = str(kwargs.get("eos_token", "</s>"))
+        unk_token = str(kwargs.get("unk_token", "<unk>"))
+        mask_token = str(kwargs.get("mask_token", "<mask>"))
+
+        vocab_list = [
+            (bos_token, 0.0),
+            (pad_token, 0.0),
+            (eos_token, 0.0),
+            (unk_token, 0.0),
+        ]
+        if vocab is not None:
+            vocab_list.extend(list(vocab)[3:])
+        vocab_list.extend((lang_code, 0.0) for lang_code in MBART_LANGUAGES)
+        vocab_list.append((mask_token, 0.0))
+        kwargs["vocab"] = vocab_list
+        return kwargs
+
 
 class MBart50Converter(SpmConverter):
     def vocab(self, proto):
@@ -867,6 +1008,27 @@ class MBart50Converter(SpmConverter):
             ],
         )
 
+    @classmethod
+    def convert_from_spm(cls, vocab=None, **kwargs):
+        cls_token = str(kwargs.get("cls_token", "<s>"))
+        pad_token = str(kwargs.get("pad_token", "<pad>"))
+        eos_token = str(kwargs.get("eos_token", "</s>"))
+        unk_token = str(kwargs.get("unk_token", "<unk>"))
+        mask_token = str(kwargs.get("mask_token", "<mask>"))
+
+        vocab_list = [
+            (cls_token, 0.0),
+            (pad_token, 0.0),
+            (eos_token, 0.0),
+            (unk_token, 0.0),
+        ]
+        if vocab is not None:
+            vocab_list.extend(list(vocab)[3:])
+        vocab_list.extend((lang_code, 0.0) for lang_code in MBART50_LANGUAGES)
+        vocab_list.append((mask_token, 0.0))
+        kwargs["vocab"] = vocab_list
+        return kwargs
+
 
 class NllbConverter(SpmConverter):
     def vocab(self, proto):
@@ -892,6 +1054,28 @@ class NllbConverter(SpmConverter):
             ],
         )
 
+    @classmethod
+    def convert_from_spm(cls, vocab=None, **kwargs):
+        bos_token = str(kwargs.get("bos_token", "<s>"))
+        pad_token = str(kwargs.get("pad_token", "<pad>"))
+        eos_token = str(kwargs.get("eos_token", "</s>"))
+        unk_token = str(kwargs.get("unk_token", "<unk>"))
+
+        reordered_vocab = {
+            bos_token: 0,
+            pad_token: 1,
+            eos_token: 2,
+            unk_token: 3,
+        }
+        if vocab is not None:
+            tokens = vocab.keys() if isinstance(vocab, dict) else [tok for tok, _ in vocab]
+            for token in tokens:
+                if token in reordered_vocab:
+                    continue
+                reordered_vocab[token] = len(reordered_vocab)
+        kwargs["vocab"] = reordered_vocab
+        return kwargs
+
 
 class SeamlessM4TConverter(SpmConverter):
     def vocab(self, proto):
@@ -944,6 +1128,26 @@ class XLMRobertaConverter(SpmConverter):
             ],
         )
 
+    @classmethod
+    def convert_from_spm(cls, vocab=None, **kwargs):
+        bos_token = str(kwargs.get("bos_token", "<s>"))
+        pad_token = str(kwargs.get("pad_token", "<pad>"))
+        eos_token = str(kwargs.get("eos_token", "</s>"))
+        unk_token = str(kwargs.get("unk_token", "<unk>"))
+        mask_token = str(kwargs.get("mask_token", "<mask>"))
+
+        vocab_list = [
+            (bos_token, 0.0),
+            (pad_token, 0.0),
+            (eos_token, 0.0),
+            (unk_token, 0.0),
+        ]
+        if vocab is not None:
+            vocab_list.extend(list(vocab)[3:])
+        vocab_list.append((mask_token, 0.0))
+        kwargs["vocab"] = vocab_list
+        return kwargs
+
 
 class XLNetConverter(SpmConverter):
     def vocab(self, proto):
@@ -1078,6 +1282,17 @@ class T5Converter(SpmConverter):
             ],
         )
 
+    @classmethod
+    def convert_from_spm(cls, vocab=None, **kwargs):
+        extra_ids = kwargs.get("extra_ids", 100)
+        extra_tokens = [f"<extra_id_{i}>" for i in range(extra_ids - 1, -1, -1)]
+        vocab_list = list(vocab) if vocab is not None else []
+        vocab_list.extend((token, 0.0) for token in extra_tokens)
+
+        kwargs.setdefault("additional_special_tokens", extra_tokens)
+        kwargs["vocab"] = vocab_list
+        return kwargs
+
 
 class UdopConverter(SpmConverter):
     def post_processor(self):
@@ -1171,7 +1386,7 @@ class CLIPConverter(Converter):
         )
         tokenizer.decoder = decoders.ByteLevel()
 
-        # Hack to have a ByteLevel and TemplaceProcessor
+        # Hack to have a ByteLevel and TemplateProcessor
         tokenizer.post_processor = processors.RobertaProcessing(
             sep=(self.original_tokenizer.eos_token, self.original_tokenizer.eos_token_id),
             cls=(self.original_tokenizer.bos_token, self.original_tokenizer.bos_token_id),