indic-tokenizer 0.1.0__py3-none-any.whl

indic_tokenizer/__init__.py

@@ -0,0 +1,5 @@
+from .bpe_tokenizer import IndicBPETokenizer
+from .preprocessor import IndicTextPreprocessor
+from .constants import SPECIAL_TOKENS, INDIC_UNICODE_RANGES
+
+__all__ = ["IndicBPETokenizer", "IndicTextPreprocessor", "SPECIAL_TOKENS", "INDIC_UNICODE_RANGES"]

indic_tokenizer/__main__.py

@@ -0,0 +1,8 @@
+"""
+Entry point for  python -m indic_tokenizer  invocation.
+"""
+
+from .bpe_tokenizer import IndicBPETokenizer
+
+print("indic_tokenizer — BPE tokenizer for Indic scripts")
+print("Usage: from indic_tokenizer import IndicBPETokenizer")

indic_tokenizer/bpe_tokenizer.py

@@ -0,0 +1,375 @@
+"""
+IndicBPETokenizer — main public class for BPE-based Indic tokenization.
+
+Space / newline convention  (Claude Sonnet style)
+--------------------------------------------------
+Spaces and newlines are kept as *literal* characters inside tokens.
+A token like " नमस्ते" contains a real leading space — no Ġ/Ċ substitution.
+Decoding is therefore a trivial string join with no post-processing.
+
+Responsibilities
+----------------
+train()     — learn BPE merges from a corpus (small / medium datasets)
+train_from_file() — convenience wrapper that auto-loads any supported format
+encode()    — text → List[int]  (with optional special-token pass-through)
+decode()    — List[int] → text  (plain string join)
+save()      — persist vocab + merges to two JSON files
+load()      — restore from those JSON files
+
+For corpora > ~1 GB use ChunkedBPETrainer (chunked_trainer.py) instead of
+train(), which stores the full token ID sequence in RAM.
+"""
+
+import json
+import re
+from pathlib import Path
+from typing import Dict, List, Optional, Set, Tuple, Union
+
+from .bpe_trainer import BPETrainer, MergesType, PairType
+from .constants import SPECIAL_TOKENS
+from .data_loader import IndicDataLoader
+from .preprocessor import pretokenize
+from .vocab_builder import VocabBuilder, InverseVocabType, VocabType
+
+
+class IndicBPETokenizer:
+    """
+    Byte-Pair Encoding tokenizer for Indian language text.
+
+    Supports all major Indic scripts: Devanagari, Bengali, Gurmukhi,
+    Gujarati, Oriya, Tamil, Telugu, Kannada, Malayalam, Sinhala, Tibetan,
+    as well as ASCII and mixed-script text.
+
+    Usage
+    -----
+    >>> tok = IndicBPETokenizer()
+    >>> tok.train(text, vocab_size=16_000)
+    >>> ids = tok.encode("नमस्ते दुनिया")
+    >>> tok.decode(ids)
+    'नमस्ते दुनिया'
+    >>> tok.save("vocab.json", "merges.json")
+
+    >>> tok2 = IndicBPETokenizer()
+    >>> tok2.load("vocab.json", "merges.json")
+    """
+
+    def __init__(self) -> None:
+        self.vocab: VocabType = {}
+        self.inverse_vocab: InverseVocabType = {}
+        self.bpe_merges: MergesType = {}
+
+        self._vocab_builder = VocabBuilder()
+        self._trainer = BPETrainer()
+
+    # ------------------------------------------------------------------
+    # Properties
+    # ------------------------------------------------------------------
+
+    @property
+    def vocab_size(self) -> int:
+        """Current number of tokens in the vocabulary."""
+        return len(self.vocab)
+
+    # ------------------------------------------------------------------
+    # Training
+    # ------------------------------------------------------------------
+
+    def train(
+        self,
+        text: str,
+        vocab_size: int,
+        special_tokens: Optional[List[str]] = None,
+        verbose: bool = True,
+    ) -> None:
+        """
+        Train BPE from *text* using sequence-based BPE.
+
+        Args:
+            text: Raw UTF-8 training corpus (any Indic language or mixed).
+            vocab_size: Target vocabulary size.
+            special_tokens: Override the default Claude-style special tokens.
+            verbose: Show tqdm progress bar during training.
+
+        Note:
+            For corpora larger than ~1 GB, use ChunkedBPETrainer instead —
+            it runs word-frequency BPE which needs <1 GB RAM even for 18 GB text.
+        """
+        if special_tokens is None:
+            special_tokens = list(SPECIAL_TOKENS)
+
+        # Seed vocab: ASCII + all Indic script characters + corpus chars + specials
+        self.vocab, self.inverse_vocab = self._vocab_builder.build_base_vocab()
+        self._vocab_builder.extend_from_text(text, self.vocab, self.inverse_vocab)
+        self._vocab_builder.add_special_tokens(
+            self.vocab, self.inverse_vocab, special_tokens
+        )
+
+        if verbose:
+            print(f"Base vocab: {self.vocab_size:,} tokens  |  Target: {vocab_size:,}")
+            if len(text) > 50_000_000:
+                print(
+                    f"WARNING: corpus is {len(text):,} chars. Sequence-based BPE "
+                    "stores the full token ID list in RAM. Consider ChunkedBPETrainer "
+                    "for large corpora (chunked_trainer.py)."
+                )
+
+        # Encode every character → its token ID
+        try:
+            from tqdm import tqdm  # type: ignore[import]
+            char_iter = tqdm(text, desc="Encoding corpus", unit=" chars",
+                             unit_scale=True, total=len(text))
+        except ImportError:
+            char_iter = text  # type: ignore[assignment]
+
+        token_ids: List[int] = [self.inverse_vocab[c] for c in char_iter]
+
+        self.bpe_merges = self._trainer.train(
+            token_ids,
+            self.vocab,
+            self.inverse_vocab,
+            target_vocab_size=vocab_size,
+            verbose=verbose,
+        )
+
+        if verbose:
+            print(f"Training complete. Final vocab size: {self.vocab_size:,}")
+
+    def train_from_file(
+        self,
+        path: Union[str, Path],
+        vocab_size: int,
+        text_column: str = "text",
+        special_tokens: Optional[List[str]] = None,
+        max_samples: Optional[int] = None,
+        verbose: bool = True,
+    ) -> None:
+        """
+        Load a corpus from *path* then call train().
+
+        Supports .parquet, .csv, .txt, .json, .jsonl.
+
+        Args:
+            path: Path to the corpus file.
+            vocab_size: Target vocabulary size.
+            text_column: Column / key for text in tabular / JSON files.
+            special_tokens: Override the default special tokens.
+            max_samples: Cap on rows / documents loaded (None → all).
+            verbose: Show training progress.
+        """
+        loader = IndicDataLoader(text_column=text_column, max_samples=max_samples)
+        text = loader.load(path)
+        self.train(text, vocab_size=vocab_size, special_tokens=special_tokens,
+                   verbose=verbose)
+
+    # ------------------------------------------------------------------
+    # Encoding
+    # ------------------------------------------------------------------
+
+    def encode(
+        self,
+        text: str,
+        allowed_special: Optional[Set[str]] = None,
+    ) -> List[int]:
+        """
+        Encode *text* into a list of token IDs.
+
+        Args:
+            text: Input string (any Indic script, ASCII, special tokens, mixed).
+            allowed_special: Set of special tokens to recognise and pass through
+                             as single IDs.  Pass set() to disable.
+                             Defaults to all tokens in SPECIAL_TOKENS.
+
+        Returns:
+            List of integer token IDs.
+        """
+        if allowed_special is None:
+            allowed_special = set(SPECIAL_TOKENS)
+
+        token_ids: List[int] = []
+
+        if allowed_special:
+            for segment, is_special in self._split_on_special_tokens(
+                text, allowed_special
+            ):
+                if is_special:
+                    tid = self.inverse_vocab.get(segment)
+                    if tid is None:
+                        raise ValueError(
+                            f"Special token {segment!r} not found in vocabulary."
+                        )
+                    token_ids.append(tid)
+                else:
+                    token_ids.extend(self._encode_ordinary(segment))
+        else:
+            token_ids = self._encode_ordinary(text)
+
+        return token_ids
+
+    def _encode_ordinary(self, text: str) -> List[int]:
+        """Encode plain text (no special-token detection)."""
+        token_ids: List[int] = []
+        for chunk in pretokenize(text):
+            tid = self.inverse_vocab.get(chunk)
+            if tid is not None:
+                token_ids.append(tid)
+            else:
+                token_ids.extend(self._apply_bpe(chunk))
+        return token_ids
+
+    # ------------------------------------------------------------------
+    # Decoding
+    # ------------------------------------------------------------------
+
+    def decode(self, token_ids: List[int]) -> str:
+        """
+        Decode a list of token IDs back into a string.
+
+        Because tokens contain literal spaces and newlines, decoding is a
+        plain concatenation — no marker substitution required.
+
+        Args:
+            token_ids: Sequence of integer token IDs.
+
+        Returns:
+            Decoded UTF-8 string.
+
+        Raises:
+            ValueError: If any token ID is not in the vocabulary.
+        """
+        parts: List[str] = []
+        for tid in token_ids:
+            token = self.vocab.get(tid)
+            if token is None:
+                raise ValueError(f"Token ID {tid} not found in vocabulary.")
+            parts.append(token)
+        return "".join(parts)
+
+    # ------------------------------------------------------------------
+    # Persistence
+    # ------------------------------------------------------------------
+
+    def save(self, vocab_path: str, merges_path: str) -> None:
+        """
+        Save vocabulary and BPE merges to JSON files.
+
+        vocab.json  format: {"<int_id>": "<token_string>", ...}
+        merges.json format: [{"pair": [id1, id2], "merged_id": id3}, ...]
+
+        Args:
+            vocab_path: Destination path for vocabulary JSON.
+            merges_path: Destination path for merges JSON.
+        """
+        with open(vocab_path, "w", encoding="utf-8") as fh:
+            json.dump(
+                {str(tid): tok for tid, tok in self.vocab.items()},
+                fh,
+                ensure_ascii=False,
+                indent=2,
+            )
+
+        merges_list = [
+            {"pair": list(pair), "merged_id": mid}
+            for pair, mid in self.bpe_merges.items()
+        ]
+        with open(merges_path, "w", encoding="utf-8") as fh:
+            json.dump(merges_list, fh, ensure_ascii=False, indent=2)
+
+    def load(self, vocab_path: str, merges_path: str) -> None:
+        """
+        Load vocabulary and BPE merges from JSON files produced by save().
+
+        Args:
+            vocab_path: Path to vocabulary JSON file.
+            merges_path: Path to merges JSON file.
+        """
+        with open(vocab_path, "r", encoding="utf-8") as fh:
+            raw = json.load(fh)
+        self.vocab = {int(k): v for k, v in raw.items()}
+        self.inverse_vocab = {v: int(k) for k, v in raw.items()}
+
+        with open(merges_path, "r", encoding="utf-8") as fh:
+            merges_list = json.load(fh)
+        self.bpe_merges = {
+            (entry["pair"][0], entry["pair"][1]): entry["merged_id"]
+            for entry in merges_list
+        }
+
+    # ------------------------------------------------------------------
+    # Vocabulary helpers
+    # ------------------------------------------------------------------
+
+    def token_to_id(self, token: str) -> Optional[int]:
+        """Return the vocabulary ID for a token string, or None."""
+        return self.inverse_vocab.get(token)
+
+    def id_to_token(self, tid: int) -> Optional[str]:
+        """Return the token string for a vocabulary ID, or None."""
+        return self.vocab.get(tid)
+
+    def special_token_id(self, token: str) -> Optional[int]:
+        """Return the vocabulary ID of a special token, or None."""
+        return self.inverse_vocab.get(token)
+
+    # ------------------------------------------------------------------
+    # Private helpers
+    # ------------------------------------------------------------------
+
+    def _apply_bpe(self, chunk: str) -> List[int]:
+        """
+        Apply BPE merges to a pre-tokenised chunk.
+
+        The chunk is first split into character IDs; then the earliest-learned
+        merge that applies is greedily applied left-to-right until no merge
+        changes the sequence.
+        """
+        unk_id = self.inverse_vocab.get("<|unk|>")
+
+        char_ids: List[int] = []
+        for char in chunk:
+            tid = self.inverse_vocab.get(char)
+            if tid is None:
+                if unk_id is None:
+                    raise ValueError(
+                        f"Character {char!r} (U+{ord(char):04X}) not in vocab "
+                        "and no <|unk|> fallback is defined."
+                    )
+                char_ids.append(unk_id)
+            else:
+                char_ids.append(tid)
+
+        changed = True
+        while changed and len(char_ids) > 1:
+            changed = False
+            merged: List[int] = []
+            i = 0
+            while i < len(char_ids):
+                if i < len(char_ids) - 1:
+                    pair: PairType = (char_ids[i], char_ids[i + 1])
+                    if pair in self.bpe_merges:
+                        merged.append(self.bpe_merges[pair])
+                        i += 2
+                        changed = True
+                        continue
+                merged.append(char_ids[i])
+                i += 1
+            char_ids = merged
+
+        return char_ids
+
+    def _split_on_special_tokens(
+        self,
+        text: str,
+        allowed_special: Set[str],
+    ) -> List[Tuple[str, bool]]:
+        """Split *text* into (segment, is_special) pairs using regex."""
+        # Sort longest-first so "<|im_start|>" matches before "<|im|>" would
+        pattern = "(" + "|".join(
+            re.escape(tok)
+            for tok in sorted(allowed_special, key=len, reverse=True)
+        ) + ")"
+        result: List[Tuple[str, bool]] = []
+        for part in re.split(pattern, text):
+            if not part:
+                continue
+            result.append((part, part in allowed_special))
+        return result

indic_tokenizer/bpe_trainer.py

@@ -0,0 +1,196 @@
+"""
+Core BPE training algorithm — CPU-optimised with numpy.
+
+BPETrainer is stateless: it receives mutable vocab / inverse_vocab dicts,
+modifies them in-place, and returns only the bpe_merges dictionary.
+
+Optimisations over the naive Python implementation
+---------------------------------------------------
+_find_most_frequent_pair
+    Uses numpy to encode every adjacent pair as a single int64, then
+    np.unique (C-level sort + scan) to count them.  For a 10 M-token
+    sequence this is ~10× faster than Python's Counter(zip(...)).
+
+_replace_pair
+    Finds all match positions with a numpy boolean mask in one vectorised
+    pass, deduplicates overlapping matches with a tiny Python loop
+    (typically k << N matches), then builds the output array in numpy —
+    no Python-level element-by-element loop.
+
+Fallback
+    If numpy is not installed both methods fall back to pure-Python
+    implementations so the module is always importable.
+"""
+
+from collections import Counter, deque
+from typing import Dict, List, Optional, Tuple
+
+# Type aliases exposed for callers
+PairType = Tuple[int, int]
+VocabType = Dict[int, str]
+InverseVocabType = Dict[str, int]
+MergesType = Dict[PairType, int]
+
+try:
+    import numpy as np
+    _NUMPY_AVAILABLE = True
+except ImportError:
+    _NUMPY_AVAILABLE = False
+
+
+class BPETrainer:
+    """Stateless, numpy-accelerated BPE training engine."""
+
+    # ------------------------------------------------------------------
+    # Public API
+    # ------------------------------------------------------------------
+
+    def train(
+        self,
+        token_ids: List[int],
+        vocab: VocabType,
+        inverse_vocab: InverseVocabType,
+        target_vocab_size: int,
+        verbose: bool = True,
+    ) -> MergesType:
+        """
+        Run BPE training until *target_vocab_size* is reached.
+
+        Mutates *vocab* and *inverse_vocab* in-place with merged tokens.
+
+        Args:
+            token_ids: Sequence of initial token IDs from the corpus.
+            vocab: Mutable id → token mapping (extended in-place).
+            inverse_vocab: Mutable token → id mapping (extended in-place).
+            target_vocab_size: Desired final vocabulary size.
+            verbose: Show tqdm progress bar (or step log if tqdm missing).
+
+        Returns:
+            bpe_merges: dict mapping (id1, id2) → merged_id in merge order.
+        """
+        bpe_merges: MergesType = {}
+        initial_size = len(vocab)
+
+        try:
+            from tqdm import tqdm  # type: ignore[import]
+            progress = tqdm(
+                total=target_vocab_size - initial_size,
+                desc="BPE merges",
+                unit="merge",
+            )
+            use_tqdm = True
+        except ImportError:
+            use_tqdm = False
+            progress = None  # type: ignore[assignment]
+
+        step = 0
+        while len(vocab) < target_vocab_size:
+            pair = self._find_most_frequent_pair(token_ids)
+            if pair is None:
+                break  # no more pairs to merge
+
+            new_id = len(vocab)
+            token_ids = self._replace_pair(token_ids, pair, new_id)
+
+            merged_token = vocab[pair[0]] + vocab[pair[1]]
+            vocab[new_id] = merged_token
+            inverse_vocab[merged_token] = new_id
+            bpe_merges[pair] = new_id
+
+            step += 1
+            if use_tqdm and progress is not None:
+                progress.update(1)
+            elif verbose and step % 100 == 0:
+                print(
+                    f"  step {step:5d} | vocab {len(vocab):6d} "
+                    f"| merged: {merged_token!r}"
+                )
+
+        if use_tqdm and progress is not None:
+            progress.close()
+
+        return bpe_merges
+
+    # ------------------------------------------------------------------
+    # Private helpers — numpy paths with pure-Python fallbacks
+    # ------------------------------------------------------------------
+
+    @staticmethod
+    def _find_most_frequent_pair(
+        token_ids: List[int],
+    ) -> Optional[PairType]:
+        """
+        Return the most frequent adjacent pair in *token_ids*.
+
+        numpy path  — O(N log N) via C-level sort in np.unique.
+        Python path — O(N) via Counter (larger constant due to dict hashing).
+        """
+        if len(token_ids) < 2:
+            return None
+
+        if _NUMPY_AVAILABLE:
+            arr = np.asarray(token_ids, dtype=np.int64)
+            max_id = int(arr.max()) + 1
+            # Encode pair (a, b) as a * max_id + b to get a single int64 key
+            flat = arr[:-1] * max_id + arr[1:]
+            unique_pairs, counts = np.unique(flat, return_counts=True)
+            best_idx = int(np.argmax(counts))
+            best = int(unique_pairs[best_idx])
+            return (best // max_id, best % max_id)
+
+        # Pure-Python fallback
+        pairs = Counter(zip(token_ids, token_ids[1:]))
+        if not pairs:
+            return None
+        return max(pairs, key=pairs.__getitem__)
+
+    @staticmethod
+    def _replace_pair(
+        token_ids: List[int],
+        pair: PairType,
+        new_id: int,
+    ) -> List[int]:
+        """
+        Replace every non-overlapping occurrence of *pair* with *new_id*.
+
+        numpy path — vectorised boolean mask + one array copy, O(N).
+        Python path — deque-based left-to-right scan, O(N).
+        """
+        a, b = pair
+
+        if _NUMPY_AVAILABLE:
+            arr = np.asarray(token_ids, dtype=np.int32)
+            match_pos = np.where((arr[:-1] == a) & (arr[1:] == b))[0]
+
+            if len(match_pos) == 0:
+                return token_ids
+
+            # Greedy dedup: skip positions adjacent to a previous match
+            valid: List[int] = [int(match_pos[0])]
+            for pos in match_pos[1:]:
+                if int(pos) > valid[-1] + 1:
+                    valid.append(int(pos))
+            matches = np.asarray(valid, dtype=np.int64)
+
+            # Remove the second element of each matched pair
+            keep = np.ones(len(arr), dtype=bool)
+            keep[matches + 1] = False
+            out = arr[keep].copy()
+
+            # Replace first elements with new_id; adjust for prior deletions
+            adjusted = matches - np.arange(len(matches), dtype=np.int64)
+            out[adjusted] = new_id
+
+            return out.tolist()
+
+        # Pure-Python fallback (deque, O(N) Python ops)
+        dq = deque(token_ids)
+        result: List[int] = []
+        while dq:
+            current = dq.popleft()
+            if dq and (current, dq[0]) == pair:
+                result.append(new_id)
+                dq.popleft()
+            else:
+                result.append(current)
+        return result

indic_tokenizer/constants.py

@@ -0,0 +1,103 @@
+"""
+Constants for the Indic Tokenizer.
+
+Covers all major Indian scripts via Unicode block ranges and provides the
+full set of modern LLM special tokens (Claude / ChatML / Llama-3 style).
+
+Space / newline handling
+------------------------
+Spaces and newlines are kept as *literal* characters inside tokens (Claude
+Sonnet style).  Decoding is therefore a plain string join with no marker
+substitution (no GPT-2 Ġ/Ċ encoding needed).
+"""
+
+from typing import Dict, FrozenSet, List, Tuple
+
+# ---------------------------------------------------------------------------
+# Indic script Unicode ranges  (start, end) — both endpoints inclusive
+# ---------------------------------------------------------------------------
+INDIC_UNICODE_RANGES: Dict[str, Tuple[int, int]] = {
+    "devanagari":     (0x0900, 0x097F),  # Hindi, Sanskrit, Marathi, Nepali
+    "bengali":        (0x0980, 0x09FF),  # Bengali, Assamese
+    "gurmukhi":       (0x0A00, 0x0A7F),  # Punjabi
+    "gujarati":       (0x0A80, 0x0AFF),  # Gujarati
+    "oriya":          (0x0B00, 0x0B7F),  # Odia
+    "tamil":          (0x0B80, 0x0BFF),  # Tamil
+    "telugu":         (0x0C00, 0x0C7F),  # Telugu
+    "kannada":        (0x0C80, 0x0CFF),  # Kannada
+    "malayalam":      (0x0D00, 0x0D7F),  # Malayalam
+    "sinhala":        (0x0D80, 0x0DFF),  # Sinhala
+    "tibetan":        (0x0F00, 0x0FFF),  # Tibetan
+    "devanagari_ext": (0xA8E0, 0xA8FF),  # Devanagari Extended
+    "vedic_ext":      (0x1CD0, 0x1CFF),  # Vedic Extensions
+}
+
+# Regex character class spanning the main South Asian block (U+0900–U+0DFF).
+# Used in preprocessor patterns as a compact alternative to listing every range.
+INDIC_REGEX_RANGE: str = r"ऀ-෿"
+
+# ---------------------------------------------------------------------------
+# Indic numeral (digit) Unicode ranges  — each script has its own digit block
+# ---------------------------------------------------------------------------
+INDIC_NUMERAL_RANGES: Dict[str, Tuple[int, int]] = {
+    "devanagari_digits": (0x0966, 0x096F),
+    "bengali_digits":    (0x09E6, 0x09EF),
+    "gurmukhi_digits":   (0x0A66, 0x0A6F),
+    "gujarati_digits":   (0x0AE6, 0x0AEF),
+    "oriya_digits":      (0x0B66, 0x0B6F),
+    "tamil_digits":      (0x0BE6, 0x0BEF),
+    "telugu_digits":     (0x0C66, 0x0C6F),
+    "kannada_digits":    (0x0CE6, 0x0CEF),
+    "malayalam_digits":  (0x0D66, 0x0D6F),
+}
+
+# ---------------------------------------------------------------------------
+# Special tokens  (Claude / ChatML / Llama-3 convention)
+# ---------------------------------------------------------------------------
+SPECIAL_TOKENS: List[str] = [
+    # ── Text boundary ──────────────────────────────────────────────────────
+    "<|endoftext|>",
+    "<|startoftext|>",
+    "<|bos|>",           # beginning-of-sequence
+    "<|eos|>",           # end-of-sequence
+    "<|pad|>",           # padding
+    "<|unk|>",           # unknown token
+
+    # ── Chat-template (ChatML / Claude style) ─────────────────────────────
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|system|>",
+    "<|user|>",
+    "<|assistant|>",
+    "<|human|>",
+
+    # ── Llama-3 / header tokens ────────────────────────────────────────────
+    "<|eot_id|>",
+    "<|start_header_id|>",
+    "<|end_header_id|>",
+
+    # ── Fill-in-the-middle (FIM / infilling) ──────────────────────────────
+    "<|fim_prefix|>",
+    "<|fim_middle|>",
+    "<|fim_suffix|>",
+
+    # ── Tool / function-calling (Claude API style) ─────────────────────────
+    "<|tool_use|>",
+    "<|tool_result|>",
+    "<|tool_call|>",
+
+    # ── Extended thinking (Claude) ─────────────────────────────────────────
+    "<|thinking|>",
+    "<|/thinking|>",
+
+    # ── Citation / retrieval ───────────────────────────────────────────────
+    "<|citation|>",
+
+    # ── General purpose / classification ──────────────────────────────────
+    "<|cls|>",
+    "<|sep|>",
+    "<|mask|>",
+]
+
+# Frozen set for fast membership tests during encoding
+DEFAULT_ALLOWED_SPECIAL: FrozenSet[str] = frozenset(SPECIAL_TOKENS)