termux-tts 1.4.0 → 1.4.2

package/termux_tts/engine_native.py

@@ -1,104 +1,104 @@
-"""
-Android OS System Native TTS Engine Bridge (Option B).
-Directly communicates with Samsung / Google Voice Engine via Termux IPC.
-Provides authentic human speech output with zero download overhead.
-"""
-
-import os
-import time
-import subprocess
-import shutil
-from dataclasses import dataclass
-from typing import Optional
-
-from .exceptions import TTSInferenceError
-
-@dataclass
-class NativeResult:
-    text: str
-    output_path: Optional[str]
-    language: str
-    pitch: float
-    rate: float
-    elapsed_ms: float
-    engine_name: str
-
-class NativeAndroidEngine:
-    """Android System Native TTS Engine Bridge (Samsung / Google Voice)."""
-
-    def __init__(
-        self,
-        language: str = "ko",
-        pitch: float = 1.0,
-        rate: float = 1.0,
-        stream: str = "MUSIC"
-    ):
-        self.language = language
-        self.pitch = pitch
-        self.rate = rate
-        self.stream = stream
-        self.binary = self._find_binary()
-
-    def _find_binary(self) -> Optional[str]:
-        bin_path = shutil.which("termux-tts-speak")
-        if bin_path and os.access(bin_path, os.X_OK):
-            return bin_path
-        default_p = "/data/data/com.termux/files/usr/bin/termux-tts-speak"
-        if os.path.exists(default_p) and os.access(default_p, os.X_OK):
-            return default_p
-        return None
-
-    def speak(self, text: str, stream: Optional[str] = None) -> NativeResult:
-        """Speak text directly through physical Android device speakers via termux-tts-speak IPC."""
-        if not text or not text.strip():
-            raise TTSInferenceError("Input text cannot be empty or whitespace only.")
-
-        if not self.binary:
-            raise TTSInferenceError(
-                "[FAIL-FAST] 'termux-tts-speak' binary not found on this system. "
-                "NativeAndroidEngine requires an Android Termux environment with 'termux-api' installed (run 'pkg install termux-api'). "
-                "If running on non-Android Linux/macOS/Windows, please use '--engine dsp' or '--engine onnx'."
-            )
-
-        t0 = time.perf_counter()
-        target_stream = stream or self.stream
-        cmd = [
-            self.binary,
-            "-l", self.language,
-            "-p", str(self.pitch),
-            "-r", str(self.rate),
-            "-s", target_stream,
-            text
-        ]
-
-        try:
-            res = subprocess.run(cmd, capture_output=True, text=True, timeout=30)
-            if res.returncode != 0:
-                err_msg = res.stderr.strip() or f"Process exited with code {res.returncode}"
-                raise TTSInferenceError(f"Native Android TTS engine execution failed: {err_msg}")
-        except subprocess.TimeoutExpired as e:
-            raise TTSInferenceError(f"Native Android TTS execution timed out (30s): {e}") from e
-        except Exception as e:
-            raise TTSInferenceError(f"Failed to execute native Android TTS: {e}") from e
-
-        elapsed_ms = (time.perf_counter() - t0) * 1000.0
-        return NativeResult(
-            text=text,
-            output_path=None,
-            language=self.language,
-            pitch=self.pitch,
-            rate=self.rate,
-            elapsed_ms=elapsed_ms,
-            engine_name="Android_Native_Voice_Engine"
-        )
-
-    def close(self) -> None:
-        pass
-
-    def __enter__(self):
-        return self
-
-    def __exit__(self, exc_type, exc_val, exc_tb):
-        self.close()
-
-
+"""
+Android OS System Native TTS Engine Bridge (Option B).
+Directly communicates with Samsung / Google Voice Engine via Termux IPC.
+Provides authentic human speech output with zero download overhead.
+"""
+
+import os
+import time
+import subprocess
+import shutil
+from dataclasses import dataclass
+from typing import Optional
+
+from .exceptions import TTSInferenceError
+
+@dataclass
+class NativeResult:
+    text: str
+    output_path: Optional[str]
+    language: str
+    pitch: float
+    rate: float
+    elapsed_ms: float
+    engine_name: str
+
+class NativeAndroidEngine:
+    """Android System Native TTS Engine Bridge (Samsung / Google Voice)."""
+
+    def __init__(
+        self,
+        language: str = "ko",
+        pitch: float = 1.0,
+        rate: float = 1.0,
+        stream: str = "MUSIC"
+    ):
+        self.language = language
+        self.pitch = pitch
+        self.rate = rate
+        self.stream = stream
+        self.binary = self._find_binary()
+
+    def _find_binary(self) -> Optional[str]:
+        bin_path = shutil.which("termux-tts-speak")
+        if bin_path and os.access(bin_path, os.X_OK):
+            return bin_path
+        default_p = "/data/data/com.termux/files/usr/bin/termux-tts-speak"
+        if os.path.exists(default_p) and os.access(default_p, os.X_OK):
+            return default_p
+        return None
+
+    def speak(self, text: str, stream: Optional[str] = None) -> NativeResult:
+        """Speak text directly through physical Android device speakers via termux-tts-speak IPC."""
+        if not text or not text.strip():
+            raise TTSInferenceError("Input text cannot be empty or whitespace only.")
+
+        if not self.binary:
+            raise TTSInferenceError(
+                "[FAIL-FAST] 'termux-tts-speak' binary not found on this system. "
+                "NativeAndroidEngine requires an Android Termux environment with 'termux-api' installed (run 'pkg install termux-api'). "
+                "If running on non-Android Linux/macOS/Windows, please use '--engine dsp' or '--engine onnx'."
+            )
+
+        t0 = time.perf_counter()
+        target_stream = stream or self.stream
+        cmd = [
+            self.binary,
+            "-l", self.language,
+            "-p", str(self.pitch),
+            "-r", str(self.rate),
+            "-s", target_stream,
+            text
+        ]
+
+        try:
+            res = subprocess.run(cmd, capture_output=True, text=True, timeout=30)
+            if res.returncode != 0:
+                err_msg = res.stderr.strip() or f"Process exited with code {res.returncode}"
+                raise TTSInferenceError(f"Native Android TTS engine execution failed: {err_msg}")
+        except subprocess.TimeoutExpired as e:
+            raise TTSInferenceError(f"Native Android TTS execution timed out (30s): {e}") from e
+        except Exception as e:
+            raise TTSInferenceError(f"Failed to execute native Android TTS: {e}") from e
+
+        elapsed_ms = (time.perf_counter() - t0) * 1000.0
+        return NativeResult(
+            text=text,
+            output_path=None,
+            language=self.language,
+            pitch=self.pitch,
+            rate=self.rate,
+            elapsed_ms=elapsed_ms,
+            engine_name="Android_Native_Voice_Engine"
+        )
+
+    def close(self) -> None:
+        pass
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self.close()
+
+

package/termux_tts/exceptions.py

@@ -1,28 +1,28 @@
-"""
-Domain Specific Exceptions for termux-tts (Strict Fail-Fast Protocol).
-Adheres to AOSF-ENG-STD-2026-V1 No-Fallback Governance.
-"""
-
-class TTSError(Exception):
-    """Base exception for all termux-tts domain errors."""
-    pass
-
-class TTSModelLoadError(TTSError):
-    """Raised when the neural TTS model file cannot be loaded or is corrupted."""
-    pass
-
-class TTSInferenceError(TTSError):
-    """Raised when tensor forward pass or audio synthesis fails."""
-    pass
-
-class VulkanInitializationError(TTSInferenceError):
-    """Raised when Vulkan GPU is explicitly requested but unavailable (Strict Fail-Fast)."""
-    pass
-
-class TTSAudioEncodingError(TTSError):
-    """Raised when raw PCM cannot be encoded to standard WAV."""
-    pass
-
-class TTSLanguageNotSupportedError(TTSError):
-    """Raised when the requested language is not supported by the current tokenizer."""
-    pass
+"""
+Domain Specific Exceptions for termux-tts (Strict Fail-Fast Protocol).
+Adheres to AOSF-ENG-STD-2026-V1 No-Fallback Governance.
+"""
+
+class TTSError(Exception):
+    """Base exception for all termux-tts domain errors."""
+    pass
+
+class TTSModelLoadError(TTSError):
+    """Raised when the neural TTS model file cannot be loaded or is corrupted."""
+    pass
+
+class TTSInferenceError(TTSError):
+    """Raised when tensor forward pass or audio synthesis fails."""
+    pass
+
+class VulkanInitializationError(TTSInferenceError):
+    """Raised when Vulkan GPU is explicitly requested but unavailable (Strict Fail-Fast)."""
+    pass
+
+class TTSAudioEncodingError(TTSError):
+    """Raised when raw PCM cannot be encoded to standard WAV."""
+    pass
+
+class TTSLanguageNotSupportedError(TTSError):
+    """Raised when the requested language is not supported by the current tokenizer."""
+    pass

package/termux_tts/tokenizer.py

@@ -1,184 +1,184 @@
-"""
-Grapheme-to-Phoneme (G2P) Phonetic Tokenizer for Korean and English.
-Includes Number-to-Speech Normalizer and Inline Expressive Tag Parser.
-"""
-
-import re
-from typing import List, Dict
-from .exceptions import TTSLanguageNotSupportedError
-from .g2p_korean import korean_text_to_phonemes
-
-HANGUL_BASE = 0xAC00
-HANGUL_END = 0xD7A3
-
-CHO = [
-    "ㄱ", "ㄲ", "ㄴ", "ㄷ", "ㄸ", "ㄹ", "ㅁ", "ㅂ", "ㅃ", "ㅅ",
-    "ㅆ", "ㅇ", "ㅈ", "ㅉ", "ㅊ", "ㅋ", "ㅌ", "ㅍ", "ㅎ"
-]
-JUNG = [
-    "ㅏ", "ㅐ", "ㅑ", "ㅒ", "ㅓ", "ㅔ", "ㅕ", "ㅖ", "ㅗ", "ㅘ",
-    "ㅙ", "ㅚ", "ㅛ", "ㅜ", "ㅝ", "ㅞ", "ㅟ", "ㅠ", "ㅡ", "ㅢ", "ㅣ"
-]
-JONG = [
-    "", "ㄱ", "ㄲ", "ㄳ", "ㄴ", "ㄵ", "ㄶ", "ㄷ", "ㄹ", "ㄺ",
-    "ㄻ", "ㄼ", "ㄽ", "ㄾ", "ㄿ", "ㅀ", "ㅁ", "ㅂ", "ㅄ", "ㅅ",
-    "ㅆ", "ㅇ", "ㅈ", "ㅊ", "ㅋ", "ㅌ", "ㅍ", "ㅎ"
-]
-
-EXPRESSIVE_TAGS = {
-    "[laugh]": 1001,
-    "[sigh]": 1002,
-    "[breath]": 1003,
-    "[uv_break]": 1004,
-    "[clears_throat]": 1005,
-    "[pause]": 1006
-}
-
-VOCAB: List[str] = [
-    "_", " ", "!", "?", ",", ".", "~", "-",
-    *CHO, *JUNG, *[j for j in JONG if j],
-    *"abcdefghijklmnopqrstuvwxyz"
-]
-VOCAB_TO_ID: Dict[str, int] = {sym: idx for idx, sym in enumerate(VOCAB)}
-PAD_ID: int = VOCAB_TO_ID["_"]
-SPACE_ID: int = VOCAB_TO_ID[" "]
-
-KOREAN_DIGITS = ["", "일", "이", "삼", "사", "오", "육", "칠", "팔", "구"]
-SMALL_UNITS = ["", "십", "백", "천"]
-BIG_UNITS = ["", "만", "억", "조", "경"]
-
-ENGLISH_WORDS = {
-    "0": "zero", "1": "one", "2": "two", "3": "three", "4": "four",
-    "5": "five", "6": "six", "7": "seven", "8": "eight", "9": "nine",
-    "10": "ten", "11": "eleven", "12": "twelve", "13": "thirteen", "14": "fourteen",
-    "15": "fifteen", "16": "sixteen", "17": "seventeen", "18": "eighteen", "19": "nineteen",
-    "20": "twenty", "30": "thirty", "40": "forty", "50": "fifty",
-    "60": "sixty", "70": "seventy", "80": "eighty", "90": "ninety",
-    "100": "hundred", "1000": "thousand"
-}
-
-def decompose_hangul(char: str) -> List[str]:
-    code = ord(char)
-    if HANGUL_BASE <= code <= HANGUL_END:
-        offset = code - HANGUL_BASE
-        cho_idx = offset // (21 * 28)
-        jung_idx = (offset % (21 * 28)) // 28
-        jong_idx = offset % 28
-        res = [CHO[cho_idx], JUNG[jung_idx]]
-        if jong_idx > 0:
-            res.append(JONG[jong_idx])
-        return res
-    return [char]
-
-def _convert_4digits_korean(chunk: str) -> str:
-    num = int(chunk)
-    if num == 0:
-        return ""
-    res = []
-    str_num = str(num).zfill(4)
-    for i, ch in enumerate(str_num):
-        d = int(ch)
-        if d > 0:
-            unit = SMALL_UNITS[3 - i]
-            if d == 1 and unit != "":
-                res.append(unit)
-            else:
-                res.append(KOREAN_DIGITS[d] + unit)
-    return "".join(res)
-
-def number_to_korean_sino(num_str: str) -> str:
-    """Convert integer string to authentic Sino-Korean place-value numerals (e.g. 1234 -> 천이백삼십사, 10000 -> 만)."""
-    try:
-        n = int(num_str)
-    except ValueError:
-        return num_str
-    if n == 0:
-        return "영"
-    rev_str = str(n)[::-1]
-    chunks = [rev_str[i:i+4][::-1] for i in range(0, len(rev_str), 4)]
-    parts = []
-    for i, chunk in enumerate(chunks):
-        c_korean = _convert_4digits_korean(chunk)
-        if c_korean:
-            unit = BIG_UNITS[i]
-            # 10000일 때 '일만' 대신 '만' (단, 210000 -> 이십일만)
-            if c_korean == "일" and unit != "" and len(chunks) == i + 1:
-                parts.append(unit)
-            else:
-                parts.append(c_korean + unit)
-    return "".join(reversed(parts))
-
-def normalize_numbers_korean(text: str) -> str:
-    """Convert digit sequences into spoken Korean place-value numerals."""
-    return re.sub(r"\d+", lambda m: number_to_korean_sino(m.group(0)), text)
-
-def normalize_numbers_english(text: str) -> str:
-    """Convert digit sequences into spoken English words."""
-    def _en_repl(m):
-        num_str = m.group(0)
-        if num_str in ENGLISH_WORDS:
-            return " " + ENGLISH_WORDS[num_str] + " "
-        # Digit-by-digit for phone numbers / codes
-        return " " + " ".join(ENGLISH_WORDS.get(d, d) for d in num_str) + " "
-    return re.sub(r"\d+", _en_repl, text)
-
-class PhoneticTokenizer:
-    def __init__(self, language: str = "ko"):
-        self.language = language.lower()
-        self.vocab = VOCAB
-        if self.language not in ["ko", "korean", "en", "english"]:
-            raise TTSLanguageNotSupportedError(f"Language '{language}' is not supported. Supported: ['ko', 'en']")
-
-    def normalize_text(self, text: str) -> str:
-        if not text or not text.strip():
-            return ""
-        # 1. Normalize linebreaks and tabs
-        text = re.sub(r"[\r\n\t]+", " ", text)
-        
-        # 2. Digits to spoken words
-        if self.language in ["ko", "korean"]:
-            text = normalize_numbers_korean(text)
-            text = korean_text_to_phonemes(text)
-        else:
-            text = normalize_numbers_english(text)
-
-        # 3. Collapse multiple spaces
-        text = re.sub(r"\s{2,}", " ", text)
-        return text.strip()
-
-    def tokenize(self, text: str) -> List[int]:
-        normalized = self.normalize_text(text)
-        if not normalized:
-            return []
-
-        # Parse expressive tags first
-        tag_pattern = re.compile(r"(\[[a-zA-Z_]+\])")
-        parts = tag_pattern.split(normalized)
-
-        tokens: List[int] = []
-        for part in parts:
-            if not part:
-                continue
-            if part in EXPRESSIVE_TAGS:
-                tokens.append(EXPRESSIVE_TAGS[part])
-                continue
-
-            if self.language in ["ko", "korean"]:
-                for char in part:
-                    if char == " ":
-                        tokens.append(SPACE_ID)
-                    elif char in [".", ",", "!", "?", "~", "-"]:
-                        if char in VOCAB_TO_ID:
-                            tokens.append(VOCAB_TO_ID[char])
-                    else:
-                        jamos = decompose_hangul(char)
-                        for j in jamos:
-                            if j in VOCAB_TO_ID:
-                                tokens.append(VOCAB_TO_ID[j])
-            else: # en
-                for char in part.lower():
-                    if char in VOCAB_TO_ID:
-                        tokens.append(VOCAB_TO_ID[char])
-
-        return tokens
-
+"""
+Grapheme-to-Phoneme (G2P) Phonetic Tokenizer for Korean and English.
+Includes Number-to-Speech Normalizer and Inline Expressive Tag Parser.
+"""
+
+import re
+from typing import List, Dict
+from .exceptions import TTSLanguageNotSupportedError
+from .g2p_korean import korean_text_to_phonemes
+
+HANGUL_BASE = 0xAC00
+HANGUL_END = 0xD7A3
+
+CHO = [
+    "ㄱ", "ㄲ", "ㄴ", "ㄷ", "ㄸ", "ㄹ", "ㅁ", "ㅂ", "ㅃ", "ㅅ",
+    "ㅆ", "ㅇ", "ㅈ", "ㅉ", "ㅊ", "ㅋ", "ㅌ", "ㅍ", "ㅎ"
+]
+JUNG = [
+    "ㅏ", "ㅐ", "ㅑ", "ㅒ", "ㅓ", "ㅔ", "ㅕ", "ㅖ", "ㅗ", "ㅘ",
+    "ㅙ", "ㅚ", "ㅛ", "ㅜ", "ㅝ", "ㅞ", "ㅟ", "ㅠ", "ㅡ", "ㅢ", "ㅣ"
+]
+JONG = [
+    "", "ㄱ", "ㄲ", "ㄳ", "ㄴ", "ㄵ", "ㄶ", "ㄷ", "ㄹ", "ㄺ",
+    "ㄻ", "ㄼ", "ㄽ", "ㄾ", "ㄿ", "ㅀ", "ㅁ", "ㅂ", "ㅄ", "ㅅ",
+    "ㅆ", "ㅇ", "ㅈ", "ㅊ", "ㅋ", "ㅌ", "ㅍ", "ㅎ"
+]
+
+EXPRESSIVE_TAGS = {
+    "[laugh]": 1001,
+    "[sigh]": 1002,
+    "[breath]": 1003,
+    "[uv_break]": 1004,
+    "[clears_throat]": 1005,
+    "[pause]": 1006
+}
+
+VOCAB: List[str] = [
+    "_", " ", "!", "?", ",", ".", "~", "-",
+    *CHO, *JUNG, *[j for j in JONG if j],
+    *"abcdefghijklmnopqrstuvwxyz"
+]
+VOCAB_TO_ID: Dict[str, int] = {sym: idx for idx, sym in enumerate(VOCAB)}
+PAD_ID: int = VOCAB_TO_ID["_"]
+SPACE_ID: int = VOCAB_TO_ID[" "]
+
+KOREAN_DIGITS = ["", "일", "이", "삼", "사", "오", "육", "칠", "팔", "구"]
+SMALL_UNITS = ["", "십", "백", "천"]
+BIG_UNITS = ["", "만", "억", "조", "경"]
+
+ENGLISH_WORDS = {
+    "0": "zero", "1": "one", "2": "two", "3": "three", "4": "four",
+    "5": "five", "6": "six", "7": "seven", "8": "eight", "9": "nine",
+    "10": "ten", "11": "eleven", "12": "twelve", "13": "thirteen", "14": "fourteen",
+    "15": "fifteen", "16": "sixteen", "17": "seventeen", "18": "eighteen", "19": "nineteen",
+    "20": "twenty", "30": "thirty", "40": "forty", "50": "fifty",
+    "60": "sixty", "70": "seventy", "80": "eighty", "90": "ninety",
+    "100": "hundred", "1000": "thousand"
+}
+
+def decompose_hangul(char: str) -> List[str]:
+    code = ord(char)
+    if HANGUL_BASE <= code <= HANGUL_END:
+        offset = code - HANGUL_BASE
+        cho_idx = offset // (21 * 28)
+        jung_idx = (offset % (21 * 28)) // 28
+        jong_idx = offset % 28
+        res = [CHO[cho_idx], JUNG[jung_idx]]
+        if jong_idx > 0:
+            res.append(JONG[jong_idx])
+        return res
+    return [char]
+
+def _convert_4digits_korean(chunk: str) -> str:
+    num = int(chunk)
+    if num == 0:
+        return ""
+    res = []
+    str_num = str(num).zfill(4)
+    for i, ch in enumerate(str_num):
+        d = int(ch)
+        if d > 0:
+            unit = SMALL_UNITS[3 - i]
+            if d == 1 and unit != "":
+                res.append(unit)
+            else:
+                res.append(KOREAN_DIGITS[d] + unit)
+    return "".join(res)
+
+def number_to_korean_sino(num_str: str) -> str:
+    """Convert integer string to authentic Sino-Korean place-value numerals (e.g. 1234 -> 천이백삼십사, 10000 -> 만)."""
+    try:
+        n = int(num_str)
+    except ValueError:
+        return num_str
+    if n == 0:
+        return "영"
+    rev_str = str(n)[::-1]
+    chunks = [rev_str[i:i+4][::-1] for i in range(0, len(rev_str), 4)]
+    parts = []
+    for i, chunk in enumerate(chunks):
+        c_korean = _convert_4digits_korean(chunk)
+        if c_korean:
+            unit = BIG_UNITS[i]
+            # 10000일 때 '일만' 대신 '만' (단, 210000 -> 이십일만)
+            if c_korean == "일" and unit != "" and len(chunks) == i + 1:
+                parts.append(unit)
+            else:
+                parts.append(c_korean + unit)
+    return "".join(reversed(parts))
+
+def normalize_numbers_korean(text: str) -> str:
+    """Convert digit sequences into spoken Korean place-value numerals."""
+    return re.sub(r"\d+", lambda m: number_to_korean_sino(m.group(0)), text)
+
+def normalize_numbers_english(text: str) -> str:
+    """Convert digit sequences into spoken English words."""
+    def _en_repl(m):
+        num_str = m.group(0)
+        if num_str in ENGLISH_WORDS:
+            return " " + ENGLISH_WORDS[num_str] + " "
+        # Digit-by-digit for phone numbers / codes
+        return " " + " ".join(ENGLISH_WORDS.get(d, d) for d in num_str) + " "
+    return re.sub(r"\d+", _en_repl, text)
+
+class PhoneticTokenizer:
+    def __init__(self, language: str = "ko"):
+        self.language = language.lower()
+        self.vocab = VOCAB
+        if self.language not in ["ko", "korean", "en", "english"]:
+            raise TTSLanguageNotSupportedError(f"Language '{language}' is not supported. Supported: ['ko', 'en']")
+
+    def normalize_text(self, text: str) -> str:
+        if not text or not text.strip():
+            return ""
+        # 1. Normalize linebreaks and tabs
+        text = re.sub(r"[\r\n\t]+", " ", text)
+        
+        # 2. Digits to spoken words
+        if self.language in ["ko", "korean"]:
+            text = normalize_numbers_korean(text)
+            text = korean_text_to_phonemes(text)
+        else:
+            text = normalize_numbers_english(text)
+
+        # 3. Collapse multiple spaces
+        text = re.sub(r"\s{2,}", " ", text)
+        return text.strip()
+
+    def tokenize(self, text: str) -> List[int]:
+        normalized = self.normalize_text(text)
+        if not normalized:
+            return []
+
+        # Parse expressive tags first
+        tag_pattern = re.compile(r"(\[[a-zA-Z_]+\])")
+        parts = tag_pattern.split(normalized)
+
+        tokens: List[int] = []
+        for part in parts:
+            if not part:
+                continue
+            if part in EXPRESSIVE_TAGS:
+                tokens.append(EXPRESSIVE_TAGS[part])
+                continue
+
+            if self.language in ["ko", "korean"]:
+                for char in part:
+                    if char == " ":
+                        tokens.append(SPACE_ID)
+                    elif char in [".", ",", "!", "?", "~", "-"]:
+                        if char in VOCAB_TO_ID:
+                            tokens.append(VOCAB_TO_ID[char])
+                    else:
+                        jamos = decompose_hangul(char)
+                        for j in jamos:
+                            if j in VOCAB_TO_ID:
+                                tokens.append(VOCAB_TO_ID[j])
+            else: # en
+                for char in part.lower():
+                    if char in VOCAB_TO_ID:
+                        tokens.append(VOCAB_TO_ID[char])
+
+        return tokens
+