math2tex 1.0.0__py3-none-any.whl

math2tex/formatter.py

@@ -0,0 +1,504 @@
+"""
+math2tex.formatter — LaTeX code generation via AST visitor.
+
+Converts a typed AST tree into a publication-ready LaTeX string.
+"""
+
+import re
+from typing import List, Optional
+
+from .config import (
+    ACCENT_FUNCTIONS, BUILTIN_LATEX_FUNCS, Config, FONT_FUNCTIONS,
+    GREEK_LETTERS, Prec, SPECIAL_CONSTANTS, WORD_ADD_OPS, WORD_MULT_OPS,
+    WORD_RELATIONS, _REL_OPS,
+)
+from .ast_nodes import (
+    Abs, BigOp, BinOp, Derivative, Factorial, Fraction, Func, Group,
+    InnerProduct, Integral, IntervalNode, MatrixNode, Node, Norm, Num,
+    PiecewiseNode, Power, Prime, SetBuilder, Subscript, TextLiteral,
+    UnOp, Var,
+)
+
+
+class LatexFormatter:
+    def __init__(self, text_map: Optional[dict] = None,
+                 align_mode: bool = False):
+        self.text_map = text_map or {}
+        self.align_mode = align_mode
+        self.in_integral = False
+
+    # ------------------------------------------------------------------
+    # Helpers
+    # ------------------------------------------------------------------
+
+    def get_vars(self, node: Node) -> List[str]:
+        vars_list: List[str] = []
+
+        def visit(n: Node) -> None:
+            if isinstance(n, Var) and len(n.name) == 1 and n.name.isalpha():
+                if n.name not in vars_list:
+                    vars_list.append(n.name)
+            elif isinstance(n, BinOp):
+                visit(n.left); visit(n.right)
+            elif isinstance(n, UnOp):
+                visit(n.expr)
+            elif isinstance(n, Fraction):
+                visit(n.num); visit(n.den)
+            elif isinstance(n, Power):
+                visit(n.base); visit(n.exp)
+            elif isinstance(n, Subscript):
+                visit(n.base); visit(n.sub)
+            elif isinstance(n, Func):
+                for a in n.args:
+                    visit(a)
+
+        visit(node)
+        return vars_list
+
+    def _is_complex(self, node: Node) -> bool:
+        if isinstance(node, Fraction):
+            return True
+        if isinstance(node, (Power, Subscript)):
+            return self._is_complex(node.base)
+        if isinstance(node, (BigOp, Integral)):
+            return True
+        if isinstance(node, BinOp) and node.op in ('+', '-', '+-', '-+'):
+            return True
+        return False
+
+    def format_child(self, parent_prec: int, child: Node) -> str:
+        child_str = self.format(child)
+        if child.precedence < parent_prec:
+            if self._is_complex(child):
+                return f'\\left({child_str}\\right)'
+            return f'({child_str})'
+        return child_str
+
+    # ------------------------------------------------------------------
+    # Entry point
+    # ------------------------------------------------------------------
+
+    def format(self, node: Node, is_root: bool = False) -> str:
+        if isinstance(node, BinOp):
+            return self.visit_BinOp(node, is_root=is_root)
+        method = getattr(self, f'visit_{type(node).__name__}',
+                         self.generic_visit)
+        return method(node)
+
+    def generic_visit(self, node: Node) -> str:
+        raise NotImplementedError(
+            f'No formatter for {type(node).__name__}'
+        )
+
+    # ------------------------------------------------------------------
+    # Atoms
+    # ------------------------------------------------------------------
+
+    def visit_Num(self, node: Num) -> str:
+        val = node.val.lower()
+        if 'e' in val:
+            base, exp = val.split('e')
+            if exp.startswith('+'):
+                exp = exp[1:]
+            return f'{base} \\times 10^{{{exp}}}'
+        if '.' not in val and len(val) >= 5:
+            parts: List[str] = []
+            v = val
+            while len(v) > 3:
+                parts.insert(0, v[-3:])
+                v = v[:-3]
+            if v:
+                parts.insert(0, v)
+            return '\\,'.join(parts)
+        return node.val
+
+    def visit_Var(self, node: Var) -> str:
+        v = node.name
+        if not v:
+            return ''
+        if v == '...':
+            return r'\ldots'
+        if v in self.text_map:
+            return f'\\text{{{self.text_map[v]}}}'
+        if v in SPECIAL_CONSTANTS:
+            return SPECIAL_CONSTANTS[v]
+        if v in GREEK_LETTERS:
+            return f'\\{v}'
+        if v in BUILTIN_LATEX_FUNCS:
+            return f'\\{v}'
+        if len(v) >= 2 and v[0] == 'd':
+            rest = v[1:]
+            if rest in GREEK_LETTERS or (len(rest) == 1 and rest.isalpha()):
+                if self.in_integral:
+                    return (f'\\dd{{\\{rest}}}'
+                            if rest in GREEK_LETTERS
+                            else f'\\dd{{{rest}}}')
+                return (f'd \\{rest}'
+                        if rest in GREEK_LETTERS
+                        else f'd {rest}')
+        return v
+
+    # ------------------------------------------------------------------
+    # Operations
+    # ------------------------------------------------------------------
+
+    def visit_BinOp(self, node: BinOp,
+                    is_root: bool = False) -> str:
+        if node.op == 'choose':
+            return (f'\\binom{{{self.format(node.left)}}}'
+                    f'{{{self.format(node.right)}}}')
+
+        left = self.format_child(node.precedence, node.left)
+        right = self.format_child(node.precedence, node.right)
+
+        # ---- Implicit / explicit multiplication ----
+        if node.op in ('*', ' '):
+            left_s = left.strip()
+            right_s = right.strip()
+            if not left_s or not right_s:
+                return f'{left_s}{right_s}'
+            if left_s == r'\exp':
+                return f'e^{{{right_s}}}'
+
+            if left_s[-1].isdigit() and right_s[0].isdigit():
+                return f'{left_s} \\cdot {right_s}'
+
+            if node.op == '*':
+                if left_s[-1].isdigit() and (
+                        right_s[0].isalpha() or right_s.startswith('\\')):
+                    return f'{left_s}{right_s}'
+                if left_s.endswith(')') or left_s.endswith('}') or right_s.startswith('(') or right_s.startswith('\\left('):
+                    return f'{left_s}{right_s}'
+                return f'{left_s} \\cdot {right_s}'
+
+            if right_s.startswith('\\dd{'):
+                return f'{left_s}{right_s}'
+            if right_s.startswith('\\ldots'):
+                return f'{left_s}{right_s}'
+            if left_s.endswith('!') and right_s[0].isalpha():
+                return f'{left_s} {right_s}'
+            if left_s.endswith('}') and (right_s[0].isalpha() or right_s.startswith('\\')) and not right_s.startswith('\\left('):
+                return f'{left_s} {right_s}'
+            if left_s.endswith(')') and right_s.startswith('\\') and not right_s.startswith('\\left('):
+                return f'{left_s} {right_s}'
+            if (left_s.endswith(')') or left_s.endswith('}')) and right_s.startswith('('):
+                return f'{left_s}{right_s}'
+            if right_s.startswith('\\'):
+                if left_s[-1].isalpha():
+                    return f'{left_s} {right_s}'
+                return f'{left_s}{right_s}'
+            if right_s.startswith('('):
+                return f'{left_s}{right_s}'
+            if left_s.endswith(')'):
+                return f'{left_s}{right_s}'
+            if left_s[-1].isalpha() and right_s[0].isalnum():
+                return f'{left_s} {right_s}'
+            return f'{left_s}{right_s}'
+
+        # ---- Plus-minus / minus-plus ----
+        if node.op in ('+-', '-+'):
+            op_str = '\\pm' if node.op == '+-' else '\\mp'
+            if isinstance(node.left, Var) and node.left.name == '':
+                return f'{op_str} {right}'
+            return f'{left} {op_str} {right}'
+
+        # ---- Word additive / multiplicative ops ----
+        if node.op in WORD_ADD_OPS:
+            return f'{left} {WORD_ADD_OPS[node.op]} {right}'
+        if node.op in WORD_MULT_OPS:
+            return f'{left} {WORD_MULT_OPS[node.op]} {right}'
+
+        # ---- Relational ops ----
+        rel_map = {
+            ':': '\\colon', '->': '\\to', '>=': '\\ge',
+            '<=': '\\le', '!=': '\\neq', '~=': '\\approx', '==': '=',
+        }
+        if node.op in _REL_OPS or node.op in WORD_RELATIONS:
+            op_str = rel_map.get(node.op, node.op)
+            if node.op in WORD_RELATIONS:
+                op_str = WORD_RELATIONS[node.op]
+
+            if (is_root and self.align_mode
+                    and Config.AUTO_ALIGN_EQUALS):
+                return f'{left} & {op_str} {right}'
+
+            if node.op in WORD_RELATIONS:
+                return f'{left} {op_str} {right}'
+            if node.op in ('->', '>=', '<=', '!=', '~='):
+                return f'{left}{op_str} {right}'
+            if node.op == ':':
+                return f'{left}\\colon {right}'
+
+            return f'{left}{op_str}{right}'
+
+        if node.op == 'if':
+            return f'{left} \\text{{if}} {right}'
+        if node.op == 'otherwise':
+            return f'{left} \\text{{otherwise}}'
+
+        # ---- Standard binary operators ----
+        if node.op in ('+', '-') and right.startswith('\\left'):
+            return f'{left}{node.op}{right}'
+
+        return f'{left}{node.op}{right}'
+
+    def visit_UnOp(self, node: UnOp) -> str:
+        if isinstance(node.expr, Power):
+            return f'{node.op}{self.format(node.expr)}'
+        return f'{node.op}{self.format_child(node.precedence, node.expr)}'
+
+    def visit_Group(self, node: Group) -> str:
+        """Emit user-written parentheses."""
+        inner = self.format(node.expr)
+        if self._is_complex(node.expr):
+            return f'\\left({inner}\\right)'
+        return f'({inner})'
+
+    def visit_Fraction(self, node: Fraction) -> str:
+        return (f'\\frac{{{self.format(node.num)}}}'
+                f'{{{self.format(node.den)}}}')
+
+    def visit_Power(self, node: Power) -> str:
+        base = self.format_child(node.precedence, node.base)
+        if isinstance(node.base, Fraction):
+            base = f'\\left({self.format(node.base)}\\right)'
+        elif isinstance(node.base, Power):
+            base = f'\\left({self.format(node.base)}\\right)'
+        elif isinstance(node.base, UnOp):
+            base = f'({self.format(node.base)})'
+        return f'{base}^{{{self.format(node.exp)}}}'
+
+    def visit_Subscript(self, node: Subscript) -> str:
+        base = self.format_child(node.precedence, node.base)
+        if isinstance(node.base, Fraction):
+            base = f'\\left({self.format(node.base)}\\right)'
+        return f'{base}_{{{self.format(node.sub)}}}'
+
+    def visit_Factorial(self, node: Factorial) -> str:
+        expr = self.format_child(node.precedence, node.expr)
+        if isinstance(node.expr, BinOp):
+            expr = f'\\left({self.format(node.expr)}\\right)'
+        return f'{expr}!'
+
+    def visit_Prime(self, node: Prime) -> str:
+        inner = self.format_child(Prec.POSTFIX, node.expr)
+        primes = "'" * node.order
+        return f'{inner}{primes}'
+
+    # ------------------------------------------------------------------
+    # Functions
+    # ------------------------------------------------------------------
+
+    def visit_Func(self, node: Func) -> str:
+        name = node.name
+
+        if name == 'sqrt' and node.args:
+            return f'\\sqrt{{{self.format(node.args[0])}}}'
+        if name == 'root' and len(node.args) >= 2:
+            return (f'\\sqrt[{self.format(node.args[0])}]'
+                    f'{{{self.format(node.args[1])}}}')
+        if name == 'exp' and node.args:
+            return f'e^{{{self.format(node.args[0])}}}'
+        if name == 'floor' and node.args:
+            return (f'\\left\\lfloor '
+                    f'{self.format(node.args[0])}\\right\\rfloor')
+        if name == 'ceil' and node.args:
+            return (f'\\left\\lceil '
+                    f'{self.format(node.args[0])}\\right\\rceil')
+        if name == 'abs' and node.args:
+            return f'\\abs{{{self.format(node.args[0])}}}'
+        if name == 'norm' and node.args:
+            return f'\\norm{{{self.format(node.args[0])}}}'
+        if name == 'binom' and len(node.args) >= 2:
+            return (f'\\binom{{{self.format(node.args[0])}}}'
+                    f'{{{self.format(node.args[1])}}}')
+        if name == 'boxed' and node.args:
+            return f'\\boxed{{{self.format(node.args[0])}}}'
+        if name == 'cancel' and node.args:
+            return f'\\cancel{{{self.format(node.args[0])}}}'
+        if name in ACCENT_FUNCTIONS and node.args:
+            cmd = ACCENT_FUNCTIONS[name]
+            base = f'\\{cmd}{{{self.format(node.args[0])}}}'
+            if node.call_args is not None:
+                inner = ','.join(self.format(a) for a in node.call_args)
+                if any(self._is_complex(a) for a in node.call_args):
+                    return f'{base}\\left({inner}\\right)'
+                return f'{base}({inner})'
+            return base
+        if name in FONT_FUNCTIONS and node.args:
+            cmd = FONT_FUNCTIONS[name]
+            base = f'\\{cmd}{{{self.format(node.args[0])}}}'
+            if node.call_args is not None:
+                inner = ','.join(self.format(a) for a in node.call_args)
+                if any(self._is_complex(a) for a in node.call_args):
+                    return f'{base}\\left({inner}\\right)'
+                return f'{base}({inner})'
+            return base
+
+        if name in self.text_map:
+            return f'\\text{{{self.text_map[name]}}}'
+
+        if name in BUILTIN_LATEX_FUNCS:
+            fmt_name = f'\\{name}'
+        elif name in GREEK_LETTERS:
+            fmt_name = f'\\{name}'
+        elif len(name) > 1:
+            fmt_name = f'\\operatorname{{{name}}}'
+        else:
+            fmt_name = name
+
+        if node.primes:
+            fmt_name = fmt_name + "'" * node.primes
+
+        if node.sub is not None:
+            fmt_name = f'{fmt_name}_{{{self.format(node.sub)}}}'
+        if node.power is not None:
+            fmt_name = f'{fmt_name}^{{{self.format(node.power)}}}'
+
+        if not node.args:
+            if fmt_name.startswith('\\') and fmt_name[-1].isalpha():
+                return fmt_name + ' '
+            return fmt_name
+
+        inner = ','.join(self.format(a) for a in node.args)
+        if any(self._is_complex(a) for a in node.args):
+            return f'{fmt_name}\\left({inner}\\right)'
+        return f'{fmt_name}({inner})'
+
+    # ------------------------------------------------------------------
+    # Calculus
+    # ------------------------------------------------------------------
+
+    def visit_Derivative(self, node: Derivative) -> str:
+        expr = self.format(node.expr)
+        order = f'[{self.format(node.order)}]' if node.order else ''
+        var_fmt = (f'\\{node.var}'
+                   if node.var in GREEK_LETTERS else node.var)
+
+        if not expr:
+            if node.is_partial:
+                if node.var2:
+                    var2_fmt = (f'\\{node.var2}'
+                                if node.var2 in GREEK_LETTERS
+                                else node.var2)
+                    return f'\\pdv{{{var_fmt}}}{{{var2_fmt}}}'
+                return f'\\pdv{order}{{{var_fmt}}}'
+            return f'\\dv{order}{{{var_fmt}}}'
+
+        if node.is_partial:
+            if node.var2:
+                var2_fmt = (f'\\{node.var2}'
+                            if node.var2 in GREEK_LETTERS
+                            else node.var2)
+                return f'\\pdv{{{expr}}}{{{var_fmt}}}{{{var2_fmt}}}'
+            return f'\\pdv{order}{{{expr}}}{{{var_fmt}}}'
+        return f'\\dv{order}{{{expr}}}{{{var_fmt}}}'
+
+    def visit_Integral(self, node: Integral) -> str:
+        op = {
+            'single': '\\int', 'double': '\\iint', 'triple': '\\iiint',
+            'contour': '\\oint', 'double_contour': '\\oiint',
+        }[node.kind]
+        bounds = ''
+        if node.lower is not None and node.upper is not None:
+            bounds = (f'_{{{self.format(node.lower)}}}'
+                      f'^{{{self.format(node.upper)}}}')
+        elif node.lower is not None:
+            bounds = f'_{{{self.format(node.lower)}}}'
+        elif node.upper is not None:
+            bounds = f'^{{{self.format(node.upper)}}}'
+
+        old_int = self.in_integral
+        self.in_integral = True
+        expr = self.format(node.expr)
+        if isinstance(node.expr, BinOp) and node.expr.op in ('+', '-', '+-', '-+'):
+            expr = f'\\left({expr}\\right)'
+        self.in_integral = old_int
+
+        if Config.ADD_DX_TO_INTEGRALS:
+            if not re.search(r'\\dd\{[a-zA-Z]\}|d\s*[a-zA-Z]', expr):
+                vars_found = self.get_vars(node.expr)
+                vars_found = [v for v in vars_found if v not in ('e', 'i', 'f', 'g', 'h')]
+                
+                num_vars = 1
+                if node.kind in ('double', 'double_contour'):
+                    num_vars = 2
+                elif node.kind == 'triple':
+                    num_vars = 3
+                
+                vars_to_add = vars_found[:num_vars]
+                defaults = ['x', 'y', 'z', 'w', 't']
+                while len(vars_to_add) < num_vars:
+                    for d in defaults:
+                        if d not in vars_to_add:
+                            vars_to_add.append(d)
+                            break
+                            
+                dd_str = ''.join(f'\\dd{{{v}}}' for v in vars_to_add)
+                expr = f"{expr.strip()}{dd_str}"
+
+        if expr.startswith('\\frac') or expr.startswith('\\left'):
+            return f'{op}{bounds}{expr}'
+        return f'{op}{bounds} {expr}'
+
+    def visit_BigOp(self, node: BigOp) -> str:
+        op = f'\\{node.op_name}'
+        bounds = ''
+        if node.lower is not None:
+            bounds += f'_{{{self.format(node.lower)}}}'
+        if node.upper is not None:
+            bounds += f'^{{{self.format(node.upper)}}}'
+        expr_str = self.format(node.expr)
+        if isinstance(node.expr, BinOp) and node.expr.op in ('+', '-', '+-', '-+'):
+            expr_str = f'\\left({expr_str}\\right)'
+        if expr_str:
+            if expr_str.startswith('\\frac') or expr_str.startswith('\\left'):
+                return f'{op}{bounds}{expr_str}'
+            return f'{op}{bounds} {expr_str}'.strip()
+        return f'{op}{bounds}'
+
+    # ------------------------------------------------------------------
+    # Delimiters & Structures
+    # ------------------------------------------------------------------
+
+    def visit_Abs(self, node: Abs) -> str:
+        return f'\\abs{{{self.format(node.expr)}}}'
+
+    def visit_Norm(self, node: Norm) -> str:
+        return f'\\norm{{{self.format(node.expr)}}}'
+
+    def visit_InnerProduct(self, node: InnerProduct) -> str:
+        return (f'\\langle {self.format(node.left)},'
+                f'{self.format(node.right)}\\rangle')
+
+    def visit_SetBuilder(self, node: SetBuilder) -> str:
+        expr = self.format(node.expr)
+        cond = self.format(node.condition)
+        return f'\\left\\{{{expr}\\mid {cond}\\right\\}}'
+
+    def visit_MatrixNode(self, node: MatrixNode) -> str:
+        lines = []
+        for row in node.rows:
+            lines.append(' & '.join(self.format(col) for col in row))
+        content = ' \\\\\n'.join(lines)
+        return f'\\begin{{{node.kind}}}\n{content}\n\\end{{{node.kind}}}'
+
+    def visit_TextLiteral(self, node: TextLiteral) -> str:
+        return f'\\text{{{node.val}}}'
+
+    def visit_IntervalNode(self, node: IntervalNode) -> str:
+        lb = '\\left[' if node.left_delim == '[' else '\\left('
+        rb = '\\right]' if node.right_delim == ']' else '\\right)'
+        return (f'{lb} {self.format(node.left_expr)}, '
+                f'{self.format(node.right_expr)} {rb}')
+
+    def visit_PiecewiseNode(self, node: PiecewiseNode) -> str:
+        lines = []
+        for expr, cond in node.conditions:
+            formatted = self.format(expr)
+            if cond:
+                lines.append(f'{formatted}, & {self.format(cond)}')
+            else:
+                lines.append(formatted)
+        content = ' \\\\\n'.join(lines)
+        return f'\\begin{{cases}}\n{content}\n\\end{{cases}}'

math2tex/lexer.py

@@ -0,0 +1,160 @@
+"""
+math2tex.lexer — Lexical analysis: preprocessing and tokenization.
+
+Responsible for converting raw plaintext input into a flat token stream.
+This module also houses the two exception types shared across the compiler.
+"""
+
+import re
+from dataclasses import dataclass
+from typing import List, Tuple
+
+from .config import SPLITTABLE_FUNCS, UNICODE_MAP
+
+# =============================================================================
+# Exception Types
+# =============================================================================
+
+class ParseError(Exception):
+    pass
+
+
+class _Backtrack(Exception):
+    pass
+
+# =============================================================================
+# Token
+# =============================================================================
+
+@dataclass(frozen=True)
+class Token:
+    type: str
+    value: str
+
+
+# =============================================================================
+# Text-literal placeholder helpers  (ISSUE 5 FIX)
+# =============================================================================
+
+_TEXT_PH_PREFIX = 'TEXTPLACEHOLDER'
+
+
+def _make_text_placeholder(idx: int) -> str:
+    return _TEXT_PH_PREFIX + chr(ord('A') + idx // 26) + chr(ord('A') + idx % 26)
+
+
+def _is_text_placeholder(value: str) -> bool:
+    return value.startswith(_TEXT_PH_PREFIX) and len(value) == len(_TEXT_PH_PREFIX) + 2
+
+
+# =============================================================================
+# Preprocessing
+# =============================================================================
+
+def _preprocess(text: str) -> Tuple[str, dict]:
+    text_map: dict = {}
+    idx = 0
+    pattern = re.compile(r'\btext\s*\(')
+    s = text
+    while True:
+        m = pattern.search(s)
+        if not m:
+            break
+        start = m.start()
+        depth = 1
+        i = m.end()
+        while i < len(s) and depth > 0:
+            if s[i] == '(':
+                depth += 1
+            elif s[i] == ')':
+                depth -= 1
+            i += 1
+        if depth != 0:
+            break
+        content = s[m.end():i - 1]
+        placeholder = _make_text_placeholder(idx)
+        text_map[placeholder] = content
+        s = s[:start] + placeholder + s[i:]
+        idx += 1
+
+    for uni_char, ascii_rep in UNICODE_MAP.items():
+        if uni_char in s:
+            if ascii_rep.isalpha():
+                s = s.replace(uni_char, f' {ascii_rep} ')
+            else:
+                s = s.replace(uni_char, ascii_rep)
+
+    s = re.sub(r'\bdouble\s+integral\b', 'iint', s)
+    s = re.sub(r'\btriple\s+integral\b', 'iiint', s)
+    s = re.sub(r'\bcontour\s+integral\b', 'oint', s)
+    s = re.sub(r'\bsurface\s+integral\b', 'iint', s)
+    s = re.sub(r'\bvolume\s+integral\b', 'iiint', s)
+    s = re.sub(r'\bline\s+integral\b', 'oint', s)
+
+    s = re.sub(r'\bapproaches\b', '->', s)
+    s = re.sub(r'\bover\b', '/', s)
+    s = re.sub(r'\bsquare\s+root\b', 'sqrt', s)
+    s = re.sub(r'\bintegrate\b', 'integral', s)
+    s = re.sub(r'\blimit\b', 'lim', s)
+
+    return s, text_map
+
+# =============================================================================
+# Tokenizer
+# =============================================================================
+
+def tokenize(text: str) -> List[Token]:
+    spec = [
+        ('NUMBER',   r'(?:\d+(?:\.\d*)?|\.\d+)(?:[eE][+-]?\d+)?'),
+        ('ELLIPSIS', r'\.\.\.'),
+        ('ARROW',    r'->'),
+        ('COLON',    r':'),
+        ('REL',      r'!=|<=|>=|~=|<|>|==|='),
+        ('DBLBAR',   r'\|\|'),
+        ('BAR',      r'\|'),
+        ('PM',       r'\+-|-\+'),
+        ('OP',       r'[+\-*/^_!]'),
+        ('PRIME',    r"'+"),
+        ('ID',       r'[A-Za-z]+'),
+        ('LPAREN',   r'\('),
+        ('RPAREN',   r'\)'),
+        ('LBRACK',   r'\['),
+        ('RBRACK',   r'\]'),
+        ('LBRACE',   r'\{'),
+        ('RBRACE',   r'\}'),
+        ('COMMA',    r','),
+        ('SEMI',     r';'),
+        ('SKIP',     r'[ \t\n\r]+'),
+        ('MISMATCH', r'.'),
+    ]
+    tok_regex = '|'.join(f'(?P<{name}>{pattern})' for name, pattern in spec)
+    tokens: List[Token] = []
+    for mo in re.finditer(tok_regex, text):
+        kind = mo.lastgroup
+        value = mo.group()
+        if kind == 'SKIP':
+            continue
+        if kind == 'MISMATCH':
+            raise ParseError(f"Unexpected character: {value!r}")
+
+        if kind == 'ID':
+            split = False
+
+            if value not in SPLITTABLE_FUNCS:
+                for pfx in SPLITTABLE_FUNCS:
+                    if value.startswith(pfx) and len(value) > len(pfx):
+                        rest = value[len(pfx):]
+
+                        if '_' not in value:
+                            tokens.append(Token('ID', pfx))
+                            tokens.append(Token('ID', rest))
+                            split = True
+                            break
+
+            if not split:
+                tokens.append(Token(kind, value))
+        else:
+            tokens.append(Token(kind, value))
+
+    tokens.append(Token('EOF', ''))
+    return tokens