iCode-TUI 0.29.1__py3-none-any.whl

chrys/app/tui/widgets/markdown/math/markdown.py

@@ -0,0 +1,291 @@
+# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved.
+
+"""Math delimiters before CommonMark escaping, with conservative dollar rules."""
+
+from __future__ import annotations
+
+import re
+from bisect import bisect_left, bisect_right
+from typing import TYPE_CHECKING
+
+from .parser import MAX_SOURCE
+
+if TYPE_CHECKING:
+    from markdown_it import MarkdownIt
+    from markdown_it.rules_block import StateBlock
+    from markdown_it.rules_core import StateCore
+    from markdown_it.rules_inline import StateInline
+    from markdown_it.utils import EnvType
+
+MATH_ENABLED = "chrys_math_enabled"
+_MATH_INDEXES = "chrys_math_delimiter_indexes"
+
+
+def _index_delimiters(source: str) -> dict[str, list[int]]:
+    """Index each source once; repeated incomplete openers never rescan its tail."""
+    positions: dict[str, list[int]] = {"$": [], "$$": [], r"\)": [], r"\]": []}
+    backslashes = 0
+    previous_dollar = -2
+    for position, character in enumerate(source):
+        if character == "\\":
+            backslashes += 1
+            continue
+        if character in ")]" and backslashes % 2:
+            positions["\\" + character].append(position - 1)
+        elif character == "$" and backslashes % 2 == 0:
+            positions["$"].append(position)
+            if previous_dollar == position - 1:
+                positions["$$"].append(previous_dollar)
+            previous_dollar = position
+        backslashes = 0
+    return positions
+
+
+def _closing(source: str, marker: str, start: int, env: EnvType) -> int:
+    indexes: dict[str, dict[str, list[int]]] = env.setdefault(_MATH_INDEXES, {})
+    if source not in indexes:
+        indexes[source] = _index_delimiters(source)
+    positions = indexes[source][marker]
+    index = bisect_left(positions, start)
+    return positions[index] if index < len(positions) else -1
+
+
+def _clear_delimiter_indexes(state: StateCore) -> None:
+    """Parsing caches must not retain old transcripts in a caller-owned env."""
+    state.env.pop(_MATH_INDEXES, None)
+
+
+def dollar_is_math(body: str) -> bool:
+    """Prefer a false negative over changing prices, paths or shell variables."""
+    if not body or body != body.strip() or "\n" in body or "`" in body:
+        return False
+    if body[-1] in "+-*/=<>" or body.startswith(("?", "#", "@", "!")):
+        return False
+    if len(body) == 1 and body.isalpha():
+        return True
+    if re.fullmatch(r"[A-Za-z_][A-Za-z_0-9]*", body):
+        return len(body) == 1 or re.fullmatch(r"[A-Za-z]_(?:[A-Za-z]|\d+)", body) is not None
+    if re.match(r"[A-Za-z_][A-Za-z_0-9]{1,}(?:/|\.|:|\[)", body):
+        return False
+    if re.fullmatch(r"\{[A-Za-z_][A-Za-z_0-9]*\}.*", body):
+        return False
+    if re.search(r"\b[A-Za-z]{2,}\b", re.sub(r"\\[A-Za-z]+", "", body)):
+        # Explicit TeX commands provide intent; bare prose does not.
+        return "\\" in body or re.search(r"[=^_{}]", body) is not None
+    return bool(
+        re.search(r"\\(?:[A-Za-z]+|[{}|])|[=+*/^_<>-]", body)
+        or body.isdecimal()
+        or re.fullmatch(r"[A-Za-z]\([A-Za-z0-9_, ]+\)", body)
+    )
+
+
+def _prose_without_links(body: str) -> str:
+    """Simplify links for detection without rescanning failed destinations."""
+    closings: dict[int, int] = {}
+    parentheses: list[int] = []
+    for index, character in enumerate(body):
+        if character == "(":
+            parentheses.append(index)
+        elif character == ")" and parentheses:
+            closings[parentheses.pop()] = index + 1
+        elif character == "\n":
+            parentheses.clear()
+    parts: list[str] = []
+    consumed = 0
+    for opening in re.finditer(r"\[([^\[\]\n]+)\]\(", body):
+        if opening.start() < consumed:
+            continue
+        closing = closings.get(opening.end() - 1)
+        if closing is None:
+            continue
+        parts.extend((body[consumed : opening.start()], opening.group(1)))
+        consumed = closing
+    parts.append(body[consumed:])
+    return "".join(parts)
+
+
+def bracket_is_math(body: str) -> bool:
+    """Explicit TeX delimiters accept notation but preserve ordinary escapes."""
+    word = body
+    if "\\" not in body:
+        if "`" in body:
+            return False
+        # Formatting wrappers must not hide ordinary prose from the heuristic.
+        # Only the detection copy is simplified; CommonMark renders the source.
+        prose = _prose_without_links(body)
+        word = prose
+        for _ in range(4):
+            unwrapped = re.sub(r"([*_~]{1,3})([A-Za-z][A-Za-z0-9]*)\1", r"\2", prose)
+            # A marker within a lone word may be multiplication, as in AB*CD*EF.
+            unwrapped_word = re.sub(r"(?<![A-Za-z0-9])([*_~]{1,3})([A-Za-z][A-Za-z0-9]*[.,:;!?]*)\1", r"\2", word)
+            if unwrapped == prose and unwrapped_word == word:
+                break
+            prose, word = unwrapped, unwrapped_word
+        if re.search(r"(?<![A-Za-z])[A-Za-z]{2,}[.,:;!?]?\s+[A-Za-z]{2,}", prose):
+            return False
+    return bool(body) and not body.isdecimal() and re.fullmatch(r"[A-Za-z]{3,}(?:\s+[A-Za-z]+)*[.,:;!?]*", word) is None
+
+
+def _emphasis_opens_before_math(state: StateInline, start: int) -> bool:
+    """Use CommonMark's flanking rules for a marker run before a formula."""
+    source = state.src
+    if not start or source[start - 1] not in "*_":
+        return False
+    marker = source[start - 1]
+    opening = start - 1
+    while opening and source[opening - 1] == marker:
+        opening -= 1
+    backslashes = 0
+    preceding = opening - 1
+    while preceding >= 0 and source[preceding] == "\\":
+        backslashes += 1
+        preceding -= 1
+    if backslashes % 2:
+        return False
+    return state.scanDelims(opening, marker == "*").can_open
+
+
+def _emphasis_wraps_math(state: StateInline, start: int, finish: int) -> bool:
+    """Recognize adjacent emphasis without treating bare shell suffixes as markup."""
+    return (
+        _emphasis_opens_before_math(state, start)
+        and finish < state.posMax
+        and state.src[finish] == state.src[start - 1]
+        and state.scanDelims(finish, state.src[finish] == "*").can_close
+    )
+
+
+def math_inline(state: StateInline, silent: bool) -> bool:
+    start = state.pos
+    source = state.src
+    bracket = source.startswith(r"\(", start)
+    display_bracket = source.startswith(r"\[", start)
+    display_dollar = source.startswith("$$", start)
+    dollar = source.startswith("$", start) and not display_dollar
+    if not (bracket or display_bracket or dollar or display_dollar):
+        return False
+    enabled = bool(state.md.options.get(MATH_ENABLED))
+    if (dollar or display_dollar) and not enabled:
+        return False
+    if bracket:
+        opening, closing = r"\(", r"\)"
+    elif display_bracket:
+        opening, closing = r"\[", r"\]"
+    else:
+        opening = closing = "$$" if display_dollar else "$"
+    end = _closing(source, closing, start + len(opening), state.env)
+    if end < 0 or end + len(closing) > state.posMax:
+        # Let CommonMark process an ordinary escape and the following inline
+        # Markdown independently while an incomplete delimiter is streaming.
+        return False
+    finish = end + len(closing)
+    body = source[start + len(opening) : end]
+    candidate = body if dollar else body.strip()
+    valid = bracket_is_math(candidate) if bracket or display_bracket else dollar_is_math(candidate)
+    if dollar or display_dollar:
+        emphasis = _emphasis_wraps_math(state, start, finish)
+        notation = (
+            re.fullmatch(r"[A-Za-z_][A-Za-z_0-9]*", candidate) is None
+            and re.search(r"\\(?:[A-Za-z]+|[{}|])|[=+*/^_<>-]", candidate) is not None
+        )
+        if start and (
+            (source[start - 1].isascii() and source[start - 1].isalnum())
+            or source[start - 1] in "$\\"
+            or (source[start - 1] == "_" and not (emphasis or (notation and _emphasis_opens_before_math(state, start))))
+        ):
+            valid = False
+        if finish < len(source):
+            following = source[finish]
+            # A real formula may end before emphasis or sentence punctuation.
+            # Bare identifiers still keep ambiguous shell suffixes as text.
+            if (
+                (following.isascii() and following.isalnum())
+                or following in "${#@("
+                or (following == "_" and not emphasis)
+                or (following in "*?!" and not (emphasis or notation))
+            ):
+                valid = False
+    if enabled and not valid:
+        # Ordinary escapes, prose, prices and shell variables belong to the
+        # CommonMark rules; rejecting a candidate must not consume its contents.
+        return False
+    if not silent:
+        compile_candidate = enabled and len(candidate) <= MAX_SOURCE
+        token = state.push("math_inline" if compile_candidate else "math_literal", "", 0)
+        token.content = candidate if compile_candidate else source[start:finish]
+        token.meta["math_source"] = source[start:finish]
+    state.pos = finish
+    return True
+
+
+def math_block(state: StateBlock, start_line: int, end_line: int, silent: bool) -> bool:
+    if state.sCount[start_line] - state.blkIndent >= 4:
+        return False
+    start = state.bMarks[start_line] + state.tShift[start_line]
+    first = state.src[start : state.eMarks[start_line]]
+    if first.startswith("$$"):
+        opening, closing = "$$", "$$"
+    elif first.startswith(r"\["):
+        opening, closing = r"\[", r"\]"
+    else:
+        return False
+    closing_position = _closing(state.src, closing, start + len(opening), state.env)
+    if closing_position < 0:
+        return False
+    closing_line = bisect_right(state.eMarks, closing_position)
+    if closing_line >= end_line:
+        return False
+    # Each matching interval is scanned once, stopping at structural boundaries.
+    # Compilation remains bounded, while larger closed formulas retain source.
+    lines: list[str] = []
+    for last in range(start_line, closing_line + 1):
+        if last > start_line and state.sCount[last] < state.blkIndent and not state.isEmpty(last):
+            return False
+        line = state.getLines(last, last + 1, state.blkIndent, False).rstrip("\n")
+        if last > start_line and (not line.strip() or re.match(r"\s*(?:`{3,}|~{3,})", line)):
+            return False
+        if last == start_line:
+            line = line.lstrip()
+        elif opening == r"\[" and line.lstrip().startswith(opening):
+            # Nested display openers cannot close this block; stopping here
+            # keeps repeated incomplete streamed formulas linear to scan.
+            return False
+        lines.append(line)
+        if last != closing_line:
+            continue
+        end = _closing(line, closing, len(opening) if last == start_line else 0, state.env)
+        if end < 0:
+            return False
+        if line[end + len(closing) :].strip():
+            return False
+        if opening == r"\[" and last == start_line:
+            body = line[len(opening) : end].strip()
+            if not bracket_is_math(body):
+                return False
+        if silent:
+            return True
+        token = state.push("math_block", "math", 0)
+        token.block = True
+        token.map = [start_line, last + 1]
+        token.meta["math_source"] = "\n".join(lines).strip()
+        lines[-1] = line[:end]
+        token.content = "\n".join(lines)[len(opening) :].strip()
+        state.line = last + 1
+        return True
+    return False
+
+
+def enable_math(parser: MarkdownIt) -> None:
+    """Install rules only on the assistant/document parser."""
+    parser.options[MATH_ENABLED] = True
+    parser.inline.ruler.before("escape", "chrys_math_inline", math_inline)
+    parser.block.ruler.before(
+        "fence", "chrys_math_block", math_block, {"alt": ["paragraph", "reference", "blockquote", "list"]}
+    )
+    parser.core.ruler.push("chrys_math_cleanup", _clear_delimiter_indexes)
+
+
+def disable_math(parser: MarkdownIt) -> None:
+    """User messages preserve their literal mathematical spelling."""
+    parser.options[MATH_ENABLED] = False
+    parser.block.ruler.disable("chrys_math_block")

chrys/app/tui/widgets/markdown/math/parser.py

@@ -0,0 +1,325 @@
+# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved.
+
+"""Bounded, notation-preserving TeX subset parser; never evaluates expressions.
+
+Unknown commands and malformed groups reject the entire formula. The caller
+retains its exact source instead of displaying a partially interpreted result.
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+from typing import Literal
+
+from .symbols import FONTS, FUNCTIONS, INTEGRAL_OPERATORS, LARGE_OPERATORS, SYMBOLS, TEXT_COMMANDS
+
+MAX_SOURCE = 8192
+MAX_NODES = 2048
+MAX_DEPTH = 48
+MAX_MATRIX_CELLS = 256
+DELIMITER_SIZES = frozenset(size + suffix for size in ("big", "Big", "bigg", "Bigg") for suffix in ("", "l", "r", "m"))
+
+type LimitPlacement = Literal["display", "above", "side"]
+
+
+class MathError(ValueError):
+    """The source cannot be represented faithfully within the supported subset."""
+
+
+@dataclass(frozen=True, slots=True)
+class Node:
+    """A notation node, independent of Markdown and terminal geometry."""
+
+    kind: str
+    text: str = ""
+    children: tuple[Node, ...] = ()
+    limits: LimitPlacement = "side"
+
+
+class Parser:
+    """Recursive descent with explicit source, recursion and allocation bounds."""
+
+    def __init__(self, source: str) -> None:
+        if not source.strip() or len(source) > MAX_SOURCE:
+            raise MathError("empty or oversized formula")
+        if any((ord(c) < 32 and c not in "\n\t\r") or 0x7F <= ord(c) < 0xA0 for c in source):
+            raise MathError("control character")
+        self.source = source
+        self.pos = 0
+        self.nodes = 0
+        self.depth = 0
+
+    def node(
+        self, kind: str, text: str = "", children: tuple[Node, ...] = (), *, limits: LimitPlacement = "side"
+    ) -> Node:
+        self.nodes += 1
+        if self.nodes > MAX_NODES:
+            raise MathError("too many nodes")
+        return Node(kind, text, children, limits)
+
+    def skip_space(self) -> None:
+        while self.pos < len(self.source) and self.source[self.pos].isspace():
+            self.pos += 1
+
+    def starts(self, text: str) -> bool:
+        self.skip_space()
+        return self.at(text)
+
+    def at(self, text: str) -> bool:
+        if not self.source.startswith(text, self.pos):
+            return False
+        end = self.pos + len(text)
+        return not (
+            text.startswith("\\")
+            and text[-1:].isalpha()
+            and end < len(self.source)
+            and self.source[end].isascii()
+            and self.source[end].isalpha()
+        )
+
+    def consume(self, text: str) -> None:
+        if not self.starts(text):
+            raise MathError(f"expected {text}")
+        self.pos += len(text)
+
+    def sequence(self, stops: tuple[str, ...] = ()) -> Node:
+        children: list[Node] = []
+        self.depth += 1
+        if self.depth > MAX_DEPTH:
+            raise MathError("nesting limit")
+        while True:
+            self.skip_space()
+            if self.pos == len(self.source) or any(self.at(stop) for stop in stops):
+                break
+            base = self.atom()
+            lower: Node | None = None
+            upper: Node | None = None
+            if self.starts(r"\limits") or self.starts(r"\nolimits"):
+                command = self.command()
+                if base.kind != "operator":
+                    raise MathError("limits on a non-operator")
+                base = self.node("operator", base.text, limits="above" if command == "limits" else "side")
+            while self.starts("_") or self.starts("^"):
+                marker = self.source[self.pos]
+                self.pos += 1
+                value = self.argument()
+                if marker == "_":
+                    if lower is not None:
+                        raise MathError("duplicate subscript")
+                    lower = value
+                else:
+                    if upper is not None:
+                        raise MathError("duplicate superscript")
+                    upper = value
+            if lower is not None or upper is not None:
+                empty = self.node("literal")
+                base = self.node("scripts", children=(base, lower or empty, upper or empty))
+            children.append(base)
+        self.depth -= 1
+        return self.node("row", children=tuple(children))
+
+    def argument(self) -> Node:
+        self.skip_space()
+        if self.starts("{"):
+            self.pos += 1
+            result = self.sequence(("}",))
+            self.consume("}")
+            return result
+        self.depth += 1
+        if self.depth > MAX_DEPTH:
+            raise MathError("nesting limit")
+        result = self.atom()
+        self.depth -= 1
+        return result
+
+    def raw_group(self) -> str:
+        self.consume("{")
+        start = self.pos
+        level = 1
+        while self.pos < len(self.source):
+            char = self.source[self.pos]
+            if char == "\\":
+                self.pos += 2
+                continue
+            if char == "{":
+                level += 1
+            elif char == "}":
+                level -= 1
+                if not level:
+                    result = self.source[start : self.pos]
+                    self.pos += 1
+                    return result
+            self.pos += 1
+        raise MathError("unclosed text group")
+
+    def command(self) -> str:
+        self.consume("\\")
+        match = re.match(r"[A-Za-z]+|.", self.source[self.pos :])
+        if match is None:
+            raise MathError("incomplete command")
+        self.pos += len(match[0])
+        return match[0]
+
+    def delimiter(self) -> str:
+        self.skip_space()
+        if self.starts("\\"):
+            name = self.command()
+            if name not in SYMBOLS:
+                raise MathError("unknown delimiter")
+            value = SYMBOLS[name]
+        else:
+            value = self.source[self.pos : self.pos + 1]
+            self.pos += 1
+        if value not in {"(", ")", "[", "]", "{", "}", "|", "‖", "⟨", "⟩", "⌊", "⌋", "⌈", "⌉", "."}:
+            raise MathError("invalid delimiter")
+        return "" if value == "." else value
+
+    def environment(self) -> Node:
+        name = self.raw_group()
+        if name not in {
+            "matrix",
+            "pmatrix",
+            "bmatrix",
+            "Bmatrix",
+            "vmatrix",
+            "Vmatrix",
+            "smallmatrix",
+            "cases",
+            "aligned",
+            "gathered",
+            "split",
+        }:
+            raise MathError("unsupported environment")
+        rows: list[Node] = []
+        cells: list[Node] = []
+        count = 0
+        while True:
+            cell = self.sequence(("&", r"\\", r"\end"))
+            count += 1
+            if count > MAX_MATRIX_CELLS:
+                raise MathError("oversized matrix")
+            cells.append(cell)
+            if self.starts("&"):
+                self.pos += 1
+                continue
+            rows.append(self.node("row", children=tuple(cells)))
+            cells = []
+            if self.starts(r"\\"):
+                self.pos += 2
+                if not self.starts(r"\end"):
+                    continue
+            self.consume(r"\end")
+            if self.raw_group() != name:
+                raise MathError("mismatched environment")
+            break
+        columns = len(rows[0].children)
+        if any(len(row.children) != columns for row in rows):
+            raise MathError("ragged matrix")
+        if name == "cases" and columns != 2:
+            raise MathError("cases needs two columns")
+        return self.node("matrix", name, tuple(rows))
+
+    def atom(self) -> Node:
+        self.skip_space()
+        if self.pos >= len(self.source):
+            raise MathError("missing argument")
+        char = self.source[self.pos]
+        if char == "{":
+            return self.argument()
+        if char in "}^_&$%#" or char == "\n":
+            raise MathError("unexpected math punctuation")
+        if char != "\\":
+            self.pos += 1
+            return self.node("literal", char)
+        command = self.command()
+        if command in DELIMITER_SIZES:
+            delimiter = self.delimiter()
+            kind = {"l": "open", "r": "close", "m": "relation"}.get(command[-1], "literal")
+            return self.node(kind if delimiter else "literal", delimiter)
+        if command == "mid":
+            # The same glyph is an ordinary delimiter when written as | or \vert.
+            return self.node("relation", "|")
+        if command == "colon":
+            return self.node("punctuation", ":")
+        if command in {"lvert", "lVert", "rvert", "rVert"}:
+            return self.node("open" if command.startswith("l") else "close", SYMBOLS[command])
+        if command == "not":
+            target = self.argument()
+            negated = {"=": "≠", "∈": "∉"}.get(target.text) if target.kind == "literal" else None
+            if negated is None:
+                raise MathError("unsupported negated symbol")
+            return self.node("literal", negated)
+        if command == "boxed":
+            return self.node("boxed", children=(self.argument(),))
+        if command in {"frac", "dfrac", "tfrac", "binom", "dbinom", "tbinom"}:
+            return self.node(
+                "binomial" if "binom" in command else "fraction", children=(self.argument(), self.argument())
+            )
+        if command == "sqrt":
+            index = self.node("literal")
+            if self.starts("["):
+                self.pos += 1
+                index = self.sequence(("]",))
+                self.consume("]")
+            return self.node("root", children=(self.argument(), index))
+        if command == "left":
+            left = self.delimiter()
+            body = self.sequence((r"\right",))
+            self.consume(r"\right")
+            return self.node("delimited", left + "\n" + self.delimiter(), (body,))
+        if command == "begin":
+            return self.environment()
+        if command in TEXT_COMMANDS:
+            raw = self.raw_group()
+            # Only escaped literal punctuation is valid in a text box. Retain
+            # unsupported nested formatting via the formula's source fallback.
+            unescaped = re.sub(r"\\[{}%_$&# ]", "", raw)
+            if any(c in unescaped for c in "\\{}"):
+                raise MathError("unsupported text command")
+            text = re.sub(r"\\([{}%_$&# ])", r"\1", raw)
+            result = self.node("function" if command == "operatorname" else "literal", re.sub(r"\s+", " ", text))
+            return result
+        if command in FONTS:
+            child = self.argument()
+            if command in {"mathrm", "mathit", "mathsf", "mathtt"}:
+                return child
+            return self.node("font", command, (child,))
+        if command in {"overline", "underline", "bar", "hat", "widehat", "tilde", "widetilde", "vec", "dot", "ddot"}:
+            return self.node("accent", command, (self.argument(),))
+        if command in {"overset", "underset", "stackrel"}:
+            annotation, base = self.argument(), self.argument()
+            if base.kind != "row":
+                base = self.node("row", children=(base,))
+            return self.node("under" if command == "underset" else "over", children=(base, annotation))
+        if command in {"displaystyle", "textstyle", "scriptstyle", "scriptscriptstyle"}:
+            return self.node("literal")
+        if command == "pmod":
+            return self.node(
+                "delimited", "(\n)", (self.node("row", children=(self.node("literal", "mod "), self.argument())),)
+            )
+        if command in {"bmod", "mod"}:
+            return self.node("literal", " mod ")
+        if command == "!":
+            # A terminal has no fractional cells: retain a zero-width boundary
+            # that suppresses automatic glue instead of shifting painted cells.
+            return self.node("glue")
+        if command in SYMBOLS or command in FUNCTIONS:
+            kind = (
+                "operator"
+                if command in LARGE_OPERATORS | INTEGRAL_OPERATORS
+                else ("function" if command in FUNCTIONS or command in {"Re", "Im"} else "literal")
+            )
+            return self.node(
+                kind, SYMBOLS.get(command, command), limits="display" if command in LARGE_OPERATORS else "side"
+            )
+        raise MathError(f"unsupported command: {command}")
+
+
+def parse_math(source: str) -> Node:
+    """Parse one complete expression or raise a bounded, expected MathError."""
+    parser = Parser(source)
+    result = parser.sequence()
+    if parser.pos != len(source) or not result.children:
+        raise MathError("incomplete expression")
+    return result