docvortex 0.2.1__py3-none-any.whl

docvortex/analyzers/native/office/rtf/lexer.py

@@ -0,0 +1,213 @@
+"""二进制安全、位置显式的 RTF lexer。"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Iterator, TypeAlias, Union
+
+from ..errors import LegacyOfficeMalformedError, LegacyOfficeResourceLimitError
+from ..limits import MAX_RECORDS
+
+MAX_RTF_GROUP_DEPTH = 256
+MAX_RTF_CONTROL_PARAMETER_DIGITS = 10
+MIN_RTF_CONTROL_PARAMETER = -(2**31)
+MAX_RTF_CONTROL_PARAMETER = 2**31 - 1
+
+
+@dataclass(frozen=True, slots=True)
+class RtfOpen:
+    """表示一个左花括号。"""
+
+    start: int
+    end: int
+
+
+@dataclass(frozen=True, slots=True)
+class RtfClose:
+    """表示一个右花括号。"""
+
+    start: int
+    end: int
+
+
+@dataclass(frozen=True, slots=True)
+class RtfControlWord:
+    """表示一个可带有有符号整数参数的 control word。"""
+
+    name: str
+    param: int | None
+    start: int
+    end: int
+
+
+@dataclass(frozen=True, slots=True)
+class RtfControlSymbol:
+    """表示反斜杠后的单字符 control symbol。"""
+
+    symbol: str
+    start: int
+    end: int
+
+
+@dataclass(frozen=True, slots=True)
+class RtfHexByte:
+    """表示一个 ``\'hh`` 十六进制字节。"""
+
+    value: int
+    start: int
+    end: int
+
+
+@dataclass(frozen=True, slots=True)
+class RtfTextBytes:
+    """表示不含 RTF 结构字符的连续原始文本字节。"""
+
+    data: bytes
+    start: int
+    end: int
+
+
+@dataclass(frozen=True, slots=True)
+class RtfBinary:
+    """表示由 ``\binN`` 声明长度的原始二进制载荷。"""
+
+    data: bytes
+    start: int
+    end: int
+
+
+RtfToken: TypeAlias = Union[
+    RtfOpen,
+    RtfClose,
+    RtfControlWord,
+    RtfControlSymbol,
+    RtfHexByte,
+    RtfTextBytes,
+    RtfBinary,
+]
+
+
+def _is_ascii_letter(value: int) -> bool:
+    """判断一个字节是否是 RTF control word 使用的 ASCII 字母。"""
+    return 65 <= value <= 90 or 97 <= value <= 122
+
+
+def _is_ascii_digit(value: int) -> bool:
+    """判断一个字节是否是 ASCII 十进制数字。"""
+    return 48 <= value <= 57
+
+
+class RtfLexer:
+    """按字节位置迭代 RTF token，并对二进制长度和资源上限负责。"""
+
+    def __init__(self, data: bytes) -> None:
+        """保存不可变输入，真正的扫描在迭代时执行。"""
+        self._data = data
+
+    def __iter__(self) -> Iterator[RtfToken]:
+        """按源顺序生成 token，允许 parser 自行恢复不平衡根组。"""
+        data = self._data
+        cursor = 0
+        depth = 0
+        token_count = 0
+        while cursor < len(data):
+            token: RtfToken
+            start = cursor
+            value = data[cursor]
+            if value == ord("{"):
+                cursor += 1
+                depth += 1
+                if depth > MAX_RTF_GROUP_DEPTH:
+                    raise LegacyOfficeResourceLimitError(f"RTF group nesting exceeds max_group_depth={MAX_RTF_GROUP_DEPTH}")
+                token = RtfOpen(start, cursor)
+            elif value == ord("}"):
+                cursor += 1
+                depth = max(depth - 1, 0)
+                token = RtfClose(start, cursor)
+            elif value != ord("\\"):
+                cursor += 1
+                while cursor < len(data) and data[cursor] not in b"{}\\":
+                    cursor += 1
+                token = RtfTextBytes(data[start:cursor], start, cursor)
+            else:
+                cursor += 1
+                if cursor >= len(data):
+                    token = RtfControlSymbol("\\", start, cursor)
+                elif data[cursor] in (10, 13):
+                    if data[cursor] == 13 and cursor + 1 < len(data) and data[cursor + 1] == 10:
+                        cursor += 2
+                    else:
+                        cursor += 1
+                    token = RtfControlSymbol("\n", start, cursor)
+                elif data[cursor] == ord("'"):
+                    cursor += 1
+                    if cursor + 2 <= len(data):
+                        raw_hex = data[cursor : cursor + 2]
+                        try:
+                            hex_value = int(raw_hex.decode("ascii"), 16)
+                        except (UnicodeDecodeError, ValueError):
+                            token = RtfControlSymbol("'", start, cursor)
+                        else:
+                            cursor += 2
+                            token = RtfHexByte(hex_value, start, cursor)
+                    else:
+                        token = RtfControlSymbol("'", start, cursor)
+                elif _is_ascii_letter(data[cursor]):
+                    name_start = cursor
+                    while cursor < len(data) and _is_ascii_letter(data[cursor]):
+                        cursor += 1
+                    name = data[name_start:cursor].decode("ascii").lower()
+                    sign = 1
+                    if cursor < len(data) and data[cursor] == ord("-"):
+                        sign = -1
+                        cursor += 1
+                    number_start = cursor
+                    while cursor < len(data) and _is_ascii_digit(data[cursor]):
+                        cursor += 1
+                    param = None
+                    if cursor > number_start:
+                        digit_count = cursor - number_start
+                        if digit_count > MAX_RTF_CONTROL_PARAMETER_DIGITS:
+                            raise LegacyOfficeResourceLimitError(
+                                f"RTF control parameter exceeds max_digits={MAX_RTF_CONTROL_PARAMETER_DIGITS}"
+                            )
+                        param = sign * int(data[number_start:cursor])
+                        if param < MIN_RTF_CONTROL_PARAMETER or param > MAX_RTF_CONTROL_PARAMETER:
+                            raise LegacyOfficeResourceLimitError("RTF control parameter exceeds signed 32-bit range")
+                    if cursor < len(data) and data[cursor] == ord(" "):
+                        cursor += 1
+                    if name == "bin":
+                        if param is None or param < 0:
+                            raise LegacyOfficeMalformedError("RTF bin control requires a non-negative length")
+                        payload_end = cursor + param
+                        if payload_end < cursor or payload_end > len(data):
+                            raise LegacyOfficeMalformedError(
+                                f"RTF bin payload is truncated: declared={param}, available={len(data) - cursor}"
+                            )
+                        token = RtfBinary(data[cursor:payload_end], start, payload_end)
+                        cursor = payload_end
+                    else:
+                        token = RtfControlWord(name, param, start, cursor)
+                else:
+                    symbol = chr(data[cursor])
+                    cursor += 1
+                    token = RtfControlSymbol(symbol, start, cursor)
+
+            token_count += 1
+            if token_count > MAX_RECORDS:
+                raise LegacyOfficeResourceLimitError(f"RTF token count exceeds max_tokens={MAX_RECORDS}")
+            yield token
+
+
+__all__ = [
+    "MAX_RTF_GROUP_DEPTH",
+    "RtfBinary",
+    "RtfClose",
+    "RtfControlSymbol",
+    "RtfControlWord",
+    "RtfHexByte",
+    "RtfLexer",
+    "RtfOpen",
+    "RtfTextBytes",
+    "RtfToken",
+]

docvortex/analyzers/native/office/rtf/math.py

@@ -0,0 +1,339 @@
+"""把 RTF Office Math destination 规范化为 OMML 并复用现有 LaTeX 转换器。"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+from lxml import etree  # type: ignore[reportAttributeAccessIssue]
+from loguru import logger
+
+from ..equation.omml import oMath2Latex
+from .lexer import RtfClose, RtfControlWord, RtfHexByte, RtfLexer, RtfOpen, RtfTextBytes
+
+_MATH_NS = "http://schemas.openxmlformats.org/officeDocument/2006/math"
+_MATH_PREFIX = f"{{{_MATH_NS}}}"
+
+# RTF Office Math control words为 OMML local name 增加一个 m 前缀；映射保留 XML 所需大小写。
+_OMML_LOCAL_NAMES = (
+    "acc",
+    "accPr",
+    "aln",
+    "alnScr",
+    "argPr",
+    "argSz",
+    "bar",
+    "barPr",
+    "baseJc",
+    "begChr",
+    "borderBox",
+    "borderBoxPr",
+    "box",
+    "boxPr",
+    "brk",
+    "brkBin",
+    "brkBinSub",
+    "cGp",
+    "cGpRule",
+    "chr",
+    "count",
+    "cSp",
+    "ctrlPr",
+    "d",
+    "defJc",
+    "deg",
+    "degHide",
+    "den",
+    "diff",
+    "dispDef",
+    "dPr",
+    "e",
+    "endChr",
+    "eqArr",
+    "eqArrPr",
+    "f",
+    "fName",
+    "fPr",
+    "func",
+    "funcPr",
+    "groupChr",
+    "groupChrPr",
+    "grow",
+    "hideBot",
+    "hideLeft",
+    "hideRight",
+    "hideTop",
+    "interSp",
+    "intLim",
+    "intraSp",
+    "jc",
+    "lim",
+    "limLoc",
+    "limLow",
+    "limLowPr",
+    "limUpp",
+    "limUppPr",
+    "lit",
+    "lMargin",
+    "m",
+    "mathFont",
+    "mathPr",
+    "maxDist",
+    "mc",
+    "mcJc",
+    "mcPr",
+    "mcs",
+    "mPr",
+    "mr",
+    "nary",
+    "naryLim",
+    "naryPr",
+    "noBreak",
+    "nor",
+    "num",
+    "objDist",
+    "oMath",
+    "oMathPara",
+    "oMathParaPr",
+    "opEmu",
+    "phant",
+    "phantPr",
+    "plcHide",
+    "pos",
+    "postSp",
+    "preSp",
+    "r",
+    "rad",
+    "radPr",
+    "rMargin",
+    "rPr",
+    "rSp",
+    "rSpRule",
+    "scr",
+    "sepChr",
+    "show",
+    "shp",
+    "smallFrac",
+    "sPre",
+    "sPrePr",
+    "sSub",
+    "sSubPr",
+    "sSubSup",
+    "sSubSupPr",
+    "sSup",
+    "sSupPr",
+    "strikeBLTR",
+    "strikeH",
+    "strikeTLBR",
+    "strikeV",
+    "sty",
+    "sub",
+    "subHide",
+    "sup",
+    "supHide",
+    "t",
+    "transp",
+    "type",
+    "vertJc",
+    "wrapIndent",
+    "wrapRight",
+    "zeroAsc",
+    "zeroDesc",
+    "zeroWid",
+)
+_CONTROL_TO_LOCAL = {f"m{name}".lower(): name for name in _OMML_LOCAL_NAMES}
+_PROPERTY_LOCAL_NAMES = {
+    "baseJc",
+    "begChr",
+    "chr",
+    "degHide",
+    "endChr",
+    "grow",
+    "jc",
+    "limLoc",
+    "nor",
+    "pos",
+    "scr",
+    "sepChr",
+    "show",
+    "sty",
+    "subHide",
+    "supHide",
+    "type",
+}
+_SCR_VALUES = {
+    0: "roman",
+    1: "script",
+    2: "fraktur",
+    3: "double-struck",
+    4: "sans-serif",
+    5: "monospace",
+}
+
+
+@dataclass(slots=True)
+class _MathGroup:
+    """保存一个 RTF group 对应的 Office Math 局部树。"""
+
+    local: str | None = None
+    text: list[str] = field(default_factory=list)
+    controls: list[tuple[str, int | None]] = field(default_factory=list)
+    children: list[_MathGroup] = field(default_factory=list)
+
+
+def _append_unicode(group: _MathGroup, value: int | None, surrogate: list[int | None]) -> int:
+    """解码有符号 UTF-16 code unit，并返回需要跳过的 fallback 字符数占位。"""
+    if value is None:
+        return 0
+    unit = value + 65536 if value < 0 else value
+    unit &= 0xFFFF
+    pending = surrogate[0]
+    if 0xD800 <= unit <= 0xDBFF:
+        surrogate[0] = unit
+        return 1
+    if 0xDC00 <= unit <= 0xDFFF and pending is not None:
+        codepoint = 0x10000 + ((pending - 0xD800) << 10) + (unit - 0xDC00)
+        group.text.append(chr(codepoint))
+        surrogate[0] = None
+        return 1
+    if pending is not None:
+        group.text.append("\ufffd")
+        surrogate[0] = None
+    group.text.append(chr(unit) if not 0xD800 <= unit <= 0xDFFF else "\ufffd")
+    return 1
+
+
+def _parse_math_groups(data: bytes, encoding: str) -> _MathGroup:
+    """把 math destination token 流构造成与 RTF group 同构的轻量树。"""
+    root = _MathGroup(local="math")
+    stack = [root]
+    uc_skip = 1
+    fallback_skip = 0
+    surrogate: list[int | None] = [None]
+    for token in RtfLexer(data):
+        if isinstance(token, RtfOpen):
+            group = _MathGroup()
+            stack[-1].children.append(group)
+            stack.append(group)
+            continue
+        if isinstance(token, RtfClose):
+            if len(stack) > 1:
+                stack.pop()
+            continue
+        group = stack[-1]
+        if isinstance(token, RtfControlWord):
+            if token.name == "uc":
+                uc_skip = max(token.param or 0, 0)
+                continue
+            if token.name == "u":
+                _append_unicode(group, token.param, surrogate)
+                fallback_skip = uc_skip
+                continue
+            local = _CONTROL_TO_LOCAL.get(token.name)
+            if local is not None:
+                if group.local is None:
+                    group.local = local
+                else:
+                    group.controls.append((local, token.param))
+            continue
+        if isinstance(token, RtfHexByte):
+            if fallback_skip:
+                fallback_skip -= 1
+                continue
+            group.text.append(bytes([token.value]).decode(encoding, errors="replace"))
+            continue
+        if isinstance(token, RtfTextBytes):
+            raw = token.data.replace(b"\r", b"").replace(b"\n", b"")
+            if fallback_skip:
+                skipped = min(fallback_skip, len(raw))
+                raw = raw[skipped:]
+                fallback_skip -= skipped
+            if raw:
+                group.text.append(raw.decode(encoding, errors="replace"))
+    if surrogate[0] is not None:
+        stack[-1].text.append("\ufffd")
+    return root
+
+
+def _value_text(group: _MathGroup) -> str:
+    """返回属性 group 的规范化直接文本。"""
+    return "".join(group.text).strip()
+
+
+def _build_omml_element(group: _MathGroup) -> etree._Element | None:
+    """把一个已命名 math group 转换为规范 OMML element。"""
+    if group.local is None or group.local == "math":
+        return None
+    element = etree.Element(f"{_MATH_PREFIX}{group.local}", nsmap={"m": _MATH_NS})
+    if group.local == "r":
+        properties: list[tuple[str, str]] = []
+        for local, param in group.controls:
+            if local == "scr" and param is not None:
+                properties.append((local, _SCR_VALUES.get(param, str(param))))
+            elif local in {"sty", "nor"}:
+                properties.append((local, str(param if param is not None else 1)))
+        if properties:
+            r_pr = etree.SubElement(element, f"{_MATH_PREFIX}rPr")
+            for local, value in properties:
+                prop = etree.SubElement(r_pr, f"{_MATH_PREFIX}{local}")
+                prop.set(f"{_MATH_PREFIX}val", value)
+        text = "".join(group.text)
+        if text:
+            etree.SubElement(element, f"{_MATH_PREFIX}t").text = text
+    elif group.local in _PROPERTY_LOCAL_NAMES:
+        value = _value_text(group)
+        if value:
+            element.set(f"{_MATH_PREFIX}val", value)
+        elif group.controls:
+            element.set(f"{_MATH_PREFIX}val", str(group.controls[-1][1] or 0))
+        else:
+            element.set(f"{_MATH_PREFIX}val", "1")
+    elif group.text:
+        text = "".join(group.text)
+        run = etree.SubElement(element, f"{_MATH_PREFIX}r")
+        etree.SubElement(run, f"{_MATH_PREFIX}t").text = text
+
+    for child in group.children:
+        child_element = _build_omml_element(child)
+        if child_element is not None:
+            element.append(child_element)
+    return element
+
+
+def _find_groups(group: _MathGroup, local: str) -> list[_MathGroup]:
+    """按深度优先顺序收集指定 OMML local name 的 group。"""
+    result: list[_MathGroup] = []
+    pending = [group]
+    while pending:
+        current = pending.pop()
+        if current.local == local:
+            result.append(current)
+        pending.extend(reversed(current.children))
+    return result
+
+
+def parse_rtf_math(data: bytes, *, encoding: str = "cp1252") -> tuple[list[str], bool]:
+    """解析一个 ``mmath`` group，返回 LaTeX 公式列表和行间公式标记。"""
+    try:
+        root = _parse_math_groups(data, encoding)
+        paragraphs = _find_groups(root, "oMathPara")
+        equation_groups: list[_MathGroup] = []
+        if paragraphs:
+            for paragraph in paragraphs:
+                equation_groups.extend(_find_groups(paragraph, "oMath"))
+        else:
+            equation_groups = _find_groups(root, "oMath")
+        formulas: list[str] = []
+        for equation_group in equation_groups:
+            element = _build_omml_element(equation_group)
+            if element is None:
+                continue
+            latex = str(oMath2Latex(element)).strip()
+            if latex:
+                formulas.append(latex)
+        return formulas, bool(paragraphs)
+    except Exception as exc:
+        logger.warning("RTF Office Math fallback: {}", exc)
+        return [], False
+
+
+__all__ = ["parse_rtf_math"]