docvortex 0.4.20__tar.gz → 0.4.22__tar.gz

{docvortex-0.4.20/src/docvortex.egg-info → docvortex-0.4.22}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: docvortex
-Version: 0.4.20
+Version: 0.4.22
 Summary: A fast, multi-format document parsing and conversion engine
 License-Expression: MIT
 Project-URL: Repository, https://github.com/myhloli/DocVortex

{docvortex-0.4.20 → docvortex-0.4.22}/src/docvortex/analyzers/native/office/docx/tables.py

@@ -185,7 +185,8 @@ class _DocxTables:
         """提取与 Mammoth 渲染规则一致的 XML 表格轻量签名。
 
         按文档顺序收集 w:t 及特殊字符元素，并且只处理 WordprocessingML
-        命名空间，以排除 Mammoth 不渲染的 OMML m:t 公式文本。
+        命名空间，以排除 Mammoth 不渲染的 OMML m:t 公式文本。纵向合并的
+        continuation 单元格会被 Mammoth 折叠掉，因此其整个子树也不参与文本签名。
         """
         w_ns = _DocxConstants._BLIP_NAMESPACES["w"]
         char_tags = (
@@ -194,7 +195,22 @@ class _DocxTables:
             f"{{{w_ns}}}softHyphen",
             f"{{{w_ns}}}sym",
         )
-        text = "".join(_DocxTables._xml_table_char_fragment(node) for node in xml_table.iter() if node.tag in char_tags)
+        continuation_nodes = set()
+        for cell in xml_table.iter(f"{{{w_ns}}}tc"):
+            properties = cell.find(f"{{{w_ns}}}tcPr")
+            vmerge = properties.find(f"{{{w_ns}}}vMerge") if properties is not None else None
+            if vmerge is None:
+                continue
+            # Mammoth 将缺少 val 或 val=continue 的单元格视为 continuation，
+            # 并在计算 rowspan 时丢弃该单元格的全部内容。
+            if vmerge.get(f"{{{w_ns}}}val") in (None, "continue"):
+                continuation_nodes.update(cell.iter())
+
+        text = "".join(
+            _DocxTables._xml_table_char_fragment(node)
+            for node in xml_table.iter()
+            if node not in continuation_nodes and node.tag in char_tags
+        )
         return {
             "row_count": len(xml_table.xpath('.//*[local-name()="tr"]')),
             "cell_count": len(xml_table.xpath('.//*[local-name()="tc"]')),

{docvortex-0.4.20 → docvortex-0.4.22}/src/docvortex/analyzers/native/office/equation/image.py

@@ -10,7 +10,7 @@ import struct
 from loguru import logger
 
 from ..errors import LegacyOfficeResourceLimitError
-from ..limits import MAX_ASSET_TOTAL_BYTES, MAX_ENTRY_BYTES, MAX_PICTURE_RECORDS
+from ..limits import MAX_ENTRY_BYTES, MAX_EQUATION_CANDIDATE_TOTAL_BYTES, MAX_PICTURE_RECORDS, MAX_RECORDS
 from .mtef import decode_mtef
 
 _PLACEABLE_WMF_MAGIC = b"\xd7\xcd\xc6\x9a"
@@ -271,14 +271,18 @@ def _wmf_mtef_candidates(image_data: bytes) -> tuple[list[bytes], bool]:
 def _gif_subblocks(
     image_data: bytes,
     cursor: int,
-    counter: list[int],
-) -> tuple[bytes, int]:
-    """读取以零长度块终止的 GIF sub-block 序列。"""
+    *,
+    collect_payload: bool,
+    subblock_counter: list[int],
+) -> tuple[bytes | None, int]:
+    """读取 GIF sub-block 序列；仅对已识别的公式扩展保留 payload。"""
 
     chunks: list[bytes] = []
     total = 0
     while True:
-        _charge_picture_record(counter)
+        subblock_counter[0] += 1
+        if subblock_counter[0] > MAX_RECORDS:
+            raise LegacyOfficeResourceLimitError(f"GIF sub-block count exceeds max_records={MAX_RECORDS}")
         if cursor >= len(image_data):
             raise _ImageEquationError("GIF sub-block length is truncated")
         size = image_data[cursor]
@@ -290,8 +294,9 @@ def _gif_subblocks(
             raise _ImageEquationError("GIF sub-block data is truncated")
         total += size
         if total > MAX_ENTRY_BYTES:
-            raise LegacyOfficeResourceLimitError(f"GIF extension exceeds max_entry_bytes={MAX_ENTRY_BYTES}")
-        chunks.append(image_data[cursor:end])
+            raise LegacyOfficeResourceLimitError(f"GIF sub-block payload exceeds max_entry_bytes={MAX_ENTRY_BYTES}")
+        if collect_payload:
+            chunks.append(image_data[cursor:end])
         cursor = end
 
 
@@ -310,6 +315,7 @@ def _gif_mtef_candidates(image_data: bytes) -> tuple[list[bytes], bool]:
     candidates: list[bytes] = []
     recognized = False
     counter = [0]
+    subblock_counter = [0]
     saw_trailer = False
     while cursor < len(image_data):
         _charge_picture_record(counter)
@@ -328,7 +334,12 @@ def _gif_mtef_candidates(image_data: bytes) -> tuple[list[bytes], bool]:
             if cursor >= len(image_data):
                 raise _ImageEquationError("GIF image data is truncated")
             cursor += 1  # LZW minimum code size
-            _image_payload, cursor = _gif_subblocks(image_data, cursor, counter)
+            _image_payload, cursor = _gif_subblocks(
+                image_data,
+                cursor,
+                collect_payload=False,
+                subblock_counter=subblock_counter,
+            )
             continue
         if marker != 0x21:
             raise _ImageEquationError("GIF block marker is invalid")
@@ -337,7 +348,12 @@ def _gif_mtef_candidates(image_data: bytes) -> tuple[list[bytes], bool]:
         extension_label = image_data[cursor]
         cursor += 1
         if extension_label != 0xFF:
-            _payload, cursor = _gif_subblocks(image_data, cursor, counter)
+            _payload, cursor = _gif_subblocks(
+                image_data,
+                cursor,
+                collect_payload=False,
+                subblock_counter=subblock_counter,
+            )
             continue
         if cursor >= len(image_data):
             raise _ImageEquationError("GIF application block size is truncated")
@@ -348,12 +364,24 @@ def _gif_mtef_candidates(image_data: bytes) -> tuple[list[bytes], bool]:
             raise _ImageEquationError("GIF application identifier is truncated")
         application = image_data[cursor:application_end]
         cursor = application_end
-        payload, cursor = _gif_subblocks(image_data, cursor, counter)
         if application_size != 11:
+            _payload, cursor = _gif_subblocks(
+                image_data,
+                cursor,
+                collect_payload=False,
+                subblock_counter=subblock_counter,
+            )
             continue
         application_id = application[:8]
         authentication = application[8:11]
-        if application_id == b"MathType" and authentication == b"001":
+        is_mathtype = application_id == b"MathType" and authentication == b"001"
+        payload, cursor = _gif_subblocks(
+            image_data,
+            cursor,
+            collect_payload=is_mathtype,
+            subblock_counter=subblock_counter,
+        )
+        if is_mathtype and payload is not None:
             recognized = True
             candidates.append(payload)
         # MathType/002 是 baseline，必须明确忽略。
@@ -406,6 +434,12 @@ class OfficeImageEquationDecoder:
     _cache: dict[tuple[str, bytes], str | None] = field(default_factory=dict)
     _warned: set[tuple[str, bytes]] = field(default_factory=set)
 
+    @property
+    def candidate_total_bytes(self) -> int:
+        """返回已计入预算的公式 candidate 总字节数。"""
+
+        return self.total_bytes
+
     def decode(
         self,
         image_data: object | None,
@@ -421,23 +455,23 @@ class OfficeImageEquationDecoder:
         if image_format is None:
             return None
         if len(image_data) > MAX_ENTRY_BYTES:
-            raise LegacyOfficeResourceLimitError(f"image equation payload exceeds max_entry_bytes={MAX_ENTRY_BYTES}")
+            raise LegacyOfficeResourceLimitError(f"office image exceeds max_entry_bytes={MAX_ENTRY_BYTES}")
         digest = hashlib.sha256(image_data).digest()
         cache_key = (image_format, digest)
         if cache_key in self._cache:
             return self._cache[cache_key]
-        if self.total_bytes + len(image_data) > MAX_ASSET_TOTAL_BYTES:
-            raise LegacyOfficeResourceLimitError(
-                f"image equation payloads exceed max_asset_total_bytes={MAX_ASSET_TOTAL_BYTES}"
-            )
-        self.total_bytes += len(image_data)
-
         recognized = False
         try:
             if image_format == "wmf":
                 candidates, recognized = _wmf_mtef_candidates(image_data)
             else:
                 candidates, recognized = _gif_mtef_candidates(image_data)
+            candidate_bytes = sum(len(candidate) for candidate in candidates)
+            if self.total_bytes + candidate_bytes > MAX_EQUATION_CANDIDATE_TOTAL_BYTES:
+                raise LegacyOfficeResourceLimitError(
+                    f"image equation candidates exceed max_equation_candidate_total_bytes={MAX_EQUATION_CANDIDATE_TOTAL_BYTES}"
+                )
+            self.total_bytes += candidate_bytes
             latex = _select_candidate_latex(candidates)
         except LegacyOfficeResourceLimitError:
             raise

{docvortex-0.4.20 → docvortex-0.4.22}/src/docvortex/analyzers/native/office/limits.py

@@ -5,6 +5,8 @@ from typing import Final
 MAX_ENTRY_BYTES: Final = 128 * 1024 * 1024
 MAX_TOTAL_BYTES: Final = 512 * 1024 * 1024
 MAX_ASSET_TOTAL_BYTES: Final = 128 * 1024 * 1024
+# 公式探测单独计费，避免普通图片耗尽嵌入素材预算；初始值与素材预算保持一致。
+MAX_EQUATION_CANDIDATE_TOTAL_BYTES: Final = 128 * 1024 * 1024
 MAX_GRID_SLOTS: Final = 4_000_000
 MAX_RECORD_DEPTH: Final = 64
 MAX_RECORDS: Final = 16_000_000

{docvortex-0.4.20 → docvortex-0.4.22}/src/docvortex/analyzers/native/office/metadata.py

@@ -5,6 +5,7 @@ from __future__ import annotations
 from io import BytesIO
 from datetime import datetime, timezone
 import posixpath
+from pathlib import Path
 from zipfile import ZipFile
 
 import lxml.etree as etree
@@ -12,7 +13,7 @@ import lxml.etree as etree
 from ....document.properties import property_count, property_date, property_text, property_values
 from ....errors import DocumentError
 from ....schema import DocumentProperties
-from .limits import MAX_ENTRY_BYTES, MAX_TOTAL_BYTES
+from .limits import MAX_ENTRY_BYTES
 
 _DC = "{http://purl.org/dc/elements/1.1/}"
 _DCT = "{http://purl.org/dc/terms/}"
@@ -43,14 +44,15 @@ def _element_value(root: etree._Element | None, tag: str) -> str | None:
     return None
 
 
-def read_ooxml_properties(data: bytes, suffix: str) -> tuple[DocumentProperties, list[str]]:
+def read_ooxml_properties(data: bytes | str | Path, suffix: str) -> tuple[DocumentProperties, list[str]]:
     """读取 OOXML 核心属性和结构计数，坏的可选属性不阻止其他读取。"""
     properties = DocumentProperties()
     warnings: list[str] = []
-    with ZipFile(BytesIO(data)) as package:
+    package_source = BytesIO(data) if isinstance(data, bytes) else data
+    with ZipFile(package_source) as package:
         infos = package.infolist()
-        if len(infos) > 100_000 or sum(info.file_size for info in infos) > MAX_TOTAL_BYTES:
-            raise DocumentError("resource_limit", "OOXML metadata package exceeds resource limits")
+        if len(infos) > 100_000:
+            raise DocumentError("resource_limit", "OOXML metadata package has too many members")
         if len({info.filename for info in infos}) != len(infos):
             raise DocumentError("open_failed", "Duplicate OOXML package members")
         if _xml_part(package, "[Content_Types].xml") is None:

{docvortex-0.4.20 → docvortex-0.4.22}/src/docvortex/analyzers/native/pdf/models.py

@@ -2,6 +2,7 @@
 
 from __future__ import annotations
 
+from collections.abc import Iterable, Iterator, MutableSet
 from dataclasses import dataclass, field
 from typing import TYPE_CHECKING, Any, Literal
 
@@ -13,6 +14,102 @@ if TYPE_CHECKING:
     from .inline.types import PDFTextScriptLine
 
 
+class _SharedLineIndexSet(MutableSet[int]):
+    """共享只读基底，仅为单个候选记录增删差集，避免复制大型行号集合。"""
+
+    __slots__ = ("base", "added", "removed")
+
+    def __init__(
+        self,
+        base: frozenset[int],
+        values: Iterable[int] = (),
+    ) -> None:
+        """以共享基底和候选已有成员初始化集合。"""
+
+        self.base = base
+        self.removed: set[int] = set()
+        if isinstance(values, _SharedLineIndexSet) and values.base is base:
+            # 与共享基底求并集时，基底成员天然全部可见，只需保留额外成员。
+            self.added = set(values.added)
+        elif isinstance(values, (set, frozenset)):
+            self.added = values.difference(base)
+        else:
+            self.added = {value for value in values if value not in base}
+
+    @classmethod
+    def from_exact_values(
+        cls,
+        base: frozenset[int],
+        values: Iterable[int],
+    ) -> _SharedLineIndexSet:
+        """以共享基底精确表示给定集合，而不是默认并入全部基底成员。"""
+
+        visible = set(values)
+        instance = cls.__new__(cls)
+        instance.base = base
+        instance.added = visible.difference(base)
+        instance.removed = set(base.difference(visible))
+        return instance
+
+    def __contains__(self, value: object) -> bool:
+        """按基底、删除集和新增集判断成员是否可见。"""
+
+        if value in self.added:
+            return True
+        return value in self.base and value not in self.removed
+
+    def __iter__(self) -> Iterator[int]:
+        """迭代当前可见成员，不展开持久化副本。"""
+
+        for value in self.base:
+            if value not in self.removed:
+                yield value
+        yield from self.added
+
+    def __len__(self) -> int:
+        """返回当前可见成员数量。"""
+
+        return len(self.base) - len(self.removed) + len(self.added)
+
+    def add(self, value: int) -> None:
+        """加入成员；基底已有成员只需撤销删除标记。"""
+
+        if value in self.base:
+            self.removed.discard(value)
+            return
+        self.added.add(value)
+
+    def discard(self, value: int) -> None:
+        """删除成员；基底成员以差集标记表示。"""
+
+        if value in self.added:
+            self.added.discard(value)
+            return
+        if value in self.base:
+            self.removed.add(value)
+
+    def update(self, values: Iterable[int]) -> None:
+        """并入成员；同共享基底的候选直接合并差集。"""
+
+        if isinstance(values, _SharedLineIndexSet) and values.base is self.base:
+            self.removed.intersection_update(values.removed)
+            for value in values.added:
+                self.add(value)
+            return
+        for value in values:
+            self.add(value)
+
+    def difference_update(self, values: Iterable[int]) -> None:
+        """移除给定成员，并保持共享基底不变。"""
+
+        if isinstance(values, (set, frozenset)):
+            self.added.difference_update(values)
+            self.removed.update(self.base.intersection(values))
+            return
+        for value in values:
+            self.discard(value)
+
+
 @dataclass(slots=True)
 class _LineItem:
     """保存单个可视文本行及其 source/ink/canonical 几何。"""
@@ -122,7 +219,7 @@ class _TableCandidate:
     angle: int
     score: float
     core_bbox: BBox | None = None
-    line_indices: set[int] = field(default_factory=set)
+    line_indices: MutableSet[int] = field(default_factory=set)
     annotations: list[_TableAnnotation] = field(default_factory=list)
 
 

{docvortex-0.4.20 → docvortex-0.4.22}/src/docvortex/analyzers/native/pdf/native_text.py

@@ -6,6 +6,7 @@ import math
 import re
 import statistics
 import unicodedata
+from bisect import bisect_left, bisect_right
 from collections import Counter
 from dataclasses import dataclass
 from typing import Any, Literal, Mapping, Sequence
@@ -818,12 +819,58 @@ def _merge_native_inline_scripts(
     reference_markers = [_INLINE_REFERENCE_MARKER_RE.fullmatch(text) is not None for text in compact_texts]
     local_bboxes = [_rotate_bbox_to_upright(line.bbox, page_size, line.angle) for line in lines]
     canonical_scales = [_native_typographic_scale(line) for line in lines]
+
+    # 上下标只能贴近主体行的左右边缘。按文本方向分别建立左右边缘索引，
+    # 先取可能相邻的安全超集，再复用下面完整判定，避免密集表格页做 O(n²) 全配对。
+    left_edge_index: dict[int, list[tuple[float, int]]] = {}
+    right_edge_index: dict[int, list[tuple[float, int]]] = {}
+    maximum_scale_by_angle: dict[int, float] = {}
+    for line_index, line in enumerate(lines):
+        left_edge_index.setdefault(line.angle, []).append((local_bboxes[line_index][0], line_index))
+        right_edge_index.setdefault(line.angle, []).append((local_bboxes[line_index][2], line_index))
+        maximum_scale_by_angle[line.angle] = max(
+            maximum_scale_by_angle.get(line.angle, 0.1),
+            canonical_scales[line_index],
+            max(0.1, line.effective_height),
+        )
+    for index in (left_edge_index, right_edge_index):
+        for values in index.values():
+            values.sort(key=lambda item: (item[0], item[1]))
+    left_edge_positions = {angle: [value for value, _index in values] for angle, values in left_edge_index.items()}
+    right_edge_positions = {angle: [value for value, _index in values] for angle, values in right_edge_index.items()}
+
     for small_index, small in enumerate(lines):
         compact_text = compact_texts[small_index]
         if not compact_text:
             continue
         small_local_bbox = local_bboxes[small_index]
-        for base_index, base in enumerate(lines):
+        maximum_base_scale = maximum_scale_by_angle.get(small.angle, 0.1)
+        overlap_padding = 0.35 * maximum_base_scale
+        gap_padding = max(1.5, 0.35 * maximum_base_scale)
+
+        left_values = left_edge_index.get(small.angle, [])
+        left_positions = left_edge_positions.get(small.angle, [])
+        right_values = right_edge_index.get(small.angle, [])
+        right_positions = right_edge_positions.get(small.angle, [])
+        candidate_base_indices = {
+            index
+            for _value, index in left_values[
+                bisect_left(left_positions, small_local_bbox[2] - overlap_padding) : bisect_right(
+                    left_positions, small_local_bbox[2] + gap_padding
+                )
+            ]
+        }
+        candidate_base_indices.update(
+            index
+            for _value, index in right_values[
+                bisect_left(right_positions, small_local_bbox[0] - gap_padding) : bisect_right(
+                    right_positions, small_local_bbox[0] + overlap_padding
+                )
+            ]
+        )
+
+        for base_index in sorted(candidate_base_indices):
+            base = lines[base_index]
             if small_index == base_index or small.angle != base.angle or small.visual_row_id == base.visual_row_id:
                 continue
             if small.effective_height <= 0 or base.effective_height <= 0: