docvortex 0.2.1__py3-none-any.whl

docvortex/document/pdf/text/__init__.py

@@ -0,0 +1,62 @@
+"""PDF 字符到文本片段及行的共享纯数据接口。"""
+
+from __future__ import annotations
+
+from .contracts import Bbox, Char, Line, Span
+from .groups import assign_scripts, get_lines
+
+
+def get_spans(chars: list[Char], superscript_height_threshold: float = 0.8, line_distance_threshold: float = 0.1) -> list[Span]:
+    """直接从自有字符记录构建片段，避免容器与数组往返转换。"""
+    spans: list[Span] = []
+    for char in chars:
+        current = spans[-1] if spans else None
+        box = char["bbox"]
+        new_span = current is None
+        if current is not None:
+            previous = current["chars"][-1]
+            height = current["bbox"].height
+            new_span = (
+                char["font"] != current["font"]
+                or char["rotation"] != current["rotation"]
+                or previous["char"] in {"\x02", "\n"}
+                or (
+                    box.y_start < current["bbox"].y_start - height * line_distance_threshold
+                    and box.y_end < height * superscript_height_threshold + current["bbox"].y_start
+                    and box.x_start > current["bbox"].x_end
+                )
+            )
+        if new_span:
+            spans.append(
+                {
+                    "bbox": box.copy(),
+                    "text": char["char"],
+                    "font": char["font"],
+                    "chars": [char],
+                    "char_start_idx": char["char_idx"],
+                    "char_end_idx": char["char_idx"],
+                    "rotation": char["rotation"],
+                    "url": "",
+                    "superscript": False,
+                    "subscript": False,
+                }
+            )
+        else:
+            current["bbox"].merge_inplace(box)
+            current["text"] += char["char"]
+            current["chars"].append(char)
+            current["char_end_idx"] = char["char_idx"]
+    return spans
+
+
+def get_lines_from_chars(
+    chars: list[Char], superscript_height_threshold: float = 0.7, line_distance_threshold: float = 0.1
+) -> list[Line]:
+    """由已物化字符生成基础文本行，不访问 PDFium 或源文档。"""
+    spans = get_spans(chars, superscript_height_threshold, line_distance_threshold)
+    lines = get_lines(spans)
+    assign_scripts(lines, superscript_height_threshold, line_distance_threshold)
+    return lines
+
+
+__all__ = ["Bbox", "Char", "Line", "Span", "get_lines_from_chars"]

docvortex/document/pdf/text/contracts.py

@@ -0,0 +1,211 @@
+# Portions derived from pdftext 0.7.1, Copyright Vik Paruchuri, Apache-2.0.
+# Changed in DocVortex: owned character types and source-index mappings replace upstream containers.
+"""DocVortex 自有 PDF 字符与几何数据，不携带 PDFium 句柄。"""
+
+from __future__ import annotations
+from typing import Any, TypedDict
+
+
+class Bbox:
+    __slots__ = ("bbox", "ensure_nonzero_area")
+
+    def __init__(self, bbox: list[float], ensure_nonzero_area: bool = False) -> None:
+        """建立独立矩形对象并按需保证面积。"""
+        if ensure_nonzero_area:
+            bbox = list(bbox)
+            bbox[2] = max(bbox[0], bbox[2] + 1)
+            bbox[3] = max(bbox[1], bbox[3] + 1)
+        self.bbox = bbox
+        self.ensure_nonzero_area = ensure_nonzero_area
+
+    def __getitem__(self, item: int | slice) -> float | list[float]:
+        """读取矩形坐标。"""
+        return self.bbox[item]
+
+    def __repr__(self) -> str:
+        """返回矩形的可读表示。"""
+        return f"Bbox({self.bbox})"
+
+    def __reduce__(self) -> tuple:
+        # ensure_nonzero_area is already applied at construction; don't re-apply on unpickle
+        """保存矩形数值以支持跨进程序列化。"""
+        return (Bbox, (self.bbox,))
+
+    def copy(self) -> Bbox:
+        """复制矩形避免共享累加状态。"""
+        return Bbox(list(self.bbox))
+
+    @property
+    def height(self) -> float:
+        """计算矩形的 height 几何属性。"""
+        return self.bbox[3] - self.bbox[1]
+
+    @property
+    def width(self) -> float:
+        """计算矩形的 width 几何属性。"""
+        return self.bbox[2] - self.bbox[0]
+
+    @property
+    def area(self) -> float:
+        """计算矩形的 area 几何属性。"""
+        return self.width * self.height
+
+    @property
+    def center(self) -> list[float]:
+        """计算矩形的 center 几何属性。"""
+        return [(self.bbox[0] + self.bbox[2]) / 2, (self.bbox[1] + self.bbox[3]) / 2]
+
+    @property
+    def size(self) -> list[float]:
+        """计算矩形的 size 几何属性。"""
+        return [self.width, self.height]
+
+    @property
+    def x_start(self) -> float:
+        """计算矩形的 x_start 几何属性。"""
+        return self.bbox[0]
+
+    @property
+    def y_start(self) -> float:
+        """计算矩形的 y_start 几何属性。"""
+        return self.bbox[1]
+
+    @property
+    def x_end(self) -> float:
+        """计算矩形的 x_end 几何属性。"""
+        return self.bbox[2]
+
+    @property
+    def y_end(self) -> float:
+        """计算矩形的 y_end 几何属性。"""
+        return self.bbox[3]
+
+    def merge(self, other: Bbox) -> Bbox:
+        """返回覆盖两个矩形的新对象。"""
+        self_bbox = self.bbox
+        other_bbox = other.bbox
+        return Bbox(
+            [
+                min(self_bbox[0], other_bbox[0]),
+                min(self_bbox[1], other_bbox[1]),
+                max(self_bbox[2], other_bbox[2]),
+                max(self_bbox[3], other_bbox[3]),
+            ]
+        )
+
+    def merge_inplace(self, other: Bbox) -> Bbox:
+        # Mutates this bbox; only safe on accumulator bboxes that aren't shared
+        """仅修改当前累加矩形。"""
+        self_bbox = self.bbox
+        other_bbox = other.bbox
+        if other_bbox[0] < self_bbox[0]:
+            self_bbox[0] = other_bbox[0]
+        if other_bbox[1] < self_bbox[1]:
+            self_bbox[1] = other_bbox[1]
+        if other_bbox[2] > self_bbox[2]:
+            self_bbox[2] = other_bbox[2]
+        if other_bbox[3] > self_bbox[3]:
+            self_bbox[3] = other_bbox[3]
+        return self
+
+    def overlap_x(self, other: Bbox) -> float:
+        """计算矩形的 overlap_x 几何属性。"""
+        return max(0, min(self.bbox[2], other.bbox[2]) - max(self.bbox[0], other.bbox[0]))
+
+    def overlap_y(self, other: Bbox) -> float:
+        """计算矩形的 overlap_y 几何属性。"""
+        return max(0, min(self.bbox[3], other.bbox[3]) - max(self.bbox[1], other.bbox[1]))
+
+    def intersection_area(self, other: Bbox) -> float:
+        """计算矩形的 intersection_area 几何属性。"""
+        return self.overlap_x(other) * self.overlap_y(other)
+
+    def intersection_pct(self, other: Bbox) -> float:
+        """计算矩形的 intersection_pct 几何属性。"""
+        if self.area <= 0:
+            return 0
+
+        intersection = self.intersection_area(other)
+        return intersection / self.area
+
+    def rotate(self, page_width: float, page_height: float, rotation: int) -> Bbox:
+        """将矩形转换到旋转后的页面坐标。"""
+        if rotation not in [0, 90, 180, 270]:
+            raise ValueError("Rotation must be one of [0, 90, 180, 270] degrees.")
+
+        x_min, y_min, x_max, y_max = self.bbox
+
+        if rotation == 0:
+            return Bbox(list(self.bbox))
+        elif rotation == 90:
+            new_x_min = page_height - y_max
+            new_y_min = x_min
+            new_x_max = page_height - y_min
+            new_y_max = x_max
+        elif rotation == 180:
+            new_x_min = page_width - x_max
+            new_y_min = page_height - y_max
+            new_x_max = page_width - x_min
+            new_y_max = page_height - y_min
+        elif rotation == 270:
+            new_x_min = y_min
+            new_y_min = page_width - x_max
+            new_x_max = y_max
+            new_y_max = page_width - x_min
+
+        # Ensure that x_min < x_max and y_min < y_max; must stay a list so
+        # merge_inplace can mutate it
+        rotated_bbox = [
+            min(new_x_min, new_x_max),
+            min(new_y_min, new_y_max),
+            max(new_x_min, new_x_max),
+            max(new_y_min, new_y_max),
+        ]
+
+        return Bbox(rotated_bbox)
+
+
+class _CharValue(TypedDict):
+    bbox: Bbox
+    char: str
+    rotation: float
+    font: dict[str, Any]
+    char_idx: int
+
+
+class Char(_CharValue, total=False):
+    """字符及原始索引映射，几何缺失保留为显式空值。"""
+
+    source_indices: tuple[int, ...]
+    raw_code: int
+    loose_bbox: tuple[float, float, float, float] | None
+    tight_bbox: tuple[float, float, float, float] | None
+    origin: tuple[float, float] | None
+
+
+class Span(TypedDict):
+    """基础字体片段，包含已解码文本及原始字符引用。"""
+
+    bbox: Bbox
+    text: str
+    font: dict[str, Any]
+    chars: list[Char]
+    char_start_idx: int
+    char_end_idx: int
+    rotation: float
+    url: str
+    superscript: bool
+    subscript: bool
+
+
+class Line(TypedDict):
+    """基础文本行，几何与片段顺序均保持可追溯。"""
+
+    spans: list[Span]
+    bbox: Bbox
+    rotation: float
+
+
+Spans = list[Span]
+Lines = list[Line]
+__all__ = ["Bbox", "Char", "Span", "Line", "Spans", "Lines"]

docvortex/document/pdf/text/extract.py

@@ -0,0 +1,165 @@
+# Portions derived from pdftext 0.7.1, Copyright Vik Paruchuri, Apache-2.0.
+# Changed in DocVortex: direct PDFium extraction collects character and extended geometry together.
+"""在一次 PDFium 字符遍历内收集原始码值、字体及可选几何。"""
+
+from __future__ import annotations
+
+from ctypes import byref, c_double, c_int, create_string_buffer
+import math
+from typing import Any
+
+import pypdfium2 as pdfium
+import pypdfium2.raw as raw
+
+from .contracts import Bbox, Char
+
+
+def transform_point(
+    point: tuple[float, float], page_bbox: tuple[float, float, float, float], rotation: int
+) -> tuple[float, float]:
+    """保留浮点页面框，将 PDF 原始坐标转换到视觉页面坐标。"""
+    left, bottom, right, top = page_bbox
+    width, height = abs(right - left), abs(top - bottom)
+    x, y = point[0] - min(left, right), max(bottom, top) - point[1]
+    rotation %= 360
+    if rotation == 90:
+        return height - y, x
+    if rotation == 180:
+        return width - x, height - y
+    if rotation == 270:
+        return y, width - x
+    return x, y
+
+
+def visual_bbox(
+    box: tuple[float, float, float, float], page_bbox: tuple[float, float, float, float], rotation: int
+) -> tuple[float, float, float, float] | None:
+    """转换原始矩形，几何缺失或零面积时返回空值而非伪造坐标。"""
+    left, bottom, right, top = box
+    points = [
+        transform_point(point, page_bbox, rotation) for point in ((left, bottom), (left, top), (right, bottom), (right, top))
+    ]
+    result = (min(p[0] for p in points), min(p[1] for p in points), max(p[0] for p in points), max(p[1] for p in points))
+    return result if all(math.isfinite(v) for v in result) and result[2] > result[0] and result[3] > result[1] else None
+
+
+def _font_name(handle: Any, index: int, buffer: Any, flags: c_int) -> tuple[str, int]:
+    """复用字体缓冲区，超长名称按 PDFium 返回长度重新读取。"""
+    try:
+        length = raw.FPDFText_GetFontInfo(handle, index, buffer, len(buffer), byref(flags))
+        if length > len(buffer):
+            buffer = create_string_buffer(length)
+            raw.FPDFText_GetFontInfo(handle, index, buffer, length, byref(flags))
+        return (buffer.value.decode("utf-8", errors="replace"), flags.value) if length > 0 else ("", 0)
+    except pdfium.PdfiumError:
+        return "", 0
+
+
+def get_chars(
+    textpage: pdfium.PdfTextPage, page_bbox: list[float], page_rotation: int, *, include_geometry: bool = False
+) -> list[Char]:
+    """读取原始字符记录；原始码值始终保留，随后统一解码和去重。"""
+    handle = textpage.raw
+    left, bottom, right, top = page_bbox
+    width, height = math.ceil(abs(right - left)), math.ceil(abs(top - bottom))
+    rect = raw.FS_RECTF()
+    tight_left, tight_right, tight_bottom, tight_top = c_double(), c_double(), c_double(), c_double()
+    origin_x, origin_y = c_double(), c_double()
+    font_buffer, font_flags = create_string_buffer(256), c_int()
+    fonts: dict[tuple[Any, ...], dict[str, Any]] = {}
+    chars: list[Char] = []
+    for index in range(textpage.count_chars()):
+        code = int(raw.FPDFText_GetUnicode(handle, index))
+        rotation = float(raw.FPDFText_GetCharAngle(handle, index))
+        loose: tuple[float, float, float, float] | None = None
+        tight: tuple[float, float, float, float] | None = None
+        if rotation == 0 or include_geometry:
+            try:
+                if raw.FPDFText_GetLooseCharBox(handle, index, rect):
+                    loose = (float(rect.left), float(rect.bottom), float(rect.right), float(rect.top))
+            except Exception:
+                if rotation == 0:
+                    raise
+        if rotation != 0 or include_geometry:
+            try:
+                if raw.FPDFText_GetCharBox(handle, index, tight_left, tight_right, tight_bottom, tight_top):
+                    tight = (tight_left.value, tight_bottom.value, tight_right.value, tight_top.value)
+            except Exception:
+                if rotation != 0:
+                    raise
+        selected = loose if rotation == 0 else tight
+        if selected is None:
+            raise pdfium.PdfiumError("Failed to get charbox.")
+        x0, y0, x1, y1 = selected
+        # 布局框保留基线的整数页面高度，原始扩展几何另用浮点页面框。
+        ys = (height - (y0 - bottom), height - (y1 - bottom))
+        box = Bbox([min(x0, x1) - left, min(ys), max(x0, x1) - left, max(ys)])
+        if page_rotation:
+            box = box.rotate(width, height, page_rotation)
+        name, flags = _font_name(handle, index, font_buffer, font_flags)
+        size, weight = raw.FPDFText_GetFontSize(handle, index), raw.FPDFText_GetFontWeight(handle, index)
+        font = fonts.setdefault((name, flags, size, weight), {"name": name, "flags": flags, "size": size, "weight": weight})
+        char: Char = {
+            "bbox": box,
+            "char": chr(code) if not 0xD800 <= code <= 0xDFFF else "\ufffd",
+            "rotation": rotation,
+            "font": font,
+            "char_idx": index,
+            "source_indices": (index,),
+            "raw_code": code,
+        }
+        if include_geometry:
+            char["loose_bbox"] = visual_bbox(loose, tuple(page_bbox), page_rotation) if loose else None
+            char["tight_bbox"] = visual_bbox(tight, tuple(page_bbox), page_rotation) if tight else None
+            char["origin"] = None
+            try:
+                if raw.FPDFText_GetCharOrigin(handle, index, origin_x, origin_y):
+                    origin = transform_point((origin_x.value, origin_y.value), tuple(page_bbox), page_rotation)
+                    if all(math.isfinite(v) for v in origin):
+                        char["origin"] = origin
+            except Exception:
+                pass
+        chars.append(char)
+    return chars
+
+
+def deduplicate_chars(chars: list[Char]) -> list[Char]:
+    """按词文本、字体、方向及取整位置去重，保留最早原始索引。"""
+    if not chars:
+        return []
+    groups: list[list[Char]] = [[chars[0]]]
+    for char in chars[1:]:
+        previous = groups[-1][-1]
+        if (
+            previous["char"] in {"\x02", "\n", " "}
+            or char["font"] != previous["font"]
+            or char["rotation"] != previous["rotation"]
+        ):
+            groups.append([])
+        groups[-1].append(char)
+    seen: dict[tuple[Any, ...], list[Char]] = {}
+    result: list[Char] = []
+    for group in groups:
+        box = group[0]["bbox"].copy()
+        for char in group[1:]:
+            box.merge_inplace(char["bbox"])
+        font = group[0]["font"]
+        key = (
+            tuple(round(float(v), 0) for v in box.bbox),
+            "".join(c["char"] for c in group),
+            group[0]["rotation"],
+            tuple(font.get(k) for k in ("name", "flags", "size", "weight")),
+        )
+        if key not in seen:
+            seen[key] = group
+            result.extend(group)
+        else:
+            for retained, duplicate in zip(seen[key], group):
+                retained["source_indices"] = (
+                    *retained.get("source_indices", (retained["char_idx"],)),
+                    *duplicate.get("source_indices", (duplicate["char_idx"],)),
+                )
+    return result
+
+
+__all__ = ["transform_point", "visual_bbox"]

docvortex/document/pdf/text/geometry.py

@@ -0,0 +1,16 @@
+"""几何载荷与独立矩形类型之间的纯值转换。"""
+
+from __future__ import annotations
+from .contracts import Bbox
+
+
+def char_bbox_values(bbox: object) -> tuple[float, float, float, float] | None:
+    """读取固定四元坐标，拒绝结构不完整的几何对象。"""
+    if isinstance(bbox, Bbox):
+        bbox = bbox.bbox
+    if isinstance(bbox, (tuple, list)) and len(bbox) == 4:
+        return tuple(float(value) for value in bbox)
+    return None
+
+
+__all__ = ["char_bbox_values"]

docvortex/document/pdf/text/groups.py

@@ -0,0 +1,162 @@
+# Portions derived from pdftext 0.7.1, Copyright Vik Paruchuri, Apache-2.0.
+# Changed in DocVortex: grouping uses owned dictionaries without upstream container adapters.
+"""基础文本行与上下标分组；保留已验证的几何判断。"""
+
+from __future__ import annotations
+import math
+import unicodedata
+from .contracts import Line, Lines, Spans
+
+
+def is_math_symbol(char: str) -> bool:
+    """判断单字符数学符号。"""
+    if len(char) != 1:
+        return False
+
+    category = unicodedata.category(char)
+    return category == "Sm"
+
+
+def _top2(values: list[float]) -> tuple[float, int, float]:
+    # Returns (max1, max1_idx, max2) so that max-excluding-index can be answered in O(1)
+    """在线性时间内找到两个最大值。"""
+    max1 = max2 = float("-inf")
+    max1_idx = -1
+    for idx, v in enumerate(values):
+        if v > max1:
+            max2 = max1
+            max1 = v
+            max1_idx = idx
+        elif v > max2:
+            max2 = v
+    return max1, max1_idx, max2
+
+
+def _bottom2(values: list[float]) -> tuple[float, int, float]:
+    """在线性时间内找到两个最小值。"""
+    min1 = min2 = float("inf")
+    min1_idx = -1
+    for idx, v in enumerate(values):
+        if v < min1:
+            min2 = min1
+            min1 = v
+            min1_idx = idx
+        elif v < min2:
+            min2 = v
+    return min1, min1_idx, min2
+
+
+def assign_scripts(lines: Lines, height_threshold: float = 0.8, line_distance_threshold: float = 0.1) -> None:
+    """根据邻接片段几何设置基础上下标提示。"""
+    for line in lines:
+        spans = line["spans"]
+        if len(spans) < 2:
+            continue
+
+        line_bbox = line["bbox"].bbox
+        line_height = line_bbox[3] - line_bbox[1]
+        # Skip vertical lines
+        if line_height > line_bbox[2] - line_bbox[0]:
+            continue
+
+        # Precompute per-span geometry once; the loop below would otherwise
+        # recompute these via Bbox properties O(n^2) times per line
+        heights = []
+        y_starts = []
+        y_ends = []
+        v_above = []
+        v_below = []
+        for s in spans:
+            bbox = s["bbox"].bbox
+            height = bbox[3] - bbox[1]
+            heights.append(height)
+            y_starts.append(bbox[1])
+            y_ends.append(bbox[3])
+            v_above.append(bbox[1] - height * line_distance_threshold)
+            v_below.append(bbox[3] + height * line_distance_threshold)
+
+        above_max1, above_max1_idx, above_max2 = _top2(v_above)
+        below_min1, below_min1_idx, below_min2 = _bottom2(v_below)
+        max_line_height = max(1, line_height)
+        last_idx = len(spans) - 1
+
+        for i, span in enumerate(spans):
+            is_first = i == 0 or not spans[i - 1]["text"].strip()
+            is_last = i == last_idx or not spans[i + 1]["text"].strip()
+            span_height = heights[i]
+            span_top = y_starts[i]
+            span_bottom = y_ends[i]
+
+            line_fullheight = span_height / max_line_height <= height_threshold
+            next_fullheight = is_last or span_height / max(1, heights[i + 1]) <= height_threshold
+            prev_fullheight = is_first or span_height / max(1, heights[i - 1]) <= height_threshold
+
+            # any(span_top < v_above[j] for j != i) == span_top < max(v_above excluding i)
+            above = span_top < (above_max2 if i == above_max1_idx else above_max1)
+            prev_above = is_first or span_top < y_starts[i - 1]
+            next_above = is_last or span_top < y_starts[i + 1]
+
+            below = span_bottom > (below_min2 if i == below_min1_idx else below_min1)
+            prev_below = is_first or span_bottom > y_ends[i - 1]
+            next_below = is_last or span_bottom > y_ends[i + 1]
+
+            span_text = span["text"].strip()
+            span_text_okay = all(
+                [
+                    (len(span_text) == 1 or span_text.isdigit()),  # Ensure that the span text is a single char or a number
+                    span_text.isalnum()
+                    or is_math_symbol(span_text),  # Ensure that the span text is an alphanumeric or a math symbol
+                ]
+            )
+
+            if all([(prev_fullheight or next_fullheight), (prev_above or next_above), above, line_fullheight, span_text_okay]):
+                span["superscript"] = True
+            elif all(
+                [(prev_fullheight or next_fullheight), (prev_below or next_below), below, line_fullheight, span_text_okay]
+            ):
+                span["subscript"] = True
+
+
+def get_lines(spans: Spans) -> Lines:
+    """按换行、角度和位置将片段聚合为行。"""
+    lines: Lines = []
+    line: Line = None
+
+    def line_break() -> None:
+        """以当前片段开始一个新的文本行。"""
+        lines.append({"spans": [span], "bbox": span["bbox"].copy(), "rotation": span["rotation"]})
+
+    for span in spans:
+        if lines:
+            line = lines[-1]
+
+        if not line:
+            line_break()
+            continue
+
+        # we break if the previous span ends with a linebreak
+        last_text = line["spans"][-1]["text"]
+        if any(last_text.endswith(suffix) for suffix in ["\n", "\x02"]):
+            line_break()
+            continue
+
+        # rotations are radians from FPDFText_GetCharAngle; compare circularly.
+        # Only break on roughly perpendicular text: pdfium reports a 180-degree
+        # flip for ordinary text rendered with negative-scale matrices, which
+        # still belongs to the same visual line
+        if span["rotation"] != line["rotation"]:
+            rotation_diff = abs(span["rotation"] - line["rotation"]) % (2 * math.pi)
+            rotation_diff = min(rotation_diff, 2 * math.pi - rotation_diff)
+            if math.radians(45) <= rotation_diff <= math.radians(135):
+                line_break()
+                continue
+
+        # sometimes pdfium doesn't inject a linebreak, so we check the span positions
+        if span["bbox"].y_start > line["bbox"].y_end:
+            line_break()
+            continue
+
+        line["spans"].append(span)
+        line["bbox"].merge_inplace(span["bbox"])
+
+    return lines