docvortex 0.2.1__py3-none-any.whl

docvortex/analyzers/native/pdf/index_blocks.py

@@ -0,0 +1,268 @@
+"""识别并构造 Flash 原生 PDF 的目录正文块。"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+import math
+import re
+import statistics
+
+
+from ....schema import BBox, BlockType
+
+from .models import _LineItem
+from .geometry import _bbox_center_x, _bbox_center_y, _bbox_overlap_in_smaller, _bbox_union_many, _rotate_bbox_to_upright
+from .line_layout import _line_effective_height, _lines_tight_output_bbox
+
+
+_INDEX_PAGE_NUMBER_RE = re.compile(
+    r"(?<![A-Za-z0-9０-９])(?:[0-9０-９]+|[ivxlcdmIVXLCDM]+)"
+    r"[\s.。．、,，;；:：)）\]】]*$"
+)
+_INDEX_MIN_ROWS = 5
+_INDEX_MIN_PAGE_NUMBER_RATIO = 0.7
+
+
+@dataclass(slots=True)
+class _IndexRow:
+    """保存目录候选视觉行的成员、局部几何和合并文本。"""
+
+    members: list[_LineItem]
+    local_member_bboxes: list[BBox]
+    local_bbox: BBox
+    content: str
+    ends_in_page_number: bool
+
+
+def _extract_index_blocks(
+    lines: list[_LineItem],
+    page_size: tuple[float, float],
+    container_bboxes: list[BBox],
+    *,
+    require_heading: bool = False,
+) -> tuple[list[dict[str, object]], list[_LineItem]]:
+    """用页码行尾和稳定版式识别目录；预判阶段可强制要求几何目录标题。"""
+
+    claimed_line_ids: set[int] = set()
+    blocks: list[dict[str, object]] = []
+    for angle in sorted({line.angle for line in lines if line.semantic_type is None}):
+        rows = _build_index_rows(lines, page_size, angle)
+        if len(rows) < _INDEX_MIN_ROWS:
+            continue
+        local_page_width = page_size[1] if angle in {90, 270} else page_size[0]
+        local_containers = [_rotate_bbox_to_upright(bbox, page_size, angle) for bbox in container_bboxes]
+        eligible_rows = [
+            row
+            for row in rows
+            if not any(_bbox_overlap_in_smaller(row.local_bbox, container_bbox) >= 0.35 for container_bbox in local_containers)
+        ]
+        for band in _split_index_bands(eligible_rows):
+            candidate = _trim_index_band_edges(band)
+            if not _index_band_has_stable_layout(candidate, local_page_width):
+                continue
+            heading_row = _find_index_heading_row(
+                eligible_rows,
+                candidate,
+                local_page_width,
+            )
+            if require_heading and heading_row is None:
+                continue
+            if heading_row is not None:
+                for line in heading_row.members:
+                    line.semantic_type = "paragraph_title"
+            for row in candidate:
+                for line in row.members:
+                    line.semantic_type = BlockType.INDEX
+                    claimed_line_ids.add(id(line))
+            candidate_lines = [line for row in candidate for line in row.members]
+            block: dict[str, object] = {
+                "type": BlockType.INDEX,
+                "bbox": _bbox_union_many([line.bbox for line in candidate_lines]),
+                "angle": angle,
+                "content": "\n".join(row.content for row in candidate),
+            }
+            tight_output_bbox = _lines_tight_output_bbox(
+                candidate_lines,
+                page_size,
+            )
+            if tight_output_bbox is not None:
+                block["_tight_output_bbox"] = tight_output_bbox
+            blocks.append(block)
+
+    remaining_lines = [line for line in lines if id(line) not in claimed_line_ids]
+    return blocks, remaining_lines
+
+
+def _build_index_rows(
+    lines: list[_LineItem],
+    page_size: tuple[float, float],
+    angle: int,
+) -> list[_IndexRow]:
+    """按 visual_row_id 复原当前方向的完整视觉行，保留左右分裂成员。"""
+
+    row_groups: dict[tuple[str, int], list[_LineItem]] = {}
+    for line in lines:
+        if line.angle != angle or line.semantic_type is not None:
+            continue
+        key = ("visual", line.visual_row_id) if line.visual_row_id is not None else ("source", line.source_index)
+        row_groups.setdefault(key, []).append(line)
+
+    rows: list[_IndexRow] = []
+    for members in row_groups.values():
+        ordered = sorted(
+            members,
+            key=lambda line: (
+                _rotate_bbox_to_upright(line.bbox, page_size, angle)[0],
+                line.run_index,
+                line.source_index,
+            ),
+        )
+        local_member_bboxes = [_rotate_bbox_to_upright(line.bbox, page_size, angle) for line in ordered]
+        local_bbox = _bbox_union_many(local_member_bboxes)
+        content = " ".join(part for line in ordered if (part := line.text.strip()))
+        if not content:
+            continue
+        rows.append(
+            _IndexRow(
+                members=ordered,
+                local_member_bboxes=local_member_bboxes,
+                local_bbox=local_bbox,
+                content=content,
+                ends_in_page_number=_index_row_ends_in_page_number(content),
+            )
+        )
+    rows.sort(
+        key=lambda row: (
+            row.local_bbox[1],
+            row.local_bbox[0],
+            min(line.source_index for line in row.members),
+        )
+    )
+    return rows
+
+
+def _index_row_ends_in_page_number(content: str) -> bool:
+    """识别行尾的半角、全角阿拉伯页码或罗马页码。"""
+
+    return _INDEX_PAGE_NUMBER_RE.search(content.rstrip()) is not None
+
+
+def _split_index_bands(rows: list[_IndexRow]) -> list[list[_IndexRow]]:
+    """按显著纵向断层拆分候选带，同时容纳目录章节之间的加大行距。"""
+
+    if not rows:
+        return []
+    median_height = statistics.median(max(_line_effective_height(line, row.local_bbox) for line in row.members) for row in rows)
+    maximum_pitch = 3.25 * max(0.1, median_height)
+    bands: list[list[_IndexRow]] = [[rows[0]]]
+    for row in rows[1:]:
+        previous = bands[-1][-1]
+        if _bbox_center_y(row.local_bbox) - _bbox_center_y(previous.local_bbox) > maximum_pitch:
+            bands.append([row])
+        else:
+            bands[-1].append(row)
+    return bands
+
+
+def _trim_index_band_edges(rows: list[_IndexRow]) -> list[_IndexRow]:
+    """移除候选带两端不带页码的标题或邻接正文，内部少量续行继续保留。"""
+
+    start = 0
+    end = len(rows)
+    while start < end and not rows[start].ends_in_page_number:
+        start += 1
+    while end > start and not rows[end - 1].ends_in_page_number:
+        end -= 1
+    return rows[start:end]
+
+
+def _index_band_has_stable_layout(
+    rows: list[_IndexRow],
+    local_page_width: float,
+) -> bool:
+    """联合页码比例、右边界、行宽、缩进和行距确认目录候选。"""
+
+    if len(rows) < _INDEX_MIN_ROWS or local_page_width <= 0:
+        return False
+    page_number_rows = [row for row in rows if row.ends_in_page_number]
+    required_page_number_rows = max(
+        4,
+        math.ceil(_INDEX_MIN_PAGE_NUMBER_RATIO * len(rows)),
+    )
+    if len(page_number_rows) < required_page_number_rows:
+        return False
+
+    row_heights = [max(0.1, row.local_bbox[3] - row.local_bbox[1]) for row in rows]
+    median_height = statistics.median(row_heights)
+    median_right = statistics.median(row.local_bbox[2] for row in page_number_rows)
+    right_tolerance = max(1.5 * median_height, 0.02 * local_page_width)
+    aligned_right_ratio = sum(abs(row.local_bbox[2] - median_right) <= right_tolerance for row in page_number_rows) / len(
+        page_number_rows
+    )
+    if median_right < 0.75 * local_page_width or aligned_right_ratio < 0.7:
+        return False
+
+    wide_row_ratio = sum(row.local_bbox[2] - row.local_bbox[0] >= 0.55 * local_page_width for row in rows) / len(rows)
+    right_sidecar_count = sum(_index_row_has_right_sidecar(row, local_page_width, median_height) for row in rows)
+    if wide_row_ratio < 0.7 and right_sidecar_count < 2:
+        return False
+
+    left_body_ratio = sum(row.local_bbox[0] <= 0.3 * local_page_width for row in rows) / len(rows)
+    if left_body_ratio < 0.7:
+        return False
+
+    pitches = [
+        _bbox_center_y(current.local_bbox) - _bbox_center_y(previous.local_bbox) for previous, current in zip(rows, rows[1:])
+    ]
+    if not pitches or min(pitches) <= 0:
+        return False
+    median_pitch = statistics.median(pitches)
+    regular_pitch_ratio = sum(0.55 * median_pitch <= pitch <= 1.75 * median_pitch for pitch in pitches) / len(pitches)
+    return regular_pitch_ratio >= 0.7
+
+
+def _find_index_heading_row(
+    all_rows: list[_IndexRow],
+    candidate: list[_IndexRow],
+    local_page_width: float,
+) -> _IndexRow | None:
+    """用候选带上方的居中、短行和垂直邻接关系保留目录标题。"""
+
+    if not candidate:
+        return None
+    first_position = next(
+        (index for index, row in enumerate(all_rows) if row is candidate[0]),
+        None,
+    )
+    if first_position is None or first_position == 0:
+        return None
+    heading = all_rows[first_position - 1]
+    if heading.ends_in_page_number:
+        return None
+    candidate_heights = [max(0.1, row.local_bbox[3] - row.local_bbox[1]) for row in candidate]
+    median_height = statistics.median(candidate_heights)
+    heading_width = heading.local_bbox[2] - heading.local_bbox[0]
+    centered = abs(_bbox_center_x(heading.local_bbox) - 0.5 * local_page_width) <= 0.15 * local_page_width
+    vertical_pitch = _bbox_center_y(candidate[0].local_bbox) - _bbox_center_y(heading.local_bbox)
+    if (
+        heading_width > 0.4 * local_page_width
+        or not centered
+        or not 1.25 * median_height <= vertical_pitch <= 5.0 * median_height
+    ):
+        return None
+    return heading
+
+
+def _index_row_has_right_sidecar(
+    row: _IndexRow,
+    local_page_width: float,
+    median_height: float,
+) -> bool:
+    """检查视觉行末尾是否存在靠近页面右侧的窄页码片段。"""
+
+    if len(row.members) < 2:
+        return False
+    last_bbox = row.local_member_bboxes[-1]
+    return last_bbox[0] >= 0.75 * local_page_width and last_bbox[2] - last_bbox[0] <= max(
+        0.12 * local_page_width, 4.0 * median_height
+    )

docvortex/analyzers/native/pdf/inline/__init__.py

@@ -0,0 +1,3 @@
+"""按职责拆分的 Flash PDF 内部实现。"""
+
+__all__: list[str] = []

docvortex/analyzers/native/pdf/inline/common.py

@@ -0,0 +1,112 @@
+"""提供行内证据共享的字符和几何规范化原语。"""
+
+from __future__ import annotations
+
+import math
+from typing import Any, Iterable, cast
+
+from .....schema import BBox
+from .types import (
+    _LIGATURE_REPLACEMENTS,
+    _PDF_CONTROL_CHAR_RE,
+    _PDF_SEPARATOR_SPACE_CHARS,
+    _PDF_ZERO_WIDTH_CHARS,
+    PDF_TEXT_STYLE_ORDER,
+    PDFTextStyle,
+    PDFTextStyleLine,
+)
+
+
+def _style_line_reading_order_key(
+    line: PDFTextStyleLine,
+) -> tuple[int, float, float]:
+    """优先使用原生 source_index 排序，重复索引时再以 bbox 保持稳定。"""
+
+    return line.source_index, line.bbox[1], line.bbox[0]
+
+
+def _coerce_bbox(value: Any) -> BBox | None:
+    """把 list、tuple 或 pdftext bbox 对象收敛为合法有限 bbox。"""
+
+    raw_bbox = getattr(value, "bbox", value)
+    try:
+        if raw_bbox is None or len(raw_bbox) != 4:
+            return None
+        bbox = tuple(float(item) for item in raw_bbox)
+    except (TypeError, ValueError):
+        return None
+    if not all(math.isfinite(item) for item in bbox):
+        return None
+    if bbox[2] <= bbox[0] or bbox[3] <= bbox[1]:
+        return None
+    return bbox  # type: ignore[return-value]
+
+
+def _ordered_line_chars(line: Any) -> list[dict[str, Any]]:
+    """按 char_idx 修复异常乱序字符，同时保留缺少索引时的来源顺序。"""
+
+    chars = [char for char in getattr(line, "chars", []) if isinstance(char, dict)]
+    indexed_chars = [char.get("char_idx") for char in chars]
+    if (
+        chars
+        and all(isinstance(index, int) for index in indexed_chars)
+        and any(first > second for first, second in zip(indexed_chars, indexed_chars[1:]))
+    ):
+        return sorted(chars, key=lambda char: int(char["char_idx"]))
+    return chars
+
+
+def _normalize_match_fragment(value: Any) -> str:
+    """把单个字符片段规范为忽略排版空白的确定性匹配文本。"""
+
+    output: list[str] = []
+    for char in str(value or ""):
+        if char in _PDF_ZERO_WIDTH_CHARS or char == "\u00ad":
+            continue
+        if char == "\x02":
+            output.append("-")
+            continue
+        if char.isspace() or char in _PDF_SEPARATOR_SPACE_CHARS:
+            continue
+        if _PDF_CONTROL_CHAR_RE.fullmatch(char):
+            continue
+        output.append(_LIGATURE_REPLACEMENTS.get(char, char))
+    return "".join(output)
+
+
+def _canonical_styles(styles: Iterable[str]) -> tuple[PDFTextStyle, ...]:
+    """按公开富文本协议顺序过滤、去重并规范样式集合。"""
+
+    style_set = set(styles)
+    return cast(
+        tuple[PDFTextStyle, ...],
+        tuple(style for style in PDF_TEXT_STYLE_ORDER if style in style_set),
+    )
+
+
+def _bbox_intersection_area(first: BBox, second: BBox) -> float:
+    """返回两个合法 bbox 的相交面积。"""
+
+    width = max(0.0, min(first[2], second[2]) - max(first[0], second[0]))
+    height = max(0.0, min(first[3], second[3]) - max(first[1], second[1]))
+    return width * height
+
+
+def _bbox_overlap_ratio(first: BBox, second: BBox) -> float:
+    """返回 first 面积中落入 second 的比例。"""
+
+    intersection_width = max(0.0, min(first[2], second[2]) - max(first[0], second[0]))
+    intersection_height = max(0.0, min(first[3], second[3]) - max(first[1], second[1]))
+    first_area = max(0.01, (first[2] - first[0]) * (first[3] - first[1]))
+    return intersection_width * intersection_height / first_area
+
+
+__all__ = [
+    "_style_line_reading_order_key",
+    "_coerce_bbox",
+    "_ordered_line_chars",
+    "_normalize_match_fragment",
+    "_canonical_styles",
+    "_bbox_intersection_area",
+    "_bbox_overlap_ratio",
+]