docvortex 0.4.19__tar.gz → 0.4.20__tar.gz

{docvortex-0.4.19/src/docvortex.egg-info → docvortex-0.4.20}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: docvortex
-Version: 0.4.19
+Version: 0.4.20
 Summary: A fast, multi-format document parsing and conversion engine
 License-Expression: MIT
 Project-URL: Repository, https://github.com/myhloli/DocVortex

{docvortex-0.4.19 → docvortex-0.4.20}/src/docvortex/analyzers/native/pdf/_script_geometry.py

@@ -30,6 +30,7 @@ SCRIPT_COMPONENT_SEED_MAX_POSITION_DISTANCE = 4
 SCRIPT_COMPONENT_MIN_OFFSET_RATIO = 0.08
 SCRIPT_COMPONENT_MAX_HEIGHT_RATIO = 1.1
 CONTROL_LINE_BREAK_CHARS = {"\r", "\n"}
+_NUMERIC_SCRIPT_SEPARATORS = frozenset({",", "，", "-", "–", "—"})
 
 ScriptRole = Literal["body", "sup", "sub"]
 ScriptMarkRole = Literal["sup", "sub"]
@@ -675,6 +676,91 @@ def paired_script_roles(
     return paired
 
 
+def _numeric_superscript_indices(
+    features: list[ScriptCharFeature],
+    roles: list[ScriptRole],
+    protected_body_indices: set[int],
+) -> set[int]:
+    """用左侧同一视觉行的稳定正文确认完整数字上标，允许中间有排版空白。"""
+    accepted: set[int] = set()
+    start = 0
+    while start < len(features):
+        if not features[start].text.isdecimal():
+            start += 1
+            continue
+        end = start + 1
+        while end < len(features) and (features[end].text.isdecimal() or features[end].text in _NUMERIC_SCRIPT_SEPARATORS):
+            end += 1
+        # 尾随标点属于正文，只有数字之间且共享上标基线的分隔符才随引用提升。
+        while end > start and not features[end - 1].text.isdecimal():
+            end -= 1
+        candidate = features[start:end]
+        if any(0 <= neighbor < len(features) and features[neighbor].text in {"/", "⁄"} for neighbor in (start - 1, end)):
+            # 不能把分式的分子或分母独立当成引用，否则会制造仅半个指数带上标的结果。
+            start = end
+            continue
+        previous = start - 1
+        while previous >= 0 and features[previous].text.isspace():
+            if features[previous].text in CONTROL_LINE_BREAK_CHARS:
+                break
+            previous -= 1
+        reference_end = previous + 1
+        while previous >= 0 and features[previous].is_valid and roles[previous] == "body":
+            if previous in protected_body_indices:
+                break
+            previous -= 1
+        reference = list(range(previous + 1, reference_end))
+        start = end
+        if not reference or any(
+            not item.is_valid or item.index in protected_body_indices or roles[item.index] == "sub" for item in candidate
+        ):
+            continue
+        clusters, _ = _cluster_baselines(features, reference)
+        body_result = _choose_body_band(features, clusters)
+        if body_result is None:
+            continue
+        band, body_cluster = body_result
+        if len(band.member_indices) < 2 or band.tight_height <= 0 or band.loose_height <= 0:
+            continue
+        edge = features[reference[-1]]
+        first = candidate[0]
+        assert first.tight_bbox is not None and edge.tight_bbox is not None
+        gap = first.tight_bbox[0] - edge.tight_bbox[2]
+        if not -0.15 * band.tight_height <= gap <= max(2.5, 1.2 * band.tight_height):
+            continue
+        digits = [item for item in candidate if item.text.isdecimal()]
+        baseline = statistics.median(item.origin[1] for item in digits if item.origin is not None)
+        shift = band.baseline - baseline
+        if (
+            not max(SCRIPT_ORIGIN_MIN_SHIFT_ABSOLUTE, SCRIPT_CONSENSUS_ORIGIN_SHIFT_RATIO * band.tight_height)
+            <= shift
+            <= (0.8 * band.tight_height)
+        ):
+            continue
+        tolerance = max(SCRIPT_BASELINE_ABSOLUTE_TOLERANCE, SCRIPT_BASELINE_LOOSE_HEIGHT_RATIO * band.loose_height)
+        if any(item.origin is None or abs(item.origin[1] - baseline) > tolerance for item in candidate):
+            continue
+        tight_ratio = max(item.tight_height for item in digits) / band.tight_height
+        if tight_ratio > SCRIPT_CONSENSUS_TIGHT_HEIGHT_RATIO:
+            # 纯小写词的 x-height 可接近缩小数字；强基线位移还须有 loose 缩小证据，不能仅放宽 tight 阈值。
+            loose_ratio = max(item.loose_height for item in digits) / band.loose_height
+            if not (
+                shift >= SCRIPT_STRONG_SHIFT_RATIO * band.tight_height
+                and tight_ratio <= SCRIPT_STRONG_MAX_HEIGHT_RATIO
+                and loose_ratio <= SCRIPT_CONSENSUS_TIGHT_HEIGHT_RATIO
+            ):
+                continue
+        digit_cluster = ScriptBaselineCluster(baseline, tuple(item.index for item in digits))
+        if (
+            _cluster_tight_center(features, digit_cluster) >= _cluster_tight_center(features, body_cluster) - 0.05
+            or _cluster_loose_center(features, digit_cluster) >= _cluster_loose_center(features, body_cluster) - 0.05
+        ):
+            continue
+        # 只补样式，不合并组件或移动字符；下一候选也不能把本次提升结果当作正文。
+        accepted.update(item.index for item in candidate)
+    return accepted
+
+
 def classify_char_script_roles(
     chars: list[Char],
     *,
@@ -689,6 +775,8 @@ def classify_char_script_roles(
     roles: list[ScriptRole] = ["body"] * len(features)
     for component_indices in split_script_visual_components(features):
         _assign_component(features, component_indices, protected, roles, font_keys)
+    for index in _numeric_superscript_indices(features, roles, protected):
+        roles[index] = "sup"
     return roles
 
 

{docvortex-0.4.19 → docvortex-0.4.20}/src/docvortex/analyzers/native/pdf/inline/scripts.py

@@ -7,7 +7,13 @@ import unicodedata
 from typing import Any, Literal, Sequence
 
 from .....schema import BBox
-from .._script_geometry import ScriptRole, classify_char_script_roles, paired_script_roles
+from .._script_geometry import (
+    ScriptRole,
+    _numeric_superscript_indices,
+    build_script_features,
+    classify_char_script_roles,
+    paired_script_roles,
+)
 from ..geometry import _rotate_bbox_to_upright
 from .common import _coerce_bbox, _normalize_match_fragment, _ordered_line_chars
 from .types import (
@@ -411,6 +417,11 @@ def _refine_math_script_tokens(
     """以最左稳定簇保护 base，并对弱单字符和复杂未分段 token 保守拒识。"""
     refined = list(roles)
     citation_indices = _citation_script_indices(chars, refined)
+    numeric_indices: set[int] = set()
+    if any(role == "sup" and _script_char_text(char).isdecimal() for char, role in zip(chars, roles)):
+        # 数字引用的基字符在 token 外；保护双 bbox/origin 已确认的整串，不能重置首位数字。
+        candidates = _numeric_superscript_indices(build_script_features(chars, tight_bboxes, origins, set()), roles, set())
+        numeric_indices = {index for index in candidates if roles[index] == "sup"}
     complex_unsegmented_token = False
     tokens = _iter_math_script_tokens(chars)
     token_alnum_positions = {
@@ -443,6 +454,9 @@ def _refine_math_script_tokens(
     for token in tokens:
         if any(index in citation_indices for index in token):
             continue
+        if all(index in numeric_indices for index in token):
+            # 只跳过纯数字引用的 base 重选，后续分式及复杂数学过滤仍按原规则执行。
+            continue
         token_key = tuple(token)
         alnum_positions = token_alnum_positions[token_key]
         if not alnum_positions or not any(refined[index] != "body" for index in token):

{docvortex-0.4.19 → docvortex-0.4.20}/src/docvortex/version.py

@@ -1,3 +1,3 @@
-__version__ = "0.4.19"
+__version__ = "0.4.20"
 
 __all__ = ["__version__"]

{docvortex-0.4.19 → docvortex-0.4.20/src/docvortex.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: docvortex
-Version: 0.4.19
+Version: 0.4.20
 Summary: A fast, multi-format document parsing and conversion engine
 License-Expression: MIT
 Project-URL: Repository, https://github.com/myhloli/DocVortex

{docvortex-0.4.19 → docvortex-0.4.20}/src/docvortex.egg-info/SOURCES.txt

@@ -458,6 +458,7 @@ tests/test_pdf_cpu_evidence.py
 tests/test_pdf_font_geometry.py
 tests/test_pdf_font_runtime.py
 tests/test_pdf_model_text_normalization.py
+tests/test_pdf_numeric_citations.py
 tests/test_pdf_output_text.py
 tests/test_pdf_render_pool.py
 tests/test_pdf_source_row_merging.py
@@ -498,6 +499,7 @@ tests/fixtures/native_pdf_table_demo_manifest.json
 tests/fixtures/native_pdf_table_script_truth.json
 tests/fixtures/pdf_char_dedup.json
 tests/fixtures/pdf_mixed_font_script_line.json
+tests/fixtures/pdf_numeric_citations_review.json
 tests/fixtures/rtf/semantic.rtf
 tests/unittest/_docx_equationxml_test_utils.py
 tests/unittest/_epub_test_utils.py
@@ -643,6 +645,7 @@ tests/unittest/test_text_utils.py
 tests/unittest/pdfs/flash_table_annotations_synthetic.pdf
 tests/unittest/pdfs/metabolic_pathway_page_3.pdf
 tests/unittest/pdfs/native_cjk_layout_synthetic.pdf
+tests/unittest/pdfs/numeric_citations_introduction.pdf
 tests/unittest/pdfs/test.pdf
 tests/unittest/pdfs/flash_layout/bloom_form_labels.pdf
 tests/unittest/pdfs/flash_layout/journal_dual_caption.pdf

{docvortex-0.4.19 → docvortex-0.4.20}/tests/fixtures/flash_reviewed_history.json

@@ -610,7 +610,7 @@
         },
         {
           "page_index": 2,
-          "fingerprint": "223f8dc3618805a8688095824d638d00348540403923ffddb74ba95a3a1961df",
+          "fingerprint": "0dfc82cf2795f0ed939b1c77e7c53c59f0788c28c5f1bef1ae4bd4e8fff8b8c7",
           "bbox_fingerprint": "c83743f89e12f6984bd9eb20f37303a668c8df982be11f86c9ae4851e035aafe"
         },
         {

docvortex-0.4.20/tests/fixtures/pdf_numeric_citations_review.json

@@ -0,0 +1,33 @@
+{
+  "accepted": [
+    {
+      "case": "klee-numeric-footnote",
+      "source": "demo/pdfs/mixed_elements_pages_07_10.pdf",
+      "source_sha256": "89076ee9cce2168ab894ee7e280872f8a3c09053b9dcdeda9c1da76d111adc1e",
+      "page": 3,
+      "anchor": "symbolic execution tool KLEE 1",
+      "before_code_sha256": "be87727e75b972fd5c409c6d6792fb8094443daa5e8a69f8044080d435f261f5",
+      "after_code_sha256": "22c18d95daf883ed457012463d5616d209e4c4c6e632815225cc116189f395f5",
+      "old_fingerprint": "223f8dc3618805a8688095824d638d00348540403923ffddb74ba95a3a1961df",
+      "new_fingerprint": "0dfc82cf2795f0ed939b1c77e7c53c59f0788c28c5f1bef1ae4bd4e8fff8b8c7",
+      "decision": "accept_improvement",
+      "reason": "原页右栏 KLEE 后的 1 明显抬高且缩小；补为上标，字符、顺序、块型和 bbox 均不变。",
+      "baseline_update": "flash_reviewed_history.json: mixed_elements_pages_07_10 page_index=2 fingerprint",
+      "evidence": [
+        "output/pdf/intro-superscripts-20260920/klee-source.png",
+        "output/pdf/intro-superscripts-20260920/klee-public.html",
+        "output/pdf/intro-superscripts-20260920/history-final/comparison.html"
+      ]
+    }
+  ],
+  "rejected": [
+    {
+      "source": "mixed_elements_pages_03_06.pdf",
+      "pages": [
+        2,
+        3
+      ],
+      "reason": "拒绝只提升分式指数分子的候选；保留原数学过滤及原基线。"
+    }
+  ]
+}

docvortex-0.4.20/tests/test_pdf_numeric_citations.py

@@ -0,0 +1,187 @@
+"""锁定宽行距论文中空白分隔及多位数字引用的完整上标。"""
+
+from copy import deepcopy
+import hashlib
+from pathlib import Path
+from types import SimpleNamespace
+
+import pytest
+
+from docvortex.analyzers.native.pdf._script_geometry import classify_char_script_roles
+from docvortex.analyzers.native.pdf.geometry import _rotate_bbox_from_upright
+from docvortex.analyzers.native.pdf.inline.scripts import _script_line_char_roles
+from docvortex.analyzers.pdf import prepare_text_evidence
+from docvortex.api import parse
+from docvortex.document.pdf import PDFDocument
+from docvortex.render import render_html, render_markdown
+
+SOURCE = Path(__file__).parent / "unittest/pdfs/numeric_citations_introduction.pdf"
+SOURCE_SHA256 = "424512a5c2c3a58c542e05f6692cb0efe3b4009b308b787db7f56806e77a7e18"
+CASES = [
+    (0, "worldwide.", ["1,2"]),
+    (1, "healthcaresystems.3", ["3"]),
+    (1, "2040.", ["4"]),
+    (1, "follow-up,", ["5,6"]),
+    (1, "productivity,", ["7,8"]),
+    (1, "fractures.", ["9,10"]),
+    (1, "conditions.", ["11"]),
+    (1, "billion.", ["12"]),
+    (2, "forAMD,", ["13,14", "15"]),
+    (2, "model,", ["16,17", "18,19"]),
+]
+EXPECTED = [value for _, _, values in CASES for value in values]
+
+
+@pytest.fixture(scope="module")
+def source_evidence():
+    """从冻结的三页原件提取证据，检查样本身份且复用只读结果。"""
+    assert hashlib.sha256(SOURCE.read_bytes()).hexdigest() == SOURCE_SHA256
+    with PDFDocument(str(SOURCE)) as document:
+        assert len(document) == 3
+        return [prepare_text_evidence(document[index]) for index in range(3)]
+
+
+@pytest.mark.parametrize("page,anchor,expected", CASES)
+def test_real_citation_evidence_is_complete(source_evidence, page, anchor, expected):
+    """逐个原文锚点验证所有引用及逗号，不以局部字符或总数代替正确性。"""
+    matches = [line for line in source_evidence[page].scripts if anchor in line.text]
+    assert len(matches) == 1
+    line = matches[0]
+    assert [(line.text[item.start : item.end], item.style) for item in line.script_ranges] == [
+        (value, "superscript") for value in expected
+    ]
+
+
+def test_real_citations_survive_public_export():
+    """公开 Model/Middle/HTML/Markdown 都保留十二组完整引用且不提升其他正文。"""
+    result = parse(SOURCE, keep_model_json=True)
+    for markup in (render_html(result.middle_json), render_markdown(result.middle_json)):
+        for value in EXPECTED:
+            assert markup.count(f"<sup>{value}</sup>") == 1
+        assert markup.count("<sup>") == len(EXPECTED)
+        assert "<sub>" not in markup
+    for document in (result.model_json.to_dict(), result.middle_json.to_dict()):
+        scripts = []
+
+        def visit(value):
+            """递归收集公共协议内的上下标，覆盖嵌套段落且不依赖块编号。"""
+            if isinstance(value, dict):
+                if set(value.get("styles", [])) & {"superscript", "subscript"}:
+                    scripts.append((value["content"], value["styles"]))
+                for child in value.values():
+                    visit(child)
+            elif isinstance(value, list):
+                for child in value:
+                    visit(child)
+
+        visit(document)
+        assert scripts == [(value, ["superscript"]) for value in EXPECTED]
+
+
+def _citation_fixture(reference="word.", citation="13,14", separator=" ", shift=-4.0):
+    """建立有明确正文参照的双 bbox 字符，引用中的标点保持同一基线。"""
+    text = reference + separator + citation + " next"
+    start, end = len(reference + separator), len(reference + separator + citation)
+    chars, tight, origins = [], {}, {}
+    for index, char in enumerate(text):
+        scripted = start <= index < end
+        x = 20.0 + index * 5.0
+        baseline = 60.0 + (shift if scripted else 0)
+        height = 6.0 if scripted else 9.0
+        box = (x, baseline - height - 3.0, x + 4.0, baseline + 2.0)
+        chars.append({"char": char, "char_idx": index, "bbox": box, "font": {"name": "arbitrary"}})
+        tight[index] = (x, baseline - height, x + 4.0, baseline)
+        origins[index] = (x, baseline)
+    return chars, tight, origins, start, end
+
+
+@pytest.mark.parametrize("citation", ["3", "11", "13,14", "18–20"])
+@pytest.mark.parametrize("separator", ["", " ", "\u00a0"])
+@pytest.mark.parametrize("angle", [0, 90, 180, 270])
+@pytest.mark.parametrize("scale,translation", [(0.75, 0), (1.0, 75), (1.6, 0)])
+def test_numeric_citations_generalize(citation, separator, angle, scale, translation):
+    """有无空白、引用长度、字号、平移和旋转均不改变正文及引用角色。"""
+    chars, tight, origins, start, end = _citation_fixture(citation=citation, separator=separator)
+    size = (800.0, 1000.0)
+
+    def transform(box):
+        """同步变换两类 bbox，保持样式判断不依赖绝对位置。"""
+        return _rotate_bbox_from_upright(tuple(v * scale + translation for v in box), size, angle)
+
+    for char in chars:
+        char["bbox"] = transform(char["bbox"])
+        char["font"]["name"] = "different-subset" if char["char_idx"] % 2 else "another-font"
+    tight = {index: transform(box) for index, box in tight.items()}
+    for index, (x, y) in origins.items():
+        x, y = x * scale + translation, y * scale + translation
+        origins[index] = {0: (x, y), 90: (size[0] - y, x), 180: (size[0] - x, size[1] - y), 270: (y, size[1] - x)}[angle]
+    before = deepcopy((chars, tight, origins))
+    _, roles, _, _ = _script_line_char_roles(
+        SimpleNamespace(chars=chars, angle=angle, inline_math_regions=[]), size, tight, origins, set()
+    )
+    assert roles == ["sup" if start <= index < end else "body" for index in range(len(chars))]
+    assert (chars, tight, origins) == before
+
+
+@pytest.mark.parametrize("mode", ["same-baseline", "weak", "newline", "column", "row", "missing", "no-anchor"])
+def test_numeric_citation_requires_local_geometry(mode):
+    """普通数字、弱偏移、跨行跨栏、缺失证据和孤立数字不可借用正文锚点。"""
+    chars, tight, origins, start, end = _citation_fixture(
+        reference="1." if mode == "no-anchor" else "word.",
+        citation="2021",
+        separator="\n" if mode == "newline" else " ",
+        shift=0 if mode == "same-baseline" else -0.5 if mode == "weak" else -25 if mode == "row" else -4,
+    )
+    for index in range(start, end):
+        if mode == "missing":
+            origins.pop(index)
+        elif mode == "column":
+            box = chars[index]["bbox"]
+            chars[index]["bbox"] = (box[0] + 100, box[1], box[2] + 100, box[3])
+            box = tight[index]
+            tight[index] = (box[0] + 100, box[1], box[2] + 100, box[3])
+            origins[index] = (origins[index][0] + 100, origins[index][1])
+    roles = classify_char_script_roles(chars, tight_bboxes=tight, origins=origins)
+    assert roles[start:end] == ["body"] * (end - start)
+
+
+def test_inline_math_retains_its_body_base():
+    """数字幂的基数留在正文，不将整个数学 token 强制保护为引用。"""
+    chars, tight, origins, start, end = _citation_fixture(reference="10", citation="12", separator="")
+    _, roles, _, _ = _script_line_char_roles(
+        SimpleNamespace(chars=chars, angle=0, inline_math_regions=[]), (800, 1000), tight, origins, set()
+    )
+    assert roles[:end] == ["body"] * start + ["sup"] * (end - start)
+
+
+@pytest.mark.parametrize("scaled_loose", [True, False])
+def test_lowercase_body_requires_complementary_loose_evidence(scaled_loose):
+    """正文 x-height 接近数字时，须同时具备强位移及较小 loose 框才能补判。"""
+    chars, tight, origins, start, end = _citation_fixture(reference="losses,", citation="18,19")
+    for index in range(start):
+        box = tight[index]
+        tight[index] = (box[0], box[3] - 6, box[2], box[3])
+    if not scaled_loose:
+        for index in range(start, end):
+            x0, _, x1, y1 = chars[index]["bbox"]
+            chars[index]["bbox"] = (x0, y1 - 14, x1, y1)
+    roles = classify_char_script_roles(chars, tight_bboxes=tight, origins=origins)
+    assert roles[start:end] == ["sup" if scaled_loose else "body"] * (end - start)
+
+
+@pytest.mark.parametrize("reference", ["SK", "σT", "word. "])
+def test_numeric_protection_does_not_bypass_fraction_filter(reference):
+    """分式指数仍由原数学过滤处理，不因保护数字而只留下分子上标。"""
+    chars, tight, origins, start, end = _citation_fixture(reference=reference, citation="1/2", separator="")
+    _, roles, _, _ = _script_line_char_roles(
+        SimpleNamespace(chars=chars, angle=0, inline_math_regions=[]), (800, 1000), tight, origins, set()
+    )
+    assert roles[start:end] == ["body"] * (end - start)
+
+
+def test_reviewed_klee_footnote_survives_public_export():
+    """锁定原页已确认的 KLEE 数字脚注改善，保持后续句号为正文。"""
+    source = Path(__file__).parents[1] / "demo/pdfs/mixed_elements_pages_07_10.pdf"
+    assert hashlib.sha256(source.read_bytes()).hexdigest() == "89076ee9cce2168ab894ee7e280872f8a3c09053b9dcdeda9c1da76d111adc1e"
+    result = parse(source)
+    assert "KLEE <sup>1</sup> . KLEE" in render_html(result.middle_json)