knowhive-pdf 0.1.0__py3-none-any.whl

knowhive_pdf/__init__.py

@@ -0,0 +1,13 @@
+"""KnowHive PDF plugin: PDF → DocumentIR JSON.
+
+The host app (KnowHive's TS sidecar) spawns `knowhive-pdf --stdio`, feeds file
+paths on stdin, and reads one JSON object per line on stdout. The IR schema is
+the TS side's DocumentIR (server/src/documentIr.ts); SCHEMA_VERSION gates
+compatibility across releases.
+"""
+
+__version__ = "0.1.0"
+
+# Bump when the emitted IR shape changes incompatibly. The host refuses to talk
+# to a plugin whose schema_version it doesn't know.
+SCHEMA_VERSION = 1

knowhive_pdf/cli.py

@@ -0,0 +1,81 @@
+"""CLI entry: stdio batch protocol (host-facing) + one-shot mode (debugging).
+
+Protocol (`knowhive-pdf --stdio`), one JSON object per line on stdout:
+
+    → {"type":"ready","schema_version":1,"plugin_version":"0.1.0","docling_version":"..."}
+    ← /path/to/file.pdf\n                (one path per stdin line)
+    → {"type":"result","path":"...","ir":{"format":"pdf","blocks":[...]}}
+    → {"type":"error","path":"...","code":"needs_ocr|bad_text_layer|parse_failed","message":"..."}
+
+Triage runs before docling, so scanned/broken files come back in milliseconds
+and never trigger the (seconds-long, model-loading) conversion pipeline.
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from pathlib import Path
+
+from . import SCHEMA_VERSION, __version__
+from .mapping import convert_pdf
+from .triage import triage_pdf
+
+
+def _emit(obj: dict) -> None:
+    sys.stdout.write(json.dumps(obj, ensure_ascii=False) + "\n")
+    sys.stdout.flush()
+
+
+def process_file(path: str) -> dict:
+    if not Path(path).is_file():
+        return {"type": "error", "path": path, "code": "parse_failed", "message": "file not found"}
+    try:
+        t = triage_pdf(path)
+        if t.code != "ok":
+            return {"type": "error", "path": path, "code": t.code, "message": t.message}
+        blocks = convert_pdf(path)
+        return {"type": "result", "path": path, "ir": {"format": "pdf", "blocks": blocks}}
+    except Exception as e:  # noqa: BLE001 — protocol boundary: everything becomes an error line
+        return {"type": "error", "path": path, "code": "parse_failed", "message": str(e)}
+
+
+def run_stdio() -> None:
+    try:
+        from docling import __version__ as docling_version
+    except Exception:  # pragma: no cover
+        docling_version = "unknown"
+    _emit(
+        {
+            "type": "ready",
+            "schema_version": SCHEMA_VERSION,
+            "plugin_version": __version__,
+            "docling_version": docling_version,
+        }
+    )
+    for line in sys.stdin:
+        path = line.strip()
+        if not path:
+            continue
+        _emit(process_file(path))
+
+
+def main() -> None:
+    ap = argparse.ArgumentParser(description="KnowHive PDF plugin: PDF → DocumentIR JSON")
+    ap.add_argument("pdf", nargs="?", help="one-shot mode: parse a single PDF and print its IR")
+    ap.add_argument("--stdio", action="store_true", help="batch mode: paths on stdin, JSONL on stdout")
+    args = ap.parse_args()
+
+    if args.stdio:
+        run_stdio()
+        return
+    if not args.pdf:
+        ap.error("provide a PDF path or --stdio")
+    result = process_file(args.pdf)
+    _emit(result)
+    if result["type"] == "error":
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

knowhive_pdf/mapping.py

@@ -0,0 +1,114 @@
+"""docling result → DocumentIR blocks (the TS side's documentIr.ts shape).
+
+Three quirks this mapping exists to absorb (found in the 2026-08-01 spike):
+
+1. Docling reports section headers at a flat tree depth, so the real hierarchy
+   lives in the numbering — "3.1 Pre-training" is level 3, "3 BERT" level 2.
+2. TableItem.export_to_markdown prepends the caption and a blank line. The TS
+   chunker's splitTable() requires line 1 = header row / line 2 = separator to
+   repeat headers when splitting oversized tables, so the caption becomes its
+   own paragraph block and the table block starts at the first pipe row.
+3. Subsetted CJK fonts map some ideographs to Kangxi-radical codepoints
+   (⼀ U+2F00 instead of 一 U+4E00) — visually identical, retrieval-fatal.
+   NFKC normalization folds them back (and full-width ASCII, ligatures too).
+"""
+from __future__ import annotations
+
+import unicodedata
+from typing import Any
+
+_converter = None
+
+
+def get_converter():
+    """Singleton DocumentConverter — model load costs seconds, pay it once."""
+    global _converter
+    if _converter is None:
+        from docling.datamodel.base_models import InputFormat
+        from docling.datamodel.pipeline_options import PdfPipelineOptions
+        from docling.document_converter import DocumentConverter, PdfFormatOption
+
+        opts = PdfPipelineOptions()
+        # v1 scope: text-layer PDFs only. Scanned files never reach docling —
+        # triage_pdf() returns needs_ocr first.
+        opts.do_ocr = False
+        opts.do_table_structure = True
+        _converter = DocumentConverter(
+            format_options={InputFormat.PDF: PdfFormatOption(pipeline_options=opts)}
+        )
+    return _converter
+
+
+def normalize_text(text: str) -> str:
+    return unicodedata.normalize("NFKC", text).strip()
+
+
+def heading_level_from_numbering(text: str) -> int:
+    """"3 BERT"→2, "3.1 Pre-training"→3; unnumbered→2 (doc title holds 1)."""
+    lead = text.split(" ", 1)[0].rstrip(".")
+    parts = lead.split(".")
+    if parts and parts[0] and all(p.isdigit() for p in parts):
+        return min(6, 1 + len(parts))
+    return 2
+
+
+def split_table_markdown(md: str) -> tuple[str, str]:
+    """Split docling's table markdown into (caption, pipe-table)."""
+    lines = md.split("\n")
+    first_pipe = next((i for i, l in enumerate(lines) if l.startswith("|")), None)
+    if first_pipe is None:
+        return md.strip(), ""
+    caption = "\n".join(lines[:first_pipe]).strip()
+    table_md = "\n".join(lines[first_pipe:]).strip()
+    return caption, table_md
+
+
+def convert_pdf(pdf_path: str) -> list[dict[str, Any]]:
+    from docling_core.types.doc import DocItemLabel, TableItem, TextItem
+
+    doc = get_converter().convert(pdf_path).document
+
+    label_map = {
+        DocItemLabel.TITLE: "heading",
+        DocItemLabel.SECTION_HEADER: "heading",
+        DocItemLabel.TEXT: "paragraph",
+        DocItemLabel.PARAGRAPH: "paragraph",
+        DocItemLabel.LIST_ITEM: "list",
+        DocItemLabel.CODE: "code",
+        DocItemLabel.FORMULA: "paragraph",
+        DocItemLabel.CAPTION: "paragraph",
+    }
+
+    blocks: list[dict[str, Any]] = []
+
+    def push(block: dict[str, Any], item: Any) -> None:
+        prov = getattr(item, "prov", None)
+        if prov:
+            block["page"] = prov[0].page_no
+            bb = prov[0].bbox
+            block["bbox"] = [round(bb.l, 1), round(bb.t, 1), round(bb.r, 1), round(bb.b, 1)]
+        block["order"] = len(blocks)
+        blocks.append(block)
+
+    for item, _level in doc.iterate_items():
+        if isinstance(item, TableItem):
+            caption, table_md = split_table_markdown(item.export_to_markdown(doc=doc))
+            if caption:
+                push({"type": "paragraph", "text": normalize_text(caption)}, item)
+            if table_md:
+                push({"type": "table", "text": normalize_text(table_md)}, item)
+        elif isinstance(item, TextItem):
+            text = normalize_text(item.text or "")
+            if not text:
+                continue
+            btype = label_map.get(item.label)
+            if btype is None:
+                continue  # page furniture (headers/footers/footnotes): skip
+            block: dict[str, Any] = {"type": btype, "text": text}
+            if btype == "heading":
+                block["level"] = (
+                    1 if item.label == DocItemLabel.TITLE else heading_level_from_numbering(text)
+                )
+            push(block, item)
+
+    return blocks

knowhive_pdf/triage.py

@@ -0,0 +1,77 @@
+"""Cheap pre-flight triage: decide whether a PDF is worth running docling on.
+
+Runs pypdfium2 text extraction only (milliseconds, no ML models). Catches the
+two failure shapes that would otherwise fail SILENTLY — a scanned PDF parsed
+with OCR off yields empty text, and a broken text layer (CID/cmap damage)
+yields garbage that would be embedded as if it were real content.
+
+Thresholds calibrated against real samples (2026-08-01):
+  - scanned page (image-only):        0 chars/page
+  - pdf.js issue9534 (broken layer):  15 chars on a visibly full page
+  - normal papers:                    thousands of chars/page, garbage ≈ 0.5%
+"""
+from __future__ import annotations
+
+from dataclasses import dataclass
+from statistics import median
+
+# A page with fewer extractable chars than this is not a text page.
+MIN_CHARS_PER_PAGE = 50
+# Share of private-use/replacement chars above which the layer is garbage.
+MAX_GARBAGE_RATIO = 0.10
+
+
+@dataclass
+class TriageResult:
+    code: str  # "ok" | "needs_ocr" | "bad_text_layer"
+    message: str
+    chars_per_page: list[int]
+
+
+def _is_garbage_char(c: str) -> bool:
+    cp = ord(c)
+    return 0xE000 <= cp <= 0xF8FF or c == "�"
+
+
+def _page_has_image(page) -> bool:
+    # FPDF_PAGEOBJ_IMAGE == 3 in pdfium's page-object enum.
+    return any(obj.type == 3 for obj in page.get_objects())
+
+
+def triage_pdf(pdf_path: str) -> TriageResult:
+    import pypdfium2 as pdfium
+
+    doc = pdfium.PdfDocument(pdf_path)
+    try:
+        texts = [doc[i].get_textpage().get_text_bounded() for i in range(len(doc))]
+        chars_per_page = [len(t.strip()) for t in texts]
+        all_text = "".join(texts)
+
+        if all_text:
+            garbage = sum(1 for c in all_text if _is_garbage_char(c)) / len(all_text)
+            if garbage > MAX_GARBAGE_RATIO:
+                return TriageResult(
+                    "bad_text_layer",
+                    f"{garbage:.0%} of extracted characters are unmapped glyphs",
+                    chars_per_page,
+                )
+
+        if median(chars_per_page) < MIN_CHARS_PER_PAGE:
+            # Distinguish "pages are pictures" from "pages should have text but
+            # the layer is broken": image-dominated pages mean a scan.
+            scannish = any(_page_has_image(doc[i]) for i in range(min(3, len(doc))))
+            if scannish:
+                return TriageResult(
+                    "needs_ocr",
+                    "pages are images with no usable text layer (scanned document)",
+                    chars_per_page,
+                )
+            return TriageResult(
+                "bad_text_layer",
+                "pages render content but the text layer is empty or broken",
+                chars_per_page,
+            )
+
+        return TriageResult("ok", "", chars_per_page)
+    finally:
+        doc.close()

knowhive_pdf-0.1.0.dist-info/METADATA

@@ -0,0 +1,54 @@
+Metadata-Version: 2.4
+Name: knowhive-pdf
+Version: 0.1.0
+Summary: KnowHive PDF plugin: docling-powered PDF → DocumentIR JSON over a stdio protocol
+License-Expression: MIT
+Requires-Python: >=3.10
+Requires-Dist: docling<3,>=2.0
+Requires-Dist: pypdfium2>=4
+Description-Content-Type: text/markdown
+
+# knowhive-pdf
+
+KnowHive 的 PDF 解析插件：用 [docling](https://github.com/docling-project/docling)
+把 PDF 解析成 KnowHive 的 DocumentIR JSON（`server/src/documentIr.ts` 的形状），
+通过 stdio 协议供主程序调用。主程序保持 single runtime——本插件经
+`uv tool install knowhive-pdf` 按需安装，不装则主程序完全无感。
+
+> 开发期暂驻主仓库 `pdf-plugin/`；发布时整体迁出为独立 repo + PyPI +
+> GitHub Actions Trusted Publishing。
+
+## 协议
+
+```
+$ knowhive-pdf --stdio
+→ {"type":"ready","schema_version":1,"plugin_version":"0.1.0","docling_version":"..."}
+← /path/to/file.pdf
+→ {"type":"result","path":"...","ir":{"format":"pdf","blocks":[...]}}
+→ {"type":"error","path":"...","code":"needs_ocr|bad_text_layer|parse_failed","message":"..."}
+```
+
+调试用一次性模式：`knowhive-pdf file.pdf`。
+
+## v1 范围与设计决定
+
+- **不含 OCR**（`do_ocr=False`）。扫描件由 triage 拦截返回 `needs_ocr`；中文 OCR 留 v2。
+- **triage 先行**（pypdfium2，毫秒级）：每页字符中位数 <50 且页面含图 → `needs_ocr`；
+  无图或乱码字符占比 >10% → `bad_text_layer`。坏文字层是最阴的静默失败——页面渲染
+  正常但抽出来是空/乱码（校准样本：pdf.js `issue9534_reduced.pdf`）。
+- **NFKC 归一化**：子集化中文字体会把部分汉字映射到康熙部首码位（⼀ U+2F00 ≠ 一
+  U+4E00），肉眼相同、检索必死。所有输出文本过 NFKC。
+- **标题层级从编号推导**（"3.1"→L3）：docling 的 section_header 是平的。
+- **表格 caption 拆分**：docling 的表格 markdown 前缀 caption；拆成独立段落块，
+  表格块从表头行开始——主程序 `splitTable()` 的表头重复逻辑依赖"首行=表头、
+  次行=分隔行"。
+
+## 开发
+
+```bash
+uv sync
+uv run pytest                      # 快速套件（不加载 docling 模型）
+KNOWHIVE_PDF_SLOW=1 uv run pytest  # 含完整 docling 解析（首跑下载版面模型）
+```
+
+`tests/fixtures/bad-textlayer.pdf` 来自 mozilla/pdf.js 测试语料（issue9534_reduced）。

knowhive_pdf-0.1.0.dist-info/RECORD

@@ -0,0 +1,8 @@
+knowhive_pdf/__init__.py,sha256=5nQP3NiICS3T8OaygLhQPQfPpBd1I7dCk6ERi9Gyx7I,492
+knowhive_pdf/cli.py,sha256=c9VHpub6DFnj7TkDMuI9D2cnFE6JnCterxK7OWCueWA,2738
+knowhive_pdf/mapping.py,sha256=yLAAmjbykhaE0kcP_lGLiaa3As775q7xi7I0iMVfQNY,4469
+knowhive_pdf/triage.py,sha256=IeYqgcbC7DxNtV9Wlargbw7MD7_mpPH3_tFjy4gDjL8,2801
+knowhive_pdf-0.1.0.dist-info/METADATA,sha256=Gsvxn28eCbsyf7jdLZhVkUApfxMCM7HBTbERpGWNhCQ,2422
+knowhive_pdf-0.1.0.dist-info/WHEEL,sha256=lCkmxWfQsSc9CfIClYeavTdQeEX2toPqufh9gI35EQA,87
+knowhive_pdf-0.1.0.dist-info/entry_points.txt,sha256=P92yRiSMVFVU5nr2E8ZRqt7Xs8Usrn5O2_VIr0fr0mM,55
+knowhive_pdf-0.1.0.dist-info/RECORD,,

knowhive_pdf-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.31.0
+Root-Is-Purelib: true
+Tag: py3-none-any

knowhive_pdf-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+knowhive-pdf = knowhive_pdf.cli:main