knowhive-pdf 0.1.0__tar.gz

knowhive_pdf-0.1.0/.github/workflows/ci.yml

@@ -0,0 +1,23 @@
+name: CI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+
+jobs:
+  test:
+    strategy:
+      matrix:
+        os: [ubuntu-latest, macos-latest]
+        python: ["3.10", "3.13"]
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: astral-sh/setup-uv@v5
+        with:
+          python-version: ${{ matrix.python }}
+      - run: uv sync
+      # Fast suite only: never downloads docling's layout models in CI.
+      # Full-parse coverage runs locally via KNOWHIVE_PDF_SLOW=1.
+      - run: uv run pytest -q

knowhive_pdf-0.1.0/.github/workflows/publish.yml

@@ -0,0 +1,35 @@
+# Publish to PyPI via Trusted Publishing (OIDC) — no API tokens stored anywhere.
+# Trigger: push a tag like v0.1.0. PyPI must have this repo + workflow registered
+# as a (pending) trusted publisher for the `knowhive-pdf` project.
+name: Publish
+
+on:
+  push:
+    tags: ["v*"]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: astral-sh/setup-uv@v5
+      - run: uv sync
+      - run: uv run pytest -q
+      - run: uv build
+      - uses: actions/upload-artifact@v4
+        with:
+          name: dist
+          path: dist/
+
+  publish:
+    needs: build
+    runs-on: ubuntu-latest
+    environment: pypi
+    permissions:
+      id-token: write # OIDC token for PyPI Trusted Publishing
+    steps:
+      - uses: actions/download-artifact@v4
+        with:
+          name: dist
+          path: dist/
+      - uses: pypa/gh-action-pypi-publish@release/v1

knowhive_pdf-0.1.0/.gitignore

@@ -0,0 +1,4 @@
+.venv/
+__pycache__/
+*.egg-info/
+dist/

knowhive_pdf-0.1.0/PKG-INFO

@@ -0,0 +1,54 @@
+Metadata-Version: 2.4
+Name: knowhive-pdf
+Version: 0.1.0
+Summary: KnowHive PDF plugin: docling-powered PDF → DocumentIR JSON over a stdio protocol
+License-Expression: MIT
+Requires-Python: >=3.10
+Requires-Dist: docling<3,>=2.0
+Requires-Dist: pypdfium2>=4
+Description-Content-Type: text/markdown
+
+# knowhive-pdf
+
+KnowHive 的 PDF 解析插件：用 [docling](https://github.com/docling-project/docling)
+把 PDF 解析成 KnowHive 的 DocumentIR JSON（`server/src/documentIr.ts` 的形状），
+通过 stdio 协议供主程序调用。主程序保持 single runtime——本插件经
+`uv tool install knowhive-pdf` 按需安装，不装则主程序完全无感。
+
+> 开发期暂驻主仓库 `pdf-plugin/`；发布时整体迁出为独立 repo + PyPI +
+> GitHub Actions Trusted Publishing。
+
+## 协议
+
+```
+$ knowhive-pdf --stdio
+→ {"type":"ready","schema_version":1,"plugin_version":"0.1.0","docling_version":"..."}
+← /path/to/file.pdf
+→ {"type":"result","path":"...","ir":{"format":"pdf","blocks":[...]}}
+→ {"type":"error","path":"...","code":"needs_ocr|bad_text_layer|parse_failed","message":"..."}
+```
+
+调试用一次性模式：`knowhive-pdf file.pdf`。
+
+## v1 范围与设计决定
+
+- **不含 OCR**（`do_ocr=False`）。扫描件由 triage 拦截返回 `needs_ocr`；中文 OCR 留 v2。
+- **triage 先行**（pypdfium2，毫秒级）：每页字符中位数 <50 且页面含图 → `needs_ocr`；
+  无图或乱码字符占比 >10% → `bad_text_layer`。坏文字层是最阴的静默失败——页面渲染
+  正常但抽出来是空/乱码（校准样本：pdf.js `issue9534_reduced.pdf`）。
+- **NFKC 归一化**：子集化中文字体会把部分汉字映射到康熙部首码位（⼀ U+2F00 ≠ 一
+  U+4E00），肉眼相同、检索必死。所有输出文本过 NFKC。
+- **标题层级从编号推导**（"3.1"→L3）：docling 的 section_header 是平的。
+- **表格 caption 拆分**：docling 的表格 markdown 前缀 caption；拆成独立段落块，
+  表格块从表头行开始——主程序 `splitTable()` 的表头重复逻辑依赖"首行=表头、
+  次行=分隔行"。
+
+## 开发
+
+```bash
+uv sync
+uv run pytest                      # 快速套件（不加载 docling 模型）
+KNOWHIVE_PDF_SLOW=1 uv run pytest  # 含完整 docling 解析（首跑下载版面模型）
+```
+
+`tests/fixtures/bad-textlayer.pdf` 来自 mozilla/pdf.js 测试语料（issue9534_reduced）。

knowhive_pdf-0.1.0/README.md

@@ -0,0 +1,44 @@
+# knowhive-pdf
+
+KnowHive 的 PDF 解析插件：用 [docling](https://github.com/docling-project/docling)
+把 PDF 解析成 KnowHive 的 DocumentIR JSON（`server/src/documentIr.ts` 的形状），
+通过 stdio 协议供主程序调用。主程序保持 single runtime——本插件经
+`uv tool install knowhive-pdf` 按需安装，不装则主程序完全无感。
+
+> 开发期暂驻主仓库 `pdf-plugin/`；发布时整体迁出为独立 repo + PyPI +
+> GitHub Actions Trusted Publishing。
+
+## 协议
+
+```
+$ knowhive-pdf --stdio
+→ {"type":"ready","schema_version":1,"plugin_version":"0.1.0","docling_version":"..."}
+← /path/to/file.pdf
+→ {"type":"result","path":"...","ir":{"format":"pdf","blocks":[...]}}
+→ {"type":"error","path":"...","code":"needs_ocr|bad_text_layer|parse_failed","message":"..."}
+```
+
+调试用一次性模式：`knowhive-pdf file.pdf`。
+
+## v1 范围与设计决定
+
+- **不含 OCR**（`do_ocr=False`）。扫描件由 triage 拦截返回 `needs_ocr`；中文 OCR 留 v2。
+- **triage 先行**（pypdfium2，毫秒级）：每页字符中位数 <50 且页面含图 → `needs_ocr`；
+  无图或乱码字符占比 >10% → `bad_text_layer`。坏文字层是最阴的静默失败——页面渲染
+  正常但抽出来是空/乱码（校准样本：pdf.js `issue9534_reduced.pdf`）。
+- **NFKC 归一化**：子集化中文字体会把部分汉字映射到康熙部首码位（⼀ U+2F00 ≠ 一
+  U+4E00），肉眼相同、检索必死。所有输出文本过 NFKC。
+- **标题层级从编号推导**（"3.1"→L3）：docling 的 section_header 是平的。
+- **表格 caption 拆分**：docling 的表格 markdown 前缀 caption；拆成独立段落块，
+  表格块从表头行开始——主程序 `splitTable()` 的表头重复逻辑依赖"首行=表头、
+  次行=分隔行"。
+
+## 开发
+
+```bash
+uv sync
+uv run pytest                      # 快速套件（不加载 docling 模型）
+KNOWHIVE_PDF_SLOW=1 uv run pytest  # 含完整 docling 解析（首跑下载版面模型）
+```
+
+`tests/fixtures/bad-textlayer.pdf` 来自 mozilla/pdf.js 测试语料（issue9534_reduced）。

knowhive_pdf-0.1.0/pyproject.toml

@@ -0,0 +1,24 @@
+[project]
+name = "knowhive-pdf"
+version = "0.1.0"
+description = "KnowHive PDF plugin: docling-powered PDF → DocumentIR JSON over a stdio protocol"
+readme = "README.md"
+license = "MIT"
+requires-python = ">=3.10"
+dependencies = [
+    "docling>=2.0,<3",
+    "pypdfium2>=4",
+]
+
+[project.scripts]
+knowhive-pdf = "knowhive_pdf.cli:main"
+
+[dependency-groups]
+dev = ["pytest>=8"]
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/knowhive_pdf"]

knowhive_pdf-0.1.0/src/knowhive_pdf/__init__.py

@@ -0,0 +1,13 @@
+"""KnowHive PDF plugin: PDF → DocumentIR JSON.
+
+The host app (KnowHive's TS sidecar) spawns `knowhive-pdf --stdio`, feeds file
+paths on stdin, and reads one JSON object per line on stdout. The IR schema is
+the TS side's DocumentIR (server/src/documentIr.ts); SCHEMA_VERSION gates
+compatibility across releases.
+"""
+
+__version__ = "0.1.0"
+
+# Bump when the emitted IR shape changes incompatibly. The host refuses to talk
+# to a plugin whose schema_version it doesn't know.
+SCHEMA_VERSION = 1

knowhive_pdf-0.1.0/src/knowhive_pdf/cli.py

@@ -0,0 +1,81 @@
+"""CLI entry: stdio batch protocol (host-facing) + one-shot mode (debugging).
+
+Protocol (`knowhive-pdf --stdio`), one JSON object per line on stdout:
+
+    → {"type":"ready","schema_version":1,"plugin_version":"0.1.0","docling_version":"..."}
+    ← /path/to/file.pdf\n                (one path per stdin line)
+    → {"type":"result","path":"...","ir":{"format":"pdf","blocks":[...]}}
+    → {"type":"error","path":"...","code":"needs_ocr|bad_text_layer|parse_failed","message":"..."}
+
+Triage runs before docling, so scanned/broken files come back in milliseconds
+and never trigger the (seconds-long, model-loading) conversion pipeline.
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from pathlib import Path
+
+from . import SCHEMA_VERSION, __version__
+from .mapping import convert_pdf
+from .triage import triage_pdf
+
+
+def _emit(obj: dict) -> None:
+    sys.stdout.write(json.dumps(obj, ensure_ascii=False) + "\n")
+    sys.stdout.flush()
+
+
+def process_file(path: str) -> dict:
+    if not Path(path).is_file():
+        return {"type": "error", "path": path, "code": "parse_failed", "message": "file not found"}
+    try:
+        t = triage_pdf(path)
+        if t.code != "ok":
+            return {"type": "error", "path": path, "code": t.code, "message": t.message}
+        blocks = convert_pdf(path)
+        return {"type": "result", "path": path, "ir": {"format": "pdf", "blocks": blocks}}
+    except Exception as e:  # noqa: BLE001 — protocol boundary: everything becomes an error line
+        return {"type": "error", "path": path, "code": "parse_failed", "message": str(e)}
+
+
+def run_stdio() -> None:
+    try:
+        from docling import __version__ as docling_version
+    except Exception:  # pragma: no cover
+        docling_version = "unknown"
+    _emit(
+        {
+            "type": "ready",
+            "schema_version": SCHEMA_VERSION,
+            "plugin_version": __version__,
+            "docling_version": docling_version,
+        }
+    )
+    for line in sys.stdin:
+        path = line.strip()
+        if not path:
+            continue
+        _emit(process_file(path))
+
+
+def main() -> None:
+    ap = argparse.ArgumentParser(description="KnowHive PDF plugin: PDF → DocumentIR JSON")
+    ap.add_argument("pdf", nargs="?", help="one-shot mode: parse a single PDF and print its IR")
+    ap.add_argument("--stdio", action="store_true", help="batch mode: paths on stdin, JSONL on stdout")
+    args = ap.parse_args()
+
+    if args.stdio:
+        run_stdio()
+        return
+    if not args.pdf:
+        ap.error("provide a PDF path or --stdio")
+    result = process_file(args.pdf)
+    _emit(result)
+    if result["type"] == "error":
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

knowhive_pdf-0.1.0/src/knowhive_pdf/mapping.py

@@ -0,0 +1,114 @@
+"""docling result → DocumentIR blocks (the TS side's documentIr.ts shape).
+
+Three quirks this mapping exists to absorb (found in the 2026-08-01 spike):
+
+1. Docling reports section headers at a flat tree depth, so the real hierarchy
+   lives in the numbering — "3.1 Pre-training" is level 3, "3 BERT" level 2.
+2. TableItem.export_to_markdown prepends the caption and a blank line. The TS
+   chunker's splitTable() requires line 1 = header row / line 2 = separator to
+   repeat headers when splitting oversized tables, so the caption becomes its
+   own paragraph block and the table block starts at the first pipe row.
+3. Subsetted CJK fonts map some ideographs to Kangxi-radical codepoints
+   (⼀ U+2F00 instead of 一 U+4E00) — visually identical, retrieval-fatal.
+   NFKC normalization folds them back (and full-width ASCII, ligatures too).
+"""
+from __future__ import annotations
+
+import unicodedata
+from typing import Any
+
+_converter = None
+
+
+def get_converter():
+    """Singleton DocumentConverter — model load costs seconds, pay it once."""
+    global _converter
+    if _converter is None:
+        from docling.datamodel.base_models import InputFormat
+        from docling.datamodel.pipeline_options import PdfPipelineOptions
+        from docling.document_converter import DocumentConverter, PdfFormatOption
+
+        opts = PdfPipelineOptions()
+        # v1 scope: text-layer PDFs only. Scanned files never reach docling —
+        # triage_pdf() returns needs_ocr first.
+        opts.do_ocr = False
+        opts.do_table_structure = True
+        _converter = DocumentConverter(
+            format_options={InputFormat.PDF: PdfFormatOption(pipeline_options=opts)}
+        )
+    return _converter
+
+
+def normalize_text(text: str) -> str:
+    return unicodedata.normalize("NFKC", text).strip()
+
+
+def heading_level_from_numbering(text: str) -> int:
+    """"3 BERT"→2, "3.1 Pre-training"→3; unnumbered→2 (doc title holds 1)."""
+    lead = text.split(" ", 1)[0].rstrip(".")
+    parts = lead.split(".")
+    if parts and parts[0] and all(p.isdigit() for p in parts):
+        return min(6, 1 + len(parts))
+    return 2
+
+
+def split_table_markdown(md: str) -> tuple[str, str]:
+    """Split docling's table markdown into (caption, pipe-table)."""
+    lines = md.split("\n")
+    first_pipe = next((i for i, l in enumerate(lines) if l.startswith("|")), None)
+    if first_pipe is None:
+        return md.strip(), ""
+    caption = "\n".join(lines[:first_pipe]).strip()
+    table_md = "\n".join(lines[first_pipe:]).strip()
+    return caption, table_md
+
+
+def convert_pdf(pdf_path: str) -> list[dict[str, Any]]:
+    from docling_core.types.doc import DocItemLabel, TableItem, TextItem
+
+    doc = get_converter().convert(pdf_path).document
+
+    label_map = {
+        DocItemLabel.TITLE: "heading",
+        DocItemLabel.SECTION_HEADER: "heading",
+        DocItemLabel.TEXT: "paragraph",
+        DocItemLabel.PARAGRAPH: "paragraph",
+        DocItemLabel.LIST_ITEM: "list",
+        DocItemLabel.CODE: "code",
+        DocItemLabel.FORMULA: "paragraph",
+        DocItemLabel.CAPTION: "paragraph",
+    }
+
+    blocks: list[dict[str, Any]] = []
+
+    def push(block: dict[str, Any], item: Any) -> None:
+        prov = getattr(item, "prov", None)
+        if prov:
+            block["page"] = prov[0].page_no
+            bb = prov[0].bbox
+            block["bbox"] = [round(bb.l, 1), round(bb.t, 1), round(bb.r, 1), round(bb.b, 1)]
+        block["order"] = len(blocks)
+        blocks.append(block)
+
+    for item, _level in doc.iterate_items():
+        if isinstance(item, TableItem):
+            caption, table_md = split_table_markdown(item.export_to_markdown(doc=doc))
+            if caption:
+                push({"type": "paragraph", "text": normalize_text(caption)}, item)
+            if table_md:
+                push({"type": "table", "text": normalize_text(table_md)}, item)
+        elif isinstance(item, TextItem):
+            text = normalize_text(item.text or "")
+            if not text:
+                continue
+            btype = label_map.get(item.label)
+            if btype is None:
+                continue  # page furniture (headers/footers/footnotes): skip
+            block: dict[str, Any] = {"type": btype, "text": text}
+            if btype == "heading":
+                block["level"] = (
+                    1 if item.label == DocItemLabel.TITLE else heading_level_from_numbering(text)
+                )
+            push(block, item)
+
+    return blocks

knowhive_pdf-0.1.0/src/knowhive_pdf/triage.py

@@ -0,0 +1,77 @@
+"""Cheap pre-flight triage: decide whether a PDF is worth running docling on.
+
+Runs pypdfium2 text extraction only (milliseconds, no ML models). Catches the
+two failure shapes that would otherwise fail SILENTLY — a scanned PDF parsed
+with OCR off yields empty text, and a broken text layer (CID/cmap damage)
+yields garbage that would be embedded as if it were real content.
+
+Thresholds calibrated against real samples (2026-08-01):
+  - scanned page (image-only):        0 chars/page
+  - pdf.js issue9534 (broken layer):  15 chars on a visibly full page
+  - normal papers:                    thousands of chars/page, garbage ≈ 0.5%
+"""
+from __future__ import annotations
+
+from dataclasses import dataclass
+from statistics import median
+
+# A page with fewer extractable chars than this is not a text page.
+MIN_CHARS_PER_PAGE = 50
+# Share of private-use/replacement chars above which the layer is garbage.
+MAX_GARBAGE_RATIO = 0.10
+
+
+@dataclass
+class TriageResult:
+    code: str  # "ok" | "needs_ocr" | "bad_text_layer"
+    message: str
+    chars_per_page: list[int]
+
+
+def _is_garbage_char(c: str) -> bool:
+    cp = ord(c)
+    return 0xE000 <= cp <= 0xF8FF or c == "�"
+
+
+def _page_has_image(page) -> bool:
+    # FPDF_PAGEOBJ_IMAGE == 3 in pdfium's page-object enum.
+    return any(obj.type == 3 for obj in page.get_objects())
+
+
+def triage_pdf(pdf_path: str) -> TriageResult:
+    import pypdfium2 as pdfium
+
+    doc = pdfium.PdfDocument(pdf_path)
+    try:
+        texts = [doc[i].get_textpage().get_text_bounded() for i in range(len(doc))]
+        chars_per_page = [len(t.strip()) for t in texts]
+        all_text = "".join(texts)
+
+        if all_text:
+            garbage = sum(1 for c in all_text if _is_garbage_char(c)) / len(all_text)
+            if garbage > MAX_GARBAGE_RATIO:
+                return TriageResult(
+                    "bad_text_layer",
+                    f"{garbage:.0%} of extracted characters are unmapped glyphs",
+                    chars_per_page,
+                )
+
+        if median(chars_per_page) < MIN_CHARS_PER_PAGE:
+            # Distinguish "pages are pictures" from "pages should have text but
+            # the layer is broken": image-dominated pages mean a scan.
+            scannish = any(_page_has_image(doc[i]) for i in range(min(3, len(doc))))
+            if scannish:
+                return TriageResult(
+                    "needs_ocr",
+                    "pages are images with no usable text layer (scanned document)",
+                    chars_per_page,
+                )
+            return TriageResult(
+                "bad_text_layer",
+                "pages render content but the text layer is empty or broken",
+                chars_per_page,
+            )
+
+        return TriageResult("ok", "", chars_per_page)
+    finally:
+        doc.close()

knowhive_pdf-0.1.0/tests/test_mapping.py

@@ -0,0 +1,33 @@
+from knowhive_pdf.mapping import (
+    heading_level_from_numbering,
+    normalize_text,
+    split_table_markdown,
+)
+
+
+def test_heading_levels_from_numbering():
+    assert heading_level_from_numbering("3 BERT") == 2
+    assert heading_level_from_numbering("3.1 Pre-training") == 3
+    assert heading_level_from_numbering("3.2.1 Scaled Dot-Product Attention") == 4
+    assert heading_level_from_numbering("Abstract") == 2
+    assert heading_level_from_numbering("A.1 Appendix") == 2  # non-numeric → default
+
+
+def test_nfkc_folds_kangxi_radicals():
+    # Subsetted CJK fonts emit ⼀ (U+2F00) for 一 (U+4E00) — retrieval-fatal.
+    assert normalize_text("⼀、⽗⼦切分") == "一、父子切分"
+    assert normalize_text("⼆、⽰例") == "二、示例"
+
+
+def test_table_caption_splits_off():
+    md = "Table 1: results overview.\n\n| a | b |\n|---|---|\n| 1 | 2 |"
+    caption, table = split_table_markdown(md)
+    assert caption == "Table 1: results overview."
+    assert table.split("\n")[0] == "| a | b |"
+    assert table.split("\n")[1] == "|---|---|"
+
+
+def test_table_without_pipes_becomes_caption_only():
+    caption, table = split_table_markdown("just a stray caption")
+    assert caption == "just a stray caption"
+    assert table == ""

knowhive_pdf-0.1.0/tests/test_stdio.py

@@ -0,0 +1,63 @@
+"""Protocol test: spawn the real CLI, feed paths, read JSONL.
+
+Uses only triage-rejected fixtures so the test never loads docling's layout
+models — the full-parse path is covered by the slow opt-in test below.
+"""
+import json
+import os
+import subprocess
+import sys
+from pathlib import Path
+
+import pytest
+
+FIXTURES = Path(__file__).parent / "fixtures"
+
+
+def run_stdio(paths: list[str]) -> list[dict]:
+    proc = subprocess.run(
+        [sys.executable, "-m", "knowhive_pdf.cli", "--stdio"],
+        input="\n".join(paths) + "\n",
+        capture_output=True,
+        text=True,
+        timeout=600,
+    )
+    assert proc.returncode == 0, proc.stderr
+    return [json.loads(line) for line in proc.stdout.strip().split("\n")]
+
+
+def test_handshake_and_error_lines():
+    lines = run_stdio(
+        [str(FIXTURES / "zh-scanned.pdf"), str(FIXTURES / "bad-textlayer.pdf"), "/nope.pdf"]
+    )
+    ready, scanned, broken, missing = lines
+    assert ready["type"] == "ready"
+    assert ready["schema_version"] == 1
+    assert scanned == {
+        "type": "error",
+        "path": str(FIXTURES / "zh-scanned.pdf"),
+        "code": "needs_ocr",
+        "message": scanned["message"],
+    }
+    assert broken["code"] == "bad_text_layer"
+    assert missing["code"] == "parse_failed"
+
+
+@pytest.mark.skipif(
+    not os.environ.get("KNOWHIVE_PDF_SLOW"),
+    reason="full docling parse (downloads layout models); set KNOWHIVE_PDF_SLOW=1",
+)
+def test_full_parse_emits_ir():
+    lines = run_stdio([str(FIXTURES / "zh-textlayer.pdf")])
+    result = lines[1]
+    assert result["type"] == "result"
+    ir = result["ir"]
+    assert ir["format"] == "pdf"
+    types = {b["type"] for b in ir["blocks"]}
+    assert "heading" in types and "table" in types
+    # NFKC applied: no Kangxi radicals survive into the IR.
+    assert not any("⼀" in b["text"] or "⽗" in b["text"] for b in ir["blocks"])
+    # Table blocks start at the header row (splitTable contract).
+    for b in ir["blocks"]:
+        if b["type"] == "table":
+            assert b["text"].startswith("|")

knowhive_pdf-0.1.0/tests/test_triage.py

@@ -0,0 +1,23 @@
+from pathlib import Path
+
+from knowhive_pdf.triage import triage_pdf
+
+FIXTURES = Path(__file__).parent / "fixtures"
+
+
+def test_text_layer_pdf_is_ok():
+    r = triage_pdf(str(FIXTURES / "zh-textlayer.pdf"))
+    assert r.code == "ok"
+    assert max(r.chars_per_page) > 50
+
+
+def test_scanned_pdf_needs_ocr():
+    r = triage_pdf(str(FIXTURES / "zh-scanned.pdf"))
+    assert r.code == "needs_ocr"
+
+
+def test_broken_text_layer_detected():
+    # pdf.js issue9534_reduced: the page renders text but extraction yields
+    # almost nothing — the silent-garbage case triage exists for.
+    r = triage_pdf(str(FIXTURES / "bad-textlayer.pdf"))
+    assert r.code == "bad_text_layer"