groundextract 0.1.2__py3-none-any.whl

groundextract/__init__.py

@@ -0,0 +1,68 @@
+"""GroundExtract-KR — value-level truth gate for Korean regulatory documents.
+
+Every extracted number is forced to (1) cite verbatim evidence, (2) satisfy
+domain arithmetic invariants; anything ungrounded or rule-violating is
+auto-discarded at confidence 0 before it reaches downstream.
+"""
+
+from .gate import run_gate, summarize
+from .grounding import ground_value, match_value
+from .models import (
+    BBox,
+    Check,
+    ExtractedValue,
+    MatchKind,
+    Verdict,
+    VerifiedField,
+)
+from .rules import RulePack, default_rules_dir, evaluate_pack, load_rule_pack
+
+__version__ = "0.1.2"
+
+
+def available_doc_types() -> list[str]:
+    """Document types that ship with a bundled rule pack, sorted by name."""
+    return sorted(p.stem for p in default_rules_dir().glob("*.yaml"))
+
+
+def load_pack(doc_type: str) -> RulePack:
+    """Load a bundled rule pack by document type, e.g. ``load_pack("tax_invoice")``.
+
+    Prefer this over ``load_rule_pack("rules/<doc_type>.yaml")``: that relative
+    path only resolves from a repository checkout, while the bundled packs are
+    located through :func:`default_rules_dir`, so this works from any working
+    directory and from a ``pip install``ed wheel.
+
+    Raises ``FileNotFoundError`` (listing the available types) when no pack
+    ships for ``doc_type``. Use :func:`load_rule_pack` for your own YAML files.
+    """
+    if not doc_type.replace("_", "").isalnum():  # no path separators / traversal
+        raise ValueError(f"invalid doc_type: {doc_type!r}")
+    path = default_rules_dir() / f"{doc_type}.yaml"
+    if not path.is_file():
+        raise FileNotFoundError(
+            f"no bundled rule pack for doc_type {doc_type!r}; "
+            f"available: {', '.join(available_doc_types()) or '(none)'}"
+        )
+    return load_rule_pack(path)
+
+
+__all__ = [
+    "BBox",
+    "Check",
+    "ExtractedValue",
+    "MatchKind",
+    "Verdict",
+    "VerifiedField",
+    "RulePack",
+    "available_doc_types",
+    "default_rules_dir",
+    "load_pack",
+    "load_rule_pack",
+    "evaluate_pack",
+    "ground_value",
+    "match_value",
+    "run_gate",
+    "summarize",
+    "__version__",
+]

groundextract/__main__.py

@@ -0,0 +1,262 @@
+"""CLI: `python -m groundextract` demos the gate, `... verify` runs it on your files.
+
+The bare command shows the core value proposition with no keys and no inputs: a
+hallucinated VAT is auto-discarded (confidence 0) while grounded,
+arithmetic-consistent numbers stay verified.
+
+`verify` is the same gate over a document and an extraction you supply. It is
+deliberately *not* an extractor — this project verifies what some other OCR/LLM
+produced, so it takes the text and the values as files rather than pretending to
+read a PDF. For the full real-document path see `adapters.DoclingAdapter` +
+`llm.OllamaExtractor`, and for agents the MCP tool `verify_extraction`.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from collections.abc import Sequence
+from pathlib import Path
+
+from . import __version__, available_doc_types, load_pack
+from .gate import run_gate, summarize
+from .llm import MockExtractor
+from .models import ExtractedValue, Verdict, VerifiedField
+from .rules import RulePack, RulePackError, load_rule_pack
+
+_DOC = (
+    "전자세금계산서\n"
+    "공급가액  1,000,000원\n"
+    "세액        100,000원\n"
+    "합계금액  1,100,000원\n"
+)
+
+_DESCRIPTION = (
+    "Run the built-in GroundExtract-KR demo: a canned Korean tax invoice is fed to a "
+    "mock extractor that hallucinates the VAT, and the gate auto-discards every value "
+    "it cannot ground in the text or reconcile with the rule pack. No API key, no "
+    "network, no randomness."
+)
+
+_EPILOG = """\
+This command takes no document arguments; it always runs the same demo.
+
+To verify your own numbers:
+  CLI     : ... verify --doc doc.txt --values values.json --doc-type tax_invoice
+  library : from groundextract import ExtractedValue, load_pack, run_gate
+  agents  : python -m groundextract.mcp_server   (MCP tool `verify_extraction`)
+  bench   : python -m groundextract.bench        (NumHall-KR regression suite)
+
+Docs: https://github.com/sos37591-prog/groundextract-kr
+"""
+
+_VERIFY_DESCRIPTION = (
+    "Run the gate over a document and an extraction you already have. This is not an "
+    "extractor: --doc is the plain text your OCR/LLM read the numbers out of, and "
+    "--values is what it claims to have found."
+)
+
+_VERIFY_EPILOG = """\
+values.json is a list of objects — the same shape the MCP tool accepts:
+
+  [
+    {"field": "supply", "raw": "1,000,000원", "number": 1000000,
+     "grounding_quote": "공급가액  1,000,000원"},
+    {"field": "vat",    "raw": "100,000원",   "number": 100000},
+    {"field": "total",  "raw": "1,100,000원", "number": 1100000}
+  ]
+
+`number` and `grounding_quote` are optional. `number` is cross-checked against
+`raw` and must agree; omit it and it is parsed from `raw`.
+
+Exit code: 0 if every field was verified, 1 if any was discarded — so a pipeline
+can gate on it directly. Usage errors exit 2.
+"""
+
+
+def _prog() -> str:
+    """Name to print in usage: the console script if that is how we were called."""
+    if Path(sys.argv[0] if sys.argv else "").stem == "groundextract":
+        return "groundextract"
+    return "python -m groundextract"
+
+
+def _build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        prog=_prog(),
+        description=_DESCRIPTION,
+        epilog=_EPILOG,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    parser.add_argument(
+        "--json",
+        dest="as_json",
+        action="store_true",
+        help="emit the demo result as JSON (fields + summary) instead of the text report",
+    )
+    parser.add_argument("--version", action="version", version=f"groundextract {__version__}")
+    # Accepted only so a file argument gets an explanatory error instead of
+    # being silently ignored (which would look like the file was processed).
+    parser.add_argument("args", nargs="*", metavar="", help=argparse.SUPPRESS)
+    return parser
+
+
+def _build_verify_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        prog=f"{_prog()} verify",
+        description=_VERIFY_DESCRIPTION,
+        epilog=_VERIFY_EPILOG,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    parser.add_argument(
+        "--doc", required=True, type=Path, metavar="PATH", help="document text file (UTF-8)"
+    )
+    parser.add_argument(
+        "--values", required=True, type=Path, metavar="PATH", help="extracted values, JSON"
+    )
+    pack = parser.add_mutually_exclusive_group(required=True)
+    pack.add_argument(
+        "--doc-type",
+        choices=available_doc_types(),
+        help="use a bundled rule pack",
+    )
+    pack.add_argument(
+        "--rules", type=Path, metavar="PATH", help="use your own rule pack (YAML)"
+    )
+    parser.add_argument(
+        "--json", dest="as_json", action="store_true", help="emit JSON instead of a text report"
+    )
+    return parser
+
+
+def _read_values(path: Path, parser: argparse.ArgumentParser) -> list[ExtractedValue]:
+    """Parse values.json, reusing the exact contract the MCP tool enforces.
+
+    The CLI and the MCP tool must accept the same value shape — two parsers would
+    drift, and the difference would show up as a document that verifies through
+    one path and not the other.
+    """
+    from .mcp_server import InvalidParamsError, _parse_values
+
+    try:
+        raw = json.loads(path.read_text(encoding="utf-8"))
+    except OSError as e:
+        parser.error(f"cannot read {path}: {e}")
+    except json.JSONDecodeError as e:
+        parser.error(f"{path}: invalid JSON ({e})")
+    try:
+        return _parse_values(raw)
+    except InvalidParamsError as e:
+        parser.error(f"{path}: {e}")
+
+
+def _verify_main(argv: Sequence[str]) -> int:
+    parser = _build_verify_parser()
+    args = parser.parse_args(argv)
+
+    try:
+        text = args.doc.read_text(encoding="utf-8")
+    except OSError as e:
+        parser.error(f"cannot read {args.doc}: {e}")
+    if not text.strip():
+        parser.error(f"{args.doc} is empty; there is nothing to ground against")
+
+    values = _read_values(args.values, parser)
+    try:
+        pack = load_pack(args.doc_type) if args.doc_type else load_rule_pack(args.rules)
+    except RulePackError as e:
+        parser.error(str(e))
+
+    fields = run_gate(values, text, pack)
+    if args.as_json:
+        payload = {"fields": [f.to_dict() for f in fields], "summary": summarize(fields, pack)}
+        print(json.dumps(payload, ensure_ascii=False, indent=2))
+    else:
+        _print_report(fields, pack, title=f"=== {args.doc.name} ===")
+    # Non-zero when anything was discarded, so `... verify && ingest` is safe.
+    return 1 if any(f.verdict is Verdict.DISCARDED for f in fields) else 0
+
+
+def _demo_fields() -> tuple[list[VerifiedField], RulePack]:
+    pack = load_pack("tax_invoice")
+
+    # Two grounded/consistent values + one hallucinated VAT (250,000 is nowhere
+    # in the document and breaks vat = supply * 10%).
+    extractor = MockExtractor(
+        [
+            ExtractedValue("supply", "1,000,000원", 1_000_000, "공급가액  1,000,000원"),
+            ExtractedValue("vat", "250,000원", 250_000, "세액  250,000원"),  # hallucinated
+            ExtractedValue("total", "1,100,000원", 1_100_000, "합계금액  1,100,000원"),
+        ]
+    )
+    return run_gate(extractor.extract(_DOC, doc_type="tax_invoice"), _DOC, pack), pack
+
+
+def _print_report(
+    fields: list[VerifiedField], pack: RulePack, title: str = "=== GroundExtract-KR demo ==="
+) -> None:
+    print(title)
+    for f in fields:
+        mark = "OK " if f.verdict.value == "verified" else "XX "
+        print(
+            f"{mark}{f.field:6} value={f.value.raw:12} "
+            f"verdict={f.verdict.value} conf={f.confidence}"
+        )
+        for c in f.failed_checks:
+            print(f"       ! {c.name}: {c.detail}")
+    print("summary:", json.dumps(summarize(fields, pack), ensure_ascii=False))
+
+
+def _survive_a_narrow_console() -> None:
+    """Never let an unencodable character turn a verdict into a traceback.
+
+    A check detail is free-form text — rule names, quoted document spans, the
+    gate's own prose — and the console it lands on is not. On a Korean Windows
+    console (cp949) the em dash in "this verifies nothing — the rule leans on …"
+    raised UnicodeEncodeError *while printing the report*, so `verify` crashed on
+    exactly the case it exists to catch: an extraction whose arithmetic passes
+    only because it leans on ungrounded values. The exit code was still 1, which
+    made it look like an ordinary failed verification rather than a crash.
+
+    Replacing unencodable characters degrades one glyph; raising loses the whole
+    report. `errors` is set without touching `encoding`, so console output stays
+    in the terminal's own codepage and Korean still renders.
+    """
+    for stream in (sys.stdout, sys.stderr):
+        reconfigure = getattr(stream, "reconfigure", None)
+        if reconfigure is not None:
+            try:
+                reconfigure(errors="replace")
+            except (ValueError, OSError):  # pragma: no cover - detached/odd stream
+                pass
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+    """Run the demo, or dispatch to `verify`. Returns the process exit code."""
+    _survive_a_narrow_console()
+    raw_args = list(sys.argv[1:] if argv is None else argv)
+    if raw_args and raw_args[0] == "verify":
+        return _verify_main(raw_args[1:])
+
+    parser = _build_parser()
+    args = parser.parse_args(raw_args)
+    if args.args:
+        parser.error(
+            f"this command runs the built-in demo only and cannot process {args.args[0]!r}. "
+            f"To verify your own document use `{_prog()} verify --doc ... --values ...` "
+            "(see `verify --help`), the library (groundextract.run_gate), or the MCP tool "
+            "`verify_extraction`."
+        )
+
+    fields, pack = _demo_fields()
+    if args.as_json:
+        payload = {"fields": [f.to_dict() for f in fields], "summary": summarize(fields, pack)}
+        print(json.dumps(payload, ensure_ascii=False, indent=2))
+    else:
+        _print_report(fields, pack)
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

groundextract/adapters/__init__.py

@@ -0,0 +1,10 @@
+"""Document conversion adapters (optional heavy backends).
+
+Heavy third-party libraries (Docling, ...) are imported lazily inside the
+adapter methods, so importing this package never requires them. Missing
+optional dependencies raise ``ImportError`` with an install hint at call time.
+"""
+
+from .docling_adapter import DoclingAdapter, SpanInfo
+
+__all__ = ["DoclingAdapter", "SpanInfo"]

groundextract/adapters/docling_adapter.py

@@ -0,0 +1,206 @@
+"""Docling PDF adapter: PDF -> full text + per-span page/bbox provenance.
+
+Docling is an *optional* dependency (``pip install groundextract[docling]``);
+it is imported lazily inside :meth:`DoclingAdapter.convert` so the rest of the
+package (gate, rules, bench) keeps working without it.
+
+The Docling API has shifted between releases, so every access to the converted
+document is defensive (``getattr``/``hasattr``). Worst case the adapter still
+returns the full text with no layout info — the gate only *needs* text; page
+and bbox are a bonus for the highlight viewer.
+
+Coordinates are copied from Docling as-is (``l/t/r/b`` -> ``x0/y0/x1/y1``);
+depending on the Docling version the origin may be top-left or bottom-left.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+from ..models import BBox, ExtractedValue
+
+_INSTALL_HINT = (
+    "docling is required for DoclingAdapter but is not installed. "
+    "Install the optional extra: pip install groundextract[docling]"
+)
+
+
+def _import_docling() -> Any:
+    """Import Docling's ``DocumentConverter`` lazily, with an install hint."""
+    try:
+        from docling.document_converter import DocumentConverter
+    except ImportError as exc:
+        raise ImportError(_INSTALL_HINT) from exc
+    return DocumentConverter
+
+
+@dataclass
+class SpanInfo:
+    """One text item from the converted document plus its provenance."""
+
+    text: str
+    page: int | None = None
+    bbox: BBox | None = None
+
+
+class DoclingAdapter:
+    """Converts a PDF with Docling and maps extracted values back to layout.
+
+    Usage::
+
+        adapter = DoclingAdapter()
+        result = adapter.convert("invoice.pdf")   # {"full_text": ..., "spans": [...]}
+        values = extractor.extract(result["full_text"], doc_type="tax_invoice")
+        adapter.enrich(values)                    # fills page/bbox where possible
+    """
+
+    def __init__(self) -> None:
+        self.full_text: str = ""
+        self.spans: list[SpanInfo] = []
+        self._converter: Any = None
+
+    # --- conversion ------------------------------------------------------------
+
+    def convert(self, pdf_path: str | Path) -> dict[str, Any]:
+        """Convert ``pdf_path`` and return ``{"full_text": str, "spans": [SpanInfo]}``.
+
+        Raises ``ImportError`` (with an install hint) when docling is missing.
+        """
+        converter_cls = _import_docling()
+        if self._converter is None:
+            self._converter = converter_cls()
+        result = self._converter.convert(str(pdf_path))
+        doc = getattr(result, "document", None)
+        if doc is None:
+            doc = result
+        self.spans = self._collect_spans(doc)
+        self.full_text = self._export_text(doc)
+        if not self.full_text and self.spans:
+            # Fallback: reconstruct the text from the individual spans.
+            self.full_text = "\n".join(s.text for s in self.spans)
+        return {"full_text": self.full_text, "spans": self.spans}
+
+    @staticmethod
+    def _export_text(doc: Any) -> str:
+        """Best-effort plain-text export across Docling versions."""
+        for method in ("export_to_text", "export_to_markdown"):
+            export = getattr(doc, method, None)
+            if not callable(export):
+                continue
+            try:
+                out = export()
+            except Exception:
+                continue
+            if isinstance(out, str) and out:
+                return out
+        return ""
+
+    @classmethod
+    def _collect_spans(cls, doc: Any) -> list[SpanInfo]:
+        """Collect text items with page/bbox provenance, tolerating API drift."""
+        spans: list[SpanInfo] = []
+        for item in cls._iter_text_items(doc):
+            text = getattr(item, "text", None)
+            if not text:
+                continue
+            page, bbox = cls._prov_page_bbox(item)
+            spans.append(SpanInfo(text=str(text), page=page, bbox=bbox))
+        return spans
+
+    @staticmethod
+    def _iter_text_items(doc: Any) -> list[Any]:
+        # TODO(W4/viewer): also collect table-cell items (doc.tables) — invoice
+        # amounts usually live inside tables, so span/bbox lookup misses them
+        # today (verified 2026-07-18: gate passes via full_text, bbox=None).
+        items = getattr(doc, "texts", None)
+        if items:
+            return list(items)
+        iterate = getattr(doc, "iterate_items", None)
+        if not callable(iterate):
+            return []
+        try:
+            # iterate_items() yields (item, level) tuples in recent docling-core.
+            return [e[0] if isinstance(e, tuple) else e for e in iterate()]
+        except Exception:
+            return []
+
+    @staticmethod
+    def _prov_page_bbox(item: Any) -> tuple[int | None, BBox | None]:
+        """Extract ``(page, BBox|None)`` from an item's first provenance entry."""
+        prov = getattr(item, "prov", None)
+        if not prov:
+            return None, None
+        try:
+            first = prov[0]
+        except (IndexError, TypeError):
+            return None, None
+
+        page: int | None = None
+        for attr in ("page_no", "page"):
+            raw_page = getattr(first, attr, None)
+            if raw_page is not None:
+                try:
+                    page = int(raw_page)
+                except (TypeError, ValueError):
+                    page = None
+                break
+
+        raw_bbox = getattr(first, "bbox", None)
+        coords: tuple[float, float, float, float] | None = None
+        if raw_bbox is not None:
+            try:
+                if all(hasattr(raw_bbox, a) for a in ("l", "t", "r", "b")):
+                    coords = (
+                        float(raw_bbox.l),
+                        float(raw_bbox.t),
+                        float(raw_bbox.r),
+                        float(raw_bbox.b),
+                    )
+                elif all(hasattr(raw_bbox, a) for a in ("x0", "y0", "x1", "y1")):
+                    coords = (
+                        float(raw_bbox.x0),
+                        float(raw_bbox.y0),
+                        float(raw_bbox.x1),
+                        float(raw_bbox.y1),
+                    )
+            except (TypeError, ValueError):
+                coords = None
+
+        if coords is None:
+            return page, None
+        return page, BBox(page if page is not None else 0, *coords)
+
+    # --- span lookup / enrichment ----------------------------------------------
+
+    def find_span(self, text: str) -> SpanInfo | None:
+        """Return the first collected span containing ``text`` (substring search)."""
+        needle = text.strip() if text else ""
+        if not needle:
+            return None
+        for span in self.spans:
+            if needle in span.text:
+                return span
+        return None
+
+    def enrich(self, values: list[ExtractedValue]) -> list[ExtractedValue]:
+        """Fill ``page``/``bbox`` on each value from its quote (in place).
+
+        The grounding quote is tried first, then ``raw``. Values whose text is
+        not found in any span — or that already carry page/bbox — are left
+        untouched. Returns the same list for chaining.
+        """
+        for value in values:
+            span = None
+            if value.grounding_quote:
+                span = self.find_span(value.grounding_quote)
+            if span is None:
+                span = self.find_span(value.raw)
+            if span is None:
+                continue
+            if value.page is None:
+                value.page = span.page
+            if value.bbox is None:
+                value.bbox = span.bbox
+        return values