table-proof-py 0.1.0__py3-none-any.whl

table_proof/docling.py

@@ -0,0 +1,168 @@
+"""Read Docling JSON and objects without a Docling runtime dependency."""
+
+from __future__ import annotations
+
+from collections.abc import Mapping, Sequence
+from typing import Any
+
+from .models import BBox, Cell, SourcePage, Table
+
+
+class DocumentFormatError(ValueError):
+    """The input does not match a supported document format."""
+
+
+def _mapping(value: Any, name: str) -> Mapping[str, Any]:
+    if not isinstance(value, Mapping):
+        raise DocumentFormatError(f"{name} must be an object.")
+    return value
+
+
+def _integer(value: Any, name: str) -> int:
+    if isinstance(value, bool) or not isinstance(value, int):
+        raise DocumentFormatError(f"{name} must be an integer.")
+    return int(value)
+
+
+def _box(value: Any, page_height: float, scale_x: float, scale_y: float) -> BBox | None:
+    if value is None:
+        return None
+    raw = _mapping(value, "bbox")
+    origin = raw.get("coord_origin", "TOPLEFT")
+    if origin not in {"TOPLEFT", "BOTTOMLEFT"}:
+        raise DocumentFormatError(f"Unsupported coordinate origin: {origin}")
+    try:
+        left, right = float(raw["l"]) * scale_x, float(raw["r"]) * scale_x
+        top, bottom = float(raw["t"]) * scale_y, float(raw["b"]) * scale_y
+    except (KeyError, TypeError, ValueError) as error:
+        raise DocumentFormatError("A Docling box needs numeric l, t, r, and b values.") from error
+    if origin == "BOTTOMLEFT":
+        top, bottom = page_height - top, page_height - bottom
+    return BBox(x0=left, top=top, x1=right, bottom=bottom)
+
+
+def _reference_text(document: Mapping[str, Any], reference: Any, visited: set[str]) -> str:
+    raw = _mapping(reference, "ref")
+    pointer = raw.get("$ref")
+    if not isinstance(pointer, str) or not pointer.startswith("#/"):
+        raise DocumentFormatError("A rich cell needs a local JSON reference.")
+    if pointer in visited or len(visited) >= 32:
+        raise DocumentFormatError("A rich cell has a cyclic or excessive reference chain.")
+    node: Any = document
+    try:
+        for token in pointer[2:].split("/"):
+            token = token.replace("~1", "/").replace("~0", "~")
+            if isinstance(node, list):
+                if not token.isdecimal():
+                    raise KeyError(token)
+                node = node[int(token)]
+            else:
+                node = node[token]
+    except (KeyError, IndexError, TypeError) as error:
+        raise DocumentFormatError(f"Unresolved rich cell reference: {pointer}") from error
+    item = _mapping(node, "Referenced item")
+    if "text" in item:
+        if not isinstance(item["text"], str):
+            raise DocumentFormatError("Referenced text must be a string.")
+        return str(item["text"])
+    children = item.get("children", [])
+    if not isinstance(children, list):
+        raise DocumentFormatError("Referenced children must be a list.")
+    return " ".join(_reference_text(document, child, visited | {pointer}) for child in children)
+
+
+def normalize_document(document: Any, pages: Mapping[int, SourcePage]) -> list[Table]:
+    """Resolve rendered rich-cell text and normalize source coordinates."""
+    if isinstance(document, Sequence) and not isinstance(document, (str, bytes)):
+        if all(isinstance(table, Table) for table in document):
+            return list(document)
+        raise DocumentFormatError("A table sequence must contain Table objects.")
+    if hasattr(document, "document"):
+        document = document.document
+    if hasattr(document, "model_dump"):
+        document = document.model_dump(mode="json")
+    root = _mapping(document, "Document")
+    if "document" in root and "tables" not in root:
+        root = _mapping(root["document"], "document")
+    raw_tables = root.get("tables")
+    if not isinstance(raw_tables, list):
+        raise DocumentFormatError("A Docling document must contain a tables list.")
+    declared_pages = _mapping(root.get("pages", {}), "pages")
+    tables: list[Table] = []
+    seen: set[str] = set()
+    for table_index, raw_table in enumerate(raw_tables):
+        raw = _mapping(raw_table, "Table")
+        table_id = str(raw.get("self_ref", f"#/tables/{table_index}"))
+        if table_id in seen:
+            raise DocumentFormatError(f"Duplicate table reference: {table_id}")
+        seen.add(table_id)
+        data = _mapping(raw.get("data"), "Table data")
+        rows = _integer(data.get("num_rows"), "num_rows")
+        cols = _integer(data.get("num_cols"), "num_cols")
+        provenance = raw.get("prov", [])
+        if not isinstance(provenance, list):
+            raise DocumentFormatError("Table provenance must be a list.")
+        reason: str | None = None
+        if len(provenance) != 1:
+            reason = "The table needs exactly one source page. Multi-page tables are unsupported."
+        prov = _mapping(provenance[0], "Provenance") if provenance else {}
+        page_no = _integer(prov.get("page_no", 1), "page_no")
+        source_page = pages.get(page_no)
+        size = _mapping(
+            _mapping(declared_pages.get(str(page_no), {}), "Page").get("size", {}), "Page size"
+        )
+        height = source_page.height if source_page else float(size.get("height", 1))
+        width = source_page.width if source_page else float(size.get("width", 1))
+        declared_width = float(size.get("width", width))
+        declared_height = float(size.get("height", height))
+        if declared_width <= 0 or declared_height <= 0:
+            raise DocumentFormatError("Declared page dimensions must be positive.")
+        sx, sy = width / declared_width, height / declared_height
+        cells: list[Cell] = []
+        raw_cells = data.get("table_cells")
+        if not isinstance(raw_cells, list):
+            raise DocumentFormatError("Table data must contain a table_cells list.")
+        for index, raw_cell in enumerate(raw_cells):
+            cell = _mapping(raw_cell, "Cell")
+            text = cell.get("text", "")
+            if not isinstance(text, str):
+                raise DocumentFormatError("Cell text must be a string.")
+            if cell.get("ref") is not None:
+                try:
+                    text = _reference_text(root, cell["ref"], set())
+                except DocumentFormatError as error:
+                    reason = str(error)
+            row = _integer(cell.get("start_row_offset_idx"), "start_row_offset_idx")
+            col = _integer(cell.get("start_col_offset_idx"), "start_col_offset_idx")
+            row_end = _integer(cell.get("end_row_offset_idx"), "end_row_offset_idx")
+            col_end = _integer(cell.get("end_col_offset_idx"), "end_col_offset_idx")
+            row_span, col_span = row_end - row, col_end - col
+            if row_span != cell.get("row_span", row_span) or col_span != cell.get(
+                "col_span", col_span
+            ):
+                raise DocumentFormatError("Cell spans disagree with their start and end offsets.")
+            cells.append(
+                Cell(
+                    id=f"{table_id}/cells/{index}",
+                    row=row,
+                    col=col,
+                    text=text,
+                    bbox=_box(cell.get("bbox"), height, sx, sy),
+                    row_span=row_span,
+                    col_span=col_span,
+                    column_header=bool(cell.get("column_header", False)),
+                    row_header=bool(cell.get("row_header", False)),
+                )
+            )
+        tables.append(
+            Table(
+                id=table_id,
+                page=page_no,
+                bbox=_box(prov.get("bbox"), height, sx, sy),
+                num_rows=rows,
+                num_cols=cols,
+                cells=cells,
+                unsupported_reason=reason,
+            )
+        )
+    return tables

table_proof/models.py

@@ -0,0 +1,111 @@
+"""Typed source evidence and audit results."""
+
+from __future__ import annotations
+
+from typing import Literal
+
+from pydantic import BaseModel, ConfigDict, Field, model_validator
+
+Status = Literal["no_findings", "review", "uncertain"]
+
+
+class Record(BaseModel):
+    model_config = ConfigDict(extra="forbid", frozen=True, allow_inf_nan=False)
+
+
+class BBox(Record):
+    """A box in PDF points, with a top-left origin."""
+
+    x0: float
+    top: float
+    x1: float
+    bottom: float
+
+    @model_validator(mode="after")
+    def positive_area(self) -> BBox:
+        if self.x1 <= self.x0 or self.bottom <= self.top:
+            raise ValueError("A box must have positive width and height.")
+        return self
+
+
+class SourceWord(Record):
+    text: str
+    bbox: BBox
+
+
+class SourcePage(Record):
+    page: int = Field(ge=1)
+    width: float = Field(gt=0)
+    height: float = Field(gt=0)
+    words: list[SourceWord]
+
+
+class Cell(Record):
+    id: str
+    row: int = Field(ge=0)
+    col: int = Field(ge=0)
+    text: str
+    bbox: BBox | None = None
+    row_span: int = Field(default=1, ge=1)
+    col_span: int = Field(default=1, ge=1)
+    column_header: bool = False
+    row_header: bool = False
+
+
+class Table(Record):
+    id: str
+    page: int = Field(ge=1)
+    bbox: BBox | None = None
+    num_rows: int = Field(ge=0)
+    num_cols: int = Field(ge=0)
+    cells: list[Cell]
+    unsupported_reason: str | None = None
+
+
+class Finding(Record):
+    code: str
+    severity: Literal["error", "uncertain"]
+    table_id: str
+    page: int
+    cell_id: str | None = None
+    message: str
+    expected_text: str | None = None
+    observed_text: str | None = None
+    bbox: BBox | None = None
+    evidence: str = "pdf_text_geometry"
+
+
+class TableResult(Record):
+    table_id: str
+    page: int
+    status: Status
+    checked_cells: int = Field(ge=0)
+    total_cells: int = Field(ge=0)
+    findings: list[Finding]
+
+
+class AuditReport(Record):
+    schema_version: str = "1.0"
+    engine_version: str = "0.1.0"
+    source_name: str
+    source_sha256: str
+    tables: list[TableResult]
+    limitations: list[str] = Field(default_factory=list)
+
+    @property
+    def status(self) -> Status:
+        if any(table.status == "review" for table in self.tables):
+            return "review"
+        if not self.tables or any(table.status == "uncertain" for table in self.tables):
+            return "uncertain"
+        return "no_findings"
+
+    def export_json(self, path: str) -> None:
+        from .report import export_json
+
+        export_json(self, path)
+
+    def export_html(self, path: str, *, source_pdf: str | None = None) -> None:
+        from .report import export_html
+
+        export_html(self, path, source_pdf=source_pdf)

table_proof/pdf.py

@@ -0,0 +1,28 @@
+"""Read digital PDF words without a model or network request."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+import pdfplumber
+
+from .models import BBox, SourcePage, SourceWord
+
+
+def read_pdf_pages(source_pdf: str | Path) -> dict[int, SourcePage]:
+    pages: dict[int, SourcePage] = {}
+    with pdfplumber.open(source_pdf) as document:
+        for index, page in enumerate(document.pages, 1):
+            left, top, _, _ = page.bbox
+            words = []
+            for word in page.dedupe_chars().extract_words(
+                x_tolerance=2, y_tolerance=2, keep_blank_chars=False, extra_attrs=["size"]
+            ):
+                x0, x1 = float(word["x0"]) - left, float(word["x1"]) - left
+                y0, y1 = float(word["top"]) - top, float(word["bottom"]) - top
+                if x1 > x0 and y1 > y0:
+                    words.append(
+                        SourceWord(text=word["text"], bbox=BBox(x0=x0, top=y0, x1=x1, bottom=y1))
+                    )
+            pages[index] = SourcePage(page=index, width=page.width, height=page.height, words=words)
+    return pages

table_proof/report.py

@@ -0,0 +1,217 @@
+"""Write local reports with escaped text and optional source images."""
+
+from __future__ import annotations
+
+import base64
+import hashlib
+import io
+import json
+from html import escape
+from pathlib import Path
+
+import pdfplumber
+
+from .models import AuditReport
+
+
+def export_json(report: AuditReport, path: str | Path) -> None:
+    data = report.model_dump(mode="json")
+    data["status"] = report.status
+    Path(path).write_text(json.dumps(data, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
+
+
+def _source_images(report: AuditReport, source_pdf: str) -> str:
+    path = Path(source_pdf)
+    with path.open("rb") as file:
+        if hashlib.file_digest(file, "sha256").hexdigest() != report.source_sha256:
+            raise ValueError("The evidence PDF does not match the audited source hash.")
+    pages = sorted({table.page for table in report.tables})
+    blocks: list[str] = []
+    with pdfplumber.open(path) as pdf:
+        for page_no in pages[:10]:
+            if page_no < 1 or page_no > len(pdf.pages):
+                continue
+            page = pdf.pages[page_no - 1]
+            resolution = min(100.0, 1600 * 72 / max(page.width, page.height))
+            img = page.to_image(resolution=resolution, force_mediabox=True).original
+            buffer = io.BytesIO()
+            img.save(buffer, format="PNG")
+            encoded = base64.b64encode(buffer.getvalue()).decode("ascii")
+            boxes = []
+            for table in report.tables:
+                if table.page != page_no:
+                    continue
+                for finding in table.findings:
+                    box = finding.bbox
+                    if box is None:
+                        continue
+                    x = max(0.0, box.x0 / page.width * 100)
+                    y = max(0.0, box.top / page.height * 100)
+                    width = min(100.0 - x, (box.x1 - box.x0) / page.width * 100)
+                    height = min(100.0 - y, (box.bottom - box.top) / page.height * 100)
+                    if width <= 0 or height <= 0:
+                        continue
+                    boxes.append(
+                        f'<span class="box {finding.severity}" '
+                        f'style="left:{x:.4f}%;top:{y:.4f}%;width:{width:.4f}%;'
+                        f'height:{height:.4f}%" title="{escape(finding.message, quote=True)}">'
+                        '<span class="sr-only">Suspected cell</span></span>'
+                    )
+            blocks.append(
+                f'<figure><figcaption>Source page {page_no}</figcaption><div class="page">'
+                f'<img alt="Source PDF page {page_no}" src="data:image/png;base64,{encoded}">'
+                f"{''.join(boxes)}</div></figure>"
+            )
+    if len(pages) > 10:
+        blocks.append("<p>The report shows the first 10 source pages.</p>")
+    return "".join(blocks)
+
+
+def export_html(report: AuditReport, path: str | Path, *, source_pdf: str | None = None) -> None:
+    destination = Path(path)
+    if source_pdf:
+        source = Path(source_pdf)
+        if destination.resolve() == source.resolve() or (
+            destination.exists() and source.exists() and destination.samefile(source)
+        ):
+            raise ValueError("The HTML report must not replace the source PDF.")
+    tables = []
+    for table in report.tables:
+        findings = []
+        for finding in table.findings:
+            values = ""
+            if finding.expected_text is not None:
+                values += f"<dt>Source evidence</dt><dd>{escape(finding.expected_text)}</dd>"
+            if finding.observed_text is not None:
+                values += f"<dt>Extracted text</dt><dd>{escape(finding.observed_text)}</dd>"
+            findings.append(
+                f'<li class="finding {finding.severity}"><strong>{escape(finding.code)}</strong>'
+                f"<p>{escape(finding.message)}</p><dl><dt>Cell</dt>"
+                f"<dd>{escape(finding.cell_id or 'Table')}</dd>{values}"
+                f"<dt>Evidence method</dt><dd>{escape(finding.evidence)}</dd></dl></li>"
+            )
+        tables.append(
+            f'<section><div class="table-title"><h2>{escape(table.table_id)}</h2>'
+            f'<span class="badge">{table.status}</span></div>'
+            f"<p>Page {table.page} · Cells with source evidence: "
+            f"{table.checked_cells} / {table.total_cells}</p>"
+            f'<ul class="findings">{"".join(findings)}</ul></section>'
+        )
+    limitations = "".join(f"<li>{escape(text)}</li>" for text in report.limitations)
+    images = _source_images(report, source_pdf) if source_pdf else ""
+    markup = f"""<!doctype html>
+<html lang="en"><head><meta charset="utf-8">
+<meta name="viewport" content="width=device-width, initial-scale=1">
+<meta http-equiv="Content-Security-Policy"
+content="default-src 'none'; img-src data:; style-src 'unsafe-inline'">
+<title>TableProof · {escape(report.source_name)}</title>
+<style>
+:root {{ color-scheme: light;
+font-family: ui-sans-serif, system-ui, sans-serif;
+color:#182936;
+background:#f2f5f4;
+}}
+body {{ margin:0;
+}} main {{ max-width:1060px;
+margin:auto;
+padding:36px 22px;
+}}
+header {{ border-top:5px solid #0a7368;
+padding:22px 0;
+}}
+.eyebrow {{ color:#0a7368;
+text-transform:uppercase;
+letter-spacing:.14em;
+font-size:12px;
+font-weight:700;
+}}
+h1 {{ font-size:34px;
+margin:10px 0;
+overflow-wrap:anywhere;
+}}
+h2 {{ font-size:19px;
+margin:0;
+overflow-wrap:anywhere;
+}} p {{ line-height:1.6;
+}}
+.badge {{ background:#dceae6;
+color:#174e43;
+border-radius:4px;
+padding:6px 10px;
+font:600 13px monospace;
+}}
+.table-title {{ display:flex;
+justify-content:space-between;
+gap:16px;
+flex-wrap:wrap;
+}}
+section, figure {{ background:white;
+border:1px solid #d4dfdc;
+padding:22px;
+margin:18px 0;
+border-radius:7px;
+}}
+.findings {{ list-style:none;
+margin:0;
+padding:0;
+}} .finding {{ padding:18px;
+margin:12px 0;
+background:#faf5ef;
+border-left:4px solid #ac5a21;
+}}
+.finding.uncertain {{ background:#f3f1e5;
+border-left-color:#877521;
+}}
+dl {{ display:grid;
+grid-template-columns:150px 1fr;
+gap:8px 14px;
+font-size:14px;
+}}
+dt {{ color:#57646c;
+}} dd {{ margin:0;
+white-space:pre-wrap;
+overflow-wrap:anywhere;
+}}
+.page {{ position:relative;
+margin:auto;
+max-width:800px;
+}} img {{ width:100%;
+display:block;
+}}
+.box {{ position:absolute;
+box-sizing:border-box;
+border:2px solid #b64222;
+background:#ff6b3625;
+}}
+.box.uncertain {{ border-color:#9a7b14;
+background:#ffdf3020;
+}}
+figcaption {{ margin-bottom:16px;
+font-weight:600;
+}}
+.sr-only {{ position:absolute;
+width:1px;
+height:1px;
+overflow:hidden;
+clip:rect(0,0,0,0);
+}}
+footer {{ font-size:12px;
+color:#5b6870;
+overflow-wrap:anywhere;
+}}
+@media(max-width:600px) {{ dl {{ grid-template-columns:1fr;
+}} section,figure {{ padding:14px;
+}} h1 {{ font-size:26px;
+}} }}
+</style></head><body><main>
+<header><div class="eyebrow">TableProof / Evidence report</div>
+<h1>{escape(report.source_name)}</h1><span class="badge">{report.status}</span>
+<p>Examine each alert against the original source. A report does not certify the table.</p></header>
+{"".join(tables) or "<section><p>No tables were supplied. Coverage is unknown.</p></section>"}
+{images}<section><h2>Limits</h2><ul>{limitations}</ul></section>
+<footer>TableProof {escape(report.engine_version)}
+· Report schema {escape(report.schema_version)}<br>
+Source SHA-256: {escape(report.source_sha256)}</footer>
+</main></body></html>
+"""
+    Path(path).write_text(markup, encoding="utf-8")