terbium-parse 0.1.0__py3-none-any.whl

terbium/__init__.py

@@ -0,0 +1,31 @@
+"""terbium - a god-level algorithmic multi-file parser that scores its own
+confidence and only reaches for AI when it is genuinely stuck.
+
+    import terbium
+    doc = terbium.parse("catalogue.pdf")
+    print(doc.stats)
+    for r in doc.records:
+        print(r.sku, r.fields)
+"""
+from __future__ import annotations
+
+from .api import parse, supported_extensions, DEFAULT_THRESHOLD
+from .harness import AI
+from .harness.vision import read_page as read_images
+from .model.document import ParsedDocument, Stats
+from .model.record import Record
+from .model.table import ExtractedTable
+
+__version__ = "0.1.0"
+
+__all__ = [
+    "parse",
+    "AI",
+    "read_images",
+    "supported_extensions",
+    "ParsedDocument",
+    "Record",
+    "ExtractedTable",
+    "Stats",
+    "__version__",
+]

terbium/api.py

@@ -0,0 +1,106 @@
+"""``terbium.parse`` - the one function most users call.
+
+Flow: adapt -> assemble tables (native, or reconstructed from PDF geometry) ->
+score confidence -> (optionally) send only the hard tables to AI -> build typed
+records -> if anything is still shaky and no key was given, attach and announce
+an escalation message.
+"""
+from __future__ import annotations
+
+import sys
+from typing import List, Optional
+
+from .documents import get_adapter, supported_extensions
+from .layout import confidence as _confidence
+from .layout import dehead, grid
+from .layout.columns import split_columns
+from .layout.lines import cluster_lines
+from .model.document import ParsedDocument, Stats
+from .model.elements import Page
+from .model.table import ExtractedTable
+from .schema import get_schema
+from .harness import arrange_tables, build_message, resolve
+
+DEFAULT_THRESHOLD = 0.72
+
+
+def _assemble_tables(pages: List[Page]) -> List[ExtractedTable]:
+    tables: List[ExtractedTable] = []
+    pdf_pages = [p for p in pages if p.source_kind == "pdf" and p.words]
+    stripper = dehead.build_stripper(pdf_pages) if pdf_pages else None
+    for p in pages:
+        if p.native_tables:
+            tables.extend(p.native_tables)
+        elif p.source_kind == "pdf" and p.words:
+            for word_group in split_columns(p):
+                lines = cluster_lines(word_group)
+                if stripper:
+                    lines = [ln for ln in lines if not stripper(ln, p)]
+                tables.extend(grid.extract_tables(lines, p))
+    return tables
+
+
+def parse(
+    path: str,
+    schema=None,
+    ai=None,
+    threshold: float = DEFAULT_THRESHOLD,
+    announce: bool = True,
+) -> ParsedDocument:
+    """Parse a PDF/PPTX/XLSX/CSV file into structured, confidence-scored records.
+
+    ``schema``: "generic" (default) or "furniture", or a Schema instance.
+    ``ai``: a ``terbium.AI(...)``, ``True`` (use env keys), or ``None`` (off).
+    ``threshold``: confidence below which a record is "ambiguous".
+    ``announce``: print the escalation message to stderr when AI could help but
+    no key is set. This is terbium telling you it is stuck.
+    """
+    adapter = get_adapter(path)
+    pages = adapter.parse(path)
+    source_kind = pages[0].source_kind if pages else "unknown"
+
+    tables = _assemble_tables(pages)
+    for t in tables:
+        _confidence.score_table(t)
+
+    ai_cfg = resolve(ai)
+    hard = [t for t in tables if t.confidence < threshold]
+    used_ai = False
+    if hard and ai_cfg is not None:
+        fixed = arrange_tables(path, pages, hard, ai_cfg)
+        used_ai = fixed > 0
+        hard = [t for t in tables if t.confidence < threshold]
+
+    schema_obj = get_schema(schema)
+    records = []
+    for t in tables:
+        recs = schema_obj.build_records([t])
+        if t.origin == "ai":
+            for r in recs:
+                r.origin = "ai"
+        records.extend(recs)
+
+    stats = Stats(
+        total=len(records),
+        confident=sum(1 for r in records if r.confidence >= threshold),
+        ambiguous=sum(1 for r in records if r.confidence < threshold),
+        threshold=threshold,
+    )
+    doc = ParsedDocument(
+        path=path,
+        source_kind=source_kind,
+        pages=pages,
+        records=records,
+        stats=stats,
+        used_ai=used_ai,
+    )
+
+    if hard:
+        doc.escalation = build_message(records, hard, threshold)
+        if announce and ai_cfg is None:
+            print(doc.escalation, file=sys.stderr)
+
+    return doc
+
+
+__all__ = ["parse", "supported_extensions", "DEFAULT_THRESHOLD"]

terbium/cli.py

@@ -0,0 +1,53 @@
+"""`terbium <file>` - parse from the command line."""
+from __future__ import annotations
+
+import argparse
+import sys
+
+from . import __version__
+from .api import parse, supported_extensions
+from .harness import AI
+
+
+def main(argv=None) -> int:
+    ap = argparse.ArgumentParser(
+        prog="terbium",
+        description="Algorithmic multi-file parser (PDF/PPTX/XLSX/CSV) that knows when it is stuck.",
+    )
+    ap.add_argument("file", help="path to a " + "/".join(supported_extensions()) + " file")
+    ap.add_argument("--schema", default="generic", help="generic (default) or furniture")
+    ap.add_argument("--json", metavar="OUT", help="write records as JSON to this path (or - for stdout)")
+    ap.add_argument("--ai", action="store_true", help="enable the AI lane using env keys")
+    ap.add_argument("--tier", choices=["haiku", "sonnet", "opus"], help="pin the AI model tier")
+    ap.add_argument("--limit", type=int, default=12, help="how many records to preview")
+    ap.add_argument("--version", action="version", version=f"terbium {__version__}")
+    args = ap.parse_args(argv)
+
+    ai = AI(force_tier=args.tier) if args.ai else None
+    doc = parse(args.file, schema=args.schema, ai=ai)
+
+    if args.json:
+        payload = doc.to_json()
+        if args.json == "-":
+            print(payload)
+        else:
+            with open(args.json, "w", encoding="utf-8") as f:
+                f.write(payload)
+            print(f"wrote {len(doc.records)} records -> {args.json}", file=sys.stderr)
+        return 0
+
+    print(f"terbium {__version__}  ·  {doc.source_kind}  ·  {len(doc.pages)} pages")
+    print(f"records: {doc.stats.total}  (confident {doc.stats.confident}, ambiguous {doc.stats.ambiguous})")
+    if doc.used_ai:
+        print("AI lane: engaged on hard tables")
+    print("-" * 60)
+    for r in doc.records[: args.limit]:
+        flag = "" if r.confidence >= doc.stats.threshold else "  [ambiguous]"
+        print(f"{r.sku or '-':>8}  {r.confidence:.2f}  {r.fields}{flag}")
+    if doc.stats.total > args.limit:
+        print(f"... and {doc.stats.total - args.limit} more")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

terbium/documents/__init__.py

@@ -0,0 +1,8 @@
+"""Importing this package registers every adapter as a side effect."""
+from .base import DocumentAdapter, get_adapter, register, supported_extensions
+from . import pdf as _pdf
+from . import pptx_adapter as _pptx
+from . import xlsx_adapter as _xlsx
+from . import csv_adapter as _csv
+
+__all__ = ["DocumentAdapter", "get_adapter", "register", "supported_extensions"]

terbium/documents/base.py

@@ -0,0 +1,46 @@
+"""Adapter interface + registry. One adapter per file format.
+
+Adapters do exactly one job: turn bytes on disk into normalized ``Page`` objects
+(words with positions, images, and - when the format exposes it natively - ready
+made tables). Everything smart happens after, on that uniform representation.
+"""
+from __future__ import annotations
+
+import os
+from abc import ABC, abstractmethod
+from typing import List
+
+from ..model.elements import Page
+
+
+class DocumentAdapter(ABC):
+    extensions: tuple = ()
+
+    @abstractmethod
+    def parse(self, path: str) -> List[Page]:
+        ...
+
+
+_REGISTRY: dict = {}
+
+
+def register(adapter_cls):
+    """Class decorator: instantiate the adapter and index it by extension."""
+    instance = adapter_cls()
+    for ext in adapter_cls.extensions:
+        _REGISTRY[ext.lower()] = instance
+    return adapter_cls
+
+
+def get_adapter(path: str) -> DocumentAdapter:
+    ext = os.path.splitext(path)[1].lower().lstrip(".")
+    if ext not in _REGISTRY:
+        raise ValueError(
+            f"terbium has no adapter for '.{ext}'. Supported: "
+            + ", ".join(sorted(_REGISTRY)) + "."
+        )
+    return _REGISTRY[ext]
+
+
+def supported_extensions() -> List[str]:
+    return sorted(_REGISTRY)

terbium/documents/csv_adapter.py

@@ -0,0 +1,78 @@
+"""CSV adapter (stdlib).
+
+Sniffs the delimiter and whether a header row exists, tolerates messy encodings,
+and hands back a single native table. The easy case, handled honestly.
+"""
+from __future__ import annotations
+
+import csv as _csv
+from typing import List, Optional
+
+from ..model.elements import Page
+from ..model.table import ExtractedTable
+from .base import DocumentAdapter, register
+
+_MAX_ROWS = 20000
+
+
+def _read_text(path: str) -> str:
+    for enc in ("utf-8-sig", "utf-8", "latin-1"):
+        try:
+            with open(path, "r", encoding=enc, newline="") as f:
+                return f.read()
+        except UnicodeDecodeError:
+            continue
+    with open(path, "r", encoding="latin-1", errors="replace", newline="") as f:
+        return f.read()
+
+
+@register
+class CsvAdapter(DocumentAdapter):
+    extensions = ("csv", "tsv")
+
+    def parse(self, path: str) -> List[Page]:
+        text = _read_text(path)
+        sample = text[:8192]
+        try:
+            dialect = _csv.Sniffer().sniff(sample, delimiters=",;\t|")
+        except _csv.Error:
+            dialect = _csv.excel
+            dialect.delimiter = "\t" if path.lower().endswith(".tsv") else ","
+        try:
+            has_header = _csv.Sniffer().has_header(sample)
+        except _csv.Error:
+            has_header = True
+
+        rows: List[List[str]] = []
+        for i, row in enumerate(_csv.reader(text.splitlines(), dialect)):
+            if i >= _MAX_ROWS:
+                break
+            rows.append([c.strip() for c in row])
+        rows = [r for r in rows if any(c for c in r)]
+        table = _rows_to_table(rows, has_header)
+        page = Page(index=0, width=0, height=0, source_kind="csv", native_tables=[table] if table else [])
+        return [page]
+
+
+def _rows_to_table(rows: List[List[str]], has_header: bool) -> Optional[ExtractedTable]:
+    if not rows:
+        return None
+    ncol = max(len(r) for r in rows)
+    rows = [r + [""] * (ncol - len(r)) for r in rows]
+    if has_header:
+        header = rows[0]
+        body = rows[1:]
+        col_headers = header[1:] if ncol > 1 else header
+    else:
+        body = rows
+        col_headers = [f"col{i + 1}" for i in range(ncol - 1 if ncol > 1 else ncol)]
+    row_headers = [r[0] for r in body] if ncol > 1 else ["" for _ in body]
+    cells = [[(v or None) for v in (r[1:] if ncol > 1 else r)] for r in body]
+    return ExtractedTable(
+        title=None,
+        row_headers=row_headers,
+        col_headers=[h or f"col{i + 1}" for i, h in enumerate(col_headers)],
+        cells=cells,
+        source_page=0,
+        kind="grid",
+    )

terbium/documents/pdf.py

@@ -0,0 +1,110 @@
+"""PDF adapter (PyMuPDF).
+
+Produces token-level words with positions AND font sizes - the two inputs the
+geometry engine needs to rebuild columns, rows, and titles. It also enumerates
+embedded images with pixel dimensions so they can be classified, and can render
+a page to PNG for the vision lane.
+"""
+from __future__ import annotations
+
+from typing import List
+
+import fitz  # PyMuPDF
+
+from ..layout.images import classify
+from ..model.elements import ImageRef, Page, Word
+from .base import DocumentAdapter, register
+
+_BOLD_FLAG = 1 << 4
+
+
+def _strip_pua(text: str) -> str:
+    """Drop private-use glyphs (U+E000..U+F8FF).
+
+    Catalogues encode material icons (FSC, oiled, varnished) as private-use font
+    glyphs that leak into extracted text as noise like 'FSC\\ue514'. The icons
+    themselves belong to the vision lane, not the text.
+    """
+    return "".join(c for c in text if not (0xE000 <= ord(c) <= 0xF8FF))
+
+
+def _span_words(span: dict) -> List[Word]:
+    """Split a span into whitespace tokens, distributing x by character offset.
+
+    Span geometry is exact; per-token x is interpolated across the span box.
+    Because catalogue article numbers are fixed-width digits, this places SKU
+    cells accurately enough to align them into columns.
+    """
+    text = _strip_pua(span.get("text", ""))
+    if not text.strip():
+        return []
+    x0, y0, x1, y1 = span["bbox"]
+    size = float(span.get("size", 0.0))
+    font = str(span.get("font", "")).lower()
+    bold = bool(span.get("flags", 0) & _BOLD_FLAG) or "bold" in font
+    width = x1 - x0
+    n = len(text)
+    words: List[Word] = []
+    idx = 0
+    for tok in text.split():
+        start = text.index(tok, idx)
+        end = start + len(tok)
+        idx = end
+        wx0 = x0 + width * (start / n) if n else x0
+        wx1 = x0 + width * (end / n) if n else x1
+        words.append(Word(text=tok, x0=wx0, y0=y0, x1=wx1, y1=y1, size=size, bold=bold))
+    return words
+
+
+@register
+class PdfAdapter(DocumentAdapter):
+    extensions = ("pdf",)
+
+    def parse(self, path: str) -> List[Page]:
+        pages: List[Page] = []
+        with fitz.open(path) as doc:
+            for i, page in enumerate(doc):
+                rect = page.rect
+                words: List[Word] = []
+                data = page.get_text("dict")
+                for block in data.get("blocks", []):
+                    for line in block.get("lines", []):
+                        for span in line.get("spans", []):
+                            words.extend(_span_words(span))
+                images: List[ImageRef] = []
+                try:
+                    for info in page.get_image_info(xrefs=True):
+                        w = int(info.get("width", 0))
+                        h = int(info.get("height", 0))
+                        if w and h:
+                            images.append(
+                                ImageRef(
+                                    page=i,
+                                    width=w,
+                                    height=h,
+                                    kind=classify(w, h),
+                                    bbox=tuple(info.get("bbox")) if info.get("bbox") else None,
+                                    xref=info.get("xref"),
+                                )
+                            )
+                except Exception:
+                    pass
+                pages.append(
+                    Page(
+                        index=i,
+                        width=rect.width,
+                        height=rect.height,
+                        words=words,
+                        images=images,
+                        source_kind="pdf",
+                    )
+                )
+        return pages
+
+
+def render_page_png(path: str, index: int, dpi: int = 120) -> bytes:
+    """Render one page to PNG bytes (used by the vision lane)."""
+    with fitz.open(path) as doc:
+        page = doc[index]
+        pix = page.get_pixmap(dpi=dpi)
+        return pix.tobytes("png")

terbium/documents/pptx_adapter.py

@@ -0,0 +1,129 @@
+"""PPTX adapter (python-pptx).
+
+PowerPoint is a gift compared to PDF: shapes carry positions, tables are native
+(rows and cells, no reconstruction), and pictures expose real pixels. So we hand
+tables straight through as high-confidence ``native_tables`` and only fall back
+to geometry for free-floating text boxes.
+"""
+from __future__ import annotations
+
+import io
+from typing import List, Optional
+
+from pptx import Presentation
+from pptx.enum.shapes import MSO_SHAPE_TYPE
+
+from ..layout.images import classify
+from ..model.elements import ImageRef, Page, Word
+from ..model.table import ExtractedTable
+from .base import DocumentAdapter, register
+
+_EMU_PER_PT = 12700.0
+
+
+def _pt(emu) -> float:
+    return float(emu) / _EMU_PER_PT if emu is not None else 0.0
+
+
+def _tokens_in_box(text: str, x0: float, y: float, x1: float, size: float, bold: bool) -> List[Word]:
+    if not text.strip():
+        return []
+    width = max(1.0, x1 - x0)
+    n = len(text)
+    out: List[Word] = []
+    idx = 0
+    for tok in text.split():
+        start = text.index(tok, idx)
+        end = start + len(tok)
+        idx = end
+        wx0 = x0 + width * (start / n)
+        wx1 = x0 + width * (end / n)
+        out.append(Word(text=tok, x0=wx0, y0=y, x1=wx1, y1=y + max(size, 8), size=size, bold=bold))
+    return out
+
+
+def _slide_title(slide) -> Optional[str]:
+    try:
+        if slide.shapes.title is not None and slide.shapes.title.text.strip():
+            return slide.shapes.title.text.strip()
+    except Exception:
+        pass
+    return None
+
+
+def _table_to_extracted(shape, page_index: int, title: Optional[str]) -> ExtractedTable:
+    tbl = shape.table
+    rows = list(tbl.rows)
+    grid = [[cell.text.strip() for cell in row.cells] for row in rows]
+    if not grid:
+        return ExtractedTable(title, [], [], [], page_index, kind="grid")
+    col_headers = grid[0]
+    body = grid[1:] if len(grid) > 1 else []
+    row_headers = [r[0] if r else "" for r in body]
+    cells = [[(v if v else None) for v in (r[1:] if len(r) > 1 else r)] for r in body]
+    col_headers = col_headers[1:] if len(col_headers) > 1 else col_headers
+    return ExtractedTable(
+        title=title,
+        row_headers=row_headers,
+        col_headers=col_headers,
+        cells=cells if cells else [[v or None for v in r] for r in body],
+        source_page=page_index,
+        kind="grid",
+    )
+
+
+@register
+class PptxAdapter(DocumentAdapter):
+    extensions = ("pptx",)
+
+    def parse(self, path: str) -> List[Page]:
+        prs = Presentation(path)
+        sw, sh = _pt(prs.slide_width), _pt(prs.slide_height)
+        pages: List[Page] = []
+        for i, slide in enumerate(prs.slides):
+            words: List[Word] = []
+            images: List[ImageRef] = []
+            native: List[ExtractedTable] = []
+            title = _slide_title(slide)
+            for shape in slide.shapes:
+                x0, y0 = _pt(shape.left), _pt(shape.top)
+                x1 = x0 + _pt(shape.width)
+                if shape.has_table:
+                    native.append(_table_to_extracted(shape, i, title))
+                    continue
+                if shape.has_text_frame:
+                    y = y0
+                    for para in shape.text_frame.paragraphs:
+                        text = "".join(run.text for run in para.runs) or para.text
+                        size = 0.0
+                        bold = False
+                        for run in para.runs:
+                            if run.font.size is not None:
+                                size = run.font.size.pt
+                            if run.font.bold:
+                                bold = True
+                        words.extend(_tokens_in_box(text, x0, y, x1, size, bold))
+                        y += max(size, 14) * 1.2
+                    continue
+                if shape.shape_type == MSO_SHAPE_TYPE.PICTURE:
+                    try:
+                        blob = shape.image.blob
+                        from PIL import Image
+
+                        with Image.open(io.BytesIO(blob)) as im:
+                            w, h = im.size
+                        images.append(ImageRef(page=i, width=w, height=h, kind=classify(w, h)))
+                    except Exception:
+                        pass
+            pages.append(
+                Page(
+                    index=i,
+                    width=sw,
+                    height=sh,
+                    words=words,
+                    images=images,
+                    source_kind="pptx",
+                    native_tables=native,
+                )
+            )
+        return pages

terbium/documents/xlsx_adapter.py

@@ -0,0 +1,88 @@
+"""XLSX adapter (openpyxl).
+
+Spreadsheets are already a grid, so the work is mostly faithful transcription:
+one sheet -> one page, merged header cells propagated across their span so
+multi-column headers survive, values coerced to strings.
+"""
+from __future__ import annotations
+
+from typing import List, Optional
+
+from openpyxl import load_workbook
+
+from ..model.elements import Page
+from ..model.table import ExtractedTable
+from .base import DocumentAdapter, register
+
+_MAX_ROWS = 5000
+_MAX_COLS = 200
+
+
+def _s(v) -> Optional[str]:
+    if v is None:
+        return None
+    s = str(v).strip()
+    return s or None
+
+
+@register
+class XlsxAdapter(DocumentAdapter):
+    extensions = ("xlsx", "xlsm")
+
+    def parse(self, path: str) -> List[Page]:
+        wb = load_workbook(path, data_only=True)
+        pages: List[Page] = []
+        for si, ws in enumerate(wb.worksheets):
+            max_row = min(ws.max_row or 0, _MAX_ROWS)
+            max_col = min(ws.max_column or 0, _MAX_COLS)
+            if max_row == 0 or max_col == 0:
+                pages.append(Page(index=si, width=0, height=0, source_kind="xlsx"))
+                continue
+            grid = [[_s(ws.cell(r, c).value) for c in range(1, max_col + 1)] for r in range(1, max_row + 1)]
+            # propagate merged header/label cells across their range
+            for rng in ws.merged_cells.ranges:
+                tl = _s(ws.cell(rng.min_row, rng.min_col).value)
+                for r in range(rng.min_row, min(rng.max_row, max_row) + 1):
+                    for c in range(rng.min_col, min(rng.max_col, max_col) + 1):
+                        grid[r - 1][c - 1] = tl
+            grid = _trim(grid)
+            table = _grid_to_table(grid, si, title=ws.title)
+            pages.append(
+                Page(index=si, width=0, height=0, source_kind="xlsx", native_tables=[table] if table else [])
+            )
+        return pages
+
+
+def _trim(grid: List[List[Optional[str]]]) -> List[List[Optional[str]]]:
+    while grid and all(v is None for v in grid[-1]):
+        grid.pop()
+    while grid and all(v is None for v in grid[0]):
+        grid.pop(0)
+    if not grid:
+        return grid
+    ncol = len(grid[0])
+    last = 0
+    for row in grid:
+        for c in range(ncol - 1, -1, -1):
+            if c < len(row) and row[c] is not None:
+                last = max(last, c)
+                break
+    return [row[: last + 1] for row in grid]
+
+
+def _grid_to_table(grid: List[List[Optional[str]]], page_index: int, title: Optional[str]) -> Optional[ExtractedTable]:
+    if not grid:
+        return None
+    header = grid[0]
+    body = grid[1:]
+    col_headers = header[1:] if len(header) > 1 else header
+    row_headers = [(r[0] if r else "") or "" for r in body]
+    cells = [r[1:] if len(r) > 1 else r for r in body]
+    return ExtractedTable(
+        title=title,
+        row_headers=row_headers,
+        col_headers=[h or f"col{i + 1}" for i, h in enumerate(col_headers)],
+        cells=cells if cells else [[v for v in r] for r in body],
+        source_page=page_index,
+        kind="grid",
+    )

terbium/harness/__init__.py

@@ -0,0 +1,7 @@
+from .ai import AI, resolve
+from .escalation import build_message
+from .arrange import arrange_tables
+from .vision import read_page
+from . import router
+
+__all__ = ["AI", "resolve", "build_message", "arrange_tables", "read_page", "router"]