tablecodec 0.0.18__py3-none-any.whl

tablecodec/codecs/_otslgrid.py

@@ -0,0 +1,318 @@
+"""Shared OTSL grid machinery for codec implementations.
+
+OTSL (Lysak et al., ICDAR 2023, arXiv 2305.03393) and the DocTags table
+subset (IBM Granite-Docling) both encode table structure with the same
+five-token cell vocabulary plus a row separator:
+
+- ``fcel`` filled-cell anchor
+- ``ecel`` empty-cell anchor
+- ``lcel`` left-merged continuation (extends the anchor's colspan)
+- ``ucel`` up-merged continuation (extends the anchor's rowspan)
+- ``xcel`` cross-merged continuation (extends both)
+- ``nl``   newline / row separator
+
+This module owns the structure↔grid conversion so OTSL and DocTags do
+not duplicate it. DocTags additionally interleaves location and content
+tokens, which it strips before calling :func:`build_anchors`, and
+re-inserts when serialising from :func:`build_token_grid`.
+
+The grid-reconstruction logic in :func:`build_anchors` (the anchor-centric
+scan, the ``check_right``/``check_down`` span runs, and the 2D-span
+registry) is adapted from docling-ibm-models' ``otsl_to_html``:
+
+    https://github.com/docling-project/docling-ibm-models
+    docling_ibm_models/tableformer/otsl.py
+    Copyright (c) 2024 International Business Machines
+    Licensed under the MIT License.
+
+It is reimplemented here for the neutral IR (it emits ``GridCell`` spans
+rather than HTML strings) and carries no third-party imports. See
+THIRD_PARTY_NOTICES.md and docs/adr/0005-port-otsl-reconstruction.md.
+
+Stdlib-only (SPEC §13).
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import cast
+
+from tablecodec.ir import GridCell, TableSample
+
+__all__ = [
+    "ANCHOR_TOKENS",
+    "CELL_TOKENS",
+    "CONTINUATION_TOKENS",
+    "VALID_TOKENS",
+    "AnchorPlacement",
+    "build_anchors",
+    "build_token_grid",
+    "cells_to_otsl",
+    "ensure_square",
+    "otsl_to_cells",
+    "split_rows",
+]
+
+ANCHOR_TOKENS = frozenset({"fcel", "ecel"})
+CONTINUATION_TOKENS = frozenset({"lcel", "ucel", "xcel"})
+CELL_TOKENS = ANCHOR_TOKENS | CONTINUATION_TOKENS
+VALID_TOKENS = CELL_TOKENS | {"nl"}
+
+
+@dataclass(slots=True)
+class AnchorPlacement:
+    """One ``fcel`` / ``ecel`` anchor mapped to its grid coordinates."""
+
+    row: int
+    col: int
+    rowspan: int = 1
+    colspan: int = 1
+    is_empty: bool = False
+
+
+def split_rows(tokens: list[str]) -> list[list[str]]:
+    """Split a flat cell-token stream on ``nl`` into per-row token lists.
+
+    Rejects any token outside :data:`VALID_TOKENS`. A trailing ``nl`` does
+    not produce an empty final row.
+    """
+    rows: list[list[str]] = [[]]
+    for tok in tokens:
+        if tok not in VALID_TOKENS:
+            msg = f"unknown OTSL token {tok!r}"
+            raise ValueError(msg)
+        if tok == "nl":
+            rows.append([])
+        else:
+            rows[-1].append(tok)
+    if rows and not rows[-1]:
+        rows.pop()
+    return rows
+
+
+def ensure_square(rows: list[list[str]]) -> int:
+    """Return the common row width, or raise if rows are jagged."""
+    if not rows:
+        return 0
+    widths = {len(r) for r in rows}
+    if len(widths) != 1:
+        msg = f"OTSL square-table assumption violated; row widths = {sorted(widths)}"
+        raise ValueError(msg)
+    return next(iter(widths))
+
+
+def _normalize_edge_continuations(rows: list[list[str]], nrows: int, ncols: int) -> list[list[str]]:
+    """Repair structurally-impossible continuations at the grid edges.
+
+    A continuation cannot merge in a direction that has no neighbour:
+    row 0 has nothing above, column 0 has nothing to the left. Real
+    encoders (and the docling OTSL decoder's "structure error correction")
+    emit ``xcel``/``ucel`` in row 0 and ``xcel``/``lcel`` in column 0 that
+    must be read as the only possible merge:
+
+    - row 0: ``ucel``/``xcel`` -> ``lcel`` (can only merge left).
+    - col 0: ``lcel``/``xcel`` -> ``ucel`` (can only merge up).
+
+    A copy is returned; the caller's rows are not mutated.
+    """
+    grid = [list(row) for row in rows]
+    for c in range(ncols):
+        if grid[0][c] in {"ucel", "xcel"}:
+            grid[0][c] = "lcel"
+    for r in range(nrows):
+        if grid[r][0] in {"lcel", "xcel"}:
+            grid[r][0] = "ucel"
+    return grid
+
+
+@dataclass(slots=True)
+class _OtslReader:
+    """Anchor-centric OTSL grid reader (logic adapted from docling, see header).
+
+    ``registry`` mirrors docling's ``registry_2d_span``: cells already
+    claimed by a 2D (``xcel``) span, so a later anchor cannot re-claim them.
+    """
+
+    grid: list[list[str]]
+    nrows: int
+    ncols: int
+    registry: list[list[bool]]
+
+    def _check_right(self, r: int, c: int) -> int:
+        # colspan: extend right over horizontal continuations (lcel/xcel);
+        # stop at an anchor, an up-merge, or the edge (docling check_right).
+        # Also stop at a cell already claimed by a 2D span above: an `xcel`
+        # there belongs to that span, not to this row's run — counting it
+        # would overlap (the irregular case real SynthTabNet rows hit).
+        dist = 1
+        x = c
+        while (
+            x + 1 < self.ncols
+            and self.grid[r][x + 1] in {"lcel", "xcel"}
+            and not self.registry[r][x + 1]
+        ):
+            x += 1
+            dist += 1
+        return dist
+
+    def _check_down(self, r: int, c: int) -> int:
+        # rowspan: extend down over vertical continuations (ucel/xcel);
+        # stop at an anchor, a left-merge, the edge, or a cell already
+        # claimed by another 2D span (symmetric to _check_right).
+        dist = 1
+        y = r
+        while (
+            y + 1 < self.nrows
+            and self.grid[y + 1][c] in {"ucel", "xcel"}
+            and not self.registry[y + 1][c]
+        ):
+            y += 1
+            dist += 1
+        return dist
+
+    def _claim_2d(self, r: int, c: int, rowspan: int, colspan: int) -> bool:
+        # Mark the rectangle in the registry iff none of it is already
+        # claimed (docling's double-count guard); returns whether it claimed.
+        for dr in range(rowspan):
+            for dc in range(colspan):
+                if self.registry[r + dr][c + dc]:
+                    return False
+        for dr in range(rowspan):
+            for dc in range(colspan):
+                self.registry[r + dr][c + dc] = True
+        return True
+
+    def span_of(self, r: int, c: int) -> tuple[int, int]:
+        """Compute (rowspan, colspan) for the anchor at (r, c) from its neighbours."""
+        colspan = rowspan = 1
+        right = self.grid[r][c + 1] if c + 1 < self.ncols else ""
+        below = self.grid[r + 1][c] if r + 1 < self.nrows else ""
+        if right == "lcel":
+            colspan = self._check_right(r, c)
+        if below == "ucel":
+            rowspan = self._check_down(r, c)
+        if right == "xcel":
+            xr = self._check_right(r, c)
+            xd = self._check_down(r, c)
+            if self._claim_2d(r, c, xd, xr):
+                colspan, rowspan = xr, xd
+        return rowspan, colspan
+
+
+def build_anchors(rows: list[list[str]]) -> tuple[int, int, list[AnchorPlacement]]:
+    """Walk the row × col grid; return (nrows, ncols, ordered anchors).
+
+    Anchor-centric reconstruction (adapted from docling's ``otsl_to_html``,
+    see the module header): each ``fcel``/``ecel`` is an anchor whose span
+    is read from its neighbouring continuation tokens — a right ``lcel`` run
+    gives colspan, a below ``ucel`` run gives rowspan, and a right ``xcel``
+    gives a 2D span guarded by the registry against double-claiming.
+    Continuation tokens carry no content and are skipped; anchors are
+    returned in row-major order — the order the source ``cells[]`` appear in.
+    """
+    nrows = len(rows)
+    ncols = ensure_square(rows)
+    reader = _OtslReader(
+        grid=_normalize_edge_continuations(rows, nrows, ncols),
+        nrows=nrows,
+        ncols=ncols,
+        registry=[[False] * ncols for _ in range(nrows)],
+    )
+    ordered: list[AnchorPlacement] = []
+    for r in range(nrows):
+        for c in range(ncols):
+            tok = reader.grid[r][c]
+            if tok not in ANCHOR_TOKENS:
+                continue
+            rowspan, colspan = reader.span_of(r, c)
+            ordered.append(
+                AnchorPlacement(
+                    row=r, col=c, rowspan=rowspan, colspan=colspan, is_empty=(tok == "ecel")
+                )
+            )
+    return nrows, ncols, ordered
+
+
+def build_token_grid(sample: TableSample) -> tuple[list[list[str]], list[GridCell]]:
+    """Lay a sample's cells onto a 2D token grid.
+
+    Returns ``(grid, anchors)`` where ``grid[r][c]`` is one of the five
+    cell tokens and ``anchors`` is the row-major list of the anchor cells
+    (so callers can attach per-cell content / bbox in the right order).
+    A cell with empty ``tokens`` becomes ``ecel``; otherwise ``fcel``.
+    """
+    grid: list[list[str]] = [[""] * sample.ncols for _ in range(sample.nrows)]
+    anchored = sorted(sample.cells, key=lambda c: (c.row, c.col))
+    for cell in anchored:
+        grid[cell.row][cell.col] = "ecel" if not cell.tokens else "fcel"
+        for dr in range(cell.rowspan):
+            for dc in range(cell.colspan):
+                if dr == 0 and dc == 0:
+                    continue
+                rr, cc = cell.row + dr, cell.col + dc
+                if dr == 0:
+                    grid[rr][cc] = "lcel"
+                elif dc == 0:
+                    grid[rr][cc] = "ucel"
+                else:
+                    grid[rr][cc] = "xcel"
+    return grid, anchored
+
+
+# ---------- OTSL payload <-> GridCells ----------
+
+
+def otsl_to_cells(
+    otsl_tokens: list[str], cell_payloads: list[dict[str, object]]
+) -> tuple[int, int, tuple[GridCell, ...]]:
+    """Map an OTSL token stream + positional ``cells[]`` to GridCells.
+
+    Returns ``(nrows, ncols, cells)``. Every cell defaults to
+    ``role="body"`` (the OTSL core has no header marker). Raises if the
+    anchor count and ``cells[]`` length disagree.
+    """
+    rows = split_rows(otsl_tokens)
+    nrows, ncols, anchors = build_anchors(rows)
+    if len(anchors) != len(cell_payloads):
+        msg = (
+            f"OTSL declares {len(anchors)} anchored cells but cells[] has "
+            f"{len(cell_payloads)} entries"
+        )
+        raise ValueError(msg)
+
+    cells: list[GridCell] = []
+    for anchor, cell_payload in zip(anchors, cell_payloads, strict=True):
+        tokens = tuple(cast("tuple[str, ...]", cell_payload.get("tokens", ())))
+        bbox_raw = cell_payload.get("bbox")
+        bbox = None
+        if bbox_raw is not None:
+            seq = cast("list[int]", bbox_raw)
+            bbox = (int(seq[0]), int(seq[1]), int(seq[2]), int(seq[3]))
+        cells.append(
+            GridCell(
+                row=anchor.row,
+                col=anchor.col,
+                rowspan=anchor.rowspan,
+                colspan=anchor.colspan,
+                tokens=tokens,
+                bbox=bbox,
+                role="body",
+            )
+        )
+    return nrows, ncols, tuple(cells)
+
+
+def cells_to_otsl(sample: TableSample) -> tuple[list[str], list[dict[str, object]]]:
+    """Serialize a sample to an OTSL token stream + positional ``cells[]``."""
+    grid, emitted_order = build_token_grid(sample)
+    tokens: list[str] = []
+    for row in grid:
+        tokens.extend(row)
+        tokens.append("nl")
+    cell_payloads: list[dict[str, object]] = []
+    for cell in emitted_order:
+        payload: dict[str, object] = {"tokens": list(cell.tokens)}
+        if cell.bbox is not None:
+            payload["bbox"] = list(cell.bbox)
+        cell_payloads.append(payload)
+    return tokens, cell_payloads

tablecodec/codecs/builtins.py

@@ -0,0 +1,36 @@
+"""The built-in codec instances, as a single source of truth.
+
+The library itself does not auto-register codecs (callers register the
+ones they need). But the CLI and the documentation generators all need
+"every codec that ships with tablecodec" — keeping that list in one
+place avoids the drift where a new codec is added to some consumers but
+not others.
+
+Order is deterministic and shapes the rendered doc tables.
+"""
+
+from __future__ import annotations
+
+from tablecodec.codecs._base import Codec
+from tablecodec.codecs.doctags import DocTagsTablesCodec
+from tablecodec.codecs.fintabnet import FinTabNetCodec
+from tablecodec.codecs.fintabnet_otsl import FinTabNetOTSLCodec
+from tablecodec.codecs.otsl import OTSL10Codec
+from tablecodec.codecs.pubtables1m import PubTables1MCodec
+from tablecodec.codecs.pubtabnet import PubTabNet10Codec, PubTabNet20Codec
+from tablecodec.codecs.tablebank import TableBankCodec
+from tablecodec.codecs.tableformer import TableFormerCodec
+
+__all__ = ["BUILTIN_CODECS"]
+
+BUILTIN_CODECS: tuple[Codec, ...] = (
+    PubTabNet10Codec(),
+    PubTabNet20Codec(),
+    FinTabNetCodec(),
+    TableFormerCodec(),
+    TableBankCodec(),
+    PubTables1MCodec(),
+    OTSL10Codec(),
+    FinTabNetOTSLCodec(),
+    DocTagsTablesCodec(),
+)

tablecodec/codecs/doctags.py

@@ -0,0 +1,278 @@
+"""DocTags table subset codec.
+
+DocTags (IBM Granite-Docling, 2026) is a document markup where tables are
+encoded as OTSL cell tokens wrapped in ``<otsl>`` ... ``</otsl>``. Each
+anchor cell is annotated with four ``<loc_n>`` tokens — a bounding box on
+a fixed 0–500 grid — followed by the cell's content tokens. Continuation
+tokens (``lcel`` / ``ucel`` / ``xcel``) carry neither loc nor content.
+
+This codec handles the **table subset** of DocTags:
+
+- read: full — structure (via the shared :mod:`._otslgrid`), bbox (from
+  the loc tokens), and cell content.
+- write: the OTSL-equivalent subset only (SPEC §7 marks this △). Header
+  / body ``role`` has no representation in the OTSL core, so it is lost
+  (``lossy_write`` declares ``role``); ``extras`` is also dropped.
+
+Canonical jsonl record shape::
+
+    {
+        "filename": "...",
+        "split": "train" | "val" | "test",  # optional
+        "imgid": 0,  # optional
+        "doctags": [
+            "<otsl>",
+            "fcel",
+            "<loc_0>",
+            "<loc_0>",
+            "<loc_250>",
+            "<loc_50>",
+            "Year",
+            ...,
+            "nl",
+            "</otsl>",
+        ],
+    }
+
+Derived from the published DocTags / OTSL description, not copied from
+upstream reference code.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+from collections.abc import Iterable, Iterator
+from dataclasses import dataclass, field
+from typing import IO, Any, cast
+
+from tablecodec.codecs._otslgrid import (
+    ANCHOR_TOKENS,
+    CELL_TOKENS,
+    build_anchors,
+    build_token_grid,
+)
+from tablecodec.ir import BBox, GridCell, TableSample
+
+__all__ = ["DocTagsTablesCodec"]
+
+_OTSL_OPEN = "<otsl>"
+_OTSL_CLOSE = "</otsl>"
+_LOC_RE = re.compile(r"^<loc_(\d+)>$")
+_LOC_PER_BBOX = 4
+
+
+@dataclass(frozen=True, slots=True)
+class DocTagsTablesCodec:
+    """Codec for the DocTags table subset (read full, write OTSL subset)."""
+
+    name: str = "doctags-tables"
+    spec_version: str = "1.0.0"
+    media_type: str = "application/jsonl"
+    writable: bool = True
+
+    def read(self, source: IO[str]) -> Iterator[TableSample]:
+        for line_no, raw in enumerate(source, start=1):
+            line = raw.strip()
+            if not line:
+                continue
+            try:
+                payload: dict[str, Any] = json.loads(line)
+            except json.JSONDecodeError as exc:
+                msg = f"invalid JSON at line {line_no}: {exc.msg}"
+                raise ValueError(msg) from exc
+            try:
+                yield _payload_to_sample(payload)
+            except (KeyError, ValueError, TypeError) as exc:
+                msg = f"malformed DocTags record at line {line_no}: {exc}"
+                raise ValueError(msg) from exc
+
+    def write(self, samples: Iterable[TableSample], sink: IO[str]) -> None:
+        for sample in samples:
+            sink.write(json.dumps(_sample_to_payload(sample), ensure_ascii=False))
+            sink.write("\n")
+
+    def lossy_read(self) -> frozenset[str]:
+        # The OTSL core has no header marker; role defaults to body.
+        return frozenset({"role"})
+
+    def lossy_write(self) -> frozenset[str]:
+        return frozenset({"extras", "role"})
+
+    def sniff(self, source: IO[str]) -> bool:
+        pos = source.tell()
+        try:
+            for raw in source:
+                line = raw.strip()
+                if not line:
+                    continue
+                try:
+                    payload: object = json.loads(line)
+                except json.JSONDecodeError:
+                    return False
+                return isinstance(payload, dict) and "doctags" in cast("dict[str, Any]", payload)
+            return False
+        finally:
+            source.seek(pos)
+
+
+# ---------- DocTags token stream parsing ----------
+
+
+@dataclass(slots=True)
+class _ParsedCell:
+    """A structural cell token plus, for anchors, its bbox and content."""
+
+    token: str
+    bbox: BBox | None = None
+    content: tuple[str, ...] = ()
+
+
+def _initial_rows() -> list[list[str]]:
+    return [[]]
+
+
+def _empty_parsed_cells() -> list[_ParsedCell]:
+    return []
+
+
+@dataclass(slots=True)
+class _StreamState:
+    rows: list[list[str]] = field(default_factory=_initial_rows)
+    anchors: list[_ParsedCell] = field(default_factory=_empty_parsed_cells)
+
+
+def _parse_loc_run(tokens: list[str], start: int) -> tuple[BBox | None, int]:
+    """Read up to four consecutive ``<loc_n>`` tokens from *start*.
+
+    Returns (bbox or None, index after the loc run).
+    """
+    coords: list[int] = []
+    j = start
+    while j < len(tokens) and len(coords) < _LOC_PER_BBOX:
+        m = _LOC_RE.match(tokens[j])
+        if m is None:
+            break
+        coords.append(int(m.group(1)))
+        j += 1
+    if len(coords) == _LOC_PER_BBOX:
+        return (coords[0], coords[1], coords[2], coords[3]), j
+    return None, start  # not a full bbox; leave tokens for content
+
+
+def _parse_content_run(tokens: list[str], start: int) -> tuple[tuple[str, ...], int]:
+    """Read content tokens until the next structural / loc / nl / wrapper token."""
+    content: list[str] = []
+    j = start
+    while j < len(tokens):
+        tok = tokens[j]
+        if tok in CELL_TOKENS or tok == "nl" or tok in (_OTSL_OPEN, _OTSL_CLOSE):
+            break
+        if _LOC_RE.match(tok) is not None:
+            break
+        content.append(tok)
+        j += 1
+    return tuple(content), j
+
+
+def _parse_doctags_stream(tokens: list[str]) -> _StreamState:
+    state = _StreamState()
+    i = 0
+    while i < len(tokens):
+        tok = tokens[i]
+        if tok in (_OTSL_OPEN, _OTSL_CLOSE):
+            i += 1
+        elif tok == "nl":
+            state.rows.append([])
+            i += 1
+        elif tok in CELL_TOKENS:
+            state.rows[-1].append(tok)
+            if tok in ANCHOR_TOKENS:
+                bbox, after_loc = _parse_loc_run(tokens, i + 1)
+                content, after_content = _parse_content_run(tokens, after_loc)
+                state.anchors.append(_ParsedCell(token=tok, bbox=bbox, content=content))
+                i = after_content
+            else:
+                i += 1
+        else:
+            msg = f"unexpected DocTags token {tok!r}"
+            raise ValueError(msg)
+    if state.rows and not state.rows[-1]:
+        state.rows.pop()
+    return state
+
+
+def _payload_to_sample(payload: dict[str, Any]) -> TableSample:
+    tokens = list(payload["doctags"])
+    state = _parse_doctags_stream(tokens)
+    _nrows, _ncols, placements = build_anchors(state.rows)
+
+    if len(placements) != len(state.anchors):
+        msg = (
+            f"DocTags declares {len(placements)} anchors but the stream parsed "
+            f"{len(state.anchors)} cell contents"
+        )
+        raise ValueError(msg)
+
+    cells = tuple(
+        GridCell(
+            row=placement.row,
+            col=placement.col,
+            rowspan=placement.rowspan,
+            colspan=placement.colspan,
+            tokens=parsed.content,
+            bbox=parsed.bbox,
+            role="body",
+        )
+        for placement, parsed in zip(placements, state.anchors, strict=True)
+    )
+
+    return TableSample(
+        filename=str(payload["filename"]),
+        nrows=_nrows,
+        ncols=_ncols,
+        cells=cells,
+        split=_normalize_split(payload.get("split")),
+        imgid=payload.get("imgid"),
+    )
+
+
+def _normalize_split(value: object) -> Any:
+    if value in ("train", "val", "test"):
+        return value
+    if value is None:
+        return None
+    msg = f"unknown split value {value!r}"
+    raise ValueError(msg)
+
+
+# ---------- IR → DocTags ----------
+
+
+def _loc_tokens(bbox: BBox) -> list[str]:
+    return [f"<loc_{v}>" for v in bbox]
+
+
+def _sample_to_payload(sample: TableSample) -> dict[str, Any]:
+    grid, anchored = build_token_grid(sample)
+    by_pos = {(c.row, c.col): c for c in anchored}
+
+    tokens: list[str] = [_OTSL_OPEN]
+    for r, row in enumerate(grid):
+        for c, structural in enumerate(row):
+            tokens.append(structural)
+            cell = by_pos.get((r, c))
+            if cell is None:
+                continue  # continuation token: no loc/content
+            if cell.bbox is not None:
+                tokens.extend(_loc_tokens(cell.bbox))
+            tokens.extend(cell.tokens)
+        tokens.append("nl")
+    tokens.append(_OTSL_CLOSE)
+
+    out: dict[str, Any] = {"filename": sample.filename, "doctags": tokens}
+    if sample.split is not None:
+        out["split"] = sample.split
+    if sample.imgid is not None:
+        out["imgid"] = sample.imgid
+    return out