tablecodec 0.0.18__py3-none-any.whl

tablecodec/codecs/fintabnet.py

@@ -0,0 +1,84 @@
+"""FinTabNet (original) codec.
+
+FinTabNet's original IBM annotations encode table structure with the
+same HTML-token scheme as PubTabNet 2.0, differing only in the
+record-level identifier: ``table_id`` instead of ``imgid``. The shared
+machinery lives in :mod:`._htmltable`; this module just sets
+``id_field="table_id"`` and a sniff discriminator.
+
+Record shape::
+
+    {
+        "filename": "...",
+        "split": "train" | "val" | "test",  # optional
+        "table_id": 0,
+        "html": {
+            "structure": {"tokens": [...]},
+            "cells": [{"tokens": [...], "bbox": [x0, y0, x1, y1]}, ...],
+        },
+    }
+"""
+
+from __future__ import annotations
+
+import json
+from collections.abc import Iterable, Iterator
+from dataclasses import dataclass
+from typing import IO, Any
+
+from tablecodec.codecs._htmltable import (
+    parse_html_table,
+    serialize_html_table,
+    sniff_html_table,
+)
+from tablecodec.ir import TableSample
+
+__all__ = ["FinTabNetCodec"]
+
+_ID_FIELD = "table_id"
+
+
+@dataclass(frozen=True, slots=True)
+class FinTabNetCodec:
+    """Codec for the FinTabNet (original) jsonl format."""
+
+    name: str = "fintabnet"
+    spec_version: str = "1.0.0"
+    media_type: str = "application/jsonl"
+    writable: bool = True
+
+    def read(self, source: IO[str]) -> Iterator[TableSample]:
+        for line_no, raw in enumerate(source, start=1):
+            line = raw.strip()
+            if not line:
+                continue
+            try:
+                payload: dict[str, Any] = json.loads(line)
+            except json.JSONDecodeError as exc:
+                msg = f"invalid JSON at line {line_no}: {exc.msg}"
+                raise ValueError(msg) from exc
+            try:
+                yield parse_html_table(payload, id_field=_ID_FIELD)
+            except (KeyError, ValueError, TypeError) as exc:
+                msg = f"malformed FinTabNet record at line {line_no}: {exc}"
+                raise ValueError(msg) from exc
+
+    def write(self, samples: Iterable[TableSample], sink: IO[str]) -> None:
+        for sample in samples:
+            sink.write(
+                json.dumps(serialize_html_table(sample, id_field=_ID_FIELD), ensure_ascii=False)
+            )
+            sink.write("\n")
+
+    def lossy_read(self) -> frozenset[str]:
+        # Same HTML-token model as PubTabNet 2.0: nothing dropped on read.
+        return frozenset()
+
+    def lossy_write(self) -> frozenset[str]:
+        # IR ``extras`` has no canonical home in the FinTabNet schema.
+        return frozenset({"extras"})
+
+    def sniff(self, source: IO[str]) -> bool:
+        # Require the table_id key so a PubTabNet (imgid) record is not
+        # mis-detected as FinTabNet.
+        return sniff_html_table(source, require_field=_ID_FIELD)

tablecodec/codecs/fintabnet_otsl.py

@@ -0,0 +1,141 @@
+"""FinTabNet_OTSL codec.
+
+FinTabNet_OTSL (Docling project, HF ``ds4sd/FinTabNet_OTSL``) is the
+FinTabNet corpus re-encoded in OTSL. Compared to the plain
+``otsl-1.0.0`` codec it adds FinTabNet provenance:
+
+- ``table_id`` as the record identifier (mapped onto the IR ``imgid``),
+  like the ``fintabnet`` codec.
+- an ``extras`` dict (carrying e.g. ``otsl_raw``, the original OTSL
+  markup string). This codec is the only one that **round-trips** IR
+  ``extras``, so ``extras`` is deliberately absent from ``lossy_write``.
+
+Structure / cell handling is shared with OTSL via :mod:`._otslgrid`.
+
+Record shape::
+
+    {
+        "filename": "...",
+        "split": "train" | "val" | "test",  # optional
+        "table_id": 0,
+        "otsl": ["fcel", "fcel", "nl", ...],
+        "cells": [{"tokens": ["a"], "bbox": [x0, y0, x1, y1]}, ...],
+        "extras": {"otsl_raw": "fcel fcel nl ...", ...}  # optional
+    }
+"""
+
+from __future__ import annotations
+
+import json
+from collections.abc import Iterable, Iterator
+from dataclasses import dataclass
+from typing import IO, Any, cast
+
+from tablecodec.codecs._otslgrid import cells_to_otsl, otsl_to_cells
+from tablecodec.ir import TableSample
+
+__all__ = ["FinTabNetOTSLCodec"]
+
+_ID_FIELD = "table_id"
+
+
+@dataclass(frozen=True, slots=True)
+class FinTabNetOTSLCodec:
+    """Codec for the FinTabNet_OTSL jsonl format (OTSL + table_id + extras)."""
+
+    name: str = "fintabnet-otsl"
+    spec_version: str = "1.0.0"
+    media_type: str = "application/jsonl"
+    writable: bool = True
+
+    def read(self, source: IO[str]) -> Iterator[TableSample]:
+        for line_no, raw in enumerate(source, start=1):
+            line = raw.strip()
+            if not line:
+                continue
+            try:
+                payload: dict[str, Any] = json.loads(line)
+            except json.JSONDecodeError as exc:
+                msg = f"invalid JSON at line {line_no}: {exc.msg}"
+                raise ValueError(msg) from exc
+            try:
+                yield _payload_to_sample(payload)
+            except (KeyError, ValueError, TypeError) as exc:
+                msg = f"malformed FinTabNet_OTSL record at line {line_no}: {exc}"
+                raise ValueError(msg) from exc
+
+    def write(self, samples: Iterable[TableSample], sink: IO[str]) -> None:
+        for sample in samples:
+            sink.write(json.dumps(_sample_to_payload(sample), ensure_ascii=False))
+            sink.write("\n")
+
+    def lossy_read(self) -> frozenset[str]:
+        # OTSL core has no header marker; role defaults to body. extras is
+        # preserved.
+        return frozenset({"role"})
+
+    def lossy_write(self) -> frozenset[str]:
+        # role is lost (OTSL core). extras is round-tripped, so unlike every
+        # other codec it is NOT listed here.
+        return frozenset({"role"})
+
+    def sniff(self, source: IO[str]) -> bool:
+        pos = source.tell()
+        try:
+            for raw in source:
+                line = raw.strip()
+                if not line:
+                    continue
+                try:
+                    payload: object = json.loads(line)
+                except json.JSONDecodeError:
+                    return False
+                if not isinstance(payload, dict):
+                    return False
+                payload_dict = cast("dict[str, Any]", payload)
+                return "otsl" in payload_dict and _ID_FIELD in payload_dict
+            return False
+        finally:
+            source.seek(pos)
+
+
+def _normalize_split(value: object) -> Any:
+    if value in ("train", "val", "test"):
+        return value
+    if value is None:
+        return None
+    msg = f"unknown split value {value!r}"
+    raise ValueError(msg)
+
+
+def _payload_to_sample(payload: dict[str, Any]) -> TableSample:
+    nrows, ncols, cells = otsl_to_cells(list(payload["otsl"]), list(payload["cells"]))
+    extras_raw: object = payload.get("extras", {})
+    extras: dict[str, object] = (
+        dict(cast("dict[str, object]", extras_raw)) if isinstance(extras_raw, dict) else {}
+    )
+    return TableSample(
+        filename=str(payload["filename"]),
+        nrows=nrows,
+        ncols=ncols,
+        cells=cells,
+        split=_normalize_split(payload.get("split")),
+        imgid=payload.get(_ID_FIELD),
+        extras=extras,
+    )
+
+
+def _sample_to_payload(sample: TableSample) -> dict[str, Any]:
+    tokens, cell_payloads = cells_to_otsl(sample)
+    out: dict[str, Any] = {
+        "filename": sample.filename,
+        "otsl": tokens,
+        "cells": cell_payloads,
+    }
+    if sample.split is not None:
+        out["split"] = sample.split
+    if sample.imgid is not None:
+        out[_ID_FIELD] = sample.imgid
+    if sample.extras:
+        out["extras"] = dict(sample.extras)
+    return out

tablecodec/codecs/otsl.py

@@ -0,0 +1,138 @@
+"""OTSL 1.0 codec.
+
+Implements the Optimized Table Structure Language (Lysak et al.,
+ICDAR 2023, arXiv 2305.03393). OTSL uses a five-token vocabulary plus
+a newline marker:
+
+- ``fcel`` filled cell anchor (body content)
+- ``ecel`` empty cell anchor
+- ``lcel`` left-merged continuation — extends the colspan of the anchor
+  to its left
+- ``ucel`` up-merged continuation — extends the rowspan of the anchor above
+- ``xcel`` cross-merged continuation — extends both row and column
+  (the anchor sits at (r-1, c-1) of this position)
+- ``nl``   newline / row separator
+
+Square-table assumption (per the paper): every row produced by ``nl``
+splits MUST have the same number of cell-position tokens. Jagged input
+is rejected with a clear error.
+
+This implementation is derived from the paper, not copied from the
+official Docling OTSL reference implementation. Cross-validation
+against the reference is wired separately in a later milestone.
+"""
+
+from __future__ import annotations
+
+import json
+from collections.abc import Iterable, Iterator
+from dataclasses import dataclass
+from typing import IO, Any, cast
+
+from tablecodec.codecs._otslgrid import cells_to_otsl, otsl_to_cells
+from tablecodec.ir import TableSample
+
+__all__ = ["OTSL10Codec"]
+
+
+@dataclass(frozen=True, slots=True)
+class OTSL10Codec:
+    """Codec for the OTSL 1.0 jsonl format."""
+
+    name: str = "otsl-1.0.0"
+    spec_version: str = "1.0.0"
+    media_type: str = "application/jsonl"
+    writable: bool = True
+
+    def read(self, source: IO[str]) -> Iterator[TableSample]:
+        for line_no, raw in enumerate(source, start=1):
+            line = raw.strip()
+            if not line:
+                continue
+            try:
+                payload: dict[str, Any] = json.loads(line)
+            except json.JSONDecodeError as exc:
+                msg = f"invalid JSON at line {line_no}: {exc.msg}"
+                raise ValueError(msg) from exc
+            try:
+                yield _payload_to_sample(payload)
+            except (KeyError, ValueError, TypeError) as exc:
+                msg = f"malformed OTSL 1.0 record at line {line_no}: {exc}"
+                raise ValueError(msg) from exc
+
+    def write(self, samples: Iterable[TableSample], sink: IO[str]) -> None:
+        for sample in samples:
+            sink.write(json.dumps(_sample_to_payload(sample), ensure_ascii=False))
+            sink.write("\n")
+
+    def lossy_read(self) -> frozenset[str]:
+        # OTSL has no header/body distinction in its grammar; reads default
+        # every cell to role="body". This is a real loss when the source
+        # carried a header/body tag.
+        return frozenset({"role"})
+
+    def lossy_write(self) -> frozenset[str]:
+        # role: collapsed to "body" on write.
+        # extras: no canonical home in OTSL.
+        return frozenset({"extras", "role"})
+
+    def sniff(self, source: IO[str]) -> bool:
+        pos = source.tell()
+        try:
+            for raw in source:
+                line = raw.strip()
+                if not line:
+                    continue
+                try:
+                    payload: object = json.loads(line)
+                except json.JSONDecodeError:
+                    return False
+                return _looks_like_otsl(payload)
+            return False
+        finally:
+            source.seek(pos)
+
+
+def _looks_like_otsl(payload: object) -> bool:
+    if not isinstance(payload, dict):
+        return False
+    payload_dict = cast("dict[str, Any]", payload)
+    return "otsl" in payload_dict and "cells" in payload_dict
+
+
+# ---------- payload <-> sample (delegates to _otslgrid) ----------
+
+
+def _payload_to_sample(payload: dict[str, Any]) -> TableSample:
+    nrows, ncols, cells = otsl_to_cells(list(payload["otsl"]), list(payload["cells"]))
+    return TableSample(
+        filename=str(payload["filename"]),
+        nrows=nrows,
+        ncols=ncols,
+        cells=cells,
+        split=_normalize_split(payload.get("split")),
+        imgid=payload.get("imgid"),
+    )
+
+
+def _normalize_split(value: object) -> Any:
+    if value in ("train", "val", "test"):
+        return value
+    if value is None:
+        return None
+    msg = f"unknown split value {value!r}"
+    raise ValueError(msg)
+
+
+def _sample_to_payload(sample: TableSample) -> dict[str, Any]:
+    tokens, cell_payloads = cells_to_otsl(sample)
+    out: dict[str, Any] = {
+        "filename": sample.filename,
+        "otsl": tokens,
+        "cells": cell_payloads,
+    }
+    if sample.split is not None:
+        out["split"] = sample.split
+    if sample.imgid is not None:
+        out["imgid"] = sample.imgid
+    return out

tablecodec/codecs/pubtables1m.py

@@ -0,0 +1,161 @@
+"""PubTables-1M codec (read-only).
+
+PubTables-1M (Microsoft, table-transformer) is an object-detection
+format: each cell carries explicit grid coordinates and a bbox, in
+detection order rather than reading order. This codec READS that into
+the IR (normalising to row-major order) and is READ-ONLY — ``write``
+raises ``NotImplementedError`` and ``writable`` is ``False`` (ADR 0002).
+
+Canonical jsonl record shape::
+
+    {
+        "filename": "...",
+        "split": "train" | "val" | "test",  # optional
+        "imgid": 0,  # optional
+        "nrows": 2,  # optional; derived from cells if absent
+        "ncols": 2,  # optional; derived from cells if absent
+        "cells": [
+            {
+                "row": 0,
+                "col": 0,
+                "rowspan": 1,
+                "colspan": 1,
+                "bbox": [x0, y0, x1, y1],
+                "tokens": ["..."],
+            },
+            ...,  # any order
+        ],
+    }
+"""
+
+from __future__ import annotations
+
+import json
+from collections.abc import Iterable, Iterator
+from dataclasses import dataclass
+from typing import IO, Any, Literal, cast
+
+from tablecodec.ir import BBox, GridCell, TableSample
+
+__all__ = ["PubTables1MCodec"]
+
+
+@dataclass(frozen=True, slots=True)
+class PubTables1MCodec:
+    """Read-only codec for the PubTables-1M object-detection format."""
+
+    name: str = "pubtables-1m"
+    spec_version: str = "1.0.0"
+    media_type: str = "application/jsonl"
+    writable: bool = False
+
+    def read(self, source: IO[str]) -> Iterator[TableSample]:
+        for line_no, raw in enumerate(source, start=1):
+            line = raw.strip()
+            if not line:
+                continue
+            try:
+                payload: dict[str, Any] = json.loads(line)
+            except json.JSONDecodeError as exc:
+                msg = f"invalid JSON at line {line_no}: {exc.msg}"
+                raise ValueError(msg) from exc
+            try:
+                yield _payload_to_sample(payload)
+            except (KeyError, ValueError, TypeError) as exc:
+                msg = f"malformed PubTables-1M record at line {line_no}: {exc}"
+                raise ValueError(msg) from exc
+
+    def write(self, samples: Iterable[TableSample], sink: IO[str]) -> None:
+        msg = "pubtables-1m is a read-only codec (object-detection format); write is unsupported"
+        raise NotImplementedError(msg)
+
+    def lossy_read(self) -> frozenset[str]:
+        # Our canonical jsonl keeps every IR field; nothing dropped.
+        return frozenset()
+
+    def lossy_write(self) -> frozenset[str]:
+        # Never consulted: analyze_loss short-circuits on writable=False.
+        return frozenset()
+
+    def sniff(self, source: IO[str]) -> bool:
+        pos = source.tell()
+        try:
+            for raw in source:
+                line = raw.strip()
+                if not line:
+                    continue
+                try:
+                    payload: object = json.loads(line)
+                except json.JSONDecodeError:
+                    return False
+                return _looks_like_pubtables1m(payload)
+            return False
+        finally:
+            source.seek(pos)
+
+
+def _looks_like_pubtables1m(payload: object) -> bool:
+    if not isinstance(payload, dict):
+        return False
+    payload_dict = cast("dict[str, Any]", payload)
+    if "html" in payload_dict:  # rules out PubTabNet/FinTabNet/TableFormer/TableBank
+        return False
+    cells: object = payload_dict.get("cells")
+    if not isinstance(cells, list) or not cells:
+        return False
+    first = cast("list[object]", cells)[0]
+    return isinstance(first, dict) and "row" in first and "col" in first
+
+
+def _normalize_split(value: object) -> Literal["train", "val", "test"] | None:
+    if value == "train":
+        return "train"
+    if value == "val":
+        return "val"
+    if value == "test":
+        return "test"
+    if value is None:
+        return None
+    msg = f"unknown split value {value!r}"
+    raise ValueError(msg)
+
+
+def _cell_from_payload(cell_payload: dict[str, Any]) -> GridCell:
+    bbox_raw = cell_payload.get("bbox")
+    bbox: BBox | None = None
+    if bbox_raw is not None:
+        bbox = (int(bbox_raw[0]), int(bbox_raw[1]), int(bbox_raw[2]), int(bbox_raw[3]))
+    role_raw = cell_payload.get("role", "body")
+    role: Literal["header", "body"] = "header" if role_raw == "header" else "body"
+    return GridCell(
+        row=int(cell_payload["row"]),
+        col=int(cell_payload["col"]),
+        rowspan=int(cell_payload.get("rowspan", 1)),
+        colspan=int(cell_payload.get("colspan", 1)),
+        tokens=tuple(cell_payload.get("tokens", ())),
+        bbox=bbox,
+        role=role,
+    )
+
+
+def _payload_to_sample(payload: dict[str, Any]) -> TableSample:
+    cell_payloads = list(payload["cells"])
+    cells = [_cell_from_payload(c) for c in cell_payloads]
+    # Object-detection order is arbitrary; the IR is row-major.
+    cells.sort(key=lambda c: (c.row, c.col))
+
+    nrows = payload.get("nrows")
+    ncols = payload.get("ncols")
+    if nrows is None:
+        nrows = max((c.row + c.rowspan for c in cells), default=0)
+    if ncols is None:
+        ncols = max((c.col + c.colspan for c in cells), default=0)
+
+    return TableSample(
+        filename=str(payload["filename"]),
+        nrows=int(nrows),
+        ncols=int(ncols),
+        cells=tuple(cells),
+        split=_normalize_split(payload.get("split")),
+        imgid=payload.get("imgid"),
+    )

tablecodec/codecs/pubtabnet.py

@@ -0,0 +1,128 @@
+"""PubTabNet codecs (1.0.0 and 2.0.0).
+
+Both share the HTML-token table machinery in :mod:`._htmltable`. The
+only difference is bbox handling:
+
+- ``pubtabnet-2.0.0`` reads and writes per-cell ``bbox``.
+- ``pubtabnet-1.0.0`` has no bbox: it drops bbox on read and omits it on
+  write (declared honestly in ``lossy_read`` / ``lossy_write``).
+
+PubTabNet jsonl record shape::
+
+    {
+        "filename": "PMC...",
+        "split": "train" | "val" | "test",  # optional
+        "imgid": 0,  # optional
+        "html": {
+            "structure": {"tokens": ["<thead>", "<tr>", "<td>", "</td>", ...]},
+            "cells": [
+                {"tokens": ["a"], "bbox": [x0, y0, x1, y1]},
+                {"tokens": []},  # empty cells may omit bbox
+                ...,
+            ],
+        },
+    }
+"""
+
+from __future__ import annotations
+
+import json
+from collections.abc import Iterable, Iterator
+from dataclasses import dataclass
+from typing import IO, Any
+
+from tablecodec.codecs._htmltable import (
+    parse_html_table,
+    serialize_html_table,
+    sniff_html_table,
+)
+from tablecodec.ir import TableSample
+
+__all__ = ["PubTabNet10Codec", "PubTabNet20Codec"]
+
+
+@dataclass(frozen=True, slots=True)
+class PubTabNet20Codec:
+    """Codec for the PubTabNet 2.0 jsonl format."""
+
+    name: str = "pubtabnet-2.0.0"
+    spec_version: str = "2.0.0"
+    media_type: str = "application/jsonl"
+    writable: bool = True
+
+    def read(self, source: IO[str]) -> Iterator[TableSample]:
+        for line_no, raw in enumerate(source, start=1):
+            line = raw.strip()
+            if not line:
+                continue
+            try:
+                payload: dict[str, Any] = json.loads(line)
+            except json.JSONDecodeError as exc:
+                msg = f"invalid JSON at line {line_no}: {exc.msg}"
+                raise ValueError(msg) from exc
+            try:
+                yield parse_html_table(payload)
+            except (KeyError, ValueError, TypeError) as exc:
+                msg = f"malformed PubTabNet 2.0 record at line {line_no}: {exc}"
+                raise ValueError(msg) from exc
+
+    def write(self, samples: Iterable[TableSample], sink: IO[str]) -> None:
+        for sample in samples:
+            sink.write(json.dumps(serialize_html_table(sample), ensure_ascii=False))
+            sink.write("\n")
+
+    def lossy_read(self) -> frozenset[str]:
+        # PubTabNet 2.0 keeps filename, split, imgid, tokens, bbox,
+        # rowspan, colspan, header/body. Nothing dropped on read.
+        return frozenset()
+
+    def lossy_write(self) -> frozenset[str]:
+        # IR ``extras`` has no canonical home in the PubTabNet schema.
+        return frozenset({"extras"})
+
+    def sniff(self, source: IO[str]) -> bool:
+        return sniff_html_table(source, require_no_bbox=False)
+
+
+@dataclass(frozen=True, slots=True)
+class PubTabNet10Codec:
+    """Codec for the PubTabNet 1.0.0 jsonl format (no bbox)."""
+
+    name: str = "pubtabnet-1.0.0"
+    spec_version: str = "1.0.0"
+    media_type: str = "application/jsonl"
+    writable: bool = True
+
+    def read(self, source: IO[str]) -> Iterator[TableSample]:
+        for line_no, raw in enumerate(source, start=1):
+            line = raw.strip()
+            if not line:
+                continue
+            try:
+                payload: dict[str, Any] = json.loads(line)
+            except json.JSONDecodeError as exc:
+                msg = f"invalid JSON at line {line_no}: {exc.msg}"
+                raise ValueError(msg) from exc
+            try:
+                yield parse_html_table(payload, drop_bbox=True)
+            except (KeyError, ValueError, TypeError) as exc:
+                msg = f"malformed PubTabNet 1.0 record at line {line_no}: {exc}"
+                raise ValueError(msg) from exc
+
+    def write(self, samples: Iterable[TableSample], sink: IO[str]) -> None:
+        for sample in samples:
+            sink.write(
+                json.dumps(serialize_html_table(sample, include_bbox=False), ensure_ascii=False)
+            )
+            sink.write("\n")
+
+    def lossy_read(self) -> frozenset[str]:
+        # bbox is not in the 1.0 source format; if the file is 2.0-shaped,
+        # bbox is dropped silently.
+        return frozenset({"bbox"})
+
+    def lossy_write(self) -> frozenset[str]:
+        return frozenset({"bbox", "extras"})
+
+    def sniff(self, source: IO[str]) -> bool:
+        return sniff_html_table(source, require_no_bbox=True)