table-stitcher 0.3.0__tar.gz → 0.4.1__tar.gz

{table_stitcher-0.3.0 → table_stitcher-0.4.1}/CHANGELOG.md

@@ -7,6 +7,45 @@ the project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html
 
 ## [Unreleased]
 
+## [0.4.1] — 2026-06-08
+
+### Fixed
+
+- **Spanning body cells duplicated across columns on multi-page merge**
+  (`adapters/docling.py`). Docling repeats a `col_span=N` cell's text across
+  every column it covers; the merge round-trip rebuilt those as `N` separate
+  `col_span=1` cells, leaking a full-width description into every value column
+  and displacing the real values (a repeated `col_span` header behaved the same
+  way). Injection now matches each merged row back to its source grid row and
+  re-emits the original spans; rows the merger transformed (stitched
+  continuations, folded overflow) fall back to the flat 1x1 rebuild. The match
+  uses the original span metadata, never value equality, so coincidentally-equal
+  adjacent values (e.g. two plan columns sharing a cap) stay separate cells.
+
+## [0.4.0] — 2026-05-29
+
+### Added
+
+- **Intervening-content guard** (`block_on_intervening_content`, default `True`).
+  Two tables that share a column schema but belong to different sections — a
+  heading sits between them in reading order — are no longer stitched into one.
+  A genuine page-split continuation has nothing but page furniture between its
+  fragments, so a section heading between them is a reliable "separate tables"
+  signal. The docling adapter computes a per-table `TableMeta.content_before`
+  signal; both merge paths (`_classify_sequential_pair` and
+  `should_force_orphan_merge`) consult it.
+  - Running headers mislabeled as headings (e.g. a journal banner labeled
+    `page_header` on one page and `section_header` on another, or a repeated
+    "Summary of benefits" banner above every page of a multi-page table) are
+    detected as furniture via near-identical (Jaccard ≥ 0.8) recurrence across
+    pages, so they do not block legitimate continuations.
+  - Only `section_header`/`title` nodes block; plain paragraphs, list items,
+    captions, footnotes and figures are deliberately ignored, since real PDFs
+    routinely scatter those between fragments of a single continued table.
+  - Fixes over-eager merging of same-schema per-section tables (e.g. an
+    insurance policy's eight `Prestige | Elite | Classic` benefit grids being
+    collapsed into one).
+
 ## [0.3.0] — 2026-05-06
 
 ### Fixed

{table_stitcher-0.3.0 → table_stitcher-0.4.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: table-stitcher
-Version: 0.3.0
+Version: 0.4.1
 Summary: Reassemble tables split across page boundaries in PDF extraction
 Project-URL: Homepage, https://github.com/pebbleroad/table-stitcher
 Project-URL: Repository, https://github.com/pebbleroad/table-stitcher

{table_stitcher-0.3.0 → table_stitcher-0.4.1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "table-stitcher"
-version = "0.3.0"
+version = "0.4.1"
 description = "Reassemble tables split across page boundaries in PDF extraction"
 readme = "README.md"
 license = "MIT"

{table_stitcher-0.3.0 → table_stitcher-0.4.1}/src/table_stitcher/adapters/docling.py

@@ -19,6 +19,7 @@ from docling_core.types.doc import (
 from ..merger import (
     first_row_has_number,
     is_numeric_like_colnames,
+    jaccard,
     normalize_col_name,
     tokenize,
 )
@@ -418,9 +419,90 @@ def _extract_original_header_rows(
     return header_rows, header_cells
 
 
+def _index_member_rows(
+    member_data: list[Optional[TableData]],
+) -> dict[tuple, list[list[TableCell]]]:
+    """
+    Index every member fragment's original grid rows by their expanded
+    text-vector, for col_span reconstruction during injection.
+
+    Docling repeats a spanning cell's text across each column it covers, so the
+    text-vector of an original grid row equals the DataFrame row that
+    ``_grid_to_dataframe`` produced for it (for any row the merger left
+    untouched). Keying on that vector lets ``_dataframe_to_docling_data``
+    recover the original col_span structure instead of flattening every cell to
+    1x1 — which duplicates a spanning cell into every column it covered on a
+    multi-page merge.
+
+    All rows are indexed, including header rows: a satellite fragment's repeated
+    header (e.g. a ``col_span=6`` banner) arrives in the merged DataFrame as a
+    body row, and must match its original spanning cell rather than duplicate
+    across the value columns. (The anchor's own header rows are reconstructed
+    separately and never looked up here.) Values are buckets because a row can
+    legitimately repeat; identical text-vectors imply identical span structure,
+    so any occurrence is interchangeable.
+    """
+    index: dict[tuple, list[list[TableCell]]] = {}
+    for data in member_data:
+        if not data or not data.grid:
+            continue
+        for row in data.grid:
+            if not row:
+                continue
+            key = tuple((getattr(c, "text", "") or "") if c else "" for c in row)
+            index.setdefault(key, []).append(row)
+    return index
+
+
+def _reemit_body_row(
+    orig_row: list[TableCell], table_row_idx: int, has_row_headers: bool
+) -> tuple[list[TableCell], list[TableCell]]:
+    """
+    Re-emit an original grid body row at a new row offset, preserving col_span.
+
+    Returns ``(grid_row, distinct_cells)`` where ``grid_row`` repeats each
+    spanning cell across the columns it covers (Docling grid convention) and
+    ``distinct_cells`` lists each origin cell once (for ``table_cells``).
+
+    row_span is intentionally clamped to 1: the merged DataFrame represents one
+    logical row per grid row, so a multi-row body span cannot be expressed
+    without desynchronizing the rebuilt grid. (Body row_spans are rare; col_span
+    is the case that corrupts multi-page merges.)
+    """
+    grid_row: list[Optional[TableCell]] = []
+    distinct: list[TableCell] = []
+    for c_idx, cell in enumerate(orig_row):
+        if cell is None:
+            grid_row.append(None)
+            continue
+        start_col = getattr(cell, "start_col_offset_idx", c_idx)
+        if start_col == c_idx:
+            col_span = getattr(cell, "col_span", 1) or 1
+            new_cell = TableCell(
+                text=getattr(cell, "text", "") or "",
+                row_span=1,
+                col_span=col_span,
+                column_header=False,
+                row_header=(c_idx == 0 and has_row_headers)
+                or bool(getattr(cell, "row_header", False)),
+                start_row_offset_idx=table_row_idx,
+                end_row_offset_idx=table_row_idx + 1,
+                start_col_offset_idx=c_idx,
+                end_col_offset_idx=c_idx + col_span,
+            )
+            distinct.append(new_cell)
+            grid_row.append(new_cell)
+        else:
+            # Continuation column of a span originating to the left: repeat the
+            # same cell object, which was already appended at ``start_col``.
+            grid_row.append(grid_row[start_col] if start_col < len(grid_row) else None)
+    return grid_row, distinct
+
+
 def _dataframe_to_docling_data(
     df: pd.DataFrame,
     original_data: Optional[TableData] = None,
+    member_data: Optional[list[Optional[TableData]]] = None,
 ) -> TableData:
     """
     Converts a pandas DataFrame back into Docling's TableData structure.
@@ -430,6 +512,12 @@ def _dataframe_to_docling_data(
     are preserved exactly.  Only the data rows are rebuilt from the DataFrame.
     This prevents the lossy roundtrip that would flatten complex headers into
     simple 1x1 cells.
+
+    When ``member_data`` (the original ``TableData`` of every fragment in the
+    logical table) is provided, body rows the merger left untouched are
+    re-emitted from their original grid cells, preserving col_span. Rows the
+    merger transformed (stitched continuations, folded overflow) fall back to a
+    flat 1x1 rebuild from the DataFrame.
     """
     if df.empty:
         cols = list(df.columns) if len(df.columns) > 0 else ["Column_0"]
@@ -498,16 +586,28 @@ def _dataframe_to_docling_data(
                     break
 
     # --- Build data rows from merged DataFrame ---
+    # Index member fragments' original body rows so spanning cells survive the
+    # round-trip (see _index_member_body_rows).
+    body_index = _index_member_rows(member_data) if member_data else {}
+
     for i, (_, row) in enumerate(df.iterrows()):
-        grid_row: list[TableCell] = []
         table_row_idx = num_header_rows + i
 
-        for j, val in enumerate(row):
-            if pd.isna(val) or val is None:
-                text_val = ""
-            else:
-                text_val = str(val)
+        row_vals = ["" if (pd.isna(v) or v is None) else str(v) for v in row]
+
+        # Re-emit untouched rows from their original grid cells (preserves
+        # col_span); only matches when widths align, so coincidentally-equal
+        # adjacent values are never fused.
+        bucket = body_index.get(tuple(row_vals))
+        if bucket:
+            orig_row = bucket.pop(0)
+            grid_row, distinct = _reemit_body_row(orig_row, table_row_idx, has_row_headers)
+            grid.append(grid_row)
+            table_cells.extend(distinct)
+            continue
 
+        grid_row: list[TableCell] = []
+        for j, text_val in enumerate(row_vals):
             row_header = j == 0 and has_row_headers
 
             cell = TableCell(
@@ -556,6 +656,176 @@ def _get_ref_pointer(ref_obj: Any) -> str:
 # -------------------------------------------------------------------
 
 
+# -------------------------------------------------------------------
+# Intervening-content detection (reading-order furniture filtering)
+#
+# Philosophy: a table split across a page break has nothing but page
+# furniture (running headers/footers, page numbers) between its fragments.
+# If *substantive* body content — a heading, paragraph, list item, or real
+# figure — sits between two fragments in reading order, they are separate
+# tables that merely share a column schema. These helpers classify the body
+# nodes between consecutive tables so the merger can refuse such merges.
+# -------------------------------------------------------------------
+
+# Only a structural section boundary reliably means "these are separate
+# tables." Plain prose, list items, captions, footnotes and figures all turn
+# up *between* fragments of genuine continuations on real-world PDFs (cell text
+# extracted as body nodes, interleaved multi-column reading order, repeated
+# legends), so blocking on them regresses legitimate merges. A new heading
+# between two tables, by contrast, is a clean separator.
+_BLOCKING_LABELS = {"section_header", "title"}
+
+_CONTINUATION_RE = re.compile(r"\bcont(?:inued|inuation|'?d|\.)?\b", re.IGNORECASE)
+
+
+def _label_str(item: Any) -> str:
+    """Normalize a docling item label (enum or str) to a lowercase string."""
+    lab = getattr(item, "label", None)
+    return str(getattr(lab, "value", lab) or "").lower()
+
+
+def _norm_text(item: Any) -> str:
+    """Whitespace-collapsed lowercase text of an item."""
+    return " ".join(str(getattr(item, "text", "") or "").split()).lower()
+
+
+def _flatten_body_refs(doc: Any) -> list[str]:
+    """Return body reference pointers in reading order (DFS through groups)."""
+    seq: list[str] = []
+    groups = getattr(doc, "groups", []) or []
+    seen_groups: set[int] = set()
+
+    def walk(node: Any) -> None:
+        children = getattr(node, "children", None) or []
+        for child in children:
+            ref = _get_ref_pointer(child)
+            if not ref:
+                continue
+            if ref.startswith("#/groups/"):
+                try:
+                    gi = int(ref.split("/")[-1])
+                except ValueError:
+                    continue
+                # Guard against malformed self-referential group cycles.
+                if gi in seen_groups or not (0 <= gi < len(groups)):
+                    continue
+                seen_groups.add(gi)
+                walk(groups[gi])
+            else:
+                seq.append(ref)
+
+    body = getattr(doc, "body", None)
+    if body is not None:
+        walk(body)
+    return seq
+
+
+def _resolve_ref(doc: Any, ref: str) -> Optional[Any]:
+    """Resolve a ``#/kind/N`` pointer to its item, or None."""
+    try:
+        _, kind, n_str = ref.split("/")
+        n = int(n_str)
+    except (ValueError, AttributeError):
+        return None
+    coll = {
+        "texts": getattr(doc, "texts", None),
+        "tables": getattr(doc, "tables", None),
+        "pictures": getattr(doc, "pictures", None),
+    }.get(kind)
+    if not coll or n >= len(coll):
+        return None
+    return coll[n]
+
+
+def _detect_running_furniture(doc: Any, cfg: MultiPageConfig) -> set[str]:
+    """
+    Identify headings that are actually running headers, not section boundaries.
+
+    Only headings (``_BLOCKING_LABELS``) can block a merge, so only headings
+    need to be exempted. A heading is a running header when near-identical text
+    appears on a *different* page — e.g. a repeated ``Summary of benefits``
+    banner above every page of a multi-page table, or a journal name that one
+    page labels ``page_header`` and another mislabels ``section_header`` (the
+    parser inconsistency this protects against).
+
+    Similarity uses symmetric Jaccard with a high threshold (near-duplicate).
+    A real, unique heading such as ``38e - Trip postponement`` has no
+    near-duplicate on another page, so it correctly stays a blocker — unlike a
+    looser containment metric, which a short subset string (a TOC entry, say)
+    would spuriously satisfy.
+    """
+    texts = getattr(doc, "texts", []) or []
+    # (ref, tokens, page, label) for every text node — the comparison pool.
+    nodes: list[tuple[str, set, Any, str]] = []
+    for i, item in enumerate(texts):
+        prov = getattr(item, "prov", None) or []
+        page = getattr(prov[0], "page_no", None) if prov else None
+        nodes.append((f"#/texts/{i}", tokenize(_norm_text(item)), page, _label_str(item)))
+
+    furniture: set[str] = set()
+    for ref, toks, page, label in nodes:
+        if label not in _BLOCKING_LABELS or not toks or page is None:
+            continue
+        for ref2, toks2, page2, _ in nodes:
+            if ref2 == ref or page2 is None or page2 == page or not toks2:
+                continue
+            if jaccard(toks, toks2) >= 0.8:
+                furniture.add(ref)
+                break
+    return furniture
+
+
+def _ref_is_blocking(doc: Any, ref: str, furniture: set[str], cfg: MultiPageConfig) -> bool:
+    """Whether a body node between two tables marks a real table boundary.
+
+    Only a non-furniture section heading qualifies (see ``_BLOCKING_LABELS``).
+    Figures, list items and plain paragraphs are deliberately ignored: on real
+    PDFs they routinely appear between fragments of a single continued table.
+    """
+    if ref in furniture or not ref.startswith("#/texts/"):
+        return False
+    item = _resolve_ref(doc, ref)
+    if item is None:
+        return False
+    if _label_str(item) not in _BLOCKING_LABELS:
+        return False
+    text = str(getattr(item, "text", "") or "")
+    if not text.strip():
+        return False
+    # A "(continued)" heading marks a continuation, not a new table.
+    if _CONTINUATION_RE.search(text):
+        return False
+    return True
+
+
+def _compute_content_before(doc: Any, cfg: MultiPageConfig) -> dict[int, bool]:
+    """
+    For each table (keyed by docling index), whether substantive body content
+    separates it from the previous table in reading order. Tables absent from
+    the body reading order (orphans) are omitted, disabling the guard for them.
+    """
+    seq = _flatten_body_refs(doc)
+    if not seq:
+        return {}
+    furniture = _detect_running_furniture(doc, cfg)
+
+    result: dict[int, bool] = {}
+    prev_table_seen = False
+    blocking_seen = False
+    for ref in seq:
+        if ref.startswith("#/tables/"):
+            try:
+                t_idx = int(ref.split("/")[-1])
+            except ValueError:
+                continue
+            result[t_idx] = prev_table_seen and blocking_seen
+            prev_table_seen = True
+            blocking_seen = False
+        elif not blocking_seen and _ref_is_blocking(doc, ref, furniture, cfg):
+            blocking_seen = True
+    return result
+
+
 class DoclingAdapter:
     """
     Table-stitcher adapter for Docling (docling-core).
@@ -569,6 +839,15 @@ class DoclingAdapter:
         total = len(doc.tables)
         skipped = 0
 
+        # Reading-order map: which tables have substantive body content before
+        # them (used by the merger's intervening-content guard). Computed once.
+        content_before_map: dict[int, bool] = {}
+        if cfg.block_on_intervening_content:
+            try:
+                content_before_map = _compute_content_before(doc, cfg)
+            except Exception as e:  # never let the guard break extraction
+                log.warning(f"Intervening-content detection failed: {e}")
+
         for idx, table in enumerate(doc.tables):
             try:
                 df = _grid_to_dataframe(table, doc)
@@ -640,6 +919,7 @@ class DoclingAdapter:
                     row_count=df.shape[0],
                     continuation_content=continuation_content,
                     is_headerless=is_headerless,
+                    content_before=content_before_map.get(idx),
                 )
             )
 
@@ -718,9 +998,18 @@ class DoclingAdapter:
 
                 original_data = getattr(anchor_table, "data", None)
 
+                # Original TableData of every fragment, captured in
+                # table_snapshots before any mutation, so injection can recover
+                # each untouched body row's col_span (see
+                # _index_member_body_rows).
+                member_data = [
+                    table_snapshots[m]["data"] for m in lt.members if m in table_snapshots
+                ]
+
                 anchor_table.data = _dataframe_to_docling_data(
                     lt.df,
                     original_data=original_data,
+                    member_data=member_data,
                 )
 
                 for satellite_idx in lt.members[1:]:

{table_stitcher-0.3.0 → table_stitcher-0.4.1}/src/table_stitcher/merger.py

@@ -312,6 +312,11 @@ def should_force_orphan_merge(h: TableMeta, d: TableMeta, cfg: MultiPageConfig)
         return False, ""
     if abs(h.width - d.width) > cfg.max_width_difference:
         return False, ""
+    # Intervening-content guard — sibling of the one in _classify_sequential_pair.
+    # Pass 2 reaches this without going through that function, so the guard must
+    # be repeated here: a heading before the data fragment means a new table.
+    if cfg.block_on_intervening_content and d.content_before:
+        return False, "content_between_tables"
 
     layout = layout_suggests_continuation(h, d, cfg)
     if h.is_header_orphan and d.is_data_orphan:
@@ -705,6 +710,17 @@ def _classify_sequential_pair(
     if page_gap < 1 or page_gap > cfg.max_page_gap:
         return False, "page_gap_out_of_range", False, []
 
+    # --- Intervening-content guard ---
+    # A genuine page-split continuation has nothing but page furniture between
+    # its fragments. Substantive body content (a heading, paragraph, list item,
+    # or figure) between tA and tB means they are separate tables that merely
+    # share a column schema, not one table split across a page break. The
+    # adapter computes this in reading order (furniture, captions and footnotes
+    # are already filtered out); ``None`` means position unknown, so we defer to
+    # the other signals rather than block.
+    if cfg.block_on_intervening_content and tB.content_before:
+        return False, "content_between_tables", False, []
+
     # --- Spillover (checked before width guards since spillover can cross
     # width boundaries legitimately: 1-col fragment follows N-col table) ---
     if is_spillover_fragment(tA, tB, cfg):

{table_stitcher-0.3.0 → table_stitcher-0.4.1}/src/table_stitcher/models.py

@@ -92,6 +92,19 @@ class MultiPageConfig:
     The structural signal (1 col following N cols) is strong enough for most cases.
     """
 
+    # --- Intervening-Content Guard ---
+    block_on_intervening_content: bool = True
+    """
+    If True, refuse to merge two fragments when substantive body content
+    (a heading, paragraph, list item, or figure) appears between them in
+    reading order. A genuine page-split continuation has nothing between its
+    fragments except page furniture (running headers/footers, page numbers),
+    which is filtered out, as are table-attached captions/footnotes and
+    ``(continued)`` markers. Requires an adapter that populates
+    ``TableMeta.content_before``; when it is left ``None`` the guard is a
+    no-op, so non-docling adapters are unaffected.
+    """
+
     # --- Cell Stitching ---
     stitch_separator: str = "\n"
     """Character(s) used to join split cell content."""
@@ -118,6 +131,13 @@ class TableMeta:
     row_count: int
     continuation_content: list[dict] = field(default_factory=list)
     is_headerless: bool = False
+    content_before: Optional[bool] = None
+    """
+    Whether substantive (non-furniture) body content immediately precedes this
+    fragment in reading order, since the previous table fragment. ``None`` means
+    the adapter could not place the table in reading order (e.g. an orphan
+    table), in which case the intervening-content guard is skipped for it.
+    """
 
 
 @dataclass

{table_stitcher-0.3.0 → table_stitcher-0.4.1}/tests/test_docling_adapter.py

@@ -300,6 +300,94 @@ class TestHeaderPreservation:
         assert td.num_rows == 3  # 1 header + 2 data
 
 
+class TestBodySpanPreservation:
+    """Body col_span cells must survive the merge round-trip, not duplicate.
+
+    Docling repeats a spanning cell's text across every column it covers, so a
+    naive grid -> DataFrame -> grid rebuild flattens a ``col_span=N`` body cell
+    into N duplicate ``col_span=1`` cells — leaking a full-width description
+    into every value column and displacing the real values. Passing the member
+    fragments' original TableData lets injection re-emit the original spans.
+    """
+
+    @staticmethod
+    def _cell(text, r, c, *, col_span=1, header=False):
+        return TableCell(
+            text=text,
+            row_span=1,
+            col_span=col_span,
+            column_header=header,
+            row_header=False,
+            start_row_offset_idx=r,
+            end_row_offset_idx=r + 1,
+            start_col_offset_idx=c,
+            end_col_offset_idx=c + col_span,
+        )
+
+    def _fragment(self) -> TableData:
+        """3-col fragment: flat header, a col_span=3 description row, a data row,
+        and a row with coincidentally-equal adjacent values (separate cells)."""
+        c = self._cell
+        # Header row 0
+        h = [
+            c("Section", 0, 0, header=True),
+            c("Plan A", 0, 1, header=True),
+            c("Plan B", 0, 2, header=True),
+        ]
+        # Row 1: description spanning all 3 cols — grid repeats the same object.
+        desc = c("See important notes below", 1, 0, col_span=3)
+        r1 = [desc, desc, desc]
+        # Row 2: ordinary data row.
+        r2 = [c("1", 2, 0), c("100", 2, 1), c("200", 2, 2)]
+        # Row 3: two value columns share a cap value, but are SEPARATE cells.
+        r3 = [c("2", 3, 0), c("150", 3, 1), c("150", 3, 2)]
+        grid = [h, r1, r2, r3]
+        flat = h + [desc, r2[0], r2[1], r2[2], r3[0], r3[1], r3[2]]
+        return TableData(num_rows=4, num_cols=3, table_cells=flat, grid=grid)
+
+    def test_body_colspan_preserved_not_duplicated(self):
+        original = self._fragment()
+        # The DataFrame as _grid_to_dataframe would produce it: the spanning
+        # description duplicated across all three columns.
+        merged_df = pd.DataFrame(
+            [
+                ["See important notes below"] * 3,
+                ["1", "100", "200"],
+                ["2", "150", "150"],
+            ],
+            columns=["Section", "Plan A", "Plan B"],
+        )
+
+        td = _dataframe_to_docling_data(merged_df, original_data=original, member_data=[original])
+
+        # Description row: one origin cell with col_span=3, repeated across the
+        # grid row — NOT three distinct duplicated cells.
+        desc_row = td.grid[1]
+        assert desc_row[0].col_span == 3
+        assert desc_row[0].text == "See important notes below"
+        assert desc_row[1] is desc_row[0] and desc_row[2] is desc_row[0]
+        distinct_desc = [
+            cell for cell in td.table_cells if cell.text == "See important notes below"
+        ]
+        assert len(distinct_desc) == 1
+
+        # Coincidentally-equal values stay as two separate col_span=1 cells.
+        last_row = td.grid[3]
+        assert last_row[1].text == last_row[2].text == "150"
+        assert last_row[1].col_span == 1 and last_row[2].col_span == 1
+        assert last_row[1] is not last_row[2]
+
+    def test_without_member_data_falls_back_to_flat(self):
+        """No member_data -> previous behaviour: flat 1x1 body cells."""
+        original = self._fragment()
+        merged_df = pd.DataFrame(
+            [["See important notes below"] * 3],
+            columns=["Section", "Plan A", "Plan B"],
+        )
+        td = _dataframe_to_docling_data(merged_df, original_data=original)
+        assert all(cell.col_span == 1 for cell in td.grid[1])
+
+
 class TestAdapterProtocol:
     """Verify DoclingAdapter satisfies the protocol."""