table2rules 0.4.1__tar.gz → 0.5.1__tar.gz

{table2rules-0.4.1/src/table2rules.egg-info → table2rules-0.5.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: table2rules
-Version: 0.4.1
+Version: 0.5.1
 Summary: Convert HTML tables to flat, LLM-friendly rules using spatial pathfinding.
 Author: PebbleRoad Pte Ltd
 License-Expression: MIT

{table2rules-0.4.1 → table2rules-0.5.1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "table2rules"
-version = "0.4.1"
+version = "0.5.1"
 description = "Convert HTML tables to flat, LLM-friendly rules using spatial pathfinding."
 readme = "README.md"
 license = "MIT"

{table2rules-0.4.1 → table2rules-0.5.1}/src/table2rules/_core.py

@@ -122,9 +122,18 @@ def _extract_cell_rows(table_html: str) -> List[List[str]]:
 def _build_rules(grid) -> List[LogicRule]:
     """Walk the parsed grid and emit one LogicRule per data cell position."""
     rules: List[LogicRule] = []
+    n_cols = len(grid[0])
+
+    # Rows that carry a *real* value — used below to decide which rows need
+    # label-only preservation. A value that merely echoes its own column header
+    # (a de-spanned or page-break-repeated header cell) carries no independent
+    # data and is dropped downstream by clean_rules; it must not mask an
+    # otherwise label-only row. Tracked at the value's *target* positions so a
+    # rowspan-filled value correctly marks every row it covers.
+    rows_with_value: set = set()
 
     for row_idx in range(len(grid)):
-        for col_idx in range(len(grid[0])):
+        for col_idx in range(n_cols):
             cell = grid[row_idx][col_idx]
 
             # Only <td> cells are data cells
@@ -144,6 +153,7 @@ def _build_rules(grid) -> List[LogicRule]:
 
             rowspan = cell.get("rowspan", 1)
             colspan = cell.get("colspan", 1)
+            outcome_norm = cell["text"].strip().lower()
 
             for r_offset in range(rowspan):
                 for c_offset in range(colspan):
@@ -166,6 +176,56 @@ def _build_rules(grid) -> List[LogicRule]:
                         )
                     )
 
+                    is_header_echo = outcome_norm in {h.strip().lower() for h in col_headers}
+                    if not is_header_echo:
+                        rows_with_value.add(target_row)
+
+    # Label-only preservation: a body row whose row-header label is present but
+    # which carries no independent value would otherwise vanish entirely — the
+    # data loop above emits nothing usable for it. This is how de-spanned
+    # section headers arrive when an OCR/HTML pipeline drops the original
+    # ``colspan``: the value column is either empty (a benefits-schedule title
+    # row "2. Public transport double indemnity") or repeats the column header
+    # (a "24. COVID-19 Coverage Extension | Sum Insured" row, whose echoed value
+    # clean_rules strips, taking the label with it). It is structurally
+    # indistinguishable from a leaf row with a genuinely missing value, so we
+    # preserve the label verbatim rather than fabricate a section breadcrumb.
+    for row_idx in range(len(grid)):
+        if row_idx in rows_with_value:
+            continue
+        # Anchor the rule at the row's data column so it satisfies the quality
+        # gate's "rules originate from <td>" invariant. A row with no <td> at
+        # all is a true full-width <th colspan> divider — already handled as a
+        # row-group ancestor upstream — so we leave it alone.
+        anchor_col = next((c for c in range(n_cols) if grid[row_idx][c]["type"] == "td"), None)
+        if anchor_col is None:
+            continue
+        label_parts: List[str] = []
+        for col_idx in range(n_cols):
+            cell = grid[row_idx][col_idx]
+            if cell["type"] != "th":
+                continue
+            if cell.get("is_thead", False) or cell.get("is_header_row", False):
+                continue
+            if cell.get("is_span_copy", False):
+                continue
+            text = (cell.get("text") or "").strip()
+            if not text:
+                continue
+            label_parts.append(text)
+        if not label_parts:
+            continue
+        rules.append(
+            LogicRule(
+                outcome=" > ".join(label_parts),
+                position=(row_idx, anchor_col),
+                row_headers=(),
+                col_headers=(),
+                origin=(row_idx, anchor_col),
+                is_label=True,
+            )
+        )
+
     return rules
 
 

{table2rules-0.4.1 → table2rules-0.5.1}/src/table2rules/models.py

@@ -12,6 +12,12 @@ class LogicRule:
     col_headers: Tuple[str, ...] = ()
     origin: Optional[Tuple[int, int]] = None
     is_footer: bool = False
+    # A label-preservation rule: the row carried a label but no independent
+    # value (empty value column, or a value that merely echoes the column
+    # header). The label is preserved verbatim as the outcome with no header
+    # relationship. The confidence gate treats these as pass-through, not a
+    # parser-confidence signal.
+    is_label: bool = False
 
     def to_string(self) -> str:
         """Descriptive format for Graph-RAG: '<rows> → <cols>: <value>'."""

{table2rules-0.4.1 → table2rules-0.5.1}/src/table2rules/quality_gate.py

@@ -74,6 +74,13 @@ def assess_confidence(grid: List[List[Dict]], rules: List[LogicRule]) -> GateRes
     if not candidates:
         return GateResult(ok=False, score=0.0, reasons=["no_candidate_data_cells"])
 
+    # Score only value rules. Label-preservation rules (a row's label kept
+    # visible when it carries no independent value) have no header relationship
+    # by design — they are pass-through, not a parser-confidence signal, so they
+    # neither help nor hurt the gate. check_invariants above still validates
+    # them (a valid <td> anchor, non-empty outcome).
+    rules = [r for r in rules if not r.is_label]
+
     rule_positions = {rule.position for rule in rules}
     coverage = len(rule_positions) / max(1, len(candidates))
 

{table2rules-0.4.1 → table2rules-0.5.1}/src/table2rules/simple_repair.py

@@ -122,6 +122,46 @@ def detect_header_block(rows):
     [0..k-1] range and are left as-is so the downstream thead-wrap naturally
     excludes them via Fix 7's contiguous-<th> chain.
 
+    Two further structural witnesses extend "clean data row" to disqualify
+    rows that look header-shaped relative to the body:
+
+    * **Fuller-than-body**: row r's non-empty cell count is strictly greater
+      than the minimum non-empty count of the non-divider rows below it,
+      AND every column where row r is non-empty has at least one body
+      row that fills the same column. A row that fills more columns
+      than at least one body row is naming columns the body sometimes
+      leaves empty — the structural signature of a column-header row
+      above an implicit-rowspan group-label column, a multi-stub
+      indentation pyramid, or an alternating coefficient/std-error
+      layout. Comparing to the minimum (rather than median or mean)
+      is intentional: the structural distinction is "exists a body
+      row with fewer non-empty cells than row r," and strictly more
+      central statistics misfire on tables where a slim majority of body
+      rows match row r's fullness. The body-coverage clause excludes
+      receipts whose row 0 is a 4-cell line item followed by 2-cell
+      totals — there cols 2 and 3 are filled only in row 0, the body
+      never uses them, and promoting row 0 to header would erase the
+      line-item data. Uniform-dense tables stay out because their min
+      equals row r's count.
+
+    * **Cell-type inversion**: row r contains at least one corner-stub
+      ``<td>`` — a column where row r is ``<td>`` while the body majority
+      is ``<th>`` — and a strict majority of compared columns invert.
+      The corner-stub clause is load-bearing: an all-``<th>`` row above
+      an all-``<td>`` body inverts every column too, but that's the
+      *normal* header pattern (and Fix 7 already wraps it in ``<thead>``
+      via the contiguous-``<th>`` chain). Inversion is only the right
+      witness when row r has a stray ``<td>`` corner cell that the body
+      makes a ``<th scope="row">`` (e.g., header row ``[td, th, th]``
+      above body rows ``[th, td, td]``). Universal: cell tags are
+      markup, not content.
+
+    Both extended witnesses apply only at r == 0 — the literal first
+    row. Beyond row 0, "fuller than the body below" describes regular
+    data rows (regression group labels, alternating coefficient/std-err
+    pairs) and "first row with col 0 non-empty" is already the existing
+    stub-column header pattern handled by the rest of the function.
+
     Returns (k, stub_cols, origin_cells, grid) on success, or None.
     """
     n = len(rows)
@@ -132,6 +172,85 @@ def detect_header_block(rows):
     if max_cols == 0:
         return None
 
+    # Pre-compute per-row non-empty counts (logical, colspan-expanded).
+    nonempty_counts = [sum(1 for c in row if c["nonempty"]) for row in grid]
+
+    def _body_min_nonempty(after_r: int) -> int:
+        """Minimum non-empty count among non-divider rows strictly after
+        ``after_r``. Returns -1 when no qualifying body row exists
+        (caller treats that as "no body to compare against" and skips
+        the witness).
+        """
+        counts = [nonempty_counts[r2] for r2 in range(after_r + 1, n) if nonempty_counts[r2] >= 2]
+        if not counts:
+            return -1
+        return min(counts)
+
+    def _row_cols_covered_by_body(r: int) -> bool:
+        """True iff every column where row r is non-empty has at least one
+        non-divider body row below that fills the same column. Excludes
+        receipts whose row 0 fills columns the body never uses (line
+        item over totals)."""
+        for c in range(max_cols):
+            if not grid[r][c]["nonempty"]:
+                continue
+            covered = False
+            for r2 in range(r + 1, n):
+                if nonempty_counts[r2] < 2:
+                    continue
+                if grid[r2][c]["nonempty"]:
+                    covered = True
+                    break
+            if not covered:
+                return False
+        return True
+
+    def _is_inverted_relative_to_body(r: int) -> bool:
+        """True iff row r contains a corner-stub ``<td>`` and inverts the
+        body majority at a strict majority of compared columns. Only
+        origin cells with non-empty text on both sides participate.
+
+        The corner-stub clause requires ≥1 column where row r is ``<td>``
+        but body majority is ``<th>``. Without it, an all-``<th>`` row
+        above an all-``<td>`` body would also "invert" every column — but
+        that's the normal header pattern, already handled by Fix 7's
+        contiguous-``<th>`` thead wrap.
+        """
+        has_corner_stub = False
+        inverted = 0
+        checked = 0
+        for c in range(max_cols):
+            row_cell = grid[r][c]
+            if row_cell["origin"] != (r, c) or not row_cell["nonempty"]:
+                continue
+            row_origin = origin_cells.get((r, c))
+            if row_origin is None or row_origin.name not in ("td", "th"):
+                continue
+            row_tag = row_origin.name
+
+            td_count = 0
+            th_count = 0
+            for r2 in range(r + 1, n):
+                body_cell = grid[r2][c]
+                if body_cell["origin"] != (r2, c) or not body_cell["nonempty"]:
+                    continue
+                body_origin = origin_cells.get((r2, c))
+                if body_origin is None:
+                    continue
+                if body_origin.name == "td":
+                    td_count += 1
+                elif body_origin.name == "th":
+                    th_count += 1
+            if td_count + th_count == 0:
+                continue
+            body_majority = "td" if td_count >= th_count else "th"
+            checked += 1
+            if body_majority != row_tag:
+                inverted += 1
+                if row_tag == "td" and body_majority == "th":
+                    has_corner_stub = True
+        return has_corner_stub and checked > 0 and inverted * 2 > checked
+
     # Find the first data row. A data row is one whose shape has no
     # structural feature that would mark it as a header:
     #   - col 0 is non-empty (typical row-label)
@@ -140,6 +259,13 @@ def detect_header_block(rows):
     #     (no rowspan copy from above pulling header content into body)
     #   - every origin cell at row r has rowspan == colspan == 1
     #     (no colspan group or rowspan marker signaling header role)
+    # Plus, only at r == 0 (the literal first row):
+    #   - row 0 is not strictly fuller than at least one body row below
+    #     while every column it fills is covered by some body row
+    #     (the fuller-than-body witness — see docstring)
+    #   - row 0's cell tags are not inverted relative to body majority,
+    #     with at least one corner-stub ``<td>`` versus body ``<th>``
+    #     (the cell-type-inversion witness — see docstring)
     #
     # The "every cell non-empty" requirement was dropped intentionally:
     # real body rows in financial tables are often gappy (a cell has a
@@ -153,7 +279,7 @@ def detect_header_block(rows):
         row = grid[r]
         if not row[0]["nonempty"]:
             continue
-        nonempty_count = sum(1 for c in row if c["nonempty"])
+        nonempty_count = nonempty_counts[r]
         if nonempty_count < 2:
             continue
         is_clean = True
@@ -165,6 +291,12 @@ def detect_header_block(rows):
             if cell["rs"] != 1 or cell["cs"] != 1:
                 is_clean = False
                 break
+        if is_clean and r == 0:
+            body_min = _body_min_nonempty(r)
+            if body_min >= 0 and nonempty_count > body_min and _row_cols_covered_by_body(r):
+                is_clean = False
+            if is_clean and _is_inverted_relative_to_body(r):
+                is_clean = False
         if is_clean:
             first_data_idx = r
             break

{table2rules-0.4.1 → table2rules-0.5.1/src/table2rules.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: table2rules
-Version: 0.4.1
+Version: 0.5.1
 Summary: Convert HTML tables to flat, LLM-friendly rules using spatial pathfinding.
 Author: PebbleRoad Pte Ltd
 License-Expression: MIT

{table2rules-0.4.1 → table2rules-0.5.1}/tests/test_correctness_oracle.py

@@ -188,6 +188,14 @@ def test_correctness_oracle(case: tuple[Path, Path]) -> None:
     matched = 0
     emitted_lines = [l for l in output.splitlines() if l.strip()]
     for line in emitted_lines:
+        # Label-preservation lines reproduce a whole source cell verbatim (a
+        # de-spanned section header whose value column is empty, e.g.
+        # "Segments: (1)"). Such a cell may itself contain ": ", which the
+        # rule-line parser would misread as a key/value split. A line equal to
+        # a full source cell is faithful preservation, not misattribution.
+        if source_tokens and _norm(line) in source_tokens:
+            matched += 1
+            continue
         parsed = _parse_rule_line(line, source_tokens)
         if parsed is None:
             continue

{table2rules-0.4.1 → table2rules-0.5.1}/tests/test_regression_golds.py

@@ -1,13 +1,20 @@
-"""Regression layer — byte-for-byte gold matching on hand-authored fixtures.
-
-Each .md file beneath tests/{adversarial,structured,headerless,smoke,
-regression}/ is a fixture containing HTML table markup. For every fixture
-we run process_tables_to_text and assert the output matches the committed
-gold file under benchmarks/gold/<format>/.
+"""Regression layer — byte-for-byte gold matching on every fixture.
+
+Each .md file beneath tests/ (except top-level docs) is a fixture containing
+HTML table markup. For every fixture we run process_tables_to_text and assert
+the output matches the committed gold file under benchmarks/gold/<format>/.
+
+This covers both the hand-authored fixtures AND the real-world corpus
+(tests/realworld/). The two suites play complementary roles: the correctness
+and robustness layers (test_correctness_oracle / test_robustness_mutations)
+assert the output is *right* (no fabricated content, correct attribution,
+stable under mutation); this layer asserts the output does not *change* unless
+a human regenerates the golds. Together they catch a silent-drop regression —
+where the parser quietly stops emitting real content — which neither the
+oracle (it only guards against fabrication) nor an un-asserted benchmark gold
+could catch on its own. See tests/README.md.
 
 This is the strictest of the three test layers — catches any output drift.
-See tests/README.md for the relationship to the correctness and robustness
-suites.
 
 Refresh gold outputs by running:  python scripts/benchmark.py --update-gold
 """
@@ -27,15 +34,14 @@ GOLD_DIR = ROOT / "benchmarks" / "gold" / DEFAULT_FORMAT
 
 
 def _discover_cases() -> list[Path]:
-    # Real-world fixtures (tests/realworld/) are checked against per-fixture
-    # oracle triples, not frozen gold text — see test_correctness_oracle.py.
-    # Top-level docs like README.md are not fixtures.
-    skip_prefixes = {"realworld"}
+    # Every fixture beneath tests/ is byte-checked, including the real-world
+    # corpus (tests/realworld/) — frozen gold text is the tripwire that makes
+    # any output change visible. Top-level docs like tests/README.md are not
+    # fixtures and are excluded.
     return [
         p
         for p in sorted(TESTS_DIR.rglob("*.md"))
-        if not (skip_prefixes & set(p.relative_to(TESTS_DIR).parts))
-        and p.parent != TESTS_DIR  # exclude tests/README.md etc.
+        if p.parent != TESTS_DIR  # exclude tests/README.md, tests/failing_table.md
     ]
 
 

{table2rules-0.4.1 → table2rules-0.5.1}/tests/test_robustness_mutations.py

@@ -95,13 +95,22 @@ def _parse_rule_line(line: str, source_tokens: frozenset[str] = frozenset()):
     return row_path, col_path, value
 
 
-def _classify(output: str) -> str:
+def _classify(output: str, source_tokens: frozenset[str] = frozenset()) -> str:
     lines = [l for l in output.splitlines() if l.strip()]
     if not lines:
         return "EMPTY"
     if any("<table" in l for l in lines):
         return "PASSTHROUGH"
-    rule_shaped = sum(1 for l in lines if _parse_rule_line(l) is not None)
+    # A label-preservation line reproduces a whole source cell verbatim (a
+    # de-spanned/echoed section header kept visible). It is not key/value
+    # shaped, but it is legitimate rules-mode output — count it as such so a
+    # table of rules plus section labels stays RULES rather than degrading to
+    # MIXED (which would skip the precision check below).
+    rule_shaped = sum(
+        1
+        for l in lines
+        if _parse_rule_line(l) is not None or (source_tokens and _norm(l) in source_tokens)
+    )
     if rule_shaped == len(lines):
         return "RULES"
     if rule_shaped == 0:
@@ -345,7 +354,7 @@ def test_robustness_under_mutation(case: tuple[Path, Path], mutation_name: str)
     mutated_html = mutator(html, rng)
 
     output = process_tables_to_text(mutated_html)
-    tier = _classify(output)
+    tier = _classify(output, source_tokens)
     if tier in {"PASSTHROUGH", "FLAT", "EMPTY", "MIXED"}:
         # Safe fallback; not a precision failure.
         pytest.skip(f"tier={tier} after mutation={mutation_name!r}")
@@ -363,6 +372,14 @@ def test_robustness_under_mutation(case: tuple[Path, Path], mutation_name: str)
     for line in output.splitlines():
         if not line.strip():
             continue
+        # Label-preservation lines reproduce a whole source cell verbatim
+        # (a de-spanned section header whose value column is empty, e.g.
+        # "Segments: (1)"). The cell text itself may contain ": ", which the
+        # rule-line parser would misread as a key/value split. A line equal to
+        # a full source cell is faithful preservation, not fabrication — the
+        # contract is "no invented content", and there is none here.
+        if source_tokens and _norm(line) in source_tokens:
+            continue
         parsed = _parse_rule_line(line, source_tokens)
         if parsed is None:
             continue