parse-bench 1.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (227) hide show
  1. parse_bench/__init__.py +3 -0
  2. parse_bench/analysis/__init__.py +6 -0
  3. parse_bench/analysis/aggregation_report.py +582 -0
  4. parse_bench/analysis/cli.py +472 -0
  5. parse_bench/analysis/comparison.py +382 -0
  6. parse_bench/analysis/comparison_core.py +357 -0
  7. parse_bench/analysis/comparison_report.py +2066 -0
  8. parse_bench/analysis/detailed_report.py +2254 -0
  9. parse_bench/analysis/leaderboard_report.py +852 -0
  10. parse_bench/analysis/metric_definitions.py +771 -0
  11. parse_bench/cli.py +267 -0
  12. parse_bench/data/__init__.py +1 -0
  13. parse_bench/data/cli.py +118 -0
  14. parse_bench/data/download.py +127 -0
  15. parse_bench/evaluation/__init__.py +11 -0
  16. parse_bench/evaluation/cli.py +435 -0
  17. parse_bench/evaluation/evaluators/__init__.py +17 -0
  18. parse_bench/evaluation/evaluators/base.py +34 -0
  19. parse_bench/evaluation/evaluators/extract.py +429 -0
  20. parse_bench/evaluation/evaluators/layoutdet.py +1682 -0
  21. parse_bench/evaluation/evaluators/parse.py +1353 -0
  22. parse_bench/evaluation/evaluators/qa.py +199 -0
  23. parse_bench/evaluation/layout_adapters/__init__.py +21 -0
  24. parse_bench/evaluation/layout_adapters/adapters.py +3180 -0
  25. parse_bench/evaluation/layout_adapters/base.py +105 -0
  26. parse_bench/evaluation/layout_adapters/registry.py +109 -0
  27. parse_bench/evaluation/layout_label_mappers/__init__.py +22 -0
  28. parse_bench/evaluation/layout_label_mappers/base.py +66 -0
  29. parse_bench/evaluation/layout_label_mappers/mappers.py +332 -0
  30. parse_bench/evaluation/layout_label_mappers/projection.py +74 -0
  31. parse_bench/evaluation/layout_label_mappers/registry.py +119 -0
  32. parse_bench/evaluation/metric_aggregation.py +56 -0
  33. parse_bench/evaluation/metrics/__init__.py +5 -0
  34. parse_bench/evaluation/metrics/attribution/__init__.py +35 -0
  35. parse_bench/evaluation/metrics/attribution/constants.py +12 -0
  36. parse_bench/evaluation/metrics/attribution/core.py +1108 -0
  37. parse_bench/evaluation/metrics/attribution/evaluate.py +446 -0
  38. parse_bench/evaluation/metrics/attribution/geometry.py +161 -0
  39. parse_bench/evaluation/metrics/attribution/text_utils.py +233 -0
  40. parse_bench/evaluation/metrics/base.py +33 -0
  41. parse_bench/evaluation/metrics/downstream/__init__.py +0 -0
  42. parse_bench/evaluation/metrics/extract/__init__.py +29 -0
  43. parse_bench/evaluation/metrics/extract/json_subset_match.py +473 -0
  44. parse_bench/evaluation/metrics/extract/json_subset_match_metric.py +81 -0
  45. parse_bench/evaluation/metrics/extract/list_unwrap.py +340 -0
  46. parse_bench/evaluation/metrics/extract/rule_based_metric.py +90 -0
  47. parse_bench/evaluation/metrics/extract/test_rules.py +409 -0
  48. parse_bench/evaluation/metrics/extract/test_types.py +11 -0
  49. parse_bench/evaluation/metrics/field_grounding/__init__.py +21 -0
  50. parse_bench/evaluation/metrics/field_grounding/core.py +437 -0
  51. parse_bench/evaluation/metrics/field_grounding/extract_adapter.py +1224 -0
  52. parse_bench/evaluation/metrics/field_grounding/parse_adapter.py +697 -0
  53. parse_bench/evaluation/metrics/field_grounding/rule_filters.py +19 -0
  54. parse_bench/evaluation/metrics/field_grounding/value_compare.py +190 -0
  55. parse_bench/evaluation/metrics/layoutdet/__init__.py +17 -0
  56. parse_bench/evaluation/metrics/layoutdet/classification_utils.py +300 -0
  57. parse_bench/evaluation/metrics/layoutdet/iou.py +76 -0
  58. parse_bench/evaluation/metrics/parse/__init__.py +5 -0
  59. parse_bench/evaluation/metrics/parse/_vendor_grits_reference.py +531 -0
  60. parse_bench/evaluation/metrics/parse/cross_page_table_consistency.py +165 -0
  61. parse_bench/evaluation/metrics/parse/emphasis_spans.py +242 -0
  62. parse_bench/evaluation/metrics/parse/fast_tree_edit.py +282 -0
  63. parse_bench/evaluation/metrics/parse/grits_metric.py +1125 -0
  64. parse_bench/evaluation/metrics/parse/grits_reference_metric.py +142 -0
  65. parse_bench/evaluation/metrics/parse/header_accuracy_metric.py +1662 -0
  66. parse_bench/evaluation/metrics/parse/llm_normalization/__init__.py +51 -0
  67. parse_bench/evaluation/metrics/parse/llm_normalization/base.py +125 -0
  68. parse_bench/evaluation/metrics/parse/llm_normalization/config.py +44 -0
  69. parse_bench/evaluation/metrics/parse/llm_normalization/postprocess.py +322 -0
  70. parse_bench/evaluation/metrics/parse/llm_normalization/strategy_judge.py +541 -0
  71. parse_bench/evaluation/metrics/parse/mermaid_graph.py +682 -0
  72. parse_bench/evaluation/metrics/parse/rule_based_judge_metric.py +56 -0
  73. parse_bench/evaluation/metrics/parse/rule_based_metric.py +434 -0
  74. parse_bench/evaluation/metrics/parse/rules_bag.py +1161 -0
  75. parse_bench/evaluation/metrics/parse/rules_base.py +751 -0
  76. parse_bench/evaluation/metrics/parse/rules_chart.py +1556 -0
  77. parse_bench/evaluation/metrics/parse/rules_diagram.py +591 -0
  78. parse_bench/evaluation/metrics/parse/rules_form.py +2274 -0
  79. parse_bench/evaluation/metrics/parse/rules_formatting.py +1500 -0
  80. parse_bench/evaluation/metrics/parse/rules_heading.py +228 -0
  81. parse_bench/evaluation/metrics/parse/rules_list.py +226 -0
  82. parse_bench/evaluation/metrics/parse/rules_page_decoration.py +276 -0
  83. parse_bench/evaluation/metrics/parse/rules_table.py +1666 -0
  84. parse_bench/evaluation/metrics/parse/rules_text.py +340 -0
  85. parse_bench/evaluation/metrics/parse/rules_watermark.py +105 -0
  86. parse_bench/evaluation/metrics/parse/structural_consistency_metric.py +251 -0
  87. parse_bench/evaluation/metrics/parse/table_extraction.py +152 -0
  88. parse_bench/evaluation/metrics/parse/table_merging.py +195 -0
  89. parse_bench/evaluation/metrics/parse/table_pairing.py +87 -0
  90. parse_bench/evaluation/metrics/parse/table_parsing.py +955 -0
  91. parse_bench/evaluation/metrics/parse/table_record_match_metric.py +1453 -0
  92. parse_bench/evaluation/metrics/parse/table_splitting.py +301 -0
  93. parse_bench/evaluation/metrics/parse/table_title_stripping.py +530 -0
  94. parse_bench/evaluation/metrics/parse/teds_metric.py +600 -0
  95. parse_bench/evaluation/metrics/parse/test_rules.py +120 -0
  96. parse_bench/evaluation/metrics/parse/test_types.py +103 -0
  97. parse_bench/evaluation/metrics/parse/text_content_projection.py +175 -0
  98. parse_bench/evaluation/metrics/parse/text_similarity_metric.py +61 -0
  99. parse_bench/evaluation/metrics/parse/utils.py +885 -0
  100. parse_bench/evaluation/metrics/qa/__init__.py +5 -0
  101. parse_bench/evaluation/metrics/qa/answer_comparison.py +380 -0
  102. parse_bench/evaluation/qa/__init__.py +5 -0
  103. parse_bench/evaluation/qa/llm_service.py +335 -0
  104. parse_bench/evaluation/reports/__init__.py +8 -0
  105. parse_bench/evaluation/reports/csv.py +64 -0
  106. parse_bench/evaluation/reports/html.py +338 -0
  107. parse_bench/evaluation/reports/markdown.py +98 -0
  108. parse_bench/evaluation/reports/rule_csv.py +22 -0
  109. parse_bench/evaluation/runner.py +1864 -0
  110. parse_bench/evaluation/stats.py +104 -0
  111. parse_bench/extensions.py +72 -0
  112. parse_bench/inference/__init__.py +33 -0
  113. parse_bench/inference/chunkr_layout_extraction.py +160 -0
  114. parse_bench/inference/cli.py +484 -0
  115. parse_bench/inference/layout_extraction.py +422 -0
  116. parse_bench/inference/pipelines/__init__.py +59 -0
  117. parse_bench/inference/pipelines/extract.py +39 -0
  118. parse_bench/inference/pipelines/layout.py +142 -0
  119. parse_bench/inference/pipelines/parse.py +2603 -0
  120. parse_bench/inference/pipelines.py +0 -0
  121. parse_bench/inference/providers/__init__.py +28 -0
  122. parse_bench/inference/providers/base.py +196 -0
  123. parse_bench/inference/providers/cancellation.py +137 -0
  124. parse_bench/inference/providers/extract/__init__.py +22 -0
  125. parse_bench/inference/providers/extract/citations.py +549 -0
  126. parse_bench/inference/providers/extract/extend.py +851 -0
  127. parse_bench/inference/providers/extract/llamaextract_v2_api.py +583 -0
  128. parse_bench/inference/providers/layoutdet/__init__.py +25 -0
  129. parse_bench/inference/providers/layoutdet/adapters.py +946 -0
  130. parse_bench/inference/providers/layoutdet/base.py +203 -0
  131. parse_bench/inference/providers/layoutdet/chandra.py +449 -0
  132. parse_bench/inference/providers/layoutdet/docling.py +125 -0
  133. parse_bench/inference/providers/layoutdet/dots_ocr.py +606 -0
  134. parse_bench/inference/providers/layoutdet/layout_v3.py +137 -0
  135. parse_bench/inference/providers/layoutdet/layout_v3_byoc.py +204 -0
  136. parse_bench/inference/providers/layoutdet/paddle.py +117 -0
  137. parse_bench/inference/providers/layoutdet/qwen3vl.py +360 -0
  138. parse_bench/inference/providers/layoutdet/surya.py +250 -0
  139. parse_bench/inference/providers/layoutdet/yolo.py +109 -0
  140. parse_bench/inference/providers/parse/__init__.py +64 -0
  141. parse_bench/inference/providers/parse/_docling_common.py +233 -0
  142. parse_bench/inference/providers/parse/_layout_utils.py +611 -0
  143. parse_bench/inference/providers/parse/amazon_nova.py +515 -0
  144. parse_bench/inference/providers/parse/anthropic.py +882 -0
  145. parse_bench/inference/providers/parse/azure_document_intelligence.py +700 -0
  146. parse_bench/inference/providers/parse/chandra2.py +633 -0
  147. parse_bench/inference/providers/parse/chunkr.py +268 -0
  148. parse_bench/inference/providers/parse/databricks_ai_parse.py +724 -0
  149. parse_bench/inference/providers/parse/datalab.py +370 -0
  150. parse_bench/inference/providers/parse/deepseekocr2.py +382 -0
  151. parse_bench/inference/providers/parse/docling.py +281 -0
  152. parse_bench/inference/providers/parse/docling_serve.py +289 -0
  153. parse_bench/inference/providers/parse/dots_ocr.py +574 -0
  154. parse_bench/inference/providers/parse/extend_parse.py +710 -0
  155. parse_bench/inference/providers/parse/falconocr.py +436 -0
  156. parse_bench/inference/providers/parse/florin_parser_nano.py +559 -0
  157. parse_bench/inference/providers/parse/gemma4.py +472 -0
  158. parse_bench/inference/providers/parse/glm_zai.py +229 -0
  159. parse_bench/inference/providers/parse/google.py +1125 -0
  160. parse_bench/inference/providers/parse/google_agentic_vision.py +819 -0
  161. parse_bench/inference/providers/parse/google_docai.py +776 -0
  162. parse_bench/inference/providers/parse/google_docai_layout_normalization.py +573 -0
  163. parse_bench/inference/providers/parse/granite_vision.py +515 -0
  164. parse_bench/inference/providers/parse/infinity_parser2.py +704 -0
  165. parse_bench/inference/providers/parse/kdl_frontier_nano.py +3327 -0
  166. parse_bench/inference/providers/parse/landingai.py +452 -0
  167. parse_bench/inference/providers/parse/liteparse.py +350 -0
  168. parse_bench/inference/providers/parse/llamaparse.py +677 -0
  169. parse_bench/inference/providers/parse/llamaparse_v2_normalization.py +1013 -0
  170. parse_bench/inference/providers/parse/markitdown.py +138 -0
  171. parse_bench/inference/providers/parse/mineru25.py +405 -0
  172. parse_bench/inference/providers/parse/mineru2605pro.py +432 -0
  173. parse_bench/inference/providers/parse/mineru_diffusion.py +371 -0
  174. parse_bench/inference/providers/parse/mistral_ocr.py +546 -0
  175. parse_bench/inference/providers/parse/nemotron_omni.py +473 -0
  176. parse_bench/inference/providers/parse/oi_parser.py +222 -0
  177. parse_bench/inference/providers/parse/openai.py +740 -0
  178. parse_bench/inference/providers/parse/opendataloader.py +152 -0
  179. parse_bench/inference/providers/parse/paddleocr.py +624 -0
  180. parse_bench/inference/providers/parse/pdf_inspector.py +142 -0
  181. parse_bench/inference/providers/parse/pulse.py +785 -0
  182. parse_bench/inference/providers/parse/pymupdf.py +207 -0
  183. parse_bench/inference/providers/parse/pymupdf4llm.py +356 -0
  184. parse_bench/inference/providers/parse/pypdf.py +179 -0
  185. parse_bench/inference/providers/parse/qwen.py +678 -0
  186. parse_bench/inference/providers/parse/rakedoc_nano.py +70 -0
  187. parse_bench/inference/providers/parse/reducto.py +546 -0
  188. parse_bench/inference/providers/parse/surya2.py +372 -0
  189. parse_bench/inference/providers/parse/tesseract.py +301 -0
  190. parse_bench/inference/providers/parse/textract.py +694 -0
  191. parse_bench/inference/providers/parse/unlimitedocr.py +346 -0
  192. parse_bench/inference/providers/parse/unstructured.py +485 -0
  193. parse_bench/inference/providers/parse/warp_ingest.py +199 -0
  194. parse_bench/inference/providers/registry.py +49 -0
  195. parse_bench/inference/renormalize.py +170 -0
  196. parse_bench/inference/runner.py +2023 -0
  197. parse_bench/layout_label_mapping.py +424 -0
  198. parse_bench/layout_projection.py +179 -0
  199. parse_bench/pipeline/__init__.py +1 -0
  200. parse_bench/pipeline/cli.py +549 -0
  201. parse_bench/schemas/__init__.py +33 -0
  202. parse_bench/schemas/evaluation.py +93 -0
  203. parse_bench/schemas/extract_output.py +36 -0
  204. parse_bench/schemas/layout_detection_output.py +545 -0
  205. parse_bench/schemas/layout_ontology.py +315 -0
  206. parse_bench/schemas/metrics.py +69 -0
  207. parse_bench/schemas/parse_output.py +152 -0
  208. parse_bench/schemas/pipeline.py +22 -0
  209. parse_bench/schemas/pipeline_io.py +106 -0
  210. parse_bench/schemas/product.py +97 -0
  211. parse_bench/test_cases/__init__.py +25 -0
  212. parse_bench/test_cases/bbox_value_strict_comparator.py +880 -0
  213. parse_bench/test_cases/extract_field_paths.py +164 -0
  214. parse_bench/test_cases/layout_attribution_generation.py +287 -0
  215. parse_bench/test_cases/loader.py +652 -0
  216. parse_bench/test_cases/parse_rule_schemas.py +1071 -0
  217. parse_bench/test_cases/rule_filters.py +32 -0
  218. parse_bench/test_cases/rule_ids.py +107 -0
  219. parse_bench/test_cases/schema.py +427 -0
  220. parse_bench/utils/__init__.py +15 -0
  221. parse_bench/utils/gemini_layout_utils.py +670 -0
  222. parse_bench/utils/text_aggregation.py +100 -0
  223. parse_bench-1.0.0.dist-info/METADATA +476 -0
  224. parse_bench-1.0.0.dist-info/RECORD +227 -0
  225. parse_bench-1.0.0.dist-info/WHEEL +4 -0
  226. parse_bench-1.0.0.dist-info/entry_points.txt +2 -0
  227. parse_bench-1.0.0.dist-info/licenses/LICENSE +201 -0
@@ -0,0 +1,251 @@
1
+ """Structural consistency metric for HTML table comparison.
2
+
3
+ A binary metric that flags when a predicted table has an inconsistent
4
+ internal structure — specifically, when rows or columns have an
5
+ inconsistent number of cells (after resolving colspan/rowspan).
6
+
7
+ This is a *self-consistency* check on the predicted table alone (not a
8
+ comparison against ground truth). A structurally consistent table has
9
+ every row spanning the same total number of columns and every column
10
+ spanning the same total number of rows.
11
+
12
+ Returns 1.0 (consistent) or 0.0 (inconsistent) per table, averaged
13
+ across all tables in the document.
14
+ """
15
+
16
+ from __future__ import annotations
17
+
18
+ from typing import Any
19
+
20
+ from bs4 import BeautifulSoup
21
+
22
+ from parse_bench.evaluation.metrics.base import Metric
23
+ from parse_bench.evaluation.metrics.parse.table_extraction import extract_html_tables
24
+ from parse_bench.schemas.evaluation import MetricValue
25
+
26
+
27
+ def _mode_value(counts: list[int]) -> int:
28
+ """Return the modal (most common) value from *counts*.
29
+
30
+ Ties are broken by first occurrence, matching the platform logic.
31
+ """
32
+ if not counts:
33
+ return 0
34
+ freq: dict[int, int] = {}
35
+ first_seen: dict[int, int] = {}
36
+ for i, c in enumerate(counts):
37
+ freq[c] = freq.get(c, 0) + 1
38
+ if c not in first_seen:
39
+ first_seen[c] = i
40
+
41
+ best_count = 0
42
+ best_val = counts[0]
43
+ for val, f in freq.items():
44
+ if f > best_count or (f == best_count and first_seen[val] < first_seen[best_val]):
45
+ best_count = f
46
+ best_val = val
47
+ return best_val
48
+
49
+
50
+ def _check_table_consistency(table_html: str) -> dict[str, Any]:
51
+ """Check structural consistency of a single HTML table.
52
+
53
+ Returns a dict with:
54
+ - consistent: bool
55
+ - num_rows: int
56
+ - num_cols: int (max column span across all rows)
57
+ - row_cell_counts: list[int] — effective column count per row
58
+ - col_cell_counts: list[int] — effective row count per column
59
+ - row_inconsistency: bool — True if rows have different widths
60
+ - col_inconsistency: bool — True if columns have different heights
61
+ - row_inconsistency_details: list[str] — per-row mismatch descriptions
62
+ - col_inconsistency_details: list[str] — per-column mismatch descriptions
63
+ """
64
+ soup = BeautifulSoup(table_html, "lxml")
65
+ table = soup.find("table")
66
+ if not table:
67
+ return {"consistent": True, "num_rows": 0, "num_cols": 0}
68
+
69
+ rows = table.find_all("tr")
70
+ if not rows:
71
+ return {"consistent": True, "num_rows": 0, "num_cols": 0}
72
+
73
+ num_rows = len(rows)
74
+
75
+ # Build an occupancy grid by resolving rowspan/colspan
76
+ occupied: dict[tuple[int, int], bool] = {}
77
+
78
+ for row_idx, row in enumerate(rows):
79
+ col_idx = 0
80
+ for cell in row.find_all(["td", "th"]):
81
+ while (row_idx, col_idx) in occupied:
82
+ col_idx += 1
83
+
84
+ rowspan = int(str(cell.get("rowspan", "1")))
85
+ colspan = int(str(cell.get("colspan", "1")))
86
+
87
+ for r in range(row_idx, row_idx + rowspan):
88
+ for c in range(col_idx, col_idx + colspan):
89
+ occupied[(r, c)] = True
90
+
91
+ col_idx += colspan
92
+
93
+ if not occupied:
94
+ return {"consistent": True, "num_rows": num_rows, "num_cols": 0}
95
+
96
+ max_row = max(r for r, c in occupied) + 1
97
+ max_col = max(c for r, c in occupied) + 1
98
+
99
+ # Count how many columns each row spans
100
+ row_cell_counts = []
101
+ for r in range(max_row):
102
+ count = sum(1 for c in range(max_col) if (r, c) in occupied)
103
+ row_cell_counts.append(count)
104
+
105
+ # Count how many rows each column spans
106
+ col_cell_counts = []
107
+ for c in range(max_col):
108
+ count = sum(1 for r in range(max_row) if (r, c) in occupied)
109
+ col_cell_counts.append(count)
110
+
111
+ # Check consistency: all rows should have the same width,
112
+ # all columns should have the same height
113
+ row_inconsistent = len(set(row_cell_counts)) > 1
114
+ col_inconsistent = len(set(col_cell_counts)) > 1
115
+
116
+ consistent = not row_inconsistent and not col_inconsistent
117
+
118
+ # Build per-row/col inconsistency details using the modal expected value
119
+ row_inconsistency_details: list[str] = []
120
+ col_inconsistency_details: list[str] = []
121
+
122
+ if row_inconsistent:
123
+ expected_cols = _mode_value(row_cell_counts)
124
+ for i, count in enumerate(row_cell_counts):
125
+ if count != expected_cols:
126
+ row_inconsistency_details.append(f"row {i + 1} has {count} cols, expected {expected_cols}")
127
+
128
+ if col_inconsistent:
129
+ expected_rows = _mode_value(col_cell_counts)
130
+ for i, count in enumerate(col_cell_counts):
131
+ if count != expected_rows:
132
+ col_inconsistency_details.append(f"col {i + 1} has {count} rows, expected {expected_rows}")
133
+
134
+ return {
135
+ "consistent": consistent,
136
+ "num_rows": max_row,
137
+ "num_cols": max_col,
138
+ "row_cell_counts": row_cell_counts,
139
+ "col_cell_counts": col_cell_counts,
140
+ "row_inconsistency": row_inconsistent,
141
+ "col_inconsistency": col_inconsistent,
142
+ "row_inconsistency_details": row_inconsistency_details,
143
+ "col_inconsistency_details": col_inconsistency_details,
144
+ }
145
+
146
+
147
+ class StructuralConsistencyMetric(Metric):
148
+ """Binary structural consistency metric for predicted HTML tables.
149
+
150
+ Checks that each predicted table is internally consistent: every row
151
+ spans the same number of columns and every column spans the same
152
+ number of rows (after resolving colspan/rowspan).
153
+
154
+ Only evaluates the *actual* (predicted) tables. Ground truth is used
155
+ only for table matching (so we report per-table diagnostics aligned
156
+ with the GT table order).
157
+
158
+ Returns a single MetricValue with value 1.0 (all tables consistent)
159
+ or 0.0 (at least one inconsistent), with per-table details in metadata.
160
+ """
161
+
162
+ @property
163
+ def name(self) -> str:
164
+ return "structural_consistency"
165
+
166
+ def compute( # type: ignore[override]
167
+ self,
168
+ expected: str,
169
+ actual: str,
170
+ **kwargs: Any,
171
+ ) -> list[MetricValue]:
172
+ actual_tables = extract_html_tables(actual)
173
+
174
+ if not actual_tables:
175
+ return [
176
+ MetricValue(
177
+ metric_name="structural_consistency",
178
+ value=1.0,
179
+ metadata={"tables_found_actual": 0, "note": "No tables to check"},
180
+ )
181
+ ]
182
+
183
+ per_table: list[dict[str, Any]] = []
184
+ scores: list[float] = []
185
+ details: list[str] = []
186
+
187
+ for idx, table_html in enumerate(actual_tables):
188
+ result = _check_table_consistency(table_html)
189
+ score = 1.0 if result["consistent"] else 0.0
190
+ scores.append(score)
191
+ per_table.append(
192
+ {
193
+ "table_index": idx,
194
+ "consistent": result["consistent"],
195
+ "num_rows": result["num_rows"],
196
+ "num_cols": result["num_cols"],
197
+ "row_inconsistency": result.get("row_inconsistency", False),
198
+ "col_inconsistency": result.get("col_inconsistency", False),
199
+ }
200
+ )
201
+
202
+ nr = result["num_rows"]
203
+ nc = result["num_cols"]
204
+
205
+ if idx > 0:
206
+ details.append("=" * 40)
207
+
208
+ if result["consistent"]:
209
+ details.append(f"Table {idx + 1}: 1.0 — consistent ({nr}×{nc})")
210
+ else:
211
+ row_details: list[str] = result.get("row_inconsistency_details", [])
212
+ col_details: list[str] = result.get("col_inconsistency_details", [])
213
+ issues = row_details + col_details
214
+ row_counts = result.get("row_cell_counts", [])
215
+ col_counts = result.get("col_cell_counts", [])
216
+ n_bad_rows = len(row_details)
217
+ n_bad_cols = len(col_details)
218
+ summary = f"Table {idx + 1}: 0.0 — inconsistent ({nr}×{nc})"
219
+ if n_bad_rows:
220
+ summary += f" {n_bad_rows}/{nr} rows"
221
+ if n_bad_cols:
222
+ summary += f" {n_bad_cols}/{nc} cols"
223
+ details.append(summary)
224
+ if row_counts:
225
+ details.append(f" row widths: {row_counts}")
226
+ if col_counts:
227
+ details.append(f" col heights: {col_counts}")
228
+ for issue in issues:
229
+ details.append(f" {issue}")
230
+
231
+ avg_score = sum(scores) / len(scores)
232
+ n_ok = sum(1 for s in scores if s == 1.0)
233
+ n_bad = sum(1 for s in scores if s == 0.0)
234
+ details.insert(
235
+ 0,
236
+ f"{avg_score:.3f} — {len(actual_tables)} table(s) checked, {n_ok} consistent, {n_bad} inconsistent",
237
+ )
238
+
239
+ return [
240
+ MetricValue(
241
+ metric_name="structural_consistency",
242
+ value=avg_score,
243
+ metadata={
244
+ "tables_found_actual": len(actual_tables),
245
+ "tables_consistent": n_ok,
246
+ "tables_inconsistent": n_bad,
247
+ "per_table_details": per_table,
248
+ },
249
+ details=details,
250
+ )
251
+ ]
@@ -0,0 +1,152 @@
1
+ """Shared table identification + parsing stage.
2
+
3
+ Run once per (expected_md, actual_md) so all table metrics consume the
4
+ same set of tables, paired the same way. Normalization stays inside each
5
+ metric — GriTS and TRM apply their own per-cell normalization downstream.
6
+
7
+ GT parse failures raise loudly (dataset bug). Pred parse failures are
8
+ dropped silently (model bug — dropped tables count as unmatched_expected
9
+ for any GT they would have paired with, so the model still pays the score).
10
+ """
11
+
12
+ import re
13
+ from dataclasses import dataclass
14
+ from typing import TYPE_CHECKING
15
+
16
+ from parse_bench.evaluation.metrics.parse.table_parsing import TableData, parse_html_tables
17
+
18
+ if TYPE_CHECKING:
19
+ from parse_bench.evaluation.metrics.parse.table_title_stripping import HeaderHints
20
+
21
+ _TABLE_OPEN_RE = re.compile(r"<table(?=[>\s])", re.IGNORECASE)
22
+ _TABLE_CLOSE_RE = re.compile(r"</table\s*>", re.IGNORECASE)
23
+
24
+
25
+ def extract_html_tables(content: str) -> list[str]:
26
+ """Extract all top-level HTML table strings from markdown/HTML content.
27
+
28
+ Uses depth-aware string scanning to correctly handle nested tables.
29
+ Nested tables (inside <td> cells) are included as part of the outer
30
+ table's HTML string, not extracted as separate entries.
31
+ """
32
+ if not content:
33
+ return []
34
+
35
+ tables: list[str] = []
36
+ search_start = 0
37
+ while True:
38
+ open_match = _TABLE_OPEN_RE.search(content, search_start)
39
+ if open_match is None:
40
+ break
41
+ start = open_match.start()
42
+
43
+ # Track nesting depth to find the matching </table>
44
+ depth = 0
45
+ pos = open_match.end()
46
+ end = -1
47
+ while pos < len(content):
48
+ next_open = _TABLE_OPEN_RE.search(content, pos)
49
+ next_close = _TABLE_CLOSE_RE.search(content, pos)
50
+ if next_close is None:
51
+ break
52
+ if next_open is not None and next_open.start() < next_close.start():
53
+ depth += 1
54
+ pos = next_open.end()
55
+ else:
56
+ if depth == 0:
57
+ end = next_close.end()
58
+ break
59
+ depth -= 1
60
+ pos = next_close.end()
61
+ if end == -1:
62
+ tables.append(content[start:])
63
+ break
64
+ tables.append(content[start:end])
65
+ search_start = end
66
+
67
+ return tables
68
+
69
+
70
+ class GroundTruthTableParseError(RuntimeError):
71
+ """Raised when a ground-truth table cannot be parsed. Dataset bug."""
72
+
73
+
74
+ @dataclass(frozen=True)
75
+ class ExtractedTable:
76
+ """One table, identified once. Cell content is NOT normalized.
77
+
78
+ - GriTS reads ``raw_html`` and runs it through its own ``html_to_cells`` +
79
+ ``normalize_cell_text`` path (P2). In P5 it switches to reading
80
+ ``table_data`` directly with upgraded normalization.
81
+ - TRM reads ``table_data`` and runs it through its own ``normalize_table``
82
+ (P3 onward).
83
+ """
84
+
85
+ raw_html: str
86
+ table_data: TableData # raw parse_html_tables output, NOT normalized
87
+ header_hints: "HeaderHints | None" = None # populated by strip_title_rows
88
+
89
+
90
+ @dataclass(frozen=True)
91
+ class TableExtractionCounts:
92
+ """Per-doc table counts surfaced as MetricValues."""
93
+
94
+ expected: int
95
+ actual: int
96
+ unparseable_pred: int # dropped pred tables (gt is always 0 — they raise)
97
+
98
+
99
+ def extract_normalized_tables(
100
+ md: str,
101
+ *,
102
+ side: str, # "expected" or "actual"
103
+ doc_id: str | None = None,
104
+ ) -> tuple[list[ExtractedTable], int]:
105
+ """Extract and parse all tables on one side of a doc.
106
+
107
+ Returns ``(tables, n_unparseable)``. ``n_unparseable`` is always 0 for the
108
+ expected side because GT parse failures raise.
109
+
110
+ Note: despite the name, this stage does **not** normalize cell content —
111
+ each metric applies its own normalization downstream. The name is kept
112
+ for backward compatibility with the plan.
113
+ """
114
+ raw_slices = extract_html_tables(md)
115
+ if not raw_slices:
116
+ return [], 0
117
+
118
+ # Parse each slice independently rather than calling parse_html_tables
119
+ # on the whole doc and zipping by index. The two parsers (depth-aware
120
+ # string scanner in extract_html_tables vs. lxml/bs4 in parse_html_tables)
121
+ # can disagree on what counts as a top-level table for malformed HTML,
122
+ # which would silently mis-pair raw_html with table_data. Parsing each
123
+ # slice individually makes the (raw_html, table_data) pairing correct
124
+ # by construction.
125
+ tables: list[ExtractedTable] = []
126
+ unparseable = 0
127
+ for i, raw in enumerate(raw_slices):
128
+ parsed_one = parse_html_tables(raw)
129
+ if not parsed_one:
130
+ if side == "expected":
131
+ raise GroundTruthTableParseError(f"Failed to parse expected table {i} in doc {doc_id!r}")
132
+ unparseable += 1
133
+ continue
134
+ tables.append(ExtractedTable(raw_html=raw, table_data=parsed_one[0]))
135
+ return tables, unparseable
136
+
137
+
138
+ def extract_table_pairs(
139
+ expected_md: str,
140
+ actual_md: str,
141
+ *,
142
+ doc_id: str | None = None,
143
+ ) -> tuple[list[ExtractedTable], list[ExtractedTable], TableExtractionCounts]:
144
+ """Extract both sides for one doc."""
145
+ expected, _ = extract_normalized_tables(expected_md, side="expected", doc_id=doc_id)
146
+ actual, n_unparseable_pred = extract_normalized_tables(actual_md, side="actual", doc_id=doc_id)
147
+ counts = TableExtractionCounts(
148
+ expected=len(expected),
149
+ actual=len(actual),
150
+ unparseable_pred=n_unparseable_pred,
151
+ )
152
+ return expected, actual, counts
@@ -0,0 +1,195 @@
1
+ """Table-merging (junction merge) metric — parse.
2
+
3
+ Scores whether a parser joins table fragments that belong to one logical table —
4
+ both within a page (section-break continuations) and across a page break — and
5
+ keeps genuinely distinct tables apart. Reads ``metadata["table_merging"]``:
6
+
7
+ atoms : ordered split GT tables (reading order = sorted by page,idx)
8
+ logical_tables : the merged grouping; ``fragments`` lists the atom keys it joins
9
+
10
+ Each consecutive atom pair ``(aᵢ, aᵢ₊₁)`` is a *junction*. Ground-truth says
11
+ "merge" iff both atoms belong to the same logical table; the parser says "merge"
12
+ iff both atoms land in the same predicted table (assignment = global argmax of
13
+ GriTS containment-recall, **no page blocking, no threshold**). Each junction is a
14
+ merge/separate decision scored over all four confusion quadrants — correct
15
+ *non*-merges (TN) count too, so over-, under-, and appropriate-merging all
16
+ register.
17
+
18
+ Metrics (rates 0..1, higher better):
19
+ - table_merge_accuracy : HEADLINE — (TP+TN)/N over all junctions (page-agnostic)
20
+ - table_merge_precision : over-merge sensitivity (TP / (TP+FP))
21
+ - table_merge_recall : under-merge sensitivity (TP / (TP+FN))
22
+ - table_merge_specificity: TN / (TN+FP)
23
+ - table_merge_f1
24
+ - table_merge_accuracy_within_page / _cross_page : per-kind slices
25
+ - table_merge_tp/fp/fn/tn: confusion-matrix counts (metadata["count"] ⇒ the
26
+ runner SUMS them at corpus level → total_table_merge_{tp,fp,fn,tn})
27
+ - table_merge_grits_con : secondary content signal (merged logical-table GriTS)
28
+
29
+ Pure over ``(parse_output, gt)``; no I/O.
30
+ """
31
+
32
+ from __future__ import annotations
33
+
34
+ from typing import Any
35
+
36
+ from bs4 import BeautifulSoup
37
+ from lxml import etree
38
+ from lxml import html as lxml_html
39
+
40
+ from parse_bench.evaluation.metrics.parse._vendor_grits_reference import grits_from_html
41
+ from parse_bench.schemas.evaluation import MetricValue
42
+
43
+
44
+ def _all_predicted_tables(parse_output: Any) -> list[str]:
45
+ """Flat, reading-ordered list of predicted ``<table>`` HTML across all pages.
46
+
47
+ Page-agnostic by design: a cross-page merge surfaces as one predicted table
48
+ (on the first page), and index identity is what "same predicted table" means.
49
+ """
50
+ out: list[str] = []
51
+ for page in getattr(parse_output, "pages", []) or []:
52
+ md = getattr(page, "markdown", "") or ""
53
+ for t in BeautifulSoup(md, "lxml").find_all("table"):
54
+ out.append(str(t))
55
+ return out
56
+
57
+
58
+ def _xml_safe(table_html: str) -> str | None:
59
+ """Normalize a table fragment to well-formed XML the vendor GriTS can parse.
60
+
61
+ ``_vendor_grits_reference.html_to_cells`` uses ``xml.etree.ElementTree``
62
+ (strict XML — raises on unclosed tags / bare ``&``), unlike the lenient lxml
63
+ used elsewhere in the bench. Re-serialize through lxml (which repairs and
64
+ escapes) and return just the ``<table>``. ``None`` if there is no table.
65
+ """
66
+ if not table_html or not table_html.strip():
67
+ return None
68
+ try:
69
+ node = lxml_html.fromstring(table_html)
70
+ except Exception:
71
+ return None
72
+ table = node if node.tag == "table" else node.find(".//table")
73
+ if table is None:
74
+ return None
75
+ return str(etree.tostring(table, encoding="unicode", method="xml"))
76
+
77
+
78
+ def _grits(true_html: str, pred_html: str, key: str) -> float:
79
+ """One GriTS sub-score between two table fragments; 0.0 on any parse failure."""
80
+ t = _xml_safe(true_html)
81
+ p = _xml_safe(pred_html)
82
+ if t is None or p is None:
83
+ return 0.0
84
+ try:
85
+ return float(grits_from_html(t, p).get(key, 0.0) or 0.0)
86
+ except Exception:
87
+ return 0.0
88
+
89
+
90
+ def _prf(tp: int, fp: int, fn: int) -> tuple[float, float, float]:
91
+ """Precision / recall / F1; a 0-denominator component is vacuously 1.0."""
92
+ prec = tp / (tp + fp) if (tp + fp) else 1.0
93
+ rec = tp / (tp + fn) if (tp + fn) else 1.0
94
+ f1 = 2 * prec * rec / (prec + rec) if (prec + rec) else 0.0
95
+ return prec, rec, f1
96
+
97
+
98
+ def _slice_accuracy(slice_name: str, b: dict[str, int]) -> list[MetricValue]:
99
+ """Per-kind accuracy slice; omitted entirely when that kind has no junctions."""
100
+ n = b["tp"] + b["fp"] + b["fn"] + b["tn"]
101
+ if n == 0:
102
+ return []
103
+ acc = (b["tp"] + b["tn"]) / n
104
+ return [
105
+ MetricValue(
106
+ metric_name=f"table_merge_accuracy_{slice_name}",
107
+ value=acc,
108
+ metadata={**b, "n_junctions": n},
109
+ )
110
+ ]
111
+
112
+
113
+ def _logical_table_grits_con(logical: list[dict[str, Any]], pred: list[str]) -> float:
114
+ """Secondary content signal: mean GriTS-Con of each merged logical table vs its
115
+ best-matching predicted table. 1.0 vacuously when there are no logical tables."""
116
+ if not logical:
117
+ return 1.0
118
+ if not pred:
119
+ return 0.0
120
+ scores = [max((_grits(lt.get("html", ""), p, "grits_con") for p in pred), default=0.0) for lt in logical]
121
+ return sum(scores) / len(scores)
122
+
123
+
124
+ def compute_table_merging_metrics(parse_output: Any, gt: dict[str, Any]) -> list[MetricValue]:
125
+ atoms: list[dict[str, Any]] = gt.get("atoms") or []
126
+ logical: list[dict[str, Any]] = gt.get("logical_tables") or []
127
+ frag_logical = {f: lt["id"] for lt in logical for f in lt.get("fragments", [])}
128
+
129
+ pred = _all_predicted_tables(parse_output)
130
+
131
+ # Assign each atom to the predicted table that best CONTAINS it (recall-oriented,
132
+ # global argmax, no page blocking, no threshold). None when the parser produced
133
+ # no tables OR no predicted table contains the atom at all (dropped fragment).
134
+ assign: dict[str, int | None] = {}
135
+ for a in atoms:
136
+ if not pred:
137
+ assign[a["key"]] = None
138
+ continue
139
+ scores = [_grits(a["html"], p, "grits_recall_con") for p in pred]
140
+ best = max(range(len(pred)), key=lambda j: scores[j])
141
+ assign[a["key"]] = best if scores[best] > 0.0 else None
142
+
143
+ if len(atoms) < 2:
144
+ return [] # junction-free doc — emit nothing, don't pollute corpus averages
145
+
146
+ tp = fp = fn = tn = 0
147
+ within = {"tp": 0, "fp": 0, "fn": 0, "tn": 0}
148
+ cross = {"tp": 0, "fp": 0, "fn": 0, "tn": 0}
149
+ for a, b in zip(atoms, atoms[1:], strict=False):
150
+ la, lb = frag_logical.get(a["key"]), frag_logical.get(b["key"])
151
+ gt_merge = la is not None and la == lb
152
+ ja, jb = assign[a["key"]], assign[b["key"]]
153
+ pred_merge = ja is not None and ja == jb
154
+ bucket = within if a["page"] == b["page"] else cross
155
+ if gt_merge and pred_merge:
156
+ tp += 1
157
+ bucket["tp"] += 1
158
+ elif gt_merge and not pred_merge:
159
+ fn += 1
160
+ bucket["fn"] += 1
161
+ elif not gt_merge and pred_merge:
162
+ fp += 1
163
+ bucket["fp"] += 1
164
+ else:
165
+ tn += 1
166
+ bucket["tn"] += 1
167
+
168
+ n = tp + fp + fn + tn # == len(atoms) - 1
169
+ acc = (tp + tn) / n
170
+ prec, rec, f1 = _prf(tp, fp, fn)
171
+ spec = tn / (tn + fp) if (tn + fp) else 1.0
172
+ grits_con = _logical_table_grits_con(logical, pred)
173
+
174
+ cm = f"TP={tp} FP={fp} FN={fn} TN={tn}"
175
+ md = {"tp": tp, "fp": fp, "fn": fn, "tn": tn, "n_atoms": len(atoms), "n_logical": len(logical)}
176
+
177
+ results = [
178
+ # HEADLINE (doc-averaged at corpus level via avg_table_merge_accuracy) + inline CM.
179
+ MetricValue(metric_name="table_merge_accuracy", value=acc, metadata=md, details=[cm]),
180
+ MetricValue(metric_name="table_merge_precision", value=prec, metadata=md),
181
+ MetricValue(metric_name="table_merge_recall", value=rec, metadata=md),
182
+ MetricValue(metric_name="table_merge_specificity", value=spec, metadata=md),
183
+ MetricValue(metric_name="table_merge_f1", value=f1, metadata=md),
184
+ # Confusion counts as first-class metrics. metadata["count"] makes the runner
185
+ # SUM them at corpus level (total_table_merge_{tp,fp,fn,tn}) rather than average,
186
+ # so over-/under-/appropriate-merging stay visible beside the headline.
187
+ MetricValue(metric_name="table_merge_tp", value=float(tp), metadata={"count": tp, "n_atoms": len(atoms)}),
188
+ MetricValue(metric_name="table_merge_fp", value=float(fp), metadata={"count": fp, "n_atoms": len(atoms)}),
189
+ MetricValue(metric_name="table_merge_fn", value=float(fn), metadata={"count": fn, "n_atoms": len(atoms)}),
190
+ MetricValue(metric_name="table_merge_tn", value=float(tn), metadata={"count": tn, "n_atoms": len(atoms)}),
191
+ MetricValue(metric_name="table_merge_grits_con", value=grits_con, metadata=md),
192
+ ]
193
+ results += _slice_accuracy("within_page", within)
194
+ results += _slice_accuracy("cross_page", cross)
195
+ return results
@@ -0,0 +1,87 @@
1
+ """Shared GT->pred table assignment for the table-similarity metrics.
2
+
3
+ GriTS and TEDS both build an ``n_expected x n_actual`` cost matrix and solve a
4
+ single document-global Hungarian assignment over it. On multi-page documents
5
+ that recur near-identical tables across pages, the global assignment mis-pairs a
6
+ GT table on one page with a prediction on another. When per-table page labels
7
+ are available, ``assign_tables`` instead solves the assignment **per page** so a
8
+ GT table only ever pairs with a prediction on the same page.
9
+
10
+ The return shape mirrors ``scipy.optimize.linear_sum_assignment`` — a
11
+ ``(row_ind, col_ind)`` pair of equal-length index lists — so callers consume it
12
+ exactly as before.
13
+ """
14
+
15
+ from __future__ import annotations
16
+
17
+ from collections import defaultdict
18
+
19
+ import numpy as np
20
+ from scipy.optimize import linear_sum_assignment
21
+
22
+
23
+ def page_blocking_active(
24
+ expected_pages: list[int] | None,
25
+ actual_pages: list[int] | None,
26
+ n_expected: int,
27
+ n_actual: int,
28
+ ) -> bool:
29
+ """Return True when page-constrained matching can and should be applied.
30
+
31
+ Requires both page-label lists to be present and length-consistent with the
32
+ table counts. Any mismatch (missing labels, wrong length) falls back to the
33
+ document-global assignment, so the feature is self-disabling.
34
+ """
35
+ return (
36
+ expected_pages is not None
37
+ and actual_pages is not None
38
+ and len(expected_pages) == n_expected
39
+ and len(actual_pages) == n_actual
40
+ )
41
+
42
+
43
+ def assign_tables(
44
+ cost_matrix: np.ndarray,
45
+ expected_pages: list[int] | None = None,
46
+ actual_pages: list[int] | None = None,
47
+ ) -> tuple[list[int], list[int]]:
48
+ """Solve the GT->pred table assignment minimizing total cost.
49
+
50
+ When ``expected_pages`` / ``actual_pages`` are provided and consistent with
51
+ ``cost_matrix.shape`` (see :func:`page_blocking_active`), the assignment is
52
+ solved independently per page: a GT table on page P can only pair with a
53
+ prediction on page P. GT tables whose page has no predictions are left
54
+ unmatched (they appear in neither returned list, so the caller scores them
55
+ as unmatched-expected). Otherwise a single global assignment is solved.
56
+
57
+ Returns ``(row_ind, col_ind)`` as index lists, mirroring
58
+ ``scipy.optimize.linear_sum_assignment``.
59
+ """
60
+ n_expected, n_actual = cost_matrix.shape
61
+
62
+ if not page_blocking_active(expected_pages, actual_pages, n_expected, n_actual):
63
+ row_ind, col_ind = linear_sum_assignment(cost_matrix)
64
+ return row_ind.tolist(), col_ind.tolist()
65
+
66
+ assert expected_pages is not None and actual_pages is not None # narrowed by the guard above
67
+
68
+ pred_by_page: dict[int, list[int]] = defaultdict(list)
69
+ for j, page in enumerate(actual_pages):
70
+ pred_by_page[page].append(j)
71
+
72
+ gt_by_page: dict[int, list[int]] = defaultdict(list)
73
+ for i, page in enumerate(expected_pages):
74
+ gt_by_page[page].append(i)
75
+
76
+ row_out: list[int] = []
77
+ col_out: list[int] = []
78
+ for page, gt_idxs in gt_by_page.items():
79
+ pred_idxs = pred_by_page.get(page)
80
+ if not pred_idxs:
81
+ continue
82
+ sub = cost_matrix[np.ix_(gt_idxs, pred_idxs)]
83
+ sub_rows, sub_cols = linear_sum_assignment(sub)
84
+ for sr, sc in zip(sub_rows.tolist(), sub_cols.tolist(), strict=True):
85
+ row_out.append(gt_idxs[sr])
86
+ col_out.append(pred_idxs[sc])
87
+ return row_out, col_out