parse-bench 1.0.0__py3-none-any.whl

parse_bench/evaluation/metrics/parse/test_rules.py

@@ -0,0 +1,120 @@
+"""Test rule implementations for parse evaluation.
+
+This module re-exports all rule classes and helpers from the split submodules
+for backward compatibility. New code should import directly from the
+specific submodule (rules_base, rules_text, rules_bag, rules_formatting,
+rules_table, rules_chart, rules_form).
+"""
+
+# Base class, helpers, and factory
+# Bag rules
+from parse_bench.evaluation.metrics.parse.rules_bag import (  # noqa: F401
+    BagOfDigitPercentRule,
+    ExtraContentRule,
+    MissingSentencePercentRule,
+    MissingSentenceRule,
+    MissingSpecificSentenceRule,
+    MissingSpecificWordRule,
+    MissingWordPercentRule,
+    MissingWordRule,
+    SentenceBagRule,
+    TooManySentenceOccurencePercentRule,
+    TooManySentenceOccurenceRule,
+    TooManyWordOccurencePercentRule,
+    TooManyWordOccurenceRule,
+    UnexpectedSentencePercentRule,
+    UnexpectedSentenceRule,
+    UnexpectedWordPercentRule,
+    UnexpectedWordRule,
+    WordBagRule,
+)
+from parse_bench.evaluation.metrics.parse.rules_base import (  # noqa: F401
+    CELL_FUZZY_MATCH_THRESHOLD,
+    AdjacentTableRuleData,
+    NoBorderTableRuleData,
+    ParseTestRule,
+    SentenceBagRuleData,
+    WordBagRuleData,
+    _augment_with_table_cell_text,
+    _dates_match,
+    _detect_csv_skip_rows,
+    _detect_header_scale,
+    _extract_table_cell_texts,
+    _normalize_date_str,
+    _strip_and_replace_latex,
+    _strip_fenced_code_blocks,
+    _strip_html_tables_and_content,
+    _unescape_html_entities,
+    create_test_rule,
+)
+
+# Chart rules
+from parse_bench.evaluation.metrics.parse.rules_chart import (  # noqa: F401
+    ChartDataArrayDataRule,
+    ChartDataArrayLabelsRule,
+    ChartDataPointRule,
+    RotateCheckRule,
+    extract_numeric_parts,
+    normalize_number_string,
+    numbers_match,
+    numeric_similarity,
+)
+
+# Form rules
+from parse_bench.evaluation.metrics.parse.rules_form import (  # noqa: F401
+    FormFieldRule,
+)
+
+# Formatting rules
+from parse_bench.evaluation.metrics.parse.rules_formatting import (  # noqa: F401
+    _FORMATTING_TEST_TYPES,
+    AbsentUnlessStrikeoutRule,
+    CodeBlockRule,
+    FormattingRule,
+    LatexRule,
+    MarkColorRule,
+    NotLatexRule,
+    PageSectionRule,
+    PresentAsStrikeoutRule,
+    TextColorRule,
+    TitleHierarchyPercentRule,
+    TitleLevelRule,
+)
+
+# List rules
+from parse_bench.evaluation.metrics.parse.rules_list import (  # noqa: F401
+    ListLevelRule,
+)
+
+# Table rules
+from parse_bench.evaluation.metrics.parse.rules_table import (  # noqa: F401
+    TableAdjacentDownRule,
+    TableAdjacentLeftRule,
+    TableAdjacentRightRule,
+    TableAdjacentRule,
+    TableAdjacentUpRule,
+    TableColspanRule,
+    TableHeaderChainRule,
+    TableLeftHeaderRule,
+    TableNoAboveRule,
+    TableNoBelowRule,
+    TableNoBorderRule,
+    TableNoLeftRule,
+    TableNoRightRule,
+    TableRowspanRule,
+    TableRule,
+    TableSameColumnRule,
+    TableSameRowRule,
+    TablesNumColsRule,
+    TablesNumRowsRule,
+    TablesValuesRule,
+    TableTopHeaderRule,
+)
+
+# Text rules
+from parse_bench.evaluation.metrics.parse.rules_text import (  # noqa: F401
+    BaselineRule,
+    TextOrderRule,
+    TextPresenceRule,
+)
+from parse_bench.evaluation.metrics.parse.rules_watermark import WatermarkRemovalRule  # noqa: F401

parse_bench/evaluation/metrics/parse/test_types.py

@@ -0,0 +1,103 @@
+"""Test type definitions for parse evaluation."""
+
+from enum import StrEnum
+
+
+class TestType(StrEnum):
+    """Test types for parse evaluation."""
+
+    BASELINE = "baseline"
+    PRESENT = "present"
+    UNEXPECTED_SENTENCE = "unexpected_sentence"
+    UNEXPECTED_SENTENCE_PERCENT = "unexpected_sentence_percent"
+    TOO_MANY_SENTENCE_OCCURENCE = "too_many_sentence_occurence"
+    TOO_MANY_SENTENCE_OCCURENCE_PERCENT = "too_many_sentence_occurence_percent"
+    MISSING_SENTENCE = "missing_sentence"
+    MISSING_SENTENCE_PERCENT = "missing_sentence_percent"
+    MISSING_SPECIFIC_SENTENCE = "missing_specific_sentence"
+    UNEXPECTED_WORD = "unexpected_word"
+    UNEXPECTED_WORD_PERCENT = "unexpected_word_percent"
+    TOO_MANY_WORD_OCCURENCE = "too_many_word_occurence"
+    TOO_MANY_WORD_OCCURENCE_PERCENT = "too_many_word_occurence_percent"
+    MISSING_WORD = "missing_word"
+    MISSING_WORD_PERCENT = "missing_word_percent"
+    MISSING_SPECIFIC_WORD = "missing_specific_word"
+    # Keep for backward compatibility with earlier combined behavior
+    EXTRA_CONTENT = "extra_content"
+    ABSENT = "absent"
+    ORDER = "order"
+    TABLE = "table"
+    TABLES_VALUES = "tables_values"
+    TABLES_NUM_ROWS = "tables_num_rows"
+    TABLES_NUM_COLS = "tables_num_cols"
+    TABLE_MARKER_CELLS = "table_marker_cells"
+    MATH = "math"
+    # Table hierarchy test types
+    TABLE_COLSPAN = "table_colspan"
+    TABLE_ROWSPAN = "table_rowspan"
+    TABLE_SAME_ROW = "table_same_row"
+    TABLE_SAME_COLUMN = "table_same_column"
+    TABLE_HEADER_CHAIN = "table_header_chain"  # Keep for backward compatibility
+    # New table adjacency test types
+    TABLE_ADJACENT_UP = "table_adjacent_up"
+    TABLE_ADJACENT_DOWN = "table_adjacent_down"
+    TABLE_ADJACENT_LEFT = "table_adjacent_left"
+    TABLE_ADJACENT_RIGHT = "table_adjacent_right"
+    TABLE_TOP_HEADER = "table_top_header"
+    TABLE_LEFT_HEADER = "table_left_header"
+    # Table border tests - verify absence of cells beyond boundaries
+    TABLE_NO_LEFT = "table_no_left"
+    TABLE_NO_RIGHT = "table_no_right"
+    TABLE_NO_ABOVE = "table_no_above"
+    TABLE_NO_BELOW = "table_no_below"
+    # Formatting test types - check if text has specific formatting
+    IS_UNDERLINE = "is_underline"
+    IS_NOT_UNDERLINE = "is_not_underline"
+    IS_BOLD = "is_bold"
+    IS_NOT_BOLD = "is_not_bold"
+    IS_STRIKEOUT = "is_strikeout"
+    IS_NOT_STRIKEOUT = "is_not_strikeout"
+    IS_ITALIC = "is_italic"
+    IS_NOT_ITALIC = "is_not_italic"
+    IS_MARK = "is_mark"
+    IS_NOT_MARK = "is_not_mark"
+    MARK_COLOR = "mark_color"
+    TEXT_COLOR = "text_color"
+    ABSENT_UNLESS_STRIKEOUT = "absent_unless_strikeout"
+    PRESENT_AS_STRIKEOUT = "present_as_strikeout"
+    IS_SUP = "is_sup"
+    IS_NOT_SUP = "is_not_sup"
+    IS_SUB = "is_sub"
+    IS_NOT_SUB = "is_not_sub"
+    IS_LATEX = "is_latex"
+    IS_NOT_LATEX = "is_not_latex"
+    IS_CODE_BLOCK = "is_code_block"
+    # Title / heading level test
+    IS_TITLE = "is_title"
+    TITLE_HIERARCHY_PERCENT = "title_hierarchy_percent"
+    HEADING_STRUCTURE = "heading_structure"
+    # List item nesting level test (CommonMark list semantics)
+    LIST_LEVEL = "list_level"
+    # Page header / footer tests
+    IS_HEADER = "is_header"
+    IS_FOOTER = "is_footer"
+    # Chart test types
+    CHART_DATA_POINT = "chart_data_point"
+    CHART_DATA_ARRAY_LABELS = "chart_data_array_labels"
+    CHART_DATA_ARRAY_DATA = "chart_data_array_data"
+    # Layout detection
+    LAYOUT = "layout"
+    # Digit bag rule
+    BAG_OF_DIGIT_PERCENT = "bag_of_digit_percent"
+    # Rotation check
+    ROTATE_CHECK = "rotate_check"
+    # Form field (key/value, checkbox, signature) extraction
+    FORM_FIELD = "form_field"
+    # Watermark suppression plus preservation of nearby document text.
+    WATERMARK_REMOVAL = "watermark_removal"
+    # Diagrams rendered as mermaid: graph semantics (nodes/edges), anchor relations, page-level count.
+    DIAGRAM_GRAPH = "diagram_graph"
+    DIAGRAM_EDGE = "diagram_edge"
+    DIAGRAM_COUNT = "diagram_count"
+    # Page furniture: running header, running footer, printed page number, and leakage into the body.
+    PAGE_DECORATION = "page_decoration"

parse_bench/evaluation/metrics/parse/text_content_projection.py

@@ -0,0 +1,175 @@
+"""Canonical plain-text projection for ParseBench Text Content rules."""
+
+from __future__ import annotations
+
+import re
+
+from bs4 import BeautifulSoup, Tag
+
+_HTML_TABLE_TAG = re.compile(r"<table(?=[\s>])[^>]*>|</table\s*>", re.IGNORECASE)
+_MARKDOWN_DELIMITER_CELL = re.compile(r"^:?-+:?$")
+
+
+def _html_table_spans(content: str) -> list[tuple[int, int]]:
+    """Return matched top-level HTML table spans, recovering after malformed starts."""
+    stack: list[int] = []
+    matched: list[tuple[int, int]] = []
+
+    for tag in _HTML_TABLE_TAG.finditer(content):
+        if tag.group().lower().startswith("</"):
+            if stack:
+                matched.append((stack.pop(), tag.end()))
+        else:
+            stack.append(tag.start())
+
+    # A valid nested table is projected through its outer table. If an unmatched
+    # outer start precedes a later valid table, the later matched span survives.
+    return [
+        span
+        for span in matched
+        if not any(outer_start < span[0] and span[1] < outer_end for outer_start, outer_end in matched)
+    ]
+
+
+def _normalize_cell_text(cell: Tag) -> str:
+    return " ".join(cell.get_text(" ", strip=True).split())
+
+
+def _html_table_lines(table: Tag) -> list[str]:
+    """Emit captions, explicit rows, and orphan cells in document order."""
+    lines: list[str] = []
+    orphan_cells: list[str] = []
+
+    def flush_orphan_cells() -> None:
+        if orphan_cells:
+            lines.append("\t".join(orphan_cells))
+            orphan_cells.clear()
+
+    def visit(container: Tag) -> None:
+        for child in container.children:
+            if not isinstance(child, Tag):
+                continue
+            if child.name == "caption":
+                flush_orphan_cells()
+                text = _normalize_cell_text(child)
+                if text:
+                    lines.append(text)
+            elif child.name == "tr":
+                flush_orphan_cells()
+                cells = [
+                    _normalize_cell_text(cell)
+                    for cell in child.find_all(["th", "td"])
+                    if cell.find_parent("tr") is child
+                ]
+                if cells:
+                    lines.append("\t".join(cells))
+            elif child.name in {"th", "td"}:
+                text = _normalize_cell_text(child)
+                if text:
+                    orphan_cells.append(text)
+            elif child.name != "table":
+                visit(child)
+
+    visit(table)
+    flush_orphan_cells()
+    return lines
+
+
+def _canonicalize_html_table(table_html: str) -> str:
+    soup = BeautifulSoup(table_html, "html.parser")
+    table = soup.find("table")
+    if table is None:
+        return table_html
+
+    return "\n".join(_html_table_lines(table))
+
+
+def _split_markdown_row(line: str) -> list[str]:
+    """Split a Markdown table row without treating escaped/code pipes as separators."""
+    cells: list[str] = []
+    buffer: list[str] = []
+    code_ticks = 0
+    index = 0
+
+    while index < len(line):
+        char = line[index]
+        if char == "\\" and index + 1 < len(line):
+            buffer.extend((char, line[index + 1]))
+            index += 2
+            continue
+        if char == "`":
+            run_end = index + 1
+            while run_end < len(line) and line[run_end] == "`":
+                run_end += 1
+            run_length = run_end - index
+            if code_ticks == 0:
+                code_ticks = run_length
+            elif code_ticks == run_length:
+                code_ticks = 0
+            buffer.extend(line[index:run_end])
+            index = run_end
+            continue
+        if char == "|" and code_ticks == 0:
+            cells.append("".join(buffer).strip())
+            buffer = []
+        else:
+            buffer.append(char)
+        index += 1
+
+    cells.append("".join(buffer).strip())
+    if line.lstrip().startswith("|") and cells and not cells[0]:
+        cells = cells[1:]
+    if line.rstrip().endswith("|") and cells and not cells[-1]:
+        cells = cells[:-1]
+    return cells
+
+
+def _is_markdown_delimiter(line: str) -> bool:
+    cells = _split_markdown_row(line)
+    return len(cells) > 0 and all(_MARKDOWN_DELIMITER_CELL.fullmatch(cell.replace(" ", "")) for cell in cells)
+
+
+def _canonicalize_markdown_tables(content: str) -> str:
+    lines = content.splitlines()
+    output: list[str] = []
+    index = 0
+    changed = False
+
+    while index < len(lines):
+        if (
+            index + 1 < len(lines)
+            and "|" in lines[index]
+            and "|" in lines[index + 1]
+            and _is_markdown_delimiter(lines[index + 1])
+        ):
+            changed = True
+            table_lines = [lines[index]]
+            index += 2
+            while index < len(lines) and "|" in lines[index] and lines[index].strip():
+                table_lines.append(lines[index])
+                index += 1
+            output.extend("\t".join(_split_markdown_row(row)) for row in table_lines)
+            continue
+
+        output.append(lines[index])
+        index += 1
+
+    return "\n".join(output) if changed else content
+
+
+def canonicalize_tables_for_text_content(markdown: str) -> str:
+    """Replace each table with one row-major text view for Text Content rules.
+
+    Every source cell is emitted once. Tabs separate cells and newlines separate
+    rows, preserving anchor continuity without adding alternate cell/row copies.
+    The caller retains the original Markdown for table and formatting metrics.
+    """
+    if not markdown:
+        return markdown
+
+    spans = _html_table_spans(markdown)
+    for start, end in reversed(spans):
+        replacement = _canonicalize_html_table(markdown[start:end])
+        markdown = f"{markdown[:start]}\n{replacement}\n{markdown[end:]}"
+
+    return _canonicalize_markdown_tables(markdown)

parse_bench/evaluation/metrics/parse/text_similarity_metric.py

@@ -0,0 +1,61 @@
+"""Text similarity metric for comparing markdown against ground truth."""
+
+from typing import Any
+
+from parse_bench.evaluation.metrics.base import Metric
+from parse_bench.schemas.evaluation import MetricValue
+
+
+class TextSimilarityMetric(Metric):
+    """Metric for comparing markdown text against ground truth using Levenshtein distance."""
+
+    @property
+    def name(self) -> str:
+        """Return the name of this metric."""
+        return "text_similarity"
+
+    def compute(self, expected: str, actual: str, **kwargs: Any) -> MetricValue:
+        """
+        Compute text similarity between expected and actual markdown.
+
+        :param expected: Expected markdown (ground truth)
+        :param actual: Actual markdown (from inference)
+        :param kwargs: Additional parameters (not used)
+        :return: MetricValue with similarity score (0.0 to 1.0)
+        """
+        from autoevals.string import Levenshtein
+
+        if not expected and not actual:
+            return MetricValue(
+                metric_name=self.name,
+                value=1.0,
+                metadata={"note": "Both empty"},
+            )
+
+        if not expected or not actual:
+            return MetricValue(
+                metric_name=self.name,
+                value=0.0,
+                metadata={"note": "One is empty"},
+            )
+
+        # Use Levenshtein distance from autoevals
+        levenshtein = Levenshtein()
+        result = levenshtein(expected, actual)
+
+        # autoevals ``Levenshtein`` already returns a normalized score in
+        # [0, 1] (1.0 identical, 0.0 fully disjoint), so use it directly.
+        # Clamp only as a defensive guard against float drift — do NOT divide
+        # by 100 (that collapsed every value by two orders of magnitude, e.g.
+        # 0.76 -> 0.0076).
+        similarity = max(0.0, min(1.0, result.score))
+
+        return MetricValue(
+            metric_name=self.name,
+            value=similarity,
+            metadata={
+                "levenshtein_score": result.score,
+                "expected_length": len(expected),
+                "actual_length": len(actual),
+            },
+        )