extract-bench 0.1.0__py3-none-any.whl

extract_bench/__init__.py

@@ -0,0 +1,42 @@
+"""Structured Extraction Evaluation Suite.
+
+A standalone package for evaluating structured extraction quality by comparing
+predicted JSON against gold JSON with per-field metrics.
+"""
+
+from .evaluation import (
+    AsyncEvaluationConfig,
+    BaseMetric,
+    EvaluationConfig,
+    EvaluationReport,
+    MetricConfig,
+    MetricRegistry,
+    MetricResult,
+    ReportBuilder,
+    ReportConfig,
+    StructuredEvaluator,
+    StructuredEvaluatorConfig,
+    global_metric_registry,
+)
+
+__version__ = "0.1.0"
+
+__all__ = [
+    # Main evaluator
+    "StructuredEvaluator",
+    "StructuredEvaluatorConfig",
+    "AsyncEvaluationConfig",
+    # Config
+    "EvaluationConfig",
+    "MetricConfig",
+    # Registry
+    "MetricRegistry",
+    "global_metric_registry",
+    # Metrics
+    "BaseMetric",
+    "MetricResult",
+    # Reporting
+    "ReportBuilder",
+    "ReportConfig",
+    "EvaluationReport",
+]

extract_bench/evaluation/__init__.py

@@ -0,0 +1,42 @@
+"""Structured extraction evaluation module."""
+
+from .evaluation_config import EvaluationConfig, MetricConfig
+from .metric_registry import MetricRegistry, global_metric_registry
+from .metrics.base_metric import BaseMetric, MetricResult
+from .reporting import EvaluationReport, ReportBuilder, ReportConfig
+from .schema_config_helpers import (
+    add_evaluation_configs_to_export,
+    get_default_evaluation_config,
+    get_evaluation_config,
+    should_evaluate,
+)
+from .structured_evaluator import (
+    AsyncEvaluationConfig,
+    StructuredEvaluator,
+    StructuredEvaluatorConfig,
+)
+
+__all__ = [
+    # Config
+    "EvaluationConfig",
+    "MetricConfig",
+    # Registry
+    "MetricRegistry",
+    "global_metric_registry",
+    # Metrics
+    "BaseMetric",
+    "MetricResult",
+    # Helpers
+    "get_evaluation_config",
+    "get_default_evaluation_config",
+    "should_evaluate",
+    "add_evaluation_configs_to_export",
+    # Evaluator
+    "AsyncEvaluationConfig",
+    "StructuredEvaluator",
+    "StructuredEvaluatorConfig",
+    # Reporting
+    "ReportBuilder",
+    "ReportConfig",
+    "EvaluationReport",
+]

extract_bench/evaluation/evaluation_config.py

@@ -0,0 +1,33 @@
+"""Evaluation config models used to configure metrics per schema node."""
+
+from typing import Any, Dict, List, Optional
+
+from pydantic import BaseModel, ConfigDict, Field
+
+
+class MetricConfig(BaseModel):
+    """Configuration for a single evaluation metric."""
+
+    metric_id: str
+    weight: Optional[float] = None
+    params: Optional[Dict[str, Any]] = Field(
+        default=None, description="Override default metric parameters"
+    )
+
+    model_config = ConfigDict(frozen=True, extra="forbid")
+
+
+class EvaluationConfig(BaseModel):
+    """Configuration attached to schema nodes to guide evaluation."""
+
+    metrics: List[MetricConfig] = Field(default_factory=list)
+    aggregation_weight: Optional[float] = None
+
+    model_config = ConfigDict(frozen=True, extra="forbid")
+
+    @classmethod
+    def from_preset(cls, preset: str) -> "EvaluationConfig":
+        """Create evaluation config from a preset string."""
+        from .presets import get_preset_config
+
+        return get_preset_config(preset)

extract_bench/evaluation/metric_id_collector.py

@@ -0,0 +1,41 @@
+"""Collect metric IDs and parameter examples from evaluation configs in a schema subtree."""
+
+from typing import Any, Dict
+
+from ..infra.visitors import AnalyzerVisitor
+from ..infra.nodes import Schema
+from .schema_config_helpers import get_evaluation_config
+
+
+def collect_metric_ids_with_params(
+    *, schema: Schema
+) -> dict[str, list[dict[str, Any]]]:
+    """Return metric_id -> list of distinct params dicts observed in the subtree."""
+    visitor = _MetricIdCollector()
+    visitor.visit(schema)
+    return visitor.metric_params
+
+
+class _MetricIdCollector(AnalyzerVisitor):
+    def __init__(self) -> None:
+        super().__init__()
+        self.metric_params: Dict[str, list[dict[str, Any]]] = {}
+        self._seen_param_sigs: Dict[str, set[str]] = {}
+
+    def _collect(self, node: Schema) -> None:
+        config = get_evaluation_config(node)
+        for metric in config.metrics:
+            metric_id = metric.metric_id
+            params = metric.params or {}
+            sig = repr(sorted(params.items()))
+            if sig in self._seen_param_sigs.setdefault(metric_id, set()):
+                continue
+            self._seen_param_sigs[metric_id].add(sig)
+            self.metric_params.setdefault(metric_id, []).append(dict(params))
+
+    def pre_visit_children_hook(self, node: Schema) -> None:
+        self._collect(node)
+
+    def generic_leaf_visit(self, node: Schema) -> Schema:
+        self._collect(node)
+        return node

extract_bench/evaluation/metric_registry.py

@@ -0,0 +1,120 @@
+"""Metric registry for structured extraction evaluation."""
+
+from collections.abc import Callable
+from typing import Dict, Mapping
+
+from .metrics.array_metrics import ArrayLlmJudgeMetric
+from .metrics.base_metric import BaseMetric
+from .metrics.boolean_metrics import BooleanExactMatchMetric
+from .metrics.llm_metrics import LlmJudgeMetric
+from .metrics.metric_descriptors import MetricPromptDescriptor
+from .metrics.number_metrics import (
+    ExactNumberMatchMetric,
+    IntegerExactMatchMetric,
+    NumberToleranceMetric,
+)
+from .metrics.string_metrics import (
+    CaseInsensitiveStringMatchMetric,
+    ExactStringMatchMetric,
+    NormalizedLevenshteinSimilarityMetric,
+    StringSemanticMetric,
+)
+
+MetricFactory = Callable[[], BaseMetric]
+
+
+class MetricRegistry:
+    """Registry storing factories for metric instances."""
+
+    def __init__(self, *, aliases: Mapping[str, str] | None = None) -> None:
+        self._factories: Dict[str, MetricFactory] = {}
+        self._aliases: Dict[str, str] = dict(aliases or {})
+        self._descriptors: Dict[str, MetricPromptDescriptor] = {}
+
+    def register_metric_factory(
+        self, metric_id: str, factory: MetricFactory, *, override: bool = False
+    ) -> None:
+        if metric_id in self._factories and not override:
+            raise ValueError(f"Metric '{metric_id}' is already registered")
+        self._factories[metric_id] = factory
+
+    def register_metric(
+        self,
+        factory: MetricFactory,
+        *,
+        override: bool = False,
+        aliases: tuple[str, ...] = (),
+    ) -> str:
+        sample = factory()
+        metric_id = sample.metric_id
+        self.register_metric_factory(metric_id, factory, override=override)
+        descriptor = getattr(sample, "prompt_descriptor", None)
+        if descriptor is not None:
+            self.register_metric_descriptor(metric_id, descriptor, override=override)
+        for alias in aliases:
+            self._aliases[alias] = metric_id
+        return metric_id
+
+    def has_metric(self, metric_id: str) -> bool:
+        resolved = self._aliases.get(metric_id, metric_id)
+        return resolved in self._factories
+
+    def create_metric(self, metric_id: str) -> BaseMetric:
+        resolved = self._aliases.get(metric_id, metric_id)
+        if resolved not in self._factories:
+            raise KeyError(f"Metric '{metric_id}' is not registered")
+        metric = self._factories[resolved]()
+        return metric
+
+    def register_metric_descriptor(
+        self,
+        metric_id: str,
+        descriptor: MetricPromptDescriptor,
+        *,
+        override: bool = False,
+    ) -> None:
+        if metric_id in self._descriptors and not override:
+            raise ValueError(f"Metric descriptor '{metric_id}' is already registered")
+        if descriptor.metric_id != metric_id:
+            raise ValueError(
+                f"Metric descriptor id mismatch: expected '{metric_id}', got '{descriptor.metric_id}'"
+            )
+        self._descriptors[metric_id] = descriptor
+
+    def get_metric_descriptor(self, metric_id: str) -> MetricPromptDescriptor | None:
+        resolved = self._aliases.get(metric_id, metric_id)
+        return self._descriptors.get(resolved)
+
+    def available_metrics(self) -> tuple[str, ...]:
+        return tuple(self._factories.keys())
+
+    def unregister_metric(self, metric_id: str) -> None:
+        resolved = self._aliases.get(metric_id, metric_id)
+        self._factories.pop(resolved, None)
+        self._descriptors.pop(resolved, None)
+        self._aliases = {
+            alias: target
+            for alias, target in self._aliases.items()
+            if alias != metric_id and target != resolved
+        }
+
+
+def _register_default_metrics(registry: MetricRegistry) -> None:
+    default_metrics = [
+        ExactStringMatchMetric,
+        CaseInsensitiveStringMatchMetric,
+        NormalizedLevenshteinSimilarityMetric,
+        StringSemanticMetric,
+        ExactNumberMatchMetric,
+        NumberToleranceMetric,
+        IntegerExactMatchMetric,
+        BooleanExactMatchMetric,
+        LlmJudgeMetric,
+        ArrayLlmJudgeMetric,
+    ]
+    for metric in default_metrics:
+        registry.register_metric(metric)
+
+
+global_metric_registry = MetricRegistry()
+_register_default_metrics(global_metric_registry)

extract_bench/evaluation/metrics/__init__.py

@@ -0,0 +1,43 @@
+"""Evaluation metrics for structured extraction."""
+
+from .array_metrics import ArrayLlmJudgeMetric
+from .base_metric import BaseMetric, MetricContext, MetricResult
+from .boolean_metrics import BooleanExactMatchMetric
+from .llm_metrics import LlmJudgeMetric
+from .metric_descriptors import MetricPromptDescriptor
+from .number_metrics import (
+    ExactNumberMatchMetric,
+    IntegerExactMatchMetric,
+    NumberToleranceMetric,
+)
+from .policy_metric import PolicyAwareMetric
+from .string_metrics import (
+    CaseInsensitiveStringMatchMetric,
+    ExactStringMatchMetric,
+    NormalizedLevenshteinSimilarityMetric,
+    StringSemanticMetric,
+)
+
+__all__ = [
+    # Base
+    "BaseMetric",
+    "MetricContext",
+    "MetricResult",
+    "MetricPromptDescriptor",
+    "PolicyAwareMetric",
+    # String
+    "ExactStringMatchMetric",
+    "CaseInsensitiveStringMatchMetric",
+    "NormalizedLevenshteinSimilarityMetric",
+    "StringSemanticMetric",
+    # Number
+    "ExactNumberMatchMetric",
+    "NumberToleranceMetric",
+    "IntegerExactMatchMetric",
+    # Boolean
+    "BooleanExactMatchMetric",
+    # Array
+    "ArrayLlmJudgeMetric",
+    # LLM
+    "LlmJudgeMetric",
+]

extract_bench/evaluation/metrics/array_metrics.py

@@ -0,0 +1,116 @@
+"""Array evaluation metrics."""
+
+import json
+from dataclasses import dataclass
+from typing import Any, ClassVar, Dict
+
+from ...infra.nodes import BaseSchema
+from ..evaluation_config import MetricConfig
+from .base_metric import MetricContext
+from .llm_metrics import LlmJudgeMetric
+from .metric_descriptors import MetricPromptDescriptor
+from .metric_prompts import ARRAY_LLM_PROMPT
+
+
+@dataclass(frozen=True, slots=True)
+class ArrayLlmJudgeMetric(LlmJudgeMetric):
+    metric_id: ClassVar[str] = "array_llm"
+    default_prompt_template: ClassVar[str] = ARRAY_LLM_PROMPT
+    prompt_descriptor = MetricPromptDescriptor(
+        metric_id="array_llm",
+        summary="LLM-based array evaluation with item matching and confusion summary.",
+        pass_rule="pass iff score >= pass_threshold (default threshold is metric-specific)",
+        score_rule="matched/(matched+missed_gold) after postprocessing",
+        params={
+            "pass_threshold": {"type": "number", "meaning": "minimum score to pass"},
+            "additional_instructions": {
+                "type": "string",
+                "meaning": "extra domain rules",
+            },
+        },
+    )
+    default_structured_output_schema: ClassVar[Dict[str, Any]] = {
+        "type": "object",
+        "properties": {
+            "reasoning": {"type": "string"},
+            "matched_items": {"type": "array", "items": {"type": "string"}},
+            "missed_gold_items": {"type": "array", "items": {"type": "string"}},
+            "spurious_pred_items": {"type": "array", "items": {"type": "string"}},
+            "matches_summary": {
+                "type": "object",
+                "properties": {
+                    "matched": {"type": "integer", "minimum": 0},
+                    "missed_gold": {"type": "integer", "minimum": 0},
+                    "spurious_pred": {"type": "integer", "minimum": 0},
+                },
+                "required": ["matched", "missed_gold", "spurious_pred"],
+            },
+        },
+        "required": ["reasoning", "matches_summary"],
+        "additionalProperties": True,
+    }
+
+    def _get_additional_prompt_context(self, node: MetricContext) -> str | None:
+        """Append metric definitions for child fields so the LLM understands per-field evaluation rules."""
+        from ..metric_id_collector import collect_metric_ids_with_params
+        from ..metric_registry import global_metric_registry
+
+        if not isinstance(node, BaseSchema):
+            return None
+        metric_params = collect_metric_ids_with_params(schema=node)
+        if not metric_params:
+            return None
+        definitions: dict[str, Any] = {}
+        for metric_id, params_examples in sorted(metric_params.items()):
+            descriptor = global_metric_registry.get_metric_descriptor(metric_id)
+            if descriptor is None:
+                definitions[metric_id] = {
+                    "metric_id": metric_id,
+                    "summary": "No descriptor registered for this metric_id.",
+                    "params_examples": params_examples,
+                }
+            else:
+                definitions[metric_id] = descriptor.describe(
+                    params_examples=params_examples
+                )
+        return (
+            f"Metric definitions (JSON):\n"
+            f"{json.dumps(definitions, ensure_ascii=True, indent=2)}"
+        )
+
+    def postprocess_parsed_result(
+        self,
+        node: MetricContext,
+        config: MetricConfig | None,
+        parsed: Dict[str, Any],
+    ) -> Dict[str, Any]:
+        parsed = dict(parsed)
+        summary = parsed["matches_summary"]
+        matched, missed, spurious = (
+            summary["matched"],
+            summary["missed_gold"],
+            summary["spurious_pred"],
+        )
+        denominator = max(1, matched + missed)
+        parsed["score"] = matched / denominator
+
+        total = matched + missed + spurious
+        gold_total = matched + missed
+        pred_total = matched + spurious
+
+        accuracy = matched / total if total > 0 else 1.0
+        precision = matched / pred_total if pred_total > 0 else 1.0
+        recall = matched / gold_total if gold_total > 0 else 1.0
+        if precision + recall == 0.0:
+            f1 = 0.0
+        else:
+            f1 = 2 * precision * recall / (precision + recall)
+
+        parsed["aggregate_metrics"] = {
+            "accuracy": accuracy,
+            "precision": precision,
+            "recall": recall,
+            "f1": f1,
+        }
+
+        return parsed

extract_bench/evaluation/metrics/base_metric.py

@@ -0,0 +1,40 @@
+"""Base interfaces for evaluation metrics."""
+
+from dataclasses import dataclass
+from typing import Any, Dict, Protocol
+
+from ..evaluation_config import MetricConfig
+
+
+@dataclass(frozen=True, slots=True)
+class MetricResult:
+    """Structured result returned by metrics."""
+
+    metric_id: str
+    score: float
+    passed: bool | None = None
+    details: dict[str, Any] | None = None
+
+
+class MetricContext(Protocol):
+    """Subset of schema node interface used by metrics."""
+
+    def get_gold_value(self) -> Any: ...
+
+    def get_extracted_value(self) -> Any: ...
+
+    def get_metadata_summary(self) -> Dict[str, Any]: ...
+
+
+class BaseMetric(Protocol):
+    """Common interface implemented by all metrics.
+
+    All metrics are async to enable parallel evaluation.
+    """
+
+    metric_id: str
+    recurse_into_children: bool = True
+
+    async def evaluate(
+        self, node: MetricContext, config: MetricConfig | None = None
+    ) -> MetricResult: ...

extract_bench/evaluation/metrics/boolean_metrics.py

@@ -0,0 +1,56 @@
+"""Boolean evaluation metrics."""
+
+from dataclasses import dataclass
+from typing import Any
+
+from ..evaluation_config import MetricConfig
+from .base_metric import BaseMetric, MetricContext, MetricResult
+from .metric_descriptors import MetricPromptDescriptor
+from .policy_metric import PolicyAwareMetric
+
+
+def _to_bool(value: Any) -> bool | None:
+    if isinstance(value, bool):
+        return value
+    return None
+
+
+@dataclass(frozen=True, slots=True)
+class BooleanExactMatchMetric(PolicyAwareMetric, BaseMetric):
+    metric_id: str = "boolean_exact"
+    prompt_descriptor = MetricPromptDescriptor(
+        metric_id="boolean_exact",
+        summary="Exact boolean equality.",
+        pass_rule="pass iff gold is True/False and predicted equals gold",
+        score_rule="1.0 if pass else 0.0",
+    )
+
+    async def _evaluate_values(
+        self,
+        *,
+        node: MetricContext,
+        gold: Any,
+        extracted: Any,
+        config: MetricConfig | None,
+    ) -> MetricResult:
+        gold_bool = _to_bool(gold)
+        extracted_bool = _to_bool(extracted)
+
+        if gold_bool is None or extracted_bool is None:
+            passed = gold_bool is extracted_bool
+            score = 1.0 if passed else 0.0
+            return MetricResult(
+                metric_id=self.metric_id,
+                score=score,
+                passed=passed,
+                details={"gold": gold, "extracted": extracted},
+            )
+
+        passed = gold_bool == extracted_bool
+        score = 1.0 if passed else 0.0
+        return MetricResult(
+            metric_id=self.metric_id,
+            score=score,
+            passed=passed,
+            details={"gold": gold_bool, "extracted": extracted_bool},
+        )