structured-eval 0.1.0__py3-none-any.whl

structured_eval/__init__.py

@@ -0,0 +1,27 @@
+"""structured_eval — field-level evaluation of structured LLM outputs.
+
+The top level exposes only the entrypoints. Everything else lives one level
+down, imported explicitly from its subsystem:
+
+- ``structured_eval.models``   — user-facing data models: ``Sample``,
+  ``EvalConfig`` (+ the ``*FieldConfig`` family & policies), ``EvalReport`` /
+  ``BatchEvalReport`` / ``ConsistencyReport``. Lower-level model pieces live in
+  precise submodules (``models.nodes`` / ``models.result`` /
+  ``models.metric_result`` / ``models.context``).
+- ``structured_eval.metrics``  — every metric plus the base hierarchy
+  (``Metric`` / ``FieldMetric`` / …), ``resolve_metric``, and the rule DSL
+  (``Rule`` / ``RulePassRate``).
+- ``structured_eval.alignment`` / ``.formats`` / ``.utils`` — supporting
+  machinery (array alignment, parsers, ``flatten`` / ``structured_diff``).
+
+``evaluate`` / ``evaluate_batch`` / ``evaluate_consistency`` are thin wrappers
+over ``engine.Evaluator``.
+"""
+
+from structured_eval.api import evaluate, evaluate_batch, evaluate_consistency
+
+__all__ = [
+    "evaluate",
+    "evaluate_batch",
+    "evaluate_consistency",
+]

structured_eval/alignment/__init__.py

@@ -0,0 +1,15 @@
+from structured_eval.alignment.base import ArrayAligner, key_value
+from structured_eval.alignment.by_index import ByIndexAligner
+from structured_eval.alignment.by_key import ByKeyAligner
+from structured_eval.alignment.factory import make_aligner
+from structured_eval.alignment.hungarian import HungarianAligner, Scorer
+
+__all__ = [
+    "ArrayAligner",
+    "ByIndexAligner",
+    "ByKeyAligner",
+    "HungarianAligner",
+    "Scorer",
+    "key_value",
+    "make_aligner",
+]

structured_eval/alignment/base.py

@@ -0,0 +1,40 @@
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from typing import TYPE_CHECKING, Any
+
+from structured_eval.utils.paths import MISSING, navigate
+
+if TYPE_CHECKING:
+    from structured_eval.models.nodes.array_node import ArrayMatchResult
+
+# Sentinel for a key that cannot be extracted (absent, or element not a dict).
+_MISSING_KEY = object()
+
+
+def key_value(element: Any, key: str | None) -> Any:
+    """The alignment key of an element: the whole element, or a named field.
+
+    Shared by every aligner that pairs on a key (``ByKeyAligner``,
+    ``HungarianAligner``). Returns ``None`` for a missing field and a private
+    sentinel when ``key`` is given but the element is not a dict.
+    """
+    if key is None:
+        return element
+    if isinstance(element, dict):
+        value = navigate(element, key)
+        return None if value is MISSING else value
+    return _MISSING_KEY
+
+
+class ArrayAligner(ABC):
+    """Maps actual array items onto expected ones (the only role of a matcher).
+
+    ``align`` returns an ``ArrayMatchResult`` with matched ``(expected_idx,
+    actual_idx)`` pairs plus the unmatched expected (missed) and actual
+    (spurious) indices. Value scoring of matched pairs happens later, in the
+    array metrics.
+    """
+
+    @abstractmethod
+    def align(self, expected: list[Any], actual: list[Any]) -> ArrayMatchResult: ...

structured_eval/alignment/by_index.py

@@ -0,0 +1,24 @@
+from __future__ import annotations
+
+from typing import Any
+
+from structured_eval.alignment.base import ArrayAligner
+from structured_eval.models.config import ArrayStrategy
+from structured_eval.models.nodes.array_node import ArrayMatchResult
+
+
+class ByIndexAligner(ArrayAligner):
+    """Pairs the i-th expected item with the i-th actual item.
+
+    For positionally significant lists (steps, time series, rankings). No key
+    comparison is performed.
+    """
+
+    def align(self, expected: list[Any], actual: list[Any]) -> ArrayMatchResult:
+        n = min(len(expected), len(actual))
+        return ArrayMatchResult(
+            strategy=ArrayStrategy.BY_INDEX,
+            matched=[(i, i) for i in range(n)],
+            missed=list(range(n, len(expected))),
+            spurious=list(range(n, len(actual))),
+        )

structured_eval/alignment/by_key.py

@@ -0,0 +1,73 @@
+from __future__ import annotations
+
+from typing import Any
+
+from structured_eval.alignment.base import ArrayAligner, key_value
+from structured_eval.metrics.base import BaseMetric, resolve_metric
+from structured_eval.metrics.exact import ExactMatch
+from structured_eval.metrics.invoker import MetricInvoker
+from structured_eval.models.config import ArrayStrategy
+from structured_eval.models.nodes.array_node import ArrayMatchResult
+
+
+class ByKeyAligner(ArrayAligner):
+    """Pairs items whose keys match, greedily best-first (generalized matching).
+
+    Extracts a key from each element (the ``key`` field, or the whole element
+    when ``key`` is None), compares keys with ``key_metric`` (default
+    ``ExactMatch``) and pairs them when the score clears ``threshold``. This
+    subsumes value- and similarity-based matching (technical_details_v3 §5).
+
+    Pairing is **globally greedy**: every candidate pair whose key score clears
+    the threshold is ranked by score (highest first) and claimed one-to-one,
+    skipping pairs whose either side is already taken. So a *soft* key picks the
+    strongest available partner rather than the first one found, and the result
+    does not depend on element order. With an exact key (all passing scores tie
+    at 1.0) this reduces to the original first-match behaviour. It is a cheap,
+    scipy-free approximation of the optimal assignment that ``HungarianAligner``
+    computes.
+    """
+
+    def __init__(
+        self,
+        key: str | None = None,
+        key_metric: str | BaseMetric | None = None,
+        threshold: float = 1.0,
+    ):
+        self.key = key
+        metric = ExactMatch() if key_metric is None else resolve_metric(key_metric)
+        self.scorer = MetricInvoker(metric)
+        self.threshold = threshold
+
+    def align(self, expected: list[Any], actual: list[Any]) -> ArrayMatchResult:
+        # Score every (expected, actual) pair on its key; keep those clearing
+        # the threshold. Generated in (ei, ai) order so a stable sort breaks
+        # score ties by that order (→ exact-key matches reproduce first-match).
+        candidates: list[tuple[float, int, int]] = []
+        for ei, e_item in enumerate(expected):
+            e_key = key_value(e_item, self.key)
+            for ai, a_item in enumerate(actual):
+                score = self.scorer.scalar_on_values(key_value(a_item, self.key), e_key)
+                if score >= self.threshold:
+                    candidates.append((score, ei, ai))
+        candidates.sort(key=lambda c: c[0], reverse=True)  # best first; ties keep order
+
+        used_e: set[int] = set()
+        used_a: set[int] = set()
+        matched: list[tuple[int, int]] = []
+        for _score, ei, ai in candidates:
+            if ei in used_e or ai in used_a:
+                continue
+            used_e.add(ei)
+            used_a.add(ai)
+            matched.append((ei, ai))
+        matched.sort()  # report pairs in expected order
+
+        missed = [ei for ei in range(len(expected)) if ei not in used_e]
+        spurious = [ai for ai in range(len(actual)) if ai not in used_a]
+        return ArrayMatchResult(
+            strategy=ArrayStrategy.BY_KEY,
+            matched=matched,
+            missed=missed,
+            spurious=spurious,
+        )

structured_eval/alignment/factory.py

@@ -0,0 +1,28 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+from structured_eval.alignment.by_index import ByIndexAligner
+from structured_eval.alignment.by_key import ByKeyAligner
+from structured_eval.alignment.hungarian import HungarianAligner
+from structured_eval.models.config import ArrayStrategy
+
+if TYPE_CHECKING:
+    from structured_eval.alignment.base import ArrayAligner
+
+
+def make_aligner(
+    strategy: ArrayStrategy = ArrayStrategy.BY_INDEX,
+    params: dict[str, Any] | None = None,
+) -> ArrayAligner:
+    """Build the aligner for an array config's ``strategy`` from its ``params``.
+
+    ``params`` keys match the chosen aligner's constructor arguments; an unknown
+    key surfaces as a ``TypeError`` from that constructor.
+    """
+    params = params or {}
+    if strategy == ArrayStrategy.BY_INDEX:
+        return ByIndexAligner()
+    if strategy == ArrayStrategy.HUNGARIAN:
+        return HungarianAligner(**params)
+    return ByKeyAligner(**params)

structured_eval/alignment/hungarian.py

@@ -0,0 +1,156 @@
+from __future__ import annotations
+
+import warnings
+from collections.abc import Callable
+from typing import Any
+
+from structured_eval.alignment.base import ArrayAligner, key_value
+from structured_eval.metrics.base import FieldMetric, Metric, resolve_metric
+from structured_eval.metrics.exact import ExactMatch
+from structured_eval.metrics.invoker import MetricInvoker
+from structured_eval.metrics.numeric_closeness import NumericCloseness
+from structured_eval.models.config import ArrayStrategy
+from structured_eval.models.nodes.array_node import ArrayMatchResult
+
+_LARGE_MATRIX_WARN = 10_000  # rows*cols beyond which we warn (quadratic scoring cost)
+
+# A per-element similarity: a Metric instance (every Metric has ``score``), its
+# registered name, or a plain ``(actual, expected) -> float`` callable.
+Scorer = Metric[Any] | str | Callable[[Any, Any], float]
+
+
+class HungarianAligner(ArrayAligner):
+    """Optimal one-to-one assignment via the Hungarian algorithm.
+
+    Builds a similarity matrix ``S[i,j] = score(expected[i], actual[j])`` and
+    solves ``min sum(1 - S)`` with ``scipy.optimize.linear_sum_assignment`` —
+    the globally optimal pairing regardless of order. A pair counts as matched
+    only when its similarity clears ``threshold`` (otherwise both sides are left
+    unmatched: a missed expected and a spurious actual).
+
+    ``scorer`` is the element similarity. Crucially our field metrics already
+    *are* scorers (``FieldMetric.score(actual, expected) -> float``), so no
+    adapter is needed — a metric, its registered name, or a plain callable is
+    used directly. It may be:
+
+    * a single ``Scorer`` — applied to the whole element;
+    * a ``dict[str, Scorer]`` — per-field scorers for arrays of objects; the
+      element score is the mean over the union of fields (a field with no entry
+      falls back to its type default);
+    * ``None`` — type-aware default (graded numeric / ``Fuzzy`` / exact), with
+      objects scored field-by-field.
+
+    ``key`` scores on a named sub-field instead of the whole element. Requires
+    the ``align`` extra (scipy).
+    """
+
+    def __init__(
+        self,
+        scorer: Scorer | dict[str, Scorer] | None = None,
+        threshold: float = 0.8,
+        key: str | None = None,
+    ):
+        self.scorer = scorer
+        self.threshold = threshold
+        self.key = key
+
+    def align(self, expected: list[Any], actual: list[Any]) -> ArrayMatchResult:
+        if not expected or not actual:
+            return ArrayMatchResult(
+                strategy=ArrayStrategy.HUNGARIAN,
+                matched=[],
+                missed=list(range(len(expected))),
+                spurious=list(range(len(actual))),
+            )
+
+        if len(expected) * len(actual) > _LARGE_MATRIX_WARN:
+            warnings.warn(
+                f"HungarianAligner: large {len(expected)}x{len(actual)} similarity "
+                "matrix; alignment may be slow.",
+                stacklevel=2,
+            )
+
+        try:
+            from scipy.optimize import linear_sum_assignment
+        except ImportError as exc:  # pragma: no cover
+            raise ImportError(
+                "scipy is required for HungarianAligner. "
+                "Install it with: pip install 'structured-eval[align]'"
+            ) from exc
+
+        cost = [[1.0 - self._score(e, a) for a in actual] for e in expected]
+        rows, cols = linear_sum_assignment(cost)
+
+        matched: list[tuple[int, int]] = []
+        used_e: set[int] = set()
+        used_a: set[int] = set()
+        for ei, ai in zip(rows, cols, strict=True):
+            if 1.0 - cost[ei][ai] >= self.threshold:
+                matched.append((int(ei), int(ai)))
+                used_e.add(int(ei))
+                used_a.add(int(ai))
+
+        missed = [ei for ei in range(len(expected)) if ei not in used_e]
+        spurious = [ai for ai in range(len(actual)) if ai not in used_a]
+        return ArrayMatchResult(
+            strategy=ArrayStrategy.HUNGARIAN,
+            matched=matched,
+            missed=missed,
+            spurious=spurious,
+        )
+
+    # ── element similarity ──────────────────────────────────────────────────
+
+    def _score(self, expected: Any, actual: Any) -> float:
+        return self._similarity(
+            key_value(expected, self.key),
+            key_value(actual, self.key),
+            self.scorer,
+        )
+
+    def _similarity(
+        self,
+        expected: Any,
+        actual: Any,
+        scorer: Scorer | dict[str, Scorer] | None,
+    ) -> float:
+        if isinstance(scorer, dict):
+            return self._object_similarity(expected, actual, scorer)
+        if scorer is not None:
+            return self._apply(scorer, expected, actual)
+        if isinstance(expected, dict) and isinstance(actual, dict):
+            return self._object_similarity(expected, actual, {})
+        return self._apply(self._default_scorer(expected, actual), expected, actual)
+
+    def _object_similarity(
+        self, expected: Any, actual: Any, scorers: dict[str, Scorer]
+    ) -> float:
+        if not isinstance(expected, dict) or not isinstance(actual, dict):
+            return 1.0 if expected == actual else 0.0
+        keys = set(expected) | set(actual)
+        if not keys:
+            return 1.0
+        total = sum(
+            self._similarity(expected.get(k), actual.get(k), scorers.get(k))
+            for k in keys
+        )
+        return total / len(keys)
+
+    @staticmethod
+    def _apply(scorer: Scorer, expected: Any, actual: Any) -> float:
+        if callable(scorer) and not isinstance(scorer, (str, Metric)):
+            return float(scorer(actual, expected))
+        return MetricInvoker(resolve_metric(scorer)).scalar_on_values(actual, expected)
+
+    @staticmethod
+    def _default_scorer(expected: Any, actual: Any) -> FieldMetric:
+        """Type-aware default similarity metric for a pair of scalar values.
+
+        ``bool`` → exact, number → graded :class:`NumericCloseness`, ``str`` →
+        :class:`Fuzzy` (or exact without rapidfuzz), everything else → exact.
+        """
+        if isinstance(expected, bool) or isinstance(actual, bool):
+            return ExactMatch()
+        if isinstance(expected, (int, float)) and isinstance(actual, (int, float)):
+            return NumericCloseness()
+        return ExactMatch()

structured_eval/api.py

@@ -0,0 +1,79 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+from structured_eval.engine.evaluator import Evaluator
+from structured_eval.models.sample import Sample
+
+if TYPE_CHECKING:
+    from structured_eval.models.config import EvalConfig
+    from structured_eval.models.result import (
+        BatchEvalReport,
+        ConsistencyReport,
+        EvalReport,
+    )
+
+
+def _is_batch(actual: Any) -> bool:
+    """A list of Samples is a batch; a bare list is a single array-root doc."""
+    return isinstance(actual, list) and all(isinstance(x, Sample) for x in actual)
+
+
+def evaluate(
+    actual: Any,
+    expected: Any = None,
+    config: EvalConfig | None = None,
+    *,
+    source: str | None = None,
+) -> EvalReport:
+    """Evaluate one document against an expected reference → ``EvalReport``.
+
+    Two call shapes:
+    - ``evaluate(actual, expected, config=...)`` — shorthand for one document;
+    - ``evaluate(sample, config=...)`` — one ``Sample``.
+
+    A bare ``list`` is a single document with an array root, not a batch. To
+    evaluate several samples use :func:`evaluate_batch`. Thin wrapper over
+    ``Evaluator``.
+    """
+    if _is_batch(actual):
+        raise TypeError(
+            "evaluate() takes a single document; pass a list of Samples to evaluate_batch()"
+        )
+    sample = (
+        actual
+        if isinstance(actual, Sample)
+        else Sample(actual=actual, expected=expected, source=source)
+    )
+    return Evaluator(config).evaluate_one(sample)
+
+
+def evaluate_batch(
+    samples: list[Sample],
+    config: EvalConfig | None = None,
+) -> BatchEvalReport:
+    """Evaluate a list of ``Sample`` s → ``BatchEvalReport``.
+
+    Each sample carries its own ``actual`` / ``expected`` / ``source``; the
+    aggregate report exposes per-sample reports plus batch-level metrics. Thin
+    wrapper over ``Evaluator``.
+    """
+    return Evaluator(config).evaluate_batch(samples)
+
+
+def evaluate_consistency(
+    runs: list[Sample],
+    config: EvalConfig | None = None,
+    *,
+    variance_threshold: float = 0.05,
+) -> ConsistencyReport:
+    """Measure run-to-run stability across repeated outputs of one prompt.
+
+    ``runs`` are several outputs for the same input (with or without a shared
+    ``expected``). Fields whose score varies at most ``variance_threshold``
+    across runs are reported as stable, the rest as unstable. Thin wrapper over
+    ``Evaluator``.
+    """
+    return Evaluator(config).evaluate_consistency(
+        runs, variance_threshold=variance_threshold
+    )

structured_eval/engine/__init__.py

@@ -0,0 +1,15 @@
+from structured_eval.engine.aggregator import BatchAggregator
+from structured_eval.engine.evaluator import Evaluator
+from structured_eval.engine.metric_runner import MetricRunner
+from structured_eval.engine.parser import Parser
+from structured_eval.engine.report_builder import ReportBuilder
+from structured_eval.engine.tree_builder import TreeBuilder
+
+__all__ = [
+    "BatchAggregator",
+    "Evaluator",
+    "MetricRunner",
+    "Parser",
+    "ReportBuilder",
+    "TreeBuilder",
+]

structured_eval/engine/aggregator.py

@@ -0,0 +1,96 @@
+"""Aggregation over multiple EvalReports: batch and consistency statistics."""
+
+from __future__ import annotations
+
+from statistics import mean, pvariance
+
+from structured_eval.models.result import (
+    BatchEvalReport,
+    ConsistencyReport,
+    EvalReport,
+    NodeType,
+)
+
+
+class BatchAggregator:
+    """Combines per-document reports into batch / consistency summaries."""
+
+    def batch(self, reports: list[EvalReport]) -> BatchEvalReport:
+        """Aggregate a list of single-document reports into a BatchEvalReport."""
+        n = len(reports)
+        errors = sum(1 for r in reports if r.parse_error)
+        ok = [r for r in reports if not r.parse_error]
+
+        scores = [r.score for r in ok if r.score is not None]
+        score = mean(scores) if scores else None
+        label = next((r.score_label for r in ok if r.score_label is not None), None)
+
+        perfect = sum(1 for r in ok if not r.failed_fields())
+
+        return BatchEvalReport(
+            per_sample=reports,
+            metrics=self._mean_metrics(reports),
+            score=score,
+            score_label=label,
+            perfect_response_rate=(perfect / n) if n else 0.0,
+            parse_error_rate=(errors / n) if n else 0.0,
+        )
+
+    def consistency(
+        self, reports: list[EvalReport], variance_threshold: float = 0.05
+    ) -> ConsistencyReport:
+        """Measure run-to-run stability across repeated outputs of one prompt."""
+        ok = [r for r in reports if not r.parse_error]
+
+        by_path: dict[str, list[float]] = {}
+        for r in ok:
+            for path, fs in r.field_scores.items():
+                # Field-level stability currently tracks leaf fields only: object/
+                # array nodes carry an aggregate representative score whose variance
+                # is just a function of its children's, so including it here would
+                # be redundant (double-counting the same wobble), non-actionable
+                # (a parent path doesn't point at a concrete field to fix) and
+                # noisy (an F1-over-children varies for different reasons than a
+                # single atomic value). Hence the leaf filter.
+                #
+                # TODO: support per-node stability regardless of node type. Some
+                # users want block-level wobble ("the whole `address` object is
+                # unstable") without drilling into leaves. The fix is NOT to drop
+                # this filter (that mixes scales) but to expose a separate,
+                # parallel view computed over non-scalar nodes (e.g.
+                # ConsistencyReport.object_variance / block_variance), keeping the
+                # leaf map clean and adding the aggregate one alongside it.
+                if fs.score is None or fs.node_type != NodeType.SCALAR:
+                    continue
+                by_path.setdefault(path, []).append(fs.score)
+
+        variance: dict[str, float] = {}
+        stable: list[str] = []
+        unstable: list[str] = []
+        for path, vals in by_path.items():
+            var = pvariance(vals) if len(vals) > 1 else 0.0
+            variance[path] = var
+            (stable if var <= variance_threshold else unstable).append(path)
+
+        scores = [r.score for r in ok if r.score is not None]
+        return ConsistencyReport(
+            per_run=reports,
+            field_variance=variance,
+            stable_fields=stable,
+            unstable_fields=unstable,
+            mean_score=mean(scores) if scores else None,
+            score_variance=(pvariance(scores) if len(scores) > 1 else 0.0)
+            if scores
+            else None,
+        )
+
+    @staticmethod
+    def _mean_metrics(reports: list[EvalReport]) -> dict[str, float]:
+        """Mean of each metric across reports that carry it (errors excluded)."""
+        buckets: dict[str, list[float]] = {}
+        for r in reports:
+            if r.parse_error:
+                continue
+            for name, coll in r.metrics.items():
+                buckets.setdefault(name, []).append(coll.representative())
+        return {name: mean(vals) for name, vals in buckets.items() if vals}

structured_eval/engine/evaluator.py

@@ -0,0 +1,72 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+from structured_eval.engine.aggregator import BatchAggregator
+from structured_eval.engine.metric_runner import MetricRunner
+from structured_eval.engine.parser import Parser
+from structured_eval.engine.report_builder import ReportBuilder
+from structured_eval.engine.tree_builder import TreeBuilder
+from structured_eval.models.config import EvalConfig
+from structured_eval.models.context import EvalContext
+from structured_eval.models.result import BatchEvalReport, ConsistencyReport, EvalReport
+from structured_eval.utils.flatten import flatten
+
+if TYPE_CHECKING:
+    from structured_eval.models.sample import Sample
+
+
+class Evaluator:
+    """Orchestrates the three evaluation phases for one config.
+
+    Holds the ``EvalConfig`` and the phase collaborators (parse → build tree →
+    run metrics → build report) and aggregates batches. The module-level
+    ``evaluate`` / ``evaluate_consistency`` functions are thin wrappers over this.
+    """
+
+    def __init__(self, config: EvalConfig | None = None):
+        self.config = config or EvalConfig()
+        self._parser = Parser()
+        self._runner = MetricRunner()
+        self._report_builder = ReportBuilder()
+        self._aggregator = BatchAggregator()
+
+    def evaluate_one(self, sample: Sample) -> EvalReport:
+        """Evaluate a single document against its expected reference."""
+        actual, actual_err = self._parser.parse(sample.actual)
+        expected, expected_err = self._parser.parse(sample.expected)
+        error = actual_err or expected_err
+        if error is not None:
+            return EvalReport(parse_error=True, parse_error_message=error)
+
+        context = EvalContext(
+            actual=actual,
+            expected=expected,
+            source=sample.source,
+            flat_actual=_flat(actual),
+            flat_expected=_flat(expected),
+            config=self.config,
+        )
+
+        root, warnings = TreeBuilder(
+            context
+        ).build()  # phase 1: structure + per-node metrics
+        self._runner.run(
+            root
+        )  # phase 2: compute post-order, each node's key_metric last
+        return self._report_builder.build(root, context, warnings)  # phase 3
+
+    def evaluate_batch(self, samples: list[Sample]) -> BatchEvalReport:
+        """Evaluate a list of documents and aggregate the results."""
+        return self._aggregator.batch([self.evaluate_one(s) for s in samples])
+
+    def evaluate_consistency(
+        self, runs: list[Sample], *, variance_threshold: float = 0.05
+    ) -> ConsistencyReport:
+        """Measure run-to-run stability across repeated outputs of one prompt."""
+        reports = [self.evaluate_one(s) for s in runs]
+        return self._aggregator.consistency(reports, variance_threshold)
+
+
+def _flat(data: Any) -> dict[str, Any]:
+    return flatten(data) if isinstance(data, (dict, list)) else {}