structured-eval 0.1.0__py3-none-any.whl

structured_eval/engine/metric_runner.py

@@ -0,0 +1,69 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+from structured_eval.metrics.invoker import MetricInvoker
+from structured_eval.models.metric_result import MetricResult
+
+if TYPE_CHECKING:
+    from structured_eval.metrics.base import BaseMetric, MetricOutput
+    from structured_eval.models.nodes.base import EvalNode
+
+
+class MetricRunner:
+    """Phase 2: compute each node's own metrics across the tree, in place.
+
+    Every node carries the metrics resolved for it by ``TreeBuilder``; this
+    phase walks the tree **post-order** (children before their parent), so an
+    aggregating parent reads its children's already-computed representative
+    scores — computation is uniform and fully recursive at any nesting depth.
+    Within a node the ``key_metric`` runs *last*: it is the representative score
+    and its logic may depend on the node's other metrics (the default
+    ``MeanScore`` averages them). A metric returning ``None`` (e.g.
+    ``Faithfulness`` without a source) is skipped.
+    """
+
+    def run(self, root: EvalNode) -> None:
+        self._visit(root)
+
+    def _visit(self, node: EvalNode) -> None:
+        for child in node.children_nodes():
+            self._visit(child)
+        key_metric = node.key_metric
+        for metric in node.metrics:
+            if metric is key_metric:
+                continue
+            self._apply(metric, node)
+        if key_metric is not None:
+            self._apply(key_metric, node)
+
+    def _apply(self, metric: BaseMetric, node: EvalNode) -> None:
+        result = MetricInvoker(metric).on_node(node)
+        node.metric_results.update(self._normalize(metric.name, result))
+
+    @staticmethod
+    def _normalize(name: str, result: MetricOutput) -> dict[str, MetricResult]:
+        """Coerce any ``compute`` return into ``{key: MetricResult}``.
+
+        Accepts ``None`` (skip), a bare value, a ``dict`` of sub-scores, a
+        ``MetricResult``, or a ``(value | dict, extra)`` tuple — so a metric can
+        attach structured ``extra`` regardless of how it shapes its score. A
+        tuple's ``extra`` is attached to every key it produces.
+        """
+        if result is None:
+            return {}
+        extra: dict[str, Any] = {}
+        if isinstance(result, tuple):
+            result, extra = result
+        if isinstance(result, dict):
+            return {
+                k: MetricResult(v, {**getattr(v, "extra", {}), **extra})
+                for k, v in result.items()
+            }
+        if isinstance(result, MetricResult):
+            return {
+                name: MetricResult(result, {**result.extra, **extra})
+                if extra
+                else result
+            }
+        return {name: MetricResult(result, extra)}

structured_eval/engine/parser.py

@@ -0,0 +1,42 @@
+from __future__ import annotations
+
+from typing import Any
+
+from structured_eval.formats.base import ParseError
+from structured_eval.formats.json_parser import JsonParser
+
+
+class Parser:
+    """Coerces raw sample input into Python values without raising.
+
+    Already-structured input (dict/list/None/scalars) passes through. A string
+    is parsed as JSON; if that fails it is retried as YAML (when PyYAML is
+    installed) and accepted only when it yields a dict or list. ``parse`` returns
+    ``(value, None)`` on success or ``(None, message)`` on failure, so the engine
+    can surface a parse error in the ``EvalReport`` rather than blowing up.
+    """
+
+    def __init__(self) -> None:
+        self._json = JsonParser()
+
+    def parse(self, raw: Any) -> tuple[Any, str | None]:
+        if not isinstance(raw, str):
+            return raw, None
+        try:
+            return self._json.parse(raw), None
+        except ParseError as json_error:
+            value = self._try_yaml(raw)
+            if value is not None:
+                return value, None
+            return None, str(json_error)
+
+    @staticmethod
+    def _try_yaml(text: str) -> Any | None:
+        """Parse ``text`` as YAML, returning a dict/list or None on any failure."""
+        from structured_eval.formats.yaml_parser import YamlParser
+
+        try:
+            value = YamlParser().parse(text)
+        except (ParseError, ImportError):
+            return None
+        return value if isinstance(value, (dict, list)) else None

structured_eval/engine/report_builder.py

@@ -0,0 +1,68 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, ClassVar
+
+from structured_eval.models.metric_result import MetricCollection
+from structured_eval.models.nodes.array_node import ArrayNode
+from structured_eval.models.nodes.object_node import ObjectNode
+from structured_eval.models.nodes.scalar import ScalarNode
+from structured_eval.models.result import EvalReport, EvalWarning, FieldScore, NodeType
+
+if TYPE_CHECKING:
+    from structured_eval.models.context import EvalContext
+    from structured_eval.models.nodes.base import EvalNode
+
+
+class ReportBuilder:
+    """Phase 3: flatten the computed node tree into an ``EvalReport``."""
+
+    _NODE_TYPE: ClassVar[dict[type, NodeType]] = {
+        ScalarNode: NodeType.SCALAR,
+        ObjectNode: NodeType.OBJECT,
+        ArrayNode: NodeType.ARRAY,
+    }
+
+    def build(
+        self, root: EvalNode, context: EvalContext, warnings: list[EvalWarning]
+    ) -> EvalReport:
+        field_scores = {}
+        array_matches = {}
+        # report.metrics is a cross-field view: each metric name → its value at
+        # every node that produced it (a MetricCollection), built as we walk.
+        # A metric's structured detail (schema errors, hallucinated paths, …)
+        # rides along on each value's ``.extra``.
+        collections: dict[str, MetricCollection] = {}
+        for node in root.walk():
+            field_scores[node.path] = self._field_score(node)
+            for name, result in node.metric_results.items():
+                coll = collections.setdefault(name, MetricCollection(name=name))
+                coll.by_path[node.path] = result
+            if isinstance(node, ArrayNode) and node.match_result is not None:
+                array_matches[node.path] = node.match_result
+
+        # The headline number is the root node's representative (key) metric.
+        score_label = root.key_metric.name if root.key_metric is not None else None
+        root_score = (
+            root.metric_results.get(score_label) if score_label is not None else None
+        )
+        score = float(root_score) if root_score is not None else None
+
+        return EvalReport(
+            score=score,
+            score_label=score_label,
+            metrics=collections,
+            field_scores=field_scores,
+            array_matches=array_matches,
+            warnings=warnings,
+        )
+
+    def _field_score(self, node: EvalNode) -> FieldScore:
+        return FieldScore(
+            path=node.path,
+            node_type=self._NODE_TYPE.get(type(node), NodeType.SCALAR),
+            actual=node.actual,
+            expected=node.expected,
+            metrics=dict(node.metric_results),
+            score=node.representative,  # the node's representative (key-metric) value
+            threshold=node.threshold,
+        )

structured_eval/engine/tree_builder.py

@@ -0,0 +1,319 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+from structured_eval.alignment import make_aligner
+from structured_eval.metrics.array_accuracy import ArrayAccuracy
+from structured_eval.metrics.base import (
+    AnyNodeMetric,
+    ArrayMetric,
+    BaseMetric,
+    FieldMetric,
+    GenericMetric,
+    Metric,
+    ObjectMetric,
+    RootMetric,
+    resolve_metric,
+)
+from structured_eval.metrics.exact import ExactMatch
+from structured_eval.metrics.invoker import GENERIC_NODE_METHOD
+from structured_eval.metrics.mean_score import MeanScore
+from structured_eval.metrics.object_accuracy import ObjectAccuracy
+from structured_eval.models.config import (
+    AnyFieldConfig,
+    ArrayFieldConfig,
+    ArrayStrategy,
+    EvalConfig,
+    ExtraKeysPolicy,
+    ObjectFieldConfig,
+    weight_of,
+)
+from structured_eval.models.nodes.array_node import ArrayNode
+from structured_eval.models.nodes.base import MISSING, EvalNode, navigate
+from structured_eval.models.nodes.object_node import ObjectNode
+from structured_eval.models.nodes.scalar import ScalarNode
+from structured_eval.models.result import EvalWarning, WarningType
+
+if TYPE_CHECKING:
+    from structured_eval.models.context import EvalContext
+
+# Metric a node falls back to when the user configured none of its type, so every
+# node always carries at least one metric for its key_metric (MeanScore) to mean.
+DEFAULT_SCALAR_METRIC = ExactMatch
+DEFAULT_OBJECT_METRIC = ObjectAccuracy
+DEFAULT_ARRAY_METRIC = ArrayAccuracy
+
+
+class TreeBuilder:
+    """Phase 1: build the EvalNode tree and resolve each node's metric list.
+
+    ``build`` returns ``(root_node, warnings)``. This phase is purely
+    structural: it shapes the tree, resolves which metrics apply to each node
+    (cascading the config's global metrics by type and adding any per-node
+    ``cfg.metrics``), and attaches them to ``node.metrics``. Computation happens
+    later, uniformly, in ``MetricRunner``. Each node carries an ``actual``-side
+    ``path`` and, when arrays reorder elements, a diverging ``expected_path`` so
+    each side navigates its own index.
+    """
+
+    def __init__(self, context: EvalContext):
+        self.context = context
+        self.config: EvalConfig = context.config
+        self.warnings: list[EvalWarning] = []
+        self._globals = self._resolve_globals()
+
+    def build(self) -> tuple[EvalNode, list[EvalWarning]]:
+        root = self.node("$", "$", self.root_config())
+        return root, self.warnings
+
+    # ── config resolution ──────────────────────────────────────────────────
+
+    def root_config(self) -> ObjectFieldConfig | ArrayFieldConfig | None:
+        if self.config.root is not None:
+            return self.config.root
+        if self.config.fields:
+            return ObjectFieldConfig(fields=dict(self.config.fields))
+        return None
+
+    @staticmethod
+    def _applies_to(metric: BaseMetric, node_cls: type, is_root: bool) -> bool:
+        """Whether ``metric`` should be resolved onto a node of ``node_cls``.
+
+        Typed metrics match their node type (a ``RootMetric`` only at the root);
+        an ``AnyNodeMetric`` matches every node; a ``GenericMetric`` matches iff
+        it defines the node's ``compute_<kind>``.
+        """
+        if isinstance(metric, RootMetric):
+            return is_root
+        if isinstance(metric, AnyNodeMetric):
+            return True
+        if isinstance(metric, FieldMetric):
+            return issubclass(node_cls, ScalarNode)
+        if isinstance(metric, ObjectMetric):
+            return issubclass(node_cls, ObjectNode)
+        if isinstance(metric, ArrayMetric):
+            return issubclass(node_cls, ArrayNode)
+        if isinstance(metric, GenericMetric):
+            method = GENERIC_NODE_METHOD.get(node_cls)
+            return method is not None and hasattr(metric, method)
+        return False
+
+    def _resolve_globals(self) -> list[BaseMetric]:
+        """The cascade set: ``config.metrics``, deduped by identity.
+
+        ``key_metric`` is *not* cascaded here — it is each node's representative
+        metric, resolved per node by ``_key_metric`` (and computed last).
+        """
+        out: list[BaseMetric] = []
+        for spec in self.config.metrics:
+            metric = resolve_metric(spec)
+            if not any(metric is seen for seen in out):
+                out.append(metric)
+        return out
+
+    def _resolve_metrics(
+        self, node_cls: type, cfg: AnyFieldConfig | None, is_root: bool
+    ) -> list[BaseMetric]:
+        """Metrics for one node: applicable globals + this node's own (additive).
+
+        Globals cascade by type (a ``RootMetric`` only at the root); per-node
+        ``cfg.metrics`` are *added* (not a replacement), deduped by identity.
+        ``out`` only ever holds metrics applicable to this node (``add`` filters
+        by ``_applies_to``); if it ends up empty the node gets the default for
+        its type so every node always carries at least one metric for its
+        ``key_metric`` to summarise (a different default is set by putting a
+        metric of that type in ``config.metrics``, which cascades).
+        """
+        out: list[BaseMetric] = []
+
+        def add(metric: BaseMetric) -> None:
+            if self._applies_to(metric, node_cls, is_root) and not any(
+                metric is s for s in out
+            ):
+                out.append(metric)
+
+        for metric in self._globals:
+            add(metric)
+        for spec in getattr(cfg, "metrics", None) or []:
+            add(resolve_metric(spec))
+
+        if not out:
+            if issubclass(node_cls, ScalarNode):
+                add(DEFAULT_SCALAR_METRIC())
+            elif issubclass(node_cls, ObjectNode):
+                add(DEFAULT_OBJECT_METRIC())
+            elif issubclass(node_cls, ArrayNode):
+                add(DEFAULT_ARRAY_METRIC())
+        return out
+
+    def _key_metric(
+        self,
+        node_cls: type,
+        cfg: AnyFieldConfig | None,
+        is_root: bool,
+        metrics: list[BaseMetric],
+    ) -> Metric[Any]:
+        """The node's representative metric (computed last).
+
+        Prefers an explicit ``cfg.key_metric``, then a distributable
+        ``config.key_metric`` (each applied only where its type fits), else the
+        default ``MeanScore`` (the mean of the node's own metrics).
+
+        A *name string* is resolved against the node's already-resolved
+        ``metrics`` first: an equally-named metric is **reused as-is** (same
+        instance, same params, no duplicate computation). It is instantiated
+        fresh only when the name is not already on the node.
+        """
+        for spec in (getattr(cfg, "key_metric", None), self.config.key_metric):
+            if spec is None:
+                continue
+            # Reuse an equally-named metric already on the node; else resolve fresh.
+            metric = next(
+                (m for m in metrics if isinstance(spec, str) and m.name == spec),
+                None,
+            ) or resolve_metric(spec)
+            if self._applies_to(metric, node_cls, is_root):
+                assert isinstance(metric, Metric)  # a key metric has compute()/score()
+                return metric
+        return MeanScore()
+
+    # ── tree construction ────────────────────────────────────────────────
+
+    def _value(self, doc: Any, path: str) -> Any:
+        value = navigate(doc, path)
+        return None if value is MISSING else value
+
+    def _child(self, path: str, key: str) -> str:
+        return key if path in ("$", "") else f"{path}.{key}"
+
+    def node(self, apath: str, epath: str, cfg: AnyFieldConfig | None) -> EvalNode:
+        actual = self._value(self.context.actual, apath)
+        expected = self._value(self.context.expected, epath)
+        ref = expected if expected is not None else actual
+
+        if isinstance(ref, dict):
+            return self._object(apath, epath, cfg, actual, expected)
+        if isinstance(ref, list):
+            return self._array(apath, epath, cfg, actual, expected)
+        return self._scalar(apath, epath, cfg)
+
+    def _object(
+        self,
+        apath: str,
+        epath: str,
+        cfg: AnyFieldConfig | None,
+        actual: Any,
+        expected: Any,
+    ) -> ObjectNode:
+        a_keys = set(actual) if isinstance(actual, dict) else set()
+        e_keys = set(expected) if isinstance(expected, dict) else set()
+        both = a_keys & e_keys
+        missing = sorted(e_keys - a_keys)  # in expected only → FN
+        extra = sorted(a_keys - e_keys)  # in actual only → FP (subject to policy)
+
+        if self.config.extra_keys == ExtraKeysPolicy.PENALIZE:
+            spurious = extra
+        else:
+            spurious = []
+            for key in extra:
+                path = self._child(apath, key)
+                self.warnings.append(
+                    EvalWarning(
+                        type=WarningType.EXTRA_KEY,
+                        path=path,
+                        message=f"{path!r} not in expected (ExtraKeysPolicy.IGNORE)",
+                    )
+                )
+
+        fields = cfg.fields if isinstance(cfg, ObjectFieldConfig) else {}
+        children: dict[str, Any] = {}
+        matched: list[Any] = []
+        for key in sorted(a_keys | e_keys):
+            child = self.node(
+                self._child(apath, key), self._child(epath, key), fields.get(key)
+            )
+            children[key] = child
+            if key in both:
+                matched.append(child)
+        for key in missing:
+            path = self._child(apath, key)
+            self.warnings.append(
+                EvalWarning(
+                    type=WarningType.MISSING_FIELD,
+                    path=path,
+                    message=f"{path!r} absent in actual",
+                )
+            )
+
+        is_root = apath == "$"
+        metrics = self._resolve_metrics(ObjectNode, cfg, is_root)
+        return ObjectNode(
+            path=apath,
+            context=self.context,
+            expected_path=epath if epath != apath else None,
+            weight=weight_of(cfg),
+            metrics=metrics,
+            key_metric=self._key_metric(ObjectNode, cfg, is_root, metrics),
+            threshold=self._threshold(cfg),
+            matched=matched,
+            missing=missing,
+            spurious=spurious,
+            children=children,
+        )
+
+    def _array(
+        self,
+        apath: str,
+        epath: str,
+        cfg: AnyFieldConfig | None,
+        actual: Any,
+        expected: Any,
+    ) -> ArrayNode:
+        a_list: list[Any] = actual if isinstance(actual, list) else []
+        e_list: list[Any] = expected if isinstance(expected, list) else []
+        if isinstance(cfg, ArrayFieldConfig):
+            aligner = make_aligner(strategy=cfg.strategy, params=cfg.params)
+            item_cfg = cfg.item
+        else:
+            aligner = make_aligner(strategy=ArrayStrategy.BY_INDEX, params=None)
+            item_cfg = None
+        result = aligner.align(e_list, a_list)
+        # TODO: with no expected list (faithfulness / schema-only mode) there are
+        # no matched pairs, so array elements get no nodes — value-on-actual
+        # metrics (FieldFaithfulness) can't reach them. Materialize actual
+        # elements directly in that mode. Roadmap.
+        items = [
+            self.node(f"{apath}[{aidx}]", f"{epath}[{eidx}]", item_cfg)
+            for eidx, aidx in result.matched
+        ]
+        is_root = apath == "$"
+        metrics = self._resolve_metrics(ArrayNode, cfg, is_root)
+        return ArrayNode(
+            path=apath,
+            context=self.context,
+            expected_path=epath if epath != apath else None,
+            weight=weight_of(cfg),
+            metrics=metrics,
+            key_metric=self._key_metric(ArrayNode, cfg, is_root, metrics),
+            threshold=self._threshold(cfg),
+            match_result=result,
+            items=items,
+        )
+
+    def _scalar(self, apath: str, epath: str, cfg: AnyFieldConfig | None) -> ScalarNode:
+        is_root = apath == "$"
+        metrics = self._resolve_metrics(ScalarNode, cfg, is_root)
+        return ScalarNode(
+            path=apath,
+            context=self.context,
+            expected_path=epath if epath != apath else None,
+            weight=weight_of(cfg),
+            metrics=metrics,
+            key_metric=self._key_metric(ScalarNode, cfg, is_root, metrics),
+            threshold=self._threshold(cfg),
+        )
+
+    @staticmethod
+    def _threshold(cfg: AnyFieldConfig | None) -> float:
+        threshold = getattr(cfg, "threshold", None)
+        return float(threshold) if threshold is not None else 1.0

structured_eval/formats/__init__.py

@@ -0,0 +1,5 @@
+from structured_eval.formats.base import ParseError, Parser
+from structured_eval.formats.json_parser import JsonlParser, JsonParser
+from structured_eval.formats.yaml_parser import YamlParser
+
+__all__ = ["JsonParser", "JsonlParser", "ParseError", "Parser", "YamlParser"]

structured_eval/formats/base.py

@@ -0,0 +1,19 @@
+from __future__ import annotations
+
+from typing import Any, Protocol, runtime_checkable
+
+
+class ParseError(ValueError):
+    """Raised when input text cannot be parsed into a structured value."""
+
+
+@runtime_checkable
+class Parser(Protocol):
+    """Parse a text string into a Python object.
+
+    Implementations must raise ParseError on malformed input.
+    Return type is Any because parsers may produce dict, list, or scalar
+    depending on the input (e.g. JSONL returns an iterator of dicts).
+    """
+
+    def parse(self, text: str) -> Any: ...

structured_eval/formats/json_parser.py

@@ -0,0 +1,44 @@
+from __future__ import annotations
+
+import json
+from typing import TYPE_CHECKING
+
+from structured_eval.formats.base import ParseError
+
+if TYPE_CHECKING:
+    from collections.abc import Iterator
+
+
+class JsonParser:
+    """Parse a JSON string into a Python object.
+
+    Accepts any valid JSON value (object, array, string, number, bool, null).
+    Raises ParseError on malformed input.
+    """
+
+    def parse(self, text: str) -> object:
+        try:
+            return json.loads(text)
+        except json.JSONDecodeError as exc:
+            raise ParseError(f"Invalid JSON: {exc}") from exc
+
+
+class JsonlParser:
+    """Parse a JSONL (JSON Lines) string into an iterator of Python objects.
+
+    Each non-empty line must be a valid JSON value. Blank lines are skipped.
+    Raises ParseError on the first malformed line, including the line number.
+    """
+
+    def parse(self, text: str) -> Iterator[object]:
+        return self._iter(text)
+
+    def _iter(self, text: str) -> Iterator[object]:
+        for lineno, raw_line in enumerate(text.splitlines(), start=1):
+            line = raw_line.strip()
+            if not line:
+                continue
+            try:
+                yield json.loads(line)
+            except json.JSONDecodeError as exc:
+                raise ParseError(f"Invalid JSON on line {lineno}: {exc}") from exc

structured_eval/formats/yaml_parser.py

@@ -0,0 +1,24 @@
+from typing import Any
+
+from structured_eval.formats.base import ParseError
+
+
+class YamlParser:
+    """Parse a YAML string into a Python object.
+
+    Uses yaml.safe_load — arbitrary Python object construction is disabled.
+    Raises ParseError on malformed input. PyYAML is imported lazily so the
+    core package stays importable without the ``yaml`` extra.
+    """
+
+    def parse(self, text: str) -> Any:
+        try:
+            import yaml
+        except ImportError as exc:
+            raise ImportError(
+                "PyYAML is required for YAML parsing. Install it with: pip install pyyaml"
+            ) from exc
+        try:
+            return yaml.safe_load(text)
+        except yaml.YAMLError as exc:
+            raise ParseError(f"Invalid YAML: {exc}") from exc

structured_eval/integrations/__init__.py

@@ -0,0 +1,11 @@
+"""Adapters that plug structured-eval into host eval frameworks.
+
+The core (``evaluate`` → ``EvalReport``) is framework-agnostic; each adapter
+lazily imports its host library so ``import structured_eval`` never requires
+deepeval/langsmith. Install with the matching extra (``structured-eval[deepeval]``
+or ``[langsmith]``).
+"""
+
+from structured_eval.integrations._adapter import reason_text, verdict
+
+__all__ = ["reason_text", "verdict"]

structured_eval/integrations/_adapter.py

@@ -0,0 +1,47 @@
+"""Framework-agnostic mapping from an EvalReport to a (score, success, reason).
+
+Shared by every integration so the host-specific classes stay thin. Tested
+directly, without any host library installed.
+"""
+
+from __future__ import annotations
+
+from structured_eval.models.result import EvalReport, NodeType
+
+_MAX_REASONS = 5
+
+
+def reason_text(report: EvalReport) -> str:
+    """Human-readable explanation of a report, focused on what failed."""
+    if report.parse_error:
+        return f"parse error: {report.parse_error_message or 'could not parse output'}"
+
+    failed = report.failed_fields()
+    if not failed:
+        return "all fields passed"
+
+    parts = []
+    for fs in list(failed.values())[:_MAX_REASONS]:
+        if fs.node_type == NodeType.SCALAR:
+            parts.append(f"{fs.path}: {fs.actual!r} != {fs.expected!r}")
+        else:
+            parts.append(
+                f"{fs.path}: score {fs.score:.2g}" if fs.score is not None else fs.path
+            )
+    if len(failed) > _MAX_REASONS:
+        parts.append(f"... +{len(failed) - _MAX_REASONS} more")
+
+    head = f"{len(failed)} field(s) failed: "
+    return head + "; ".join(parts)
+
+
+def verdict(report: EvalReport, threshold: float) -> tuple[float | None, bool, str]:
+    """Reduce a report to (score, success, reason) for a host framework.
+
+    ``score`` is the key-metric value (``None`` when no key metric / no ground
+    truth). ``success`` requires a parsed document and ``score >= threshold``;
+    when ``score`` is ``None`` the pass/fail bar cannot be applied → ``False``.
+    """
+    score = report.score
+    success = not report.parse_error and score is not None and score >= threshold
+    return score, success, reason_text(report)