structured-eval 0.1.0__py3-none-any.whl

structured_eval/models/result.py

@@ -0,0 +1,361 @@
+from __future__ import annotations
+
+from enum import StrEnum
+from pathlib import Path
+from statistics import mean
+from typing import Any
+
+from pydantic import BaseModel, ConfigDict, Field
+
+from structured_eval.models.metric_result import (  # noqa: TC001
+    MetricCollection,
+    MetricResult,
+)
+from structured_eval.models.nodes.array_node import ArrayMatchResult  # noqa: TC001
+
+
+class NodeType(StrEnum):
+    """The kind of tree node a ``FieldScore`` describes."""
+
+    SCALAR = "scalar"
+    OBJECT = "object"
+    ARRAY = "array"
+
+
+def _percentile(values: list[float], q: float) -> float:
+    """Linear-interpolation percentile (q in [0, 1]) over a non-empty list."""
+    ordered = sorted(values)
+    if len(ordered) == 1:
+        return ordered[0]
+    pos = q * (len(ordered) - 1)
+    lo = int(pos)
+    hi = min(lo + 1, len(ordered) - 1)
+    frac = pos - lo
+    return ordered[lo] + (ordered[hi] - ordered[lo]) * frac
+
+
+# ── Warnings ────────────────────────────────────────────────────────────────
+
+
+class WarningType(StrEnum):
+    """The kind of structural warning the engine raised while building the tree."""
+
+    EXTRA_KEY = (
+        "extra_key"  # key present in actual but not expected (ExtraKeysPolicy.IGNORE)
+    )
+    MISSING_FIELD = "missing_field"  # key present in expected but absent in actual
+
+
+class EvalWarning(BaseModel):
+    """A structural warning, typed by ``WarningType`` and located by ``path``."""
+
+    type: WarningType
+    path: str
+    message: str = ""
+
+    def __str__(self) -> str:
+        tag = f"[{self.type.name}]"
+        return f"{tag} {self.message}" if self.message else f"{tag} {self.path}"
+
+
+# ── Rules ─────────────────────────────────────────────────────────────────────
+
+
+class RuleResult(BaseModel):
+    """Result of evaluating a single business rule."""
+
+    name: str
+    passed: bool
+    message: str = ""
+
+
+# ── Per-path score ────────────────────────────────────────────────────────────
+
+
+class FieldScore(BaseModel):
+    """Evaluation result for one node of the tree (flat, dot-notation path).
+
+    ``metrics`` holds only the metrics that were requested and applied to this
+    node (e.g. ``{"exact_match": 0.0, "token_f1": 0.62}``). Each value is a
+    ``MetricResult`` — a ``float`` that also carries ``.extra`` (structured
+    detail the metric chose to surface). ``score`` is the value of the key metric
+    at this path, ``threshold`` the bar applied to it.
+    """
+
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+
+    path: str
+    node_type: NodeType
+    actual: Any = None
+    expected: Any = None
+    metrics: dict[str, MetricResult] = Field(default_factory=dict)
+    # TODO: Should be required by default key metric
+    score: float | None = None
+    # TODO: Reconsider default arguments - all possible should be defined in model as defaults
+    threshold: float | None = None
+
+
+# ── Regression diff ─────────────────────────────────────────────────────────
+
+
+class RegressionDiff(BaseModel):
+    """Metric deltas between two EvalReports (self minus other).
+
+    ``deltas`` are per-metric changes in the aggregate; ``field_deltas`` maps
+    each field path to its own per-metric changes. Positive means improvement.
+    """
+
+    deltas: dict[str, float] = Field(default_factory=dict)
+    field_deltas: dict[str, dict[str, float]] = Field(default_factory=dict)
+
+
+# ── Eval report ───────────────────────────────────────────────────────────────
+
+
+class EvalReport(BaseModel):
+    """Full evaluation result for a single document.
+
+    ``metrics`` maps each metric name to a ``MetricCollection`` — its value at
+    every node that produced it, plus that metric's structured detail (schema
+    errors, hallucinated paths, per-rule outcomes, …) on each value's ``.extra``.
+    ``field_scores`` is a flat map of every tree node keyed by its path. On a
+    parse error, ``parse_error`` is True and the metrics are left empty.
+    """
+
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+
+    score: float | None = None
+    score_label: str | None = None
+    metrics: dict[str, MetricCollection] = Field(default_factory=dict)
+    field_scores: dict[str, FieldScore] = Field(default_factory=dict)
+    array_matches: dict[str, ArrayMatchResult] = Field(default_factory=dict)
+    parse_error: bool = False
+    parse_error_message: str | None = None
+    warnings: list[EvalWarning] = Field(default_factory=list)
+
+    # ── Queries ───────────────────────────────────────────────────────────
+
+    def failed_fields(self, threshold: float | None = None) -> dict[str, FieldScore]:
+        """Return fields whose score falls below the applicable threshold.
+
+        Keyed by field path (the same keys as ``field_scores``). Precedence per
+        field: the ``threshold`` argument, else the field's own ``threshold``,
+        else a perfect-match bar of 1.0. Fields without a score (no key metric
+        applied) are skipped.
+        """
+        failed: dict[str, FieldScore] = {}
+        for path, fs in self.field_scores.items():
+            if fs.score is None:
+                continue
+            bar = threshold if threshold is not None else fs.threshold
+            if bar is None:
+                bar = 1.0
+            if fs.score < bar:
+                failed[path] = fs
+        return failed
+
+    # ── Reporting / serialization ─────────────────────────────────────────
+
+    def print_summary(self) -> None:
+        """Print a field-level summary table to stdout."""
+        from structured_eval.reporting import render
+
+        print(render(self))  # noqa: T201
+
+    def to_dict(self) -> dict[str, Any]:
+        """Return a JSON-friendly dict of the full report."""
+        return self.model_dump(mode="json")
+
+    def to_json(self, path: str) -> None:
+        """Serialize the report to a JSON file."""
+        with Path(path).open("w", encoding="utf-8") as fh:
+            fh.write(self.model_dump_json(indent=2))
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> EvalReport:
+        """Reconstruct a report from a dict produced by ``to_dict``."""
+        return cls.model_validate(data)
+
+    @classmethod
+    def from_json(cls, path: str) -> EvalReport:
+        """Load a report from a JSON file."""
+        with Path(path).open(encoding="utf-8") as fh:
+            return cls.model_validate_json(fh.read())
+
+    def diff_from(
+        self, other: EvalReport, metrics: list[str] | None = None
+    ) -> RegressionDiff:
+        """Compute metric deltas relative to ``other`` (self minus other).
+
+        ``deltas`` covers document-level metrics present in both reports (or the
+        subset named in ``metrics``); ``field_deltas`` covers per-field metrics
+        for paths present in both. Positive means improvement.
+        """
+        names = metrics if metrics is not None else sorted(self.metrics)
+        deltas = {
+            name: self.metrics[name].representative()
+            - other.metrics[name].representative()
+            for name in names
+            if name in self.metrics and name in other.metrics
+        }
+
+        field_deltas: dict[str, dict[str, float]] = {}
+        for path, fs in self.field_scores.items():
+            other_fs = other.field_scores.get(path)
+            if other_fs is None:
+                continue
+            per: dict[str, float] = {
+                m: fs.metrics[m] - other_fs.metrics[m]
+                for m in fs.metrics
+                if m in other_fs.metrics
+            }
+            if fs.score is not None and other_fs.score is not None:
+                per["score"] = fs.score - other_fs.score
+            if per:
+                field_deltas[path] = per
+
+        return RegressionDiff(deltas=deltas, field_deltas=field_deltas)
+
+    # ── Assertions (pytest-style: raise AssertionError, else None) ────────
+
+    def assert_no_parse_errors(self) -> None:
+        """Fail if the document could not be parsed."""
+        if self.parse_error:
+            raise AssertionError(
+                f"parse error: {self.parse_error_message or 'could not parse document'}"
+            )
+
+    def assert_score(self, min_score: float) -> None:
+        """Fail if the key-metric score is below ``min_score``."""
+        self.assert_no_parse_errors()
+        if self.score is None:
+            raise AssertionError(
+                "no score available (no key metric configured); use assert_metric() instead"
+            )
+        if self.score < min_score:
+            label = self.score_label or "score"
+            raise AssertionError(f"{label} {self.score:.4g} < required {min_score:.4g}")
+
+    def assert_field(self, path: str, min_score: float) -> None:
+        """Fail if the field at ``path`` scores below ``min_score``."""
+        fs = self.field_scores.get(path)
+        if fs is None:
+            raise AssertionError(f"no field at path {path!r}")
+        if fs.score is None:
+            raise AssertionError(f"field {path!r} has no score (no key metric applied)")
+        if fs.score < min_score:
+            raise AssertionError(
+                f"field {path!r} scored {fs.score:.4g} < required {min_score:.4g} "
+                f"(actual={fs.actual!r}, expected={fs.expected!r})"
+            )
+
+    def assert_metric(self, metric_name: str, min_value: float) -> None:
+        """Fail if metric ``metric_name`` is missing or below ``min_value``.
+
+        Compares the metric's document-level value (the root, else its mean
+        across the tree).
+        """
+        if metric_name not in self.metrics:
+            available = ", ".join(sorted(self.metrics)) or "none"
+            raise AssertionError(
+                f"metric {metric_name!r} not computed (available: {available})"
+            )
+        value = self.metrics[metric_name].representative()
+        if value < min_value:
+            raise AssertionError(
+                f"metric {metric_name!r} {value:.4g} < required {min_value:.4g}"
+            )
+
+    def assert_schema_valid(self) -> None:
+        """Fail if schema validation produced errors."""
+        coll = self.metrics.get("schema_validity")
+        if coll is None:
+            return
+        errors = coll.extra_values("schema_errors")
+        if coll.representative() == 0.0 or errors:
+            message = "; ".join(errors) or "schema validation failed"
+            raise AssertionError(f"schema invalid: {message}")
+
+
+# ── Batch / consistency reports ───────────────────────────────────────────────
+
+
+class BatchEvalReport(BaseModel):
+    """Aggregate result over a list of documents (``evaluate(list[Sample])``).
+
+    ``metrics`` is the mean of each document-level metric across successfully
+    parsed samples; ``score`` is the mean key-metric score. ``perfect_response_rate``
+    is the fraction of samples that parsed and had no failing field;
+    ``parse_error_rate`` the fraction that failed to parse.
+    """
+
+    per_sample: list[EvalReport] = Field(default_factory=list)
+    metrics: dict[str, float] = Field(default_factory=dict)
+    score: float | None = None
+    score_label: str | None = None
+    perfect_response_rate: float = 0.0
+    parse_error_rate: float = 0.0
+
+    def field_breakdown(
+        self, threshold: float | None = None
+    ) -> dict[str, dict[str, float]]:
+        """Per-path statistics across the batch: mean/min/max/p95/fail_rate.
+
+        Only nodes with a score (a key metric applied) are counted. ``fail_rate``
+        is the fraction of samples where the field scored below its bar (the
+        ``threshold`` argument, else the field's own threshold, else 1.0).
+        """
+        scores: dict[str, list[float]] = {}
+        fails: dict[str, int] = {}
+        for r in self.per_sample:
+            if r.parse_error:
+                continue
+            for path, fs in r.field_scores.items():
+                if fs.score is None:
+                    continue
+                scores.setdefault(path, []).append(fs.score)
+                bar = threshold if threshold is not None else fs.threshold
+                if bar is None:
+                    bar = 1.0
+                if fs.score < bar:
+                    fails[path] = fails.get(path, 0) + 1
+
+        return {
+            path: {
+                "mean": mean(vals),
+                "min": min(vals),
+                "max": max(vals),
+                "p95": _percentile(vals, 0.95),
+                "fail_rate": fails.get(path, 0) / len(vals),
+            }
+            for path, vals in scores.items()
+        }
+
+    def print_summary(self) -> None:
+        """Print a batch summary (aggregate metrics + field breakdown)."""
+        from structured_eval.reporting import render
+
+        print(render(self))  # noqa: T201
+
+
+class ConsistencyReport(BaseModel):
+    """Stability of repeated runs of one prompt (``evaluate_consistency``).
+
+    ``field_variance`` is the variance of each field's score across runs;
+    fields with variance at or below ``variance_threshold`` are ``stable_fields``,
+    the rest ``unstable_fields``. ``score_variance`` is the variance of the
+    document-level key-metric score.
+    """
+
+    per_run: list[EvalReport] = Field(default_factory=list)
+    field_variance: dict[str, float] = Field(default_factory=dict)
+    stable_fields: list[str] = Field(default_factory=list)
+    unstable_fields: list[str] = Field(default_factory=list)
+    mean_score: float | None = None
+    score_variance: float | None = None
+
+    def print_summary(self) -> None:
+        """Print a consistency summary (stable/unstable fields + variance)."""
+        from structured_eval.reporting import render
+
+        print(render(self))  # noqa: T201

structured_eval/models/sample.py

@@ -0,0 +1,19 @@
+from __future__ import annotations
+
+from typing import Any
+
+from pydantic import BaseModel
+
+
+class Sample(BaseModel):
+    """One document to evaluate.
+
+    Wrapping in ``Sample`` removes the ambiguity of a bare ``list``: a list
+    passed as ``actual`` is a single document whose root is an array, whereas
+    ``list[Sample]`` is a batch of documents.
+    """
+
+    actual: dict[str, Any] | list[Any] | str
+    expected: dict[str, Any] | list[Any] | str | None = None
+    source: str | None = None  # original text, for Faithfulness
+    id: str | None = None  # identifier in a BatchEvalReport

structured_eval/reporting/__init__.py

@@ -0,0 +1,5 @@
+"""Report rendering: plain-text console summaries for the report types."""
+
+from structured_eval.reporting.console import render
+
+__all__ = ["render"]

structured_eval/reporting/console.py

@@ -0,0 +1,194 @@
+"""Plain-text console rendering for the report types (no hard dependency).
+
+``ConsoleRenderer().render(report)`` returns a string; the module-level
+``render`` is a thin convenience over it, used by ``EvalReport.print_summary``.
+The layout is pure stdlib so it works out of the box; Rich can be layered on
+for colour later.
+"""
+
+from __future__ import annotations
+
+from structured_eval.models.result import (
+    BatchEvalReport,
+    ConsistencyReport,
+    EvalReport,
+)
+
+_RULE = "─"
+_BAR = "━"
+_WIDTH = 60
+
+
+class ConsoleRenderer:
+    """Renders ``EvalReport`` / ``BatchEvalReport`` / ``ConsistencyReport``."""
+
+    def render(self, report: EvalReport | BatchEvalReport | ConsistencyReport) -> str:
+        if isinstance(report, EvalReport):
+            return self._render_eval(report)
+        if isinstance(report, BatchEvalReport):
+            return self._render_batch(report)
+        if isinstance(report, ConsistencyReport):
+            return self._render_consistency(report)
+        raise NotImplementedError
+
+    # ── EvalReport ──────────────────────────────────────────────────────────
+
+    def _render_eval(self, report: EvalReport) -> str:
+        out: list[str] = [_BAR * _WIDTH]
+
+        if report.parse_error:
+            out.append(f"  PARSE ERROR: {report.parse_error_message}")
+            out.append(_BAR * _WIDTH)
+            return "\n".join(out)
+
+        bar = _BAR * _WIDTH
+        if report.score is not None:
+            verdict = "✓ PASS" if report.score >= 1.0 else "✗ FAIL"
+            label = report.score_label or ""
+            out.append(f"  OVERALL   {report.score:.2f}   {verdict}        {label}")
+        else:
+            out.append("  OVERALL   —   (no ground truth)")
+
+        # Document-level metrics: those a metric produced at the root ("$").
+        doc_metrics: dict[str, float] = {}
+        for name, coll in report.metrics.items():
+            v = coll.root()
+            if v is not None:
+                doc_metrics[name] = v
+        grid = self._metric_grid(doc_metrics, skip=report.score_label)
+        if grid:
+            out += ["", *grid]
+        out.append(bar)
+
+        # scalar leaves with a key-metric score
+        rows = []
+        for fs in report.field_scores.values():
+            if fs.score is None:
+                continue
+            metric_name = next(
+                (k for k, v in fs.metrics.items() if v == fs.score), "score"
+            )
+            rows.append(
+                [
+                    fs.path,
+                    metric_name,
+                    self._num(fs.score),
+                    self._num(fs.threshold),
+                    self._mark(fs.score, fs.threshold),
+                ]
+            )
+        if rows:
+            out += self._table(
+                ["Field", "Metric", "Score", "Threshold", "Mark"],
+                rows,
+                ["<", "<", ">", ">", "^"],
+            )
+            out.append(bar)
+
+        return "\n".join(out)
+
+    # ── BatchEvalReport ───────────────────────────────────────────────────
+
+    def _render_batch(self, report: BatchEvalReport) -> str:
+        bar = _BAR * _WIDTH
+        n = len(report.per_sample)
+        out = [bar, f"  BATCH   {n} samples"]
+        if report.score is not None:
+            out.append(f"  mean {report.score_label or 'score'}   {report.score:.2f}")
+        out.append(f"  perfect_response_rate   {report.perfect_response_rate:.2f}")
+        out.append(f"  parse_error_rate        {report.parse_error_rate:.2f}")
+        grid = self._metric_grid(report.metrics, skip=report.score_label)
+        if grid:
+            out += ["", *grid]
+        out.append(bar)
+
+        bd = report.field_breakdown()
+        ranked = sorted(bd.items(), key=lambda kv: kv[1]["fail_rate"], reverse=True)
+        if ranked:
+            rows = [
+                [
+                    path,
+                    self._num(s["mean"]),
+                    self._num(s["p95"]),
+                    self._num(s["fail_rate"]),
+                ]
+                for path, s in ranked
+            ]
+            out.append("  Field breakdown (worst first)")
+            out += self._table(
+                ["Field", "mean", "p95", "fail_rate"], rows, ["<", ">", ">", ">"]
+            )
+            out.append(bar)
+        return "\n".join(out)
+
+    # ── ConsistencyReport ─────────────────────────────────────────────────
+
+    def _render_consistency(self, report: ConsistencyReport) -> str:
+        bar = _BAR * _WIDTH
+        out = [bar, f"  CONSISTENCY   {len(report.per_run)} runs"]
+        if report.mean_score is not None:
+            out.append(f"  mean score       {report.mean_score:.2f}")
+        if report.score_variance is not None:
+            out.append(f"  score variance   {report.score_variance:.4f}")
+        out += [
+            f"  stable           {', '.join(report.stable_fields) or '—'}",
+            f"  unstable         {', '.join(report.unstable_fields) or '—'}",
+            bar,
+        ]
+        ranked = sorted(
+            report.field_variance.items(), key=lambda kv: kv[1], reverse=True
+        )
+        if ranked:
+            rows = [[path, f"{var:.4f}"] for path, var in ranked]
+            out += self._table(["Field", "variance"], rows, ["<", ">"])
+            out.append(bar)
+        return "\n".join(out)
+
+    # ── formatting helpers ─────────────────────────────────────────────────
+
+    @staticmethod
+    def _num(value: float | None) -> str:
+        return "—" if value is None else f"{value:.2f}"
+
+    @staticmethod
+    def _mark(score: float | None, bar: float | None) -> str:
+        if score is None or bar is None:
+            return " "
+        return "✓" if score >= bar else "✗"
+
+    @staticmethod
+    def _table(
+        headers: list[str], rows: list[list[str]], aligns: list[str] | None = None
+    ) -> list[str]:
+        """Render a simple monospace table as a list of lines."""
+        cols = (
+            list(zip(*([headers, *rows]), strict=False))
+            if rows
+            else [[h] for h in headers]
+        )
+        widths = [max(len(c) for c in col) for col in cols]
+        aligns = aligns or ["<"] * len(headers)
+
+        def fmt(cells: list[str]) -> str:
+            return "  ".join(
+                f"{c:{a}{w}}" for c, w, a in zip(cells, widths, aligns, strict=False)
+            )
+
+        lines = [fmt(headers), fmt([_RULE * w for w in widths])]
+        lines += [fmt(r) for r in rows]
+        return ["  " + line for line in lines]
+
+    @staticmethod
+    def _metric_grid(metrics: dict[str, float], skip: str | None = None) -> list[str]:
+        """Two-per-line key/value grid of document metrics."""
+        items = [(k, v) for k, v in metrics.items() if k != skip]
+        if not items:
+            return []
+        width = max(len(k) for k, _ in items)
+        cells = [f"{k:<{width}} {ConsoleRenderer._num(v)}" for k, v in items]
+        return ["  " + "     ".join(cells[i : i + 2]) for i in range(0, len(cells), 2)]
+
+
+def render(report: EvalReport | BatchEvalReport | ConsistencyReport) -> str:
+    """Render any of the report types to a printable string."""
+    return ConsoleRenderer().render(report)

structured_eval/utils/__init__.py

@@ -0,0 +1,16 @@
+from structured_eval.utils.flatten import extract_paths, flatten
+from structured_eval.utils.structured_diff import (
+    DiffEntry,
+    DiffType,
+    StructuredDiff,
+    structured_diff,
+)
+
+__all__ = [
+    "DiffEntry",
+    "DiffType",
+    "StructuredDiff",
+    "extract_paths",
+    "flatten",
+    "structured_diff",
+]

structured_eval/utils/flatten.py

@@ -0,0 +1,66 @@
+from __future__ import annotations
+
+from typing import Any
+
+
+def flatten(obj: dict[str, Any] | list[Any], prefix: str = "") -> dict[str, Any]:
+    """Recursively flatten a nested dict/list into dot-and-bracket key paths.
+
+    Dict keys use dot notation:     {"a": {"b": 1}}         → {"a.b": 1}
+    List indices use brackets:      {"a": [1, 2]}           → {"a[0]": 1, "a[1]": 2}
+    Empty containers are left as-is: {"a": {}}              → {"a": {}}
+
+    Args:
+        obj: Dict or list to flatten.
+        prefix: Internal prefix for recursive calls; do not pass externally.
+
+    Returns:
+        Flat dict mapping string paths to primitive (or empty container) values.
+
+    Example:
+        >>> flatten({"invoice": {"id": "1", "items": [{"price": 100}]}})
+        {"invoice.id": "1", "invoice.items[0].price": 100}
+    """
+    result: dict[str, Any] = {}
+    if isinstance(obj, dict):
+        for key, value in obj.items():
+            path = f"{prefix}.{key}" if prefix else key
+            if isinstance(value, (dict, list)) and value:
+                result.update(flatten(value, path))
+            else:
+                result[path] = value
+    elif isinstance(obj, list):
+        for i, item in enumerate(obj):
+            path = f"{prefix}[{i}]"
+            if isinstance(item, (dict, list)) and item:
+                result.update(flatten(item, path))
+            else:
+                result[path] = item
+    return result
+
+
+def extract_paths(value: Any, prefix: str = "") -> set[str]:
+    """Every structural path in a JSON-like value — order- and value-blind.
+
+    Yields the path of each container *and* each leaf, so the set captures the
+    whole skeleton: dict keys (``a``, ``a.b``), list indices (``a[0]``) and the
+    leaf paths beneath them. Values themselves are ignored — only the shape.
+    Unlike :func:`flatten`, intermediate container paths are included, not just
+    leaves, and the result is a set of paths rather than a path→value mapping.
+
+    Example:
+        >>> sorted(extract_paths({"a": {"b": 1}, "c": [2]}))
+        ['a', 'a.b', 'c', 'c[0]']
+    """
+    paths: set[str] = set()
+    if isinstance(value, dict):
+        for key, child in value.items():
+            here = f"{prefix}.{key}" if prefix else str(key)
+            paths.add(here)
+            paths |= extract_paths(child, here)
+    elif isinstance(value, list):
+        for index, child in enumerate(value):
+            here = f"{prefix}[{index}]"
+            paths.add(here)
+            paths |= extract_paths(child, here)
+    return paths