structured-eval 0.1.0__py3-none-any.whl

structured_eval/metrics/base.py

@@ -0,0 +1,144 @@
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from typing import Any
+
+from structured_eval.models.metric_result import MetricResult
+from structured_eval.models.nodes.array_node import ArrayNode
+from structured_eval.models.nodes.base import EvalNode
+from structured_eval.models.nodes.object_node import ObjectNode
+from structured_eval.models.nodes.scalar import ScalarNode
+
+# What a metric's ``compute`` may return; ``MetricRunner._apply`` normalizes any
+# of these to a ``MetricResult``. A bare value / dict of sub-scores, optionally
+# paired with structured ``extra`` via a tuple, or a ready ``MetricResult``.
+MetricOutput = (
+    float
+    | dict[str, float]
+    | tuple[float | dict[str, float], dict[str, Any]]
+    | MetricResult
+    | None
+)
+
+# Name → metric class. Populated automatically as BaseMetric subclasses are
+# declared; used by EvalConfig.from_yaml() to resolve string names (Stage 10).
+_METRIC_REGISTRY: dict[str, type] = {}
+
+
+class BaseMetric(ABC):  # noqa: B024 — registry root; subclasses define the interface
+    """Registry root for every metric — no evaluation interface of its own.
+
+    ``name`` is the key under which a scalar result lands in ``report.metrics``
+    and ``FieldScore.metrics``. A metric that returns a ``dict`` instead writes
+    each of its keys directly (the ``name`` is then only a registry handle).
+    Declaring a subclass with a ``name`` registers it automatically.
+    """
+
+    name: str = ""
+
+    def __init_subclass__(cls, **kwargs: Any) -> None:
+        super().__init_subclass__(**kwargs)
+        if n := getattr(cls, "name", None):
+            _METRIC_REGISTRY[n] = cls
+
+
+class Metric[NodeT: EvalNode](BaseMetric):
+    """The unified metric interface: ``compute(node)`` + ``score(actual, expected)``.
+
+    Every concrete metric is a ``Metric`` and therefore *has* a ``score`` — a
+    pure value-level comparison ``(actual, expected) -> float | dict`` reused by
+    array alignment. ``compute(node)`` is the node-level entry point; by default
+    it delegates to ``score`` on the node's values, so a leaf comparison need
+    only implement ``score``. Aggregating metrics override ``compute`` and leave
+    ``score`` at its default (callers that require a scalar verdict check the
+    result type where it matters). The type parameter ``NodeT`` pins the node
+    type a subtype operates on (``ScalarNode`` for fields, ``ObjectNode`` …).
+    """
+
+    def compute(self, node: NodeT) -> MetricOutput:
+        return self.score(node.actual, node.expected)
+
+    def score(self, actual: Any, expected: Any) -> float | dict[str, float]:
+        raise NotImplementedError
+
+
+class FieldMetric(Metric[ScalarNode]):
+    """A leaf comparison applied to each ScalarNode.
+
+    Implements ``score(actual, expected)`` and relies on the inherited
+    ``compute``; metrics that need node context (e.g. ``Presence``) override
+    ``compute`` directly. Also the marker the engine dispatches on for scalars.
+    """
+
+
+class ObjectMetric(Metric[ObjectNode]):
+    """Applies to each ObjectNode (root and nested)."""
+
+    @abstractmethod
+    def compute(self, node: ObjectNode) -> MetricOutput: ...
+
+
+class ArrayMetric(Metric[ArrayNode]):
+    """Applies to each ArrayNode."""
+
+    @abstractmethod
+    def compute(self, node: ArrayNode) -> MetricOutput: ...
+
+
+class RootMetric(Metric[EvalNode]):
+    """Applies only to the root node (path == "$"); receives any EvalNode."""
+
+    @abstractmethod
+    def compute(self, node: EvalNode) -> MetricOutput: ...
+
+
+class AnyNodeMetric(Metric[EvalNode]):
+    """Applies uniformly to *every* node — same ``compute`` regardless of kind.
+
+    The node-agnostic branch of the hierarchy: unlike the typed metrics
+    (``FieldMetric`` / ``ObjectMetric`` / ``ArrayMetric``) it is not pinned to
+    one node type, and unlike ``GenericMetric`` it does not dispatch per kind —
+    it runs one uniform computation on any ``EvalNode``. ``RootMetric`` is the
+    sibling that is *also* ``Metric[EvalNode]`` but admitted only at the root;
+    an ``AnyNodeMetric`` is admitted everywhere. ``MeanScore`` (the default
+    representative) lives here, and a custom uniform metric can be cascaded via
+    ``config.metrics`` or chosen as a ``key_metric``.
+    """
+
+    @abstractmethod
+    def compute(self, node: EvalNode) -> MetricOutput: ...
+
+
+class GenericMetric(BaseMetric):
+    """Metrics spanning several node types, outside the single-``compute`` shape.
+
+    Override whichever per-kind methods apply: ``compute_scalar`` /
+    ``compute_object`` / ``compute_array`` for node mode, and (optionally)
+    ``score_scalar`` / ``score_object`` / ``score_array`` for value mode.
+    ``MetricInvoker`` dispatches by kind; ``TreeBuilder`` admits the metric onto
+    a node only when the matching ``compute_<kind>`` exists. (Replaces the former
+    ``NodeMetric``.)
+    """
+
+
+def get_metric_class(name: str) -> type:
+    """Resolve a metric class by its ``name`` (e.g. ``"object_f1"``)."""
+    if name not in _METRIC_REGISTRY:
+        raise KeyError(f"Unknown metric: {name!r}. Known: {sorted(_METRIC_REGISTRY)}")
+    return _METRIC_REGISTRY[name]
+
+
+def resolve_metric(spec: str | BaseMetric) -> BaseMetric:
+    """Coerce a metric spec to a ``BaseMetric`` instance.
+
+    Accepts an instance as-is or a registered name string (instantiated with no
+    args). The single resolver shared by the engine, array alignment, and the
+    match-criterion helper. ``None`` is *not* handled here — callers supply
+    their own default. Score-needing call sites narrow the result to ``Metric``.
+    """
+    if isinstance(spec, str):
+        instance = get_metric_class(spec)()
+        assert isinstance(instance, BaseMetric)
+        return instance
+    assert isinstance(spec, BaseMetric)
+    return spec

structured_eval/metrics/character_f1.py

@@ -0,0 +1,50 @@
+from __future__ import annotations
+
+import re
+from collections import Counter
+from typing import Any
+
+from structured_eval.metrics.base import FieldMetric
+
+_NON_WORD = re.compile(r"[^\w\s]")
+
+
+def _characters(value: Any) -> list[str]:
+    """Lowercase, drop punctuation and whitespace, split into characters."""
+    normalized = _NON_WORD.sub("", str(value).lower())
+    normalized = "".join(normalized.split())  # remove all whitespace
+    return list(normalized)
+
+
+class CharacterF1(FieldMetric):
+    """Character-overlap F1 for short free-text fields.
+
+    Characters are matched as a **multiset** (``Counter``), so repeated
+    characters contribute only as many times as they appear on both sides.
+    Precision and recall are computed over character counts, and their
+    harmonic mean is returned. String-only: if either side is not a ``str``
+    the score is ``0.0`` (no coercion).
+    """
+
+    name = "character_f1"
+
+    def score(self, actual: Any, expected: Any) -> float:
+        if not (isinstance(actual, str) and isinstance(expected, str)):
+            return 0.0
+
+        a = _characters(actual)
+        e = _characters(expected)
+
+        if not a and not e:
+            return 1.0
+        if not a or not e:
+            return 0.0
+
+        same = sum((Counter(a) & Counter(e)).values())
+        if not same:
+            return 0.0
+
+        precision = same / len(a)
+        recall = same / len(e)
+
+        return 2 * precision * recall / (precision + recall)

structured_eval/metrics/composite_score.py

@@ -0,0 +1,46 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from structured_eval.metrics.base import AnyNodeMetric
+
+if TYPE_CHECKING:
+    from structured_eval.models.nodes.base import EvalNode
+
+
+class CompositeScore(AnyNodeMetric):
+    """Weighted blend of other metrics already computed on the same node.
+
+    Given ``weights={metric_name: weight}``, the score is the weighted mean of
+    those metrics' values on the node::
+
+        score = Σ wᵢ · metric_resultsᵢ        (weights normalized to sum 1.0)
+
+    The referenced metrics must already be present in ``node.metric_results``,
+    so list them in the node's ``metrics`` (or as cascaded ``config.metrics``)
+    alongside ``CompositeScore``. As a representative it is best used as the
+    node's ``key_metric``, which the engine runs **last** — by then every other
+    metric on the node is computed.
+
+    Only the metrics named in ``weights`` contribute; any other metric on the
+    node is ignored, and a named metric that is absent contributes ``0``. The
+    result is clamped to ``[0, 1]`` (each input is expected in ``[0, 1]``).
+    """
+
+    name = "composite_score"
+
+    def __init__(self, weights: dict[str, float]) -> None:
+        if not weights:
+            raise ValueError("CompositeScore requires at least one metric weight")
+        total = sum(weights.values())
+        if total <= 0:
+            raise ValueError("Sum of weights must be > 0")
+        self.weights: dict[str, float] = {m: w / total for m, w in weights.items()}
+
+    def compute(self, node: EvalNode) -> float:
+        total = sum(
+            weight * float(node.metric_results[name])
+            for name, weight in self.weights.items()
+            if name in node.metric_results
+        )
+        return min(1.0, max(0.0, total))

structured_eval/metrics/coverage_leaf_score.py

@@ -0,0 +1,29 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from structured_eval.metrics.base import RootMetric
+
+if TYPE_CHECKING:
+    from structured_eval.models.nodes.base import EvalNode
+
+
+class CoverageLeafScore(RootMetric):
+    """Fraction of expected leaf fields that are present (non-null) in actual.
+
+    Completeness across the whole document, independent of value correctness.
+    Counts only leaves expected to have a value; a document expecting nothing
+    is vacuously 1.0. (Array elements missed during alignment have no leaf
+    node and are covered by the array metrics instead.)
+    """
+
+    name = "coverage_leaf_score"
+
+    def compute(self, node: EvalNode) -> float:
+        expected = covered = 0
+        for leaf in node.leaves():
+            if leaf.expected is not None:
+                expected += 1
+                if leaf.actual is not None:
+                    covered += 1
+        return covered / expected if expected else 1.0

structured_eval/metrics/date_distance_score.py

@@ -0,0 +1,63 @@
+from __future__ import annotations
+
+from datetime import date, datetime
+from typing import Any
+
+from pydantic import TypeAdapter
+
+from structured_eval.metrics.base import FieldMetric
+
+
+def _to_date(value: Any) -> date | None:
+    try:
+        adapter = TypeAdapter(date)
+        return adapter.validate_python(value)
+    except Exception:
+        return None
+
+
+class DateDistanceScore(FieldMetric):
+    """Linear similarity for date and datetime fields.
+
+    The score is computed as::
+
+        max(0, 1 - days_difference / max_days)
+
+    yielding:
+
+    - ``1.0`` for identical dates;
+    - a linear decrease as the difference in days grows;
+    - ``0.0`` once the difference reaches or exceeds ``max_days``.
+
+    Both ``date`` and ``datetime`` values are supported, and ISO-8601 strings
+    (e.g. ``"2026-06-29"``) are coerced via pydantic. Datetime values are
+    compared by their calendar date only (time-of-day is ignored).
+
+    If either side cannot be read as a date — ``None``, an unparseable string,
+    or any non-date type — the score is ``0.0``.
+    """
+
+    name = "date_distance_score"
+
+    def __init__(self, max_days: int = 30) -> None:
+        if max_days <= 0:
+            raise ValueError("max_days must be greater than 0")
+        self.max_days = max_days
+
+    def score(self, actual: Any, expected: Any) -> float:
+        if not isinstance(actual, (date, datetime)):
+            actual = _to_date(actual)
+        if not isinstance(expected, (date, datetime)):
+            expected = _to_date(expected)
+        if not (
+            isinstance(actual, (date, datetime))
+            and isinstance(expected, (date, datetime))
+        ):
+            return 0.0
+
+        actual_date = actual.date() if isinstance(actual, datetime) else actual
+        expected_date = expected.date() if isinstance(expected, datetime) else expected
+
+        days = abs((actual_date - expected_date).days)
+
+        return max(0.0, 1.0 - days / self.max_days)

structured_eval/metrics/exact.py

@@ -0,0 +1,21 @@
+from __future__ import annotations
+
+from typing import Any
+
+from structured_eval.metrics.base import FieldMetric
+
+
+class ExactMatch(FieldMetric):
+    """Strict equality: ``actual == expected`` → 1.0, else 0.0.
+
+    The default scalar comparison, and the default key comparison in ``by_key``
+    array alignment. It does *not* score whole objects/arrays: object metrics
+    read each child's representative, and array alignment defaults are
+    type-aware — ExactMatch only ever touches a dict/list through the
+    value-level ``score`` path.
+    """
+
+    name = "exact_match"
+
+    def score(self, actual: Any, expected: Any) -> float:
+        return 1.0 if actual == expected else 0.0

structured_eval/metrics/exponential_numeric_score.py

@@ -0,0 +1,47 @@
+from __future__ import annotations
+
+import math
+from typing import Any
+
+from structured_eval.metrics.base import FieldMetric
+from structured_eval.metrics.utils.number import parse_number
+
+
+class ExponentialNumericScore(FieldMetric):
+    """Exponentially decaying similarity for numeric fields.
+
+    The score is computed as::
+
+        exp(-abs(actual - expected) / scale)
+
+    yielding:
+
+    - ``1.0`` for an exact match;
+    - a smooth exponential decay as the absolute error increases;
+    - values always in the range ``(0.0, 1.0]``.
+
+    The ``scale`` parameter controls how quickly the score decreases. Larger
+    values make the metric more tolerant to numeric differences. Unlike the
+    ratio-based :class:`NumericCloseness`, the decay is on the **absolute**
+    error, so it is scale-aware — pick ``scale`` to match the field's units.
+
+    Values are read with the same lenient parser as :class:`Numeric` /
+    :class:`NumericCloseness`, so numeric strings are graded too. The metric
+    applies **only to numbers**: if either side isn't numeric (``None``, a
+    non-numeric string, or a ``bool`` — ``True`` is not ``1``) the score is
+    ``0.0``.
+    """
+
+    name = "exponential_numeric_score"
+
+    def __init__(self, scale: float = 1.0) -> None:
+        if scale <= 0:
+            raise ValueError("scale must be greater than 0")
+        self.scale = scale
+
+    def score(self, actual: Any, expected: Any) -> float:
+        a = parse_number(actual)
+        e = parse_number(expected)
+        if a is None or e is None:
+            return 0.0
+        return math.exp(-abs(a - e) / self.scale)

structured_eval/metrics/field_faithfulness.py

@@ -0,0 +1,38 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from structured_eval.metrics.base import FieldMetric
+
+if TYPE_CHECKING:
+    from structured_eval.models.nodes.scalar import ScalarNode
+
+
+class FieldFaithfulness(FieldMetric):
+    """Is this leaf value grounded in the sample's ``source``? (L1 substring.)
+
+    A per-field faithfulness check, true to the framework's "comparison is a
+    metric" core: each scalar leaf scores ``1.0`` if its string form appears
+    (case-insensitively) verbatim in ``source``, else ``0.0`` (a hallucination).
+    Cascade it via ``EvalConfig(metrics=[FieldFaithfulness()])`` and the engine
+    does the rest — aggregation is just the usual leaf roll-up
+    (``MeanScore`` / ``OverallLeafScore``), and the hallucinated fields are the
+    leaves scoring ``0.0`` (``report.metrics["field_faithfulness"].by_path``).
+
+    Requires a grounding ``source`` on the sample — faithfulness is undefined
+    without one, so a missing ``source`` is a configuration error
+    (``ValueError``) rather than a silently omitted metric.
+    """
+
+    name = "field_faithfulness"
+
+    def compute(self, node: ScalarNode) -> float | None:
+        source = node.context.source
+        if source is None:
+            raise ValueError(
+                "Faithfulness requires a grounding `source`; pass source=... to evaluate()"
+            )
+        actual = node.actual
+        if actual is None:
+            return None
+        return 1.0 if str(actual).lower() in source.lower() else 0.0

structured_eval/metrics/fuzzy.py

@@ -0,0 +1,64 @@
+from __future__ import annotations
+
+from enum import StrEnum
+from typing import Any
+
+from structured_eval.metrics.base import FieldMetric
+
+
+class FuzzyMethod(StrEnum):
+    """RapidFuzz scorer used by :class:`Fuzzy`."""
+
+    RATIO = "ratio"  # plain normalized Levenshtein ratio
+    PARTIAL_RATIO = "partial_ratio"  # best matching substring
+    TOKEN_SORT_RATIO = "token_sort_ratio"  # order-insensitive (default)
+    TOKEN_SET_RATIO = "token_set_ratio"  # set-based, ignores duplicate tokens
+
+
+class Fuzzy(FieldMetric):
+    """Fuzzy string similarity via RapidFuzz (optional dependency).
+
+    ``method`` selects the RapidFuzz scorer:
+
+    * ``ratio`` — plain normalized Levenshtein ratio;
+    * ``partial_ratio`` — best matching substring;
+    * ``token_sort_ratio`` (default) — order-insensitive, sorts tokens;
+    * ``token_set_ratio`` — set-based, ignores duplicate/extra tokens.
+
+    ``normalize`` strips surrounding whitespace and lowercases before comparison.
+    String-only: if either side is not a ``str`` the score is 0.0 (no coercion),
+    consistent with the other text metrics.
+    """
+
+    name = "fuzzy"
+
+    def __init__(
+        self,
+        method: FuzzyMethod = FuzzyMethod.TOKEN_SORT_RATIO,
+        normalize: bool = True,
+    ):
+        self.method = FuzzyMethod(method)
+        self.normalize = normalize
+
+    def score(self, actual: Any, expected: Any) -> float:
+        if not (isinstance(actual, str) and isinstance(expected, str)):
+            return 0.0
+        try:
+            from rapidfuzz import fuzz
+        except ImportError as exc:  # pragma: no cover
+            raise ImportError(
+                "rapidfuzz is required for the 'fuzzy' metric. "
+                "Install it with: pip install 'structured-eval[fuzzy]'"
+            ) from exc
+
+        scorer = {
+            "ratio": fuzz.ratio,
+            "partial_ratio": fuzz.partial_ratio,
+            "token_sort_ratio": fuzz.token_sort_ratio,
+            "token_set_ratio": fuzz.token_set_ratio,
+        }[self.method]
+
+        a, e = actual, expected
+        if self.normalize:
+            a, e = a.strip().lower(), e.strip().lower()
+        return float(scorer(a, e)) / 100.0

structured_eval/metrics/invoker.py

@@ -0,0 +1,90 @@
+"""The single way to run a metric, whatever input is available.
+
+Every metric is invoked through ``MetricInvoker`` — never by calling ``compute``
+/ ``compute_<kind>`` / ``score`` directly. Two input modes:
+
+* ``on_node`` — a node is available: grade it. A ``Metric`` uses ``compute``; a
+  ``GenericMetric`` dispatches to the ``compute_<kind>`` for the node's type.
+* ``on_values`` — only raw ``actual`` / ``expected`` (array alignment, before any
+  node exists): compare them. A ``Metric`` uses ``score``; a ``GenericMetric``
+  dispatches to the ``score_<kind>`` for the kind inferred from the value's shape.
+
+Each mode has a ``scalar_*`` variant that narrows the result to a single
+``float`` (rejecting a dict of sub-scores) — that narrowing is the caller's
+contract, hence its own method.
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+from structured_eval.metrics.base import BaseMetric, GenericMetric, Metric, MetricOutput
+from structured_eval.models.nodes.array_node import ArrayNode
+from structured_eval.models.nodes.object_node import ObjectNode
+from structured_eval.models.nodes.scalar import ScalarNode
+
+if TYPE_CHECKING:
+    from structured_eval.models.nodes.base import EvalNode
+
+# A GenericMetric's per-kind method names, by node class, for each input mode.
+GENERIC_NODE_METHOD: dict[type, str] = {
+    ScalarNode: "compute_scalar",
+    ObjectNode: "compute_object",
+    ArrayNode: "compute_array",
+}
+GENERIC_SCORE_METHOD: dict[type, str] = {
+    ScalarNode: "score_scalar",
+    ObjectNode: "score_object",
+    ArrayNode: "score_array",
+}
+
+
+def _kind_of(actual: Any, expected: Any) -> type:
+    """The node class a raw value pair would build (mirrors ``TreeBuilder``)."""
+    ref = expected if expected is not None else actual
+    if isinstance(ref, dict):
+        return ObjectNode
+    if isinstance(ref, list):
+        return ArrayNode
+    return ScalarNode
+
+
+class MetricInvoker:
+    """Runs ``self.metric`` in either input mode; see module docstring."""
+
+    def __init__(self, metric: BaseMetric):
+        self.metric = metric
+
+    def on_node(self, node: EvalNode) -> MetricOutput:
+        metric = self.metric
+        if isinstance(metric, GenericMetric):
+            return self._dispatch_generic(GENERIC_NODE_METHOD.get(type(node)), node)
+        assert isinstance(metric, Metric)  # every non-generic metric has compute(node)
+        return metric.compute(node)
+
+    def on_values(self, actual: Any, expected: Any) -> MetricOutput:
+        metric = self.metric
+        if isinstance(metric, GenericMetric):
+            method = GENERIC_SCORE_METHOD.get(_kind_of(actual, expected))
+            return self._dispatch_generic(method, actual, expected)
+        assert isinstance(metric, Metric)  # value comparison needs score()
+        return metric.score(actual, expected)
+
+    def scalar_on_node(self, node: EvalNode) -> float:
+        return self._scalar(self.on_node(node), node.path)
+
+    def scalar_on_values(self, actual: Any, expected: Any) -> float:
+        return self._scalar(self.on_values(actual, expected), "<values>")
+
+    def _dispatch_generic(self, method: str | None, *args: Any) -> MetricOutput:
+        if method is None or not hasattr(self.metric, method):
+            return None
+        result: MetricOutput = getattr(self.metric, method)(*args)
+        return result
+
+    def _scalar(self, result: Any, where: str) -> float:
+        assert isinstance(result, (int, float)), (
+            f"metric {self.metric.name!r} must yield a scalar at {where}, "
+            f"got {type(result).__name__}"
+        )
+        return float(result)

structured_eval/metrics/levenshtein.py

@@ -0,0 +1,16 @@
+from __future__ import annotations
+
+from structured_eval.metrics.fuzzy import Fuzzy, FuzzyMethod
+
+
+class Levenshtein(Fuzzy):
+    """Normalized Levenshtein ratio — a thin alias over ``Fuzzy(RATIO)``.
+
+    RapidFuzz's ``ratio`` *is* the normalized Levenshtein similarity; this class
+    exists only for discoverability. All arithmetic lives in ``Fuzzy``.
+    """
+
+    name = "levenshtein"
+
+    def __init__(self, method: FuzzyMethod = FuzzyMethod.RATIO, normalize: bool = True):
+        super().__init__(method=method, normalize=normalize)

structured_eval/metrics/mean_score.py

@@ -0,0 +1,31 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from structured_eval.metrics.base import AnyNodeMetric
+
+if TYPE_CHECKING:
+    from structured_eval.models.nodes.base import EvalNode
+
+
+class MeanScore(AnyNodeMetric):
+    """A node's representative score: the arithmetic mean of its own metrics.
+
+    The default ``key_metric`` of every node — the single number that bubbles up
+    to a parent's aggregation and, at the root, to ``report.score``. It is
+    computed **last**, so by the time it runs the node's other metrics already
+    populate ``metric_results``; it averages those (excluding itself), without
+    recursing into children — any cross-child aggregation is the job of the
+    node's *own* metrics (``ObjectAccuracy`` / ``ObjectF1`` / ``ArrayAccuracy``),
+    which the engine guarantees by defaulting one onto every node. A node with
+    no other computed metric (e.g. a leaf whose only metric opted out by
+    returning ``None``) scores ``0.0`` — every node always has a representative.
+    """
+
+    name = "mean_score"
+
+    def compute(self, node: EvalNode) -> float:
+        values = [
+            float(v) for name, v in node.metric_results.items() if name != self.name
+        ]
+        return sum(values) / len(values) if values else 0.0