structured-eval 0.1.0__py3-none-any.whl

structured_eval/integrations/deepeval.py

@@ -0,0 +1,74 @@
+"""deepeval adapter: structured-eval as a ``BaseMetric``.
+
+Usage (requires ``structured-eval[deepeval]``)::
+
+    from structured_eval.integrations.deepeval import StructuredMetric
+    from deepeval import assert_test
+    from deepeval.test_case import LLMTestCase
+
+    metric = StructuredMetric(config=cfg, threshold=0.85)
+    assert_test(LLMTestCase(input=..., actual_output=raw, expected_output=ref), [metric])
+
+``actual_output``/``expected_output`` may be JSON strings or already-parsed
+objects — ``evaluate`` handles both. ``report.score`` becomes ``metric.score``;
+failing fields are summarised into ``metric.reason``. Importing this module
+requires deepeval to be installed (the ``[deepeval]`` extra).
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+try:
+    from deepeval.metrics import BaseMetric
+except ImportError as exc:  # pragma: no cover
+    raise ImportError(
+        "deepeval is required for this integration. "
+        "Install it with: pip install structured-eval[deepeval]"
+    ) from exc
+
+from structured_eval.api import evaluate
+from structured_eval.integrations._adapter import verdict
+from structured_eval.models.config import EvalConfig
+
+if TYPE_CHECKING:
+    from structured_eval.models.result import EvalReport
+
+
+class StructuredMetric(BaseMetric):
+    """Field-level structured-output metric for deepeval."""
+
+    def __init__(
+        self,
+        config: EvalConfig | None = None,
+        threshold: float = 0.5,
+        *,
+        include_reason: bool = True,
+    ) -> None:
+        self.config = config or EvalConfig()
+        self.threshold = threshold
+        self.include_reason = include_reason
+        self.score: float = 0.0
+        self.success: bool = False
+        self.reason: str | None = None
+        self.report: EvalReport | None = None
+
+    def measure(self, test_case: Any, *args: Any, **kwargs: Any) -> float:
+        self.report = evaluate(
+            test_case.actual_output, test_case.expected_output, self.config
+        )
+        score, success, reason = verdict(self.report, self.threshold)
+        self.score = 0.0 if score is None else score
+        self.success = success
+        self.reason = reason if self.include_reason else None
+        return self.score
+
+    async def a_measure(self, test_case: Any, *args: Any, **kwargs: Any) -> float:
+        return self.measure(test_case, *args, **kwargs)
+
+    def is_successful(self) -> bool:
+        return self.success
+
+    @property
+    def __name__(self) -> str:  # shown in deepeval output
+        return "Structured Eval"

structured_eval/integrations/langsmith.py

@@ -0,0 +1,90 @@
+"""LangSmith adapter: structured-eval as an evaluator function.
+
+Usage (requires ``structured-eval[langsmith]``)::
+
+    from langsmith import evaluate
+    from structured_eval.integrations.langsmith import structured_evaluator
+
+    evaluator = structured_evaluator(config=cfg, threshold=0.85)
+    evaluate(target, data=dataset, evaluators=[evaluator])
+
+The returned callable follows LangSmith's ``(run, example) -> dict`` contract and
+emits a single feedback key with ``report.score`` plus a ``comment`` summarising
+failures. By default the actual output is read from ``run.outputs`` and the
+reference from ``example.outputs``; pass ``extract_actual``/``extract_expected``
+to point at a nested field or adapt a different object shape.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Callable
+from typing import Any
+
+from structured_eval.api import evaluate
+from structured_eval.integrations._adapter import verdict
+from structured_eval.models.config import EvalConfig
+from structured_eval.models.result import EvalReport
+
+Extractor = Callable[[Any], Any]
+
+
+def _outputs(obj: Any) -> Any:
+    """Default extraction: the ``outputs`` payload of a run/example."""
+    if obj is None:
+        return None
+    if isinstance(obj, dict):
+        return obj.get("outputs", obj)
+    return getattr(obj, "outputs", obj)
+
+
+class StructuredEvaluator:
+    """A LangSmith evaluator that scores structured outputs field-by-field.
+
+    Instances are callable with LangSmith's ``(run, example) -> dict`` contract.
+    ``key`` is the feedback key recorded in LangSmith; ``threshold`` decides the
+    boolean only for the ``comment`` — LangSmith stores the numeric
+    ``report.score`` itself. ``extract_actual`` / ``extract_expected`` adapt the
+    run/example shape (default: their ``outputs`` payload).
+    """
+
+    def __init__(
+        self,
+        config: EvalConfig | None = None,
+        *,
+        key: str = "structured_eval",
+        threshold: float = 0.5,
+        extract_actual: Extractor | None = None,
+        extract_expected: Extractor | None = None,
+    ) -> None:
+        self.config = config or EvalConfig()
+        self.key = key
+        self.threshold = threshold
+        self._get_actual = extract_actual or _outputs
+        self._get_expected = extract_expected or _outputs
+        self.__name__ = key
+
+    def __call__(self, run: Any, example: Any) -> dict[str, Any]:
+        report = evaluate(
+            self._get_actual(run), self._get_expected(example), self.config
+        )
+        assert isinstance(report, EvalReport)  # single-document evaluation
+        score, _success, reason = verdict(report, self.threshold)
+        return {"key": self.key, "score": score, "comment": reason}
+
+
+def structured_evaluator(
+    config: EvalConfig | None = None,
+    *,
+    key: str = "structured_eval",
+    threshold: float = 0.5,
+    extract_actual: Extractor | None = None,
+    extract_expected: Extractor | None = None,
+) -> StructuredEvaluator:
+    """Convenience factory returning a ``StructuredEvaluator`` instance."""
+    return StructuredEvaluator(
+        config,
+        key=key,
+        threshold=threshold,
+        extract_actual=extract_actual,
+        extract_expected=extract_expected,
+    )

structured_eval/metrics/__init__.py

@@ -0,0 +1,101 @@
+from structured_eval.metrics.array_accuracy import ArrayAccuracy
+from structured_eval.metrics.array_cardinality import ArrayCardinality
+from structured_eval.metrics.array_exact_match import ArrayExactMatch
+from structured_eval.metrics.array_f1 import ArrayF1
+from structured_eval.metrics.array_jaccard_similarity import ArrayJaccardSimilarity
+from structured_eval.metrics.array_precision import ArrayPrecision
+from structured_eval.metrics.array_prf1 import ArrayPRF1
+from structured_eval.metrics.array_recall import ArrayRecall
+from structured_eval.metrics.base import (
+    ArrayMetric,
+    BaseMetric,
+    FieldMetric,
+    GenericMetric,
+    Metric,
+    ObjectMetric,
+    RootMetric,
+    get_metric_class,
+    resolve_metric,
+)
+from structured_eval.metrics.character_f1 import CharacterF1
+from structured_eval.metrics.composite_score import CompositeScore
+from structured_eval.metrics.coverage_leaf_score import CoverageLeafScore
+from structured_eval.metrics.date_distance_score import DateDistanceScore
+from structured_eval.metrics.exact import ExactMatch
+from structured_eval.metrics.exponential_numeric_score import ExponentialNumericScore
+from structured_eval.metrics.field_faithfulness import FieldFaithfulness
+from structured_eval.metrics.fuzzy import Fuzzy
+from structured_eval.metrics.levenshtein import Levenshtein
+from structured_eval.metrics.mean_score import MeanScore
+from structured_eval.metrics.numeric import Numeric
+from structured_eval.metrics.numeric_closeness import NumericCloseness
+from structured_eval.metrics.object_accuracy import ObjectAccuracy
+from structured_eval.metrics.object_exact_match import ObjectExactMatch
+from structured_eval.metrics.object_f1 import ObjectF1
+from structured_eval.metrics.object_precision import ObjectPrecision
+from structured_eval.metrics.object_prf1 import ObjectPRF1
+from structured_eval.metrics.object_recall import ObjectRecall
+from structured_eval.metrics.object_type_validity import ObjectTypeValidity
+from structured_eval.metrics.overall_leaf_score import OverallLeafScore
+from structured_eval.metrics.presence import Presence
+from structured_eval.metrics.regex_match import RegexMatch
+from structured_eval.metrics.rule_pass_rate import Rule, RulePassRate
+from structured_eval.metrics.schema_validity import SchemaValidity
+from structured_eval.metrics.structural_similarity import StructuralSimilarity
+from structured_eval.metrics.token_f1 import TokenF1
+from structured_eval.metrics.type_match import TypeMatch
+
+__all__ = [
+    # array metrics
+    "ArrayAccuracy",
+    "ArrayCardinality",
+    "ArrayExactMatch",
+    "ArrayF1",
+    "ArrayJaccardSimilarity",
+    "ArrayMetric",
+    "ArrayPRF1",
+    "ArrayPrecision",
+    "ArrayRecall",
+    # base hierarchy
+    "BaseMetric",
+    # field metrics
+    "CharacterF1",
+    # any-node metrics
+    "CompositeScore",
+    "CoverageLeafScore",
+    "DateDistanceScore",
+    "ExactMatch",
+    "ExponentialNumericScore",
+    "FieldFaithfulness",
+    "FieldMetric",
+    "Fuzzy",
+    "GenericMetric",
+    "Levenshtein",
+    "MeanScore",
+    "Metric",
+    "Numeric",
+    "NumericCloseness",
+    # object metrics
+    "ObjectAccuracy",
+    "ObjectExactMatch",
+    "ObjectF1",
+    "ObjectMetric",
+    "ObjectPRF1",
+    "ObjectPrecision",
+    "ObjectRecall",
+    "ObjectTypeValidity",
+    # root metrics
+    "OverallLeafScore",
+    "Presence",
+    "RegexMatch",
+    "RootMetric",
+    # rules DSL
+    "Rule",
+    "RulePassRate",
+    "SchemaValidity",
+    "StructuralSimilarity",
+    "TokenF1",
+    "TypeMatch",
+    "get_metric_class",
+    "resolve_metric",
+]

structured_eval/metrics/array_accuracy.py

@@ -0,0 +1,28 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from structured_eval.metrics.base import ArrayMetric
+
+if TYPE_CHECKING:
+    from structured_eval.models.nodes.array_node import ArrayNode
+
+
+class ArrayAccuracy(ArrayMetric):
+    """Mean element score over the aligned items (soft).
+
+    How good the matched elements are, regardless of how many were produced:
+    the mean of each matched item's representative score over (items + missed).
+    Missed expected items count as 0.0; an empty/fully-missed array is vacuously
+    1.0. The default array metric, and the array branch of the old
+    ``structural_score``.
+    """
+
+    name = "array_accuracy"
+
+    def compute(self, node: ArrayNode) -> float:
+        n_missing = len(node.match_result.missed) if node.match_result else 0
+        denom = len(node.items) + n_missing
+        if denom == 0:
+            return 1.0
+        return sum(item.representative for item in node.items) / denom

structured_eval/metrics/array_cardinality.py

@@ -0,0 +1,27 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from structured_eval.metrics.base import ArrayMetric
+
+if TYPE_CHECKING:
+    from structured_eval.models.nodes.array_node import ArrayNode
+
+
+class ArrayCardinality(ArrayMetric):
+    """Count agreement: ``min(|actual|, |expected|) / max(...)``.
+
+    A cheap length-ratio check independent of element correctness. Two empty
+    arrays are vacuously 1.0.
+    """
+
+    name = "array_cardinality"
+
+    def compute(self, node: ArrayNode) -> float:
+        mr = node.match_result
+        if mr is None:
+            return 1.0
+        actual_count = len(mr.matched) + len(mr.spurious)
+        expected_count = len(mr.matched) + len(mr.missed)
+        hi = max(actual_count, expected_count)
+        return 1.0 if hi == 0 else min(actual_count, expected_count) / hi

structured_eval/metrics/array_exact_match.py

@@ -0,0 +1,48 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+from structured_eval.metrics.base import ArrayMetric
+
+if TYPE_CHECKING:
+    from structured_eval.models.nodes.array_node import ArrayNode
+
+
+class ArrayExactMatch(ArrayMetric):
+    """Strict whole-array equality: identical lists → 1.0, else 0.0.
+
+    Compares the raw ``actual`` / ``expected`` lists element-by-element,
+    **order-sensitively** and recursively (nested dicts/lists are deep-compared).
+    No alignment, no partial credit — the array as a whole is either right or
+    wrong. Use it when element order is part of correctness; for set-style or
+    value-aware scoring reach for :class:`ArrayJaccardSimilarity` or the
+    aligned ``Array*`` P/R/F1 metrics instead.
+    """
+
+    name = "array_exact_match"
+
+    def compute(self, node: ArrayNode) -> float:
+        return self.score(node.actual, node.expected)
+
+    def score(self, actual: Any, expected: Any) -> float:
+        return 1.0 if self._array_equal(actual, expected) else 0.0
+
+    def _array_equal(self, a: Any, b: Any) -> bool:
+        """Strict order-sensitive array comparison."""
+        if not (isinstance(a, list) and isinstance(b, list)):
+            return False
+        if len(a) != len(b):
+            return False
+        return all(self._deep_equal(x, y) for x, y in zip(a, b, strict=False))
+
+    def _deep_equal(self, a: Any, b: Any) -> bool:
+        """Shared recursive equality helper."""
+        if type(a) is not type(b):
+            return False
+        if isinstance(a, dict):
+            if set(a.keys()) != set(b.keys()):
+                return False
+            return all(self._deep_equal(a[k], b[k]) for k in a)
+        if isinstance(a, list):
+            return self._array_equal(a, b)
+        return bool(a == b)

structured_eval/metrics/array_f1.py

@@ -0,0 +1,34 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from structured_eval.metrics.base import ArrayMetric
+from structured_eval.metrics.utils import array as astats
+from structured_eval.metrics.utils import calculate as stats
+
+if TYPE_CHECKING:
+    from structured_eval.models.nodes.array_node import ArrayNode
+
+
+class ArrayF1(ArrayMetric):
+    """Harmonic mean of array precision and recall over aligned elements.
+
+    Threshold and ``mode`` behave as for ``ArrayPrecision``.
+    """
+
+    name = "array_f1"
+
+    def __init__(
+        self, threshold: float = 1.0, mode: stats.GradingMode = stats.GradingMode.HARD
+    ):
+        self.threshold = threshold
+        self.mode = stats.GradingMode(mode)
+
+    def compute(self, node: ArrayNode) -> float:
+        n_missing, n_spurious = astats.missing_spurious(node)
+        tp, predicted, expected = stats.prf_counts(
+            astats.verdicts(node, self.threshold), n_missing, n_spurious, self.mode
+        )
+        p = stats.precision(tp, predicted, expected)
+        r = stats.recall(tp, predicted, expected)
+        return stats.f1(p, r)

structured_eval/metrics/array_jaccard_similarity.py

@@ -0,0 +1,60 @@
+from __future__ import annotations
+
+import json
+from typing import TYPE_CHECKING, Any
+
+from structured_eval.metrics.base import ArrayMetric
+
+if TYPE_CHECKING:
+    from structured_eval.models.nodes.array_node import ArrayNode
+
+
+def _member(value: Any) -> Any:
+    """A hashable, comparison-stable set key for one element.
+
+    Scalars are used as-is; an unhashable element (dict/list) is keyed by its
+    canonical JSON so set membership still works without a TypeError.
+    """
+    if isinstance(value, (dict, list)):
+        return json.dumps(value, sort_keys=True, default=str)
+    return value
+
+
+class ArrayJaccardSimilarity(ArrayMetric):
+    """Set-overlap (Jaccard) similarity for arrays, order- and count-insensitive.
+
+    ``|A ∩ B| / |A ∪ B|`` over the two lists treated as **sets** (duplicates
+    collapse, order is ignored):
+
+    - ``1.0`` when the sets are identical (both empty → vacuously ``1.0``);
+    - ``0.0`` when there is no overlap (or exactly one side is empty);
+    - a value in ``(0, 1)`` otherwise.
+
+    Built for arrays of scalars — tags, labels, categories. Membership is exact
+    equality (no partial credit); for value-aware element matching use the
+    aligned ``Array*`` P/R/F1 metrics instead.
+    """
+
+    name = "array_jaccard_similarity"
+
+    def compute(self, node: ArrayNode) -> float:
+        return self.score(node.actual, node.expected)
+
+    def score(self, actual: Any, expected: Any) -> float:
+        a = self._to_set(actual)
+        e = self._to_set(expected)
+
+        if not a and not e:
+            return 1.0
+        if not a or not e:
+            return 0.0
+
+        return len(a & e) / len(a | e)
+
+    def _to_set(self, value: Any) -> set[Any]:
+        """Convert a value to a set of hashable members."""
+        if value is None:
+            return set()
+        if isinstance(value, (set, list, tuple)):
+            return {_member(item) for item in value}
+        return {_member(value)}

structured_eval/metrics/array_precision.py

@@ -0,0 +1,34 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from structured_eval.metrics.base import ArrayMetric
+from structured_eval.metrics.utils import array as astats
+from structured_eval.metrics.utils import calculate as stats
+
+if TYPE_CHECKING:
+    from structured_eval.models.nodes.array_node import ArrayNode
+
+
+class ArrayPrecision(ArrayMetric):
+    """TP / (TP + FP) over aligned array elements.
+
+    An aligned item is a TP when its ``element_score`` clears ``threshold``
+    (``mode="soft"`` instead adds the score fractionally); ``spurious`` items
+    are FP. So a wrong-but-aligned element lowers precision.
+    """
+
+    name = "array_precision"
+
+    def __init__(
+        self, threshold: float = 1.0, mode: stats.GradingMode = stats.GradingMode.HARD
+    ):
+        self.threshold = threshold
+        self.mode = stats.GradingMode(mode)
+
+    def compute(self, node: ArrayNode) -> float:
+        n_missing, n_spurious = astats.missing_spurious(node)
+        tp, predicted, expected = stats.prf_counts(
+            astats.verdicts(node, self.threshold), n_missing, n_spurious, self.mode
+        )
+        return stats.precision(tp, predicted, expected)

structured_eval/metrics/array_prf1.py

@@ -0,0 +1,40 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from structured_eval.metrics.base import ArrayMetric
+from structured_eval.metrics.utils import array as astats
+from structured_eval.metrics.utils import calculate as stats
+
+if TYPE_CHECKING:
+    from structured_eval.models.nodes.array_node import ArrayNode
+
+
+class ArrayPRF1(ArrayMetric):
+    """Array precision, recall and F1 in one pass.
+
+    Returns a dict; the engine writes ``array_precision`` / ``array_recall`` /
+    ``array_f1`` into ``report.metrics`` directly. Threshold and ``mode`` behave
+    as for ``ArrayPrecision``.
+    """
+
+    name = "array_prf1"
+
+    def __init__(
+        self, threshold: float = 1.0, mode: stats.GradingMode = stats.GradingMode.HARD
+    ):
+        self.threshold = threshold
+        self.mode = stats.GradingMode(mode)
+
+    def compute(self, node: ArrayNode) -> dict[str, float]:
+        n_missing, n_spurious = astats.missing_spurious(node)
+        tp, predicted, expected = stats.prf_counts(
+            astats.verdicts(node, self.threshold), n_missing, n_spurious, self.mode
+        )
+        p = stats.precision(tp, predicted, expected)
+        r = stats.recall(tp, predicted, expected)
+        return {
+            "array_precision": p,
+            "array_recall": r,
+            "array_f1": stats.f1(p, r),
+        }

structured_eval/metrics/array_recall.py

@@ -0,0 +1,33 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from structured_eval.metrics.base import ArrayMetric
+from structured_eval.metrics.utils import array as astats
+from structured_eval.metrics.utils import calculate as stats
+
+if TYPE_CHECKING:
+    from structured_eval.models.nodes.array_node import ArrayNode
+
+
+class ArrayRecall(ArrayMetric):
+    """TP / (TP + FN) over aligned array elements.
+
+    ``missed`` expected items are FN; threshold and ``mode`` behave as for
+    ``ArrayPrecision``.
+    """
+
+    name = "array_recall"
+
+    def __init__(
+        self, threshold: float = 1.0, mode: stats.GradingMode = stats.GradingMode.HARD
+    ):
+        self.threshold = threshold
+        self.mode = stats.GradingMode(mode)
+
+    def compute(self, node: ArrayNode) -> float:
+        n_missing, n_spurious = astats.missing_spurious(node)
+        tp, predicted, expected = stats.prf_counts(
+            astats.verdicts(node, self.threshold), n_missing, n_spurious, self.mode
+        )
+        return stats.recall(tp, predicted, expected)