structured-eval 0.1.0__py3-none-any.whl

structured_eval/metrics/numeric.py

@@ -0,0 +1,83 @@
+from __future__ import annotations
+
+from enum import StrEnum
+from typing import Any
+
+from structured_eval.metrics.base import FieldMetric
+from structured_eval.metrics.utils.number import parse_number
+
+
+class NumericMode(StrEnum):
+    """Tolerance band for the single-band form of :class:`Numeric`."""
+
+    RELATIVE = "relative"  # |a - e| / |e|
+    ABSOLUTE = "absolute"  # |a - e|
+
+
+class Numeric(FieldMetric):
+    """Numeric equality within a tolerance band → 1.0, otherwise 0.0.
+
+    Values are parsed leniently: currency symbols and thousands separators are
+    stripped (``"$1,234.50"`` → ``1234.50``), accounting notation is honored
+    (``"(123)"`` → ``-123``), and scientific notation is supported
+    (``"1e3"`` → ``1000``). A percent sign is only stripped, **not** interpreted
+    (``"50%"`` → ``50``, not ``0.5``). US format is assumed (``,`` = thousands,
+    ``.`` = decimal); other shapes that don't parse cleanly yield 0.0.
+
+    Tolerance can be given two ways:
+
+    * ``tolerance`` + ``mode`` (``"relative"`` | ``"absolute"``) — the original
+      single-band form; ``relative`` measures ``|a - e| / |e|``, ``absolute``
+      measures ``|a - e|``. A tolerance of 0 means exact numeric equality.
+    * ``relative_tolerance`` and/or ``absolute_tolerance`` — explicit bands; a
+      value matches if it falls within *either* band. When either is supplied it
+      takes precedence over ``tolerance``/``mode``.
+    """
+
+    name = "numeric"
+
+    def __init__(
+        self,
+        tolerance: float = 0.01,
+        mode: NumericMode = NumericMode.RELATIVE,
+        relative_tolerance: float | None = None,
+        absolute_tolerance: float | None = None,
+    ):
+        self.tolerance = tolerance
+        self.mode = NumericMode(mode)
+        self.relative_tolerance = relative_tolerance
+        self.absolute_tolerance = absolute_tolerance
+
+    def score(self, actual: Any, expected: Any) -> float:
+        a = parse_number(actual)
+        e = parse_number(expected)
+        if a is None or e is None:
+            return 0.0
+        return 1.0 if self._within_tolerance(a, e) else 0.0
+
+    def _within_tolerance(self, a: float, e: float) -> bool:
+        if a == e:
+            return True
+
+        # Explicit bands take precedence; match within either.
+        if self.relative_tolerance is not None or self.absolute_tolerance is not None:
+            if self.relative_tolerance is not None:
+                if e == 0:
+                    if a == 0:
+                        return True
+                elif abs(a - e) / abs(e) <= self.relative_tolerance:
+                    return True
+            return (
+                self.absolute_tolerance is not None
+                and abs(a - e) <= self.absolute_tolerance
+            )
+
+        # Single-band form (tolerance + mode).
+        if self.mode == NumericMode.RELATIVE:
+            if e == 0:
+                deviation = 0.0 if a == 0 else float("inf")
+            else:
+                deviation = abs(a - e) / abs(e)
+        else:
+            deviation = abs(a - e)
+        return deviation <= self.tolerance

structured_eval/metrics/numeric_closeness.py

@@ -0,0 +1,35 @@
+from __future__ import annotations
+
+from typing import Any
+
+from structured_eval.metrics.base import FieldMetric
+from structured_eval.metrics.utils.number import parse_number
+
+
+class NumericCloseness(FieldMetric):
+    """Graded numeric similarity in ``[0, 1]`` (not a pass/fail tolerance).
+
+    ``1 - |actual - expected| / max(|actual|, |expected|)`` — the ratio of the
+    smaller magnitude to the larger (``min/max`` for same-sign values): equal
+    values score 1.0, opposite signs trend toward 0.0, and ``0/0`` is 1.0.
+    Unlike :class:`Numeric` (a hard 0/1 verdict against a tolerance), this yields
+    a continuous score, making it the default element scorer for numbers under
+    the Hungarian array aligner where a graded cost matrix matters.
+
+    Values are parsed with the shared lenient numeric parser (same as
+    :class:`Numeric`), so numeric strings are graded too. The metric applies
+    **only to numbers**: if either side isn't numeric (``None``, a non-numeric
+    string, or a ``bool`` — ``True`` is not ``1``) the score is 0.0.
+    """
+
+    name = "numeric_closeness"
+
+    def score(self, actual: Any, expected: Any) -> float:
+        a = parse_number(actual)
+        e = parse_number(expected)
+        if a is None or e is None:
+            return 0.0
+        if a == e:
+            return 1.0
+        denom = max(abs(a), abs(e))
+        return max(0.0, 1.0 - abs(a - e) / denom) if denom else 1.0

structured_eval/metrics/object_accuracy.py

@@ -0,0 +1,47 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+from structured_eval.metrics.base import ObjectMetric
+from structured_eval.metrics.utils import calculate as stats
+from structured_eval.metrics.utils import object_utils as obj
+
+if TYPE_CHECKING:
+    from structured_eval.models.nodes.object_node import ObjectNode
+
+
+class ObjectAccuracy(ObjectMetric):
+    """Weighted soft mean of field correctness over an object's expected fields.
+
+    Equivalent to **soft recall**: ``Σ weight·score / (matched_weight +
+    missing_weight)``. Each matched field contributes its ``representative``
+    (any child kind — a nested object/array counts via its representative, not
+    only scalars), or a ``score_policy`` override. Missing expected fields count
+    as 0.0. **Spurious (extra) fields are not penalized** — the denominator is
+    the expected side only (use ``ObjectF1`` for a precision-aware score). An
+    object with no expected fields is vacuously 1.0.
+
+    ``weight_mode`` (default ``PROPORTIONAL``) makes this a weighted mean by each
+    child's configured ``weight``; ``NONE`` restores the plain mean.
+    """
+
+    name = "object_accuracy"
+
+    def __init__(
+        self,
+        score_policy: dict[str, Any] | None = None,
+        weight_mode: stats.WeightMode = stats.WeightMode.PROPORTIONAL,
+    ):
+        self.score_policy = score_policy
+        self.weight_mode = stats.WeightMode(weight_mode)
+
+    def compute(self, node: ObjectNode) -> float:
+        verdicts = obj.matched_verdicts(
+            node, self.score_policy, weight_mode=self.weight_mode
+        )
+        denom = sum(weight for _, _, weight in verdicts) + obj.missing_weight(
+            node, self.weight_mode
+        )
+        if denom == 0:
+            return 1.0
+        return sum(weight * score for score, _, weight in verdicts) / denom

structured_eval/metrics/object_exact_match.py

@@ -0,0 +1,41 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+from structured_eval.metrics.base import ObjectMetric
+
+if TYPE_CHECKING:
+    from structured_eval.models.nodes.object_node import ObjectNode
+
+
+class ObjectExactMatch(ObjectMetric):
+    """Strict deep equality for objects: identical dicts → 1.0, else 0.0.
+
+    Compares the raw ``actual`` / ``expected`` mappings recursively — same keys,
+    and every value deep-equal (nested dicts and lists included). No partial
+    credit and no coercion: the object as a whole is either right or wrong. For
+    field-level partial credit use the aggregating ``Object*`` metrics
+    (``ObjectAccuracy`` / ``ObjectF1`` …) instead.
+    """
+
+    name = "object_exact_match"
+
+    def compute(self, node: ObjectNode) -> float:
+        return self.score(node.actual, node.expected)
+
+    def score(self, actual: Any, expected: Any) -> float:
+        return 1.0 if self._object_equal(actual, expected) else 0.0
+
+    def _object_equal(self, a: Any, b: Any) -> bool:
+        """Deep strict equality for JSON-like structures."""
+        if type(a) is not type(b):
+            return False
+        if isinstance(a, dict):
+            if set(a.keys()) != set(b.keys()):
+                return False
+            return all(self._object_equal(a[k], b[k]) for k in a)
+        if isinstance(a, list):
+            if len(a) != len(b):
+                return False
+            return all(self._object_equal(x, y) for x, y in zip(a, b, strict=False))
+        return bool(a == b)

structured_eval/metrics/object_f1.py

@@ -0,0 +1,47 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+from structured_eval.metrics.base import ObjectMetric
+from structured_eval.metrics.utils import calculate as stats
+from structured_eval.metrics.utils import object_utils as obj
+
+if TYPE_CHECKING:
+    from structured_eval.models.nodes.object_node import ObjectNode
+
+
+class ObjectF1(ObjectMetric):
+    """Harmonic mean of object precision and recall over an object's fields.
+
+    Slot-filling F1: matched-and-correct → TP, missing → FN, extra → FP. Match
+    criterion and ``mode`` behave as for ``ObjectPrecision`` (counts all child
+    kinds via their representative).
+    """
+
+    name = "object_f1"
+
+    def __init__(
+        self,
+        score_policy: dict[str, Any] | None = None,
+        threshold: float | None = None,
+        mode: stats.GradingMode = stats.GradingMode.HARD,
+        weight_mode: stats.WeightMode = stats.WeightMode.PROPORTIONAL,
+    ):
+        self.score_policy = score_policy
+        self.threshold = threshold
+        self.mode = stats.GradingMode(mode)
+        self.weight_mode = stats.WeightMode(weight_mode)
+
+    def compute(self, node: ObjectNode) -> float:
+        verdicts = obj.matched_verdicts(
+            node, self.score_policy, self.threshold, self.weight_mode
+        )
+        tp, predicted, expected = stats.prf_counts(
+            verdicts,
+            obj.missing_weight(node, self.weight_mode),
+            obj.spurious_weight(node, self.weight_mode),
+            self.mode,
+        )
+        p = stats.precision(tp, predicted, expected)
+        r = stats.recall(tp, predicted, expected)
+        return stats.f1(p, r)

structured_eval/metrics/object_precision.py

@@ -0,0 +1,49 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+from structured_eval.metrics.base import ObjectMetric
+from structured_eval.metrics.utils import calculate as stats
+from structured_eval.metrics.utils import object_utils as obj
+
+if TYPE_CHECKING:
+    from structured_eval.models.nodes.object_node import ObjectNode
+
+
+class ObjectPrecision(ObjectMetric):
+    """TP / (TP + FP) over an object's fields (slot-filling precision).
+
+    Each matched field is a TP when its ``representative`` clears its threshold
+    (any child kind — nested objects/arrays count via their representative);
+    extra fields are FP. The per-field score comes from ``score_policy`` →
+    the child's ``key_metric`` → ``ExactMatch``. Default ``mode=HARD`` with the
+    field threshold (``1.0`` unless configured), so a field counts only when its
+    score is a perfect match; ``mode="soft"`` drops the threshold and uses the
+    field score fractionally.
+    """
+
+    name = "object_precision"
+
+    def __init__(
+        self,
+        score_policy: dict[str, Any] | None = None,
+        threshold: float | None = None,
+        mode: stats.GradingMode = stats.GradingMode.HARD,
+        weight_mode: stats.WeightMode = stats.WeightMode.PROPORTIONAL,
+    ):
+        self.score_policy = score_policy
+        self.threshold = threshold
+        self.mode = stats.GradingMode(mode)
+        self.weight_mode = stats.WeightMode(weight_mode)
+
+    def compute(self, node: ObjectNode) -> float:
+        verdicts = obj.matched_verdicts(
+            node, self.score_policy, self.threshold, self.weight_mode
+        )
+        tp, predicted, expected = stats.prf_counts(
+            verdicts,
+            obj.missing_weight(node, self.weight_mode),
+            obj.spurious_weight(node, self.weight_mode),
+            self.mode,
+        )
+        return stats.precision(tp, predicted, expected)

structured_eval/metrics/object_prf1.py

@@ -0,0 +1,51 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+from structured_eval.metrics.base import ObjectMetric
+from structured_eval.metrics.utils import calculate as stats
+from structured_eval.metrics.utils import object_utils as obj
+
+if TYPE_CHECKING:
+    from structured_eval.models.nodes.object_node import ObjectNode
+
+
+class ObjectPRF1(ObjectMetric):
+    """Precision, recall and F1 in one pass.
+
+    Returns a dict; the engine writes each key (``object_precision``,
+    ``object_recall``, ``object_f1``) into ``report.metrics`` directly. Match
+    criterion and ``mode`` behave as for ``ObjectPrecision``.
+    """
+
+    name = "object_prf1"
+
+    def __init__(
+        self,
+        score_policy: dict[str, Any] | None = None,
+        threshold: float | None = None,
+        mode: stats.GradingMode = stats.GradingMode.HARD,
+        weight_mode: stats.WeightMode = stats.WeightMode.PROPORTIONAL,
+    ):
+        self.score_policy = score_policy
+        self.threshold = threshold
+        self.mode = stats.GradingMode(mode)
+        self.weight_mode = stats.WeightMode(weight_mode)
+
+    def compute(self, node: ObjectNode) -> dict[str, float]:
+        verdicts = obj.matched_verdicts(
+            node, self.score_policy, self.threshold, self.weight_mode
+        )
+        tp, predicted, expected = stats.prf_counts(
+            verdicts,
+            obj.missing_weight(node, self.weight_mode),
+            obj.spurious_weight(node, self.weight_mode),
+            self.mode,
+        )
+        p = stats.precision(tp, predicted, expected)
+        r = stats.recall(tp, predicted, expected)
+        return {
+            "object_precision": p,
+            "object_recall": r,
+            "object_f1": stats.f1(p, r),
+        }

structured_eval/metrics/object_recall.py

@@ -0,0 +1,44 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+from structured_eval.metrics.base import ObjectMetric
+from structured_eval.metrics.utils import calculate as stats
+from structured_eval.metrics.utils import object_utils as obj
+
+if TYPE_CHECKING:
+    from structured_eval.models.nodes.object_node import ObjectNode
+
+
+class ObjectRecall(ObjectMetric):
+    """TP / (TP + FN) over an object's fields (slot-filling recall).
+
+    Missing expected fields are FN. Match criterion and ``mode`` behave as for
+    ``ObjectPrecision`` (counts all child kinds via their representative).
+    """
+
+    name = "object_recall"
+
+    def __init__(
+        self,
+        score_policy: dict[str, Any] | None = None,
+        threshold: float | None = None,
+        mode: stats.GradingMode = stats.GradingMode.HARD,
+        weight_mode: stats.WeightMode = stats.WeightMode.PROPORTIONAL,
+    ):
+        self.score_policy = score_policy
+        self.threshold = threshold
+        self.mode = stats.GradingMode(mode)
+        self.weight_mode = stats.WeightMode(weight_mode)
+
+    def compute(self, node: ObjectNode) -> float:
+        verdicts = obj.matched_verdicts(
+            node, self.score_policy, self.threshold, self.weight_mode
+        )
+        tp, predicted, expected = stats.prf_counts(
+            verdicts,
+            obj.missing_weight(node, self.weight_mode),
+            obj.spurious_weight(node, self.weight_mode),
+            self.mode,
+        )
+        return stats.recall(tp, predicted, expected)

structured_eval/metrics/object_type_validity.py

@@ -0,0 +1,34 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from structured_eval.metrics.base import ObjectMetric
+from structured_eval.metrics.invoker import MetricInvoker
+from structured_eval.metrics.type_match import TypeMatch
+
+if TYPE_CHECKING:
+    from structured_eval.models.nodes.object_node import ObjectNode
+
+
+class ObjectTypeValidity(ObjectMetric):
+    """Fraction of present fields that are type-valid.
+
+    A structural sanity check independent of value correctness: of the fields
+    present in both, how many carry the right JSON type. ``TypeMatch`` covers
+    every JSON type, so this validates scalars (``"100"`` vs ``100``) *and*
+    containers (a ``list`` where an object was expected) alike — a basic
+    type check, not a deep one. An object with no present fields is vacuously
+    1.0.
+    """
+
+    name = "object_type_validity"
+
+    def __init__(self) -> None:
+        self._type_match = MetricInvoker(TypeMatch())
+
+    def compute(self, node: ObjectNode) -> float:
+        present = node.matched
+        if not present:
+            return 1.0
+        valid = sum(self._type_match.scalar_on_node(n) for n in present)
+        return valid / len(present)

structured_eval/metrics/overall_leaf_score.py

@@ -0,0 +1,32 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from structured_eval.metrics.base import RootMetric
+from structured_eval.models.nodes.scalar import ScalarNode
+
+if TYPE_CHECKING:
+    from structured_eval.models.nodes.base import EvalNode
+
+
+class OverallLeafScore(RootMetric):
+    """Weighted mean of leaf match-criterion scores over the whole document.
+
+    Each scalar field contributes its match-criterion verdict weighted by its
+    configured ``weight`` (``FieldConfig.weight``). Missing expected leaves
+    score 0; a document with no leaves is vacuously 1.0. The headline number.
+
+    Leaf-style: it flattens to scalar leaves across the whole tree (unlike the
+    hierarchical object metrics, which aggregate a node's direct children).
+    """
+
+    name = "overall_leaf_score"
+
+    def compute(self, node: EvalNode) -> float:
+        total_weight = 0.0
+        weighted = 0.0
+        for leaf in node.leaves():
+            assert isinstance(leaf, ScalarNode)
+            total_weight += leaf.weight
+            weighted += leaf.weight * leaf.representative
+        return weighted / total_weight if total_weight else 1.0

structured_eval/metrics/presence.py

@@ -0,0 +1,22 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from structured_eval.metrics.base import FieldMetric
+
+if TYPE_CHECKING:
+    from structured_eval.models.nodes.scalar import ScalarNode
+
+
+class Presence(FieldMetric):
+    """Was the field populated? 1.0 if present and non-null, else 0.0.
+
+    A single-value check — it ignores ``expected`` and looks only at ``actual``
+    (a missing key surfaces as ``None`` through the node). Overrides ``compute``
+    rather than ``score`` since it is not a comparison of two values.
+    """
+
+    name = "presence"
+
+    def compute(self, node: ScalarNode) -> float:
+        return 1.0 if node.actual is not None else 0.0

structured_eval/metrics/regex_match.py

@@ -0,0 +1,51 @@
+from __future__ import annotations
+
+import re
+from typing import Any
+
+from structured_eval.metrics.base import FieldMetric
+
+
+class RegexMatch(FieldMetric):
+    """String equality after an optional regex rewrite → 1.0, else 0.0.
+
+    A **string-only** metric: if either side is not a ``str`` the score is
+    ``0.0`` (use ``Numeric`` for numbers, ``ExactMatch`` for verbatim
+    equality). For two strings it applies, in order, optional ``lower`` and
+    ``strip``, then substitutes every match of ``pattern`` with ``repl``, and
+    compares the results exactly.
+
+    The default ``pattern=r"\\s+", repl=" "`` (with ``lower``/``strip`` on)
+    collapses whitespace and ignores casing. Tune the rewrite, e.g.::
+
+        RegexMatch(pattern=r"[^\\w\\s]", repl="")  # drop punctuation
+        RegexMatch(pattern=r"[-_]", repl=" ")       # dashes/underscores → spaces
+        RegexMatch(lower=False)                      # case-sensitive
+    """
+
+    name = "regex_match"
+
+    def __init__(
+        self,
+        pattern: str | re.Pattern[str] = r"\s+",
+        repl: str = " ",
+        lower: bool = True,
+        strip: bool = True,
+    ):
+        self.pattern = re.compile(pattern) if isinstance(pattern, str) else pattern
+        self.repl = repl
+        self.lower = lower
+        self.strip = strip
+
+    def _normalize(self, value: str) -> str:
+        if self.lower:
+            value = value.lower()
+        if self.strip:
+            value = value.strip()
+        value = self.pattern.sub(self.repl, value)
+        return value.strip() if self.strip else value
+
+    def score(self, actual: Any, expected: Any) -> float:
+        if not (isinstance(actual, str) and isinstance(expected, str)):
+            return 0.0
+        return 1.0 if self._normalize(actual) == self._normalize(expected) else 0.0

structured_eval/metrics/rule_pass_rate/__init__.py

@@ -0,0 +1,5 @@
+from structured_eval.metrics.rule_pass_rate.dsl import Rule
+from structured_eval.metrics.rule_pass_rate.engine import RuleProcessor
+from structured_eval.metrics.rule_pass_rate.metric import RulePassRate
+
+__all__ = ["Rule", "RulePassRate", "RuleProcessor"]