structured-eval 0.1.0__py3-none-any.whl

structured_eval/metrics/rule_pass_rate/dsl.py

@@ -0,0 +1,224 @@
+from __future__ import annotations
+
+import ast
+import operator
+import re
+from typing import TYPE_CHECKING, Any
+
+from structured_eval.models.result import RuleResult
+
+if TYPE_CHECKING:
+    from collections.abc import Callable
+
+# Matches a bare JSONPath like "$.field" or "$.nested.child"
+_PLAIN_PATH_RE = re.compile(r"^\$(?:\.[a-zA-Z_][a-zA-Z0-9_]*)+$")
+# Matches any JSONPath fragment inside a larger expression
+_PATH_IN_EXPR_RE = re.compile(r"\$(?:\.[a-zA-Z_][a-zA-Z0-9_]*)+")
+
+_ARITH_OPS: dict[type[ast.operator], Any] = {
+    ast.Add: operator.add,
+    ast.Sub: operator.sub,
+    ast.Mult: operator.mul,
+    ast.Div: operator.truediv,
+}
+
+
+# ── jsonpath-ng lazy import ────────────────────────────────────────────────────
+
+
+def _ensure_jsonpath() -> None:
+    try:
+        import jsonpath_ng  # noqa: F401
+    except ImportError as e:
+        raise ImportError(
+            "jsonpath-ng is required for the Rule DSL. "
+            "Install it with: pip install 'structured-eval[rules]'"
+        ) from e
+
+
+def _resolve_path(path: str, document: dict[str, Any]) -> Any:
+    _ensure_jsonpath()
+    from jsonpath_ng import parse
+
+    matches = parse(path).find(document)
+    if not matches:
+        raise KeyError(f"Path {path!r} not found in document")
+    return matches[0].value
+
+
+# ── Arithmetic expression evaluator ───────────────────────────────────────────
+
+
+def _eval_arithmetic(expr: str, document: dict[str, Any]) -> Any:
+    """Resolve JSONPath fragments inside expr, then evaluate safe arithmetic."""
+
+    def _replace(m: re.Match[str]) -> str:
+        return repr(_resolve_path(m.group(), document))
+
+    resolved = _PATH_IN_EXPR_RE.sub(_replace, expr)
+    tree = ast.parse(resolved, mode="eval")
+    return _eval_node(tree.body)
+
+
+def _eval_node(node: ast.expr) -> Any:
+    if isinstance(node, ast.Constant):
+        return node.value
+    if isinstance(node, ast.UnaryOp) and isinstance(node.op, ast.USub):
+        return -_eval_node(node.operand)
+    if isinstance(node, ast.BinOp):
+        op_fn = _ARITH_OPS.get(type(node.op))
+        if op_fn is None:
+            raise ValueError(f"Unsupported operator: {type(node.op).__name__}")
+        return op_fn(_eval_node(node.left), _eval_node(node.right))
+    raise ValueError(f"Unsupported expression node: {type(node).__name__}")
+
+
+# ── CustomRule ────────────────────────────────────────────────────────────────
+
+
+class _CustomRule:
+    """Returned by Rule.custom(). Evaluates an arbitrary function over a document."""
+
+    def __init__(self, fn: Callable[[dict[str, Any]], bool], *, name: str = "") -> None:
+        self._fn = fn
+        self._name = name
+
+    @property
+    def name(self) -> str:
+        return self._name or "custom"
+
+    def evaluate(self, document: dict[str, Any]) -> RuleResult:
+        try:
+            passed = bool(self._fn(document))
+        except Exception as exc:
+            return RuleResult(name=self.name, passed=False, message=str(exc))
+        return RuleResult(
+            name=self.name,
+            passed=passed,
+            message="" if passed else "custom rule failed",
+        )
+
+
+# ── Rule ──────────────────────────────────────────────────────────────────────
+
+
+class Rule:
+    """JSONPath-based document constraint.
+
+    Chain a comparison method to create a bound rule, then pass it to
+    EvalConfig.rules or call evaluate() directly.
+
+    Examples::
+
+        Rule("$.status").eq("paid")
+        Rule("$.total").gte(0)
+        Rule("$.total").eq("$.subtotal + $.tax")
+        Rule("$.currency").in_(["USD", "EUR"])
+        Rule.custom(lambda doc: doc["amount"] > 0, name="positive_amount")
+    """
+
+    def __init__(self, path: str, *, name: str = "") -> None:
+        self._path = path
+        self._name = name
+        self._op: str | None = None
+        self._rhs: Any = None
+
+    # ── Builder ───────────────────────────────────────────────────────────────
+
+    def _bind(self, op: str, rhs: Any) -> Rule:
+        r = Rule(self._path, name=self._name)
+        r._op = op
+        r._rhs = rhs
+        return r
+
+    def eq(self, rhs: Any) -> Rule:
+        return self._bind("eq", rhs)
+
+    def lt(self, rhs: Any) -> Rule:
+        return self._bind("lt", rhs)
+
+    def gt(self, rhs: Any) -> Rule:
+        return self._bind("gt", rhs)
+
+    def lte(self, rhs: Any) -> Rule:
+        return self._bind("lte", rhs)
+
+    def gte(self, rhs: Any) -> Rule:
+        return self._bind("gte", rhs)
+
+    def in_(self, collection: Any) -> Rule:
+        return self._bind("in", collection)
+
+    @classmethod
+    def custom(
+        cls, fn: Callable[[dict[str, Any]], bool], *, name: str = ""
+    ) -> _CustomRule:
+        """Wrap an arbitrary function as a rule.
+
+        Args:
+            fn: Callable(document) -> bool. Receives the full document dict.
+            name: Human-readable name shown in reports.
+        """
+        return _CustomRule(fn=fn, name=name)
+
+    # ── Evaluation ────────────────────────────────────────────────────────────
+
+    @property
+    def name(self) -> str:
+        if self._name:
+            return self._name
+        if self._op is not None:
+            rhs_str = self._rhs if isinstance(self._rhs, str) else repr(self._rhs)
+            return f"{self._path} {self._op} {rhs_str}"
+        return self._path
+
+    def evaluate(self, document: dict[str, Any]) -> RuleResult:
+        if self._op is None:
+            raise ValueError(
+                f"Rule {self._path!r} has no comparison — call .eq(), .lt(), etc."
+            )
+
+        try:
+            lhs = _resolve_path(self._path, document)
+        except (KeyError, ImportError) as exc:
+            return RuleResult(name=self.name, passed=False, message=str(exc))
+
+        try:
+            rhs = self._resolve_rhs(self._rhs, document)
+        except Exception as exc:
+            return RuleResult(name=self.name, passed=False, message=str(exc))
+
+        try:
+            passed = self._compare(lhs, rhs)
+        except TypeError as exc:
+            return RuleResult(name=self.name, passed=False, message=str(exc))
+
+        msg = (
+            ""
+            if passed
+            else f"{self._path!r} ({lhs!r}) does not satisfy {self._op}({rhs!r})"
+        )
+        return RuleResult(name=self.name, passed=passed, message=msg)
+
+    def _resolve_rhs(self, rhs: Any, document: dict[str, Any]) -> Any:
+        if isinstance(rhs, str) and "$" in rhs:
+            stripped = rhs.strip()
+            if _PLAIN_PATH_RE.match(stripped):
+                return _resolve_path(stripped, document)
+            return _eval_arithmetic(stripped, document)
+        return rhs
+
+    def _compare(self, lhs: Any, rhs: Any) -> bool:
+        if self._op == "eq":
+            return bool(lhs == rhs)
+        if self._op == "lt":
+            return bool(lhs < rhs)
+        if self._op == "gt":
+            return bool(lhs > rhs)
+        if self._op == "lte":
+            return bool(lhs <= rhs)
+        if self._op == "gte":
+            return bool(lhs >= rhs)
+        if self._op == "in":
+            return lhs in rhs
+        raise ValueError(f"Unknown operator: {self._op!r}")

structured_eval/metrics/rule_pass_rate/engine.py

@@ -0,0 +1,24 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+    from structured_eval.models.result import RuleResult
+
+
+class RuleProcessor:
+    """Evaluates a list of business rules against a document.
+
+    Each rule must expose ``evaluate(document) -> RuleResult`` (satisfied by both
+    ``Rule`` and the result of ``Rule.custom()``). ``run`` returns the per-rule
+    results and the pass rate (1.0 when there are no rules).
+    """
+
+    def run(
+        self, rules: list[Any], document: dict[str, Any]
+    ) -> tuple[list[RuleResult], float]:
+        results: list[RuleResult] = [rule.evaluate(document) for rule in rules]
+        if not results:
+            return results, 1.0
+        pass_rate = sum(1 for r in results if r.passed) / len(results)
+        return results, pass_rate

structured_eval/metrics/rule_pass_rate/metric.py

@@ -0,0 +1,33 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+from structured_eval.metrics.base import RootMetric
+from structured_eval.metrics.rule_pass_rate.engine import RuleProcessor
+
+if TYPE_CHECKING:
+    from structured_eval.models.nodes.base import EvalNode
+
+
+class RulePassRate(RootMetric):
+    """Fraction of business rules that hold for the document.
+
+    ``rules`` is a list of ``Rule`` (DSL) or ``Rule.custom(...)`` objects, each
+    exposing ``evaluate(document) -> RuleResult``. Per-rule outcomes are returned
+    as the result's ``extra["rule_results"]`` (serialized ``RuleResult`` dicts) —
+    read via ``report.metrics["rule_pass_rate"].extra_values("rule_results")``. An
+    empty rule list scores 1.0 (vacuously true).
+    """
+
+    name = "rule_pass_rate"
+
+    def __init__(self, rules: list[Any]):
+        self.rules = rules
+        self.processor = RuleProcessor()
+
+    def compute(self, node: EvalNode) -> tuple[float, dict[str, Any]]:
+        document = node.actual
+        results, pass_rate = self.processor.run(
+            self.rules, document if isinstance(document, dict) else {}
+        )
+        return pass_rate, {"rule_results": [r.model_dump() for r in results]}

structured_eval/metrics/schema_validity/__init__.py

@@ -0,0 +1,7 @@
+from structured_eval.metrics.schema_validity.metric import SchemaValidity
+from structured_eval.metrics.schema_validity.validator import (
+    SchemaResult,
+    SchemaValidator,
+)
+
+__all__ = ["SchemaResult", "SchemaValidator", "SchemaValidity"]

structured_eval/metrics/schema_validity/metric.py

@@ -0,0 +1,35 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+from structured_eval.metrics.base import RootMetric
+from structured_eval.metrics.schema_validity.validator import SchemaValidator
+
+if TYPE_CHECKING:
+    from pydantic import BaseModel
+
+    from structured_eval.models.nodes.base import EvalNode
+
+
+class SchemaValidity(RootMetric):
+    """Does the actual document validate against ``schema``? 1.0 / 0.0.
+
+    ``schema`` is a Pydantic model class or a JSON Schema dict. Validation
+    errors are returned as the result's ``extra["schema_errors"]`` — read via
+    ``report.metrics["schema_validity"].extra_values("schema_errors")``.
+    """
+
+    name = "schema_validity"
+
+    def __init__(self, schema: type[BaseModel] | dict[str, Any]):
+        self.validator = SchemaValidator(schema)
+
+    def compute(self, node: EvalNode) -> tuple[float, dict[str, Any]]:
+        result = self.validator.validate(node.actual)
+        return (1.0 if result.valid else 0.0), {
+            "schema_errors": {
+                "type_errors": result.type_errors,
+                "missing_required": result.missing_required,
+                "extra_fields": result.extra_fields,
+            }
+        }

structured_eval/metrics/schema_validity/validator.py

@@ -0,0 +1,119 @@
+from __future__ import annotations
+
+from typing import Any
+
+from pydantic import BaseModel, ValidationError, computed_field
+
+
+class SchemaResult(BaseModel):
+    """Outcome of validating actual against a schema."""
+
+    valid: bool
+    type_errors: list[str] = []
+    missing_required: list[str] = []
+    extra_fields: list[str] = []
+    total_fields: int = 0
+
+    @computed_field  # type: ignore[prop-decorator]
+    @property
+    def type_error_rate(self) -> float | None:
+        if self.total_fields == 0:
+            return None
+        return len(self.type_errors) / self.total_fields
+
+
+class SchemaValidator:
+    """Validates a document against a Pydantic model class or JSON Schema dict.
+
+    Constructed with the ``schema`` once; ``validate(actual)`` returns a
+    ``SchemaResult`` describing type errors, missing required and extra fields.
+    """
+
+    def __init__(self, schema: type[BaseModel] | dict[str, Any]):
+        self.schema = schema
+
+    def validate(self, actual: Any) -> SchemaResult:
+        schema = self.schema
+        if isinstance(schema, type) and issubclass(schema, BaseModel):
+            return self._validate_pydantic(actual, schema)
+        if isinstance(schema, dict):
+            return self._validate_jsonschema(actual, schema)
+        raise TypeError(
+            f"schema must be a Pydantic BaseModel subclass or a dict, got {type(schema)!r}"
+        )
+
+    # ── Pydantic ──────────────────────────────────────────────────────────
+
+    def _validate_pydantic(self, actual: Any, model: type[BaseModel]) -> SchemaResult:
+        total = len(model.model_fields)
+        try:
+            model.model_validate(actual)
+            return SchemaResult(valid=True, total_fields=total)
+        except ValidationError as exc:
+            type_errors: list[str] = []
+            missing_required: list[str] = []
+            extra_fields: list[str] = []
+
+            for err in exc.errors():
+                loc = ".".join(str(p) for p in err["loc"])
+                kind = err["type"]
+                if kind == "missing":
+                    missing_required.append(loc)
+                elif kind == "extra_forbidden":
+                    extra_fields.append(loc)
+                else:
+                    type_errors.append(loc)
+
+            return SchemaResult(
+                valid=False,
+                type_errors=type_errors,
+                missing_required=missing_required,
+                extra_fields=extra_fields,
+                total_fields=total,
+            )
+
+    # ── JSON Schema ─────────────────────────────────────────────────────────
+
+    def _validate_jsonschema(self, actual: Any, schema: dict[str, Any]) -> SchemaResult:
+        try:
+            from jsonschema import Draft7Validator
+        except ImportError as exc:  # pragma: no cover
+            raise ImportError(
+                "jsonschema is required for dict-schema validation. "
+                "Install it with: pip install 'structured-eval[jsonschema]'"
+            ) from exc
+
+        total = len(schema.get("properties", {}))
+        validator = Draft7Validator(schema)
+        errors = list(validator.iter_errors(actual))
+
+        if not errors:
+            return SchemaResult(valid=True, total_fields=total)
+
+        type_errors: list[str] = []
+        missing_required: list[str] = []
+        extra_fields: list[str] = []
+
+        for err in errors:
+            loc = (
+                ".".join(str(p) for p in err.absolute_path)
+                if err.absolute_path
+                else err.json_path
+            )
+            if err.validator == "required":
+                # err.message names the missing field directly
+                missing_field = err.message.split("'")[1] if "'" in err.message else loc
+                missing_required.append(missing_field)
+            elif err.validator == "additionalProperties":
+                extra_fields.append(loc or err.message)
+            else:
+                # type and other validators (pattern, minLength, …) → type errors
+                type_errors.append(loc)
+
+        return SchemaResult(
+            valid=False,
+            type_errors=type_errors,
+            missing_required=missing_required,
+            extra_fields=extra_fields,
+            total_fields=total,
+        )

structured_eval/metrics/structural_similarity.py

@@ -0,0 +1,40 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from structured_eval.metrics.base import RootMetric
+from structured_eval.utils.flatten import extract_paths
+
+if TYPE_CHECKING:
+    from structured_eval.metrics.base import MetricOutput
+    from structured_eval.models.nodes.base import EvalNode
+
+
+class StructuralSimilarity(RootMetric):
+    """Structural similarity of two documents — Jaccard over their paths.
+
+    Compares the *shape* of ``actual`` and ``expected``, ignoring values::
+
+        |paths_actual ∩ paths_expected| / |paths_actual ∪ paths_expected|
+
+    where a path is enumerated for every dict key, list index and nested
+    sub-path (containers and leaves alike — see
+    :func:`~structured_eval.utils.flatten.extract_paths`). Returns ``1.0``
+    for identical structure (both empty → vacuously ``1.0``), ``0.0`` for no
+    shared path, and a value in ``(0, 1)`` otherwise. A complement to the
+    value-aware metrics: it answers "did the model produce the right skeleton"
+    regardless of whether the values are correct.
+    """
+
+    name = "structural_similarity"
+
+    def compute(self, node: EvalNode) -> MetricOutput:
+        paths_a = extract_paths(node.context.actual)
+        paths_e = extract_paths(node.context.expected)
+
+        if not paths_a and not paths_e:
+            return 1.0
+        if not paths_a or not paths_e:
+            return 0.0
+
+        return len(paths_a & paths_e) / len(paths_a | paths_e)

structured_eval/metrics/token_f1.py

@@ -0,0 +1,44 @@
+from __future__ import annotations
+
+import re
+from collections import Counter
+from typing import Any
+
+from structured_eval.metrics.base import FieldMetric
+
+_NON_WORD = re.compile(r"[^\w\s]")
+
+
+def _tokenize(value: Any) -> list[str]:
+    """Lowercase, drop punctuation, split on whitespace."""
+    return _NON_WORD.sub(" ", str(value).lower()).split()
+
+
+class TokenF1(FieldMetric):
+    """SQuAD-style token-overlap F1 — a default for free-text fields.
+
+    Tokens are matched as a **multiset** (``Counter``), counting shared tokens
+    with multiplicity exactly like the official SQuAD F1 — so a repeated token
+    only helps as often as it appears on both sides (``"the the cat"`` vs
+    ``"the cat"`` is 0.8, not 1.0). Precision and recall are over the token
+    *counts*; their harmonic mean is the score. String-only: if either side is
+    not a ``str`` the score is 0.0 (no coercion).
+    """
+
+    name = "token_f1"
+
+    def score(self, actual: Any, expected: Any) -> float:
+        if not (isinstance(actual, str) and isinstance(expected, str)):
+            return 0.0
+        a = _tokenize(actual)
+        e = _tokenize(expected)
+        if not a and not e:
+            return 1.0
+        if not a or not e:
+            return 0.0
+        same = sum((Counter(a) & Counter(e)).values())
+        if not same:
+            return 0.0
+        precision = same / len(a)
+        recall = same / len(e)
+        return 2 * precision * recall / (precision + recall)

structured_eval/metrics/type_match.py

@@ -0,0 +1,35 @@
+from __future__ import annotations
+
+from typing import Any
+
+from structured_eval.metrics.base import FieldMetric
+
+
+def _json_type(value: Any) -> str:
+    """Map a Python value to its JSON type name (bool before int)."""
+    if isinstance(value, bool):
+        return "bool"
+    if isinstance(value, (int, float)):
+        return "number"
+    if isinstance(value, str):
+        return "string"
+    if isinstance(value, list):
+        return "array"
+    if isinstance(value, dict):
+        return "object"
+    if value is None:
+        return "null"
+    return type(value).__name__
+
+
+class TypeMatch(FieldMetric):
+    """Right JSON type? 1.0 if actual and expected share a type, else 0.0.
+
+    Catches a common LLM error — emitting ``"100"`` (string) where ``100``
+    (number) is expected — independently of value correctness.
+    """
+
+    name = "type_match"
+
+    def score(self, actual: Any, expected: Any) -> float:
+        return 1.0 if _json_type(actual) == _json_type(expected) else 0.0

structured_eval/metrics/utils/__init__.py

@@ -0,0 +1,10 @@
+"""Utilities shared across metric implementations (metric-layer only).
+
+Clearly-scoped modules:
+
+* ``calculate`` — the precision / recall / F1 arithmetic (and the ``GradingMode``
+  hard/soft enum) used by every P/R/F1 metric, object and array alike.
+* ``object_utils`` — turning an object's matched fields into the
+  ``(score, threshold)`` pairs that ``calculate.prf_counts`` consumes.
+* ``array`` — the same for an array's aligned items, plus missing/spurious counts.
+"""

structured_eval/metrics/utils/array.py

@@ -0,0 +1,31 @@
+"""Verdicts for array metrics: aligned items → ``(score, threshold, weight)``.
+
+An aligned item is graded by its representative score against a single
+``threshold`` (hard) or contributes that score fractionally (soft) — mirroring
+how object fields are graded. ``missed`` items are FN, ``spurious`` items FP.
+The verdicts feed ``calculate.prf_counts``.
+
+Array elements share one ``item`` config, so they carry no individual weights:
+every item (and every missed/spurious slot) weighs ``1.0`` and array metrics are
+effectively count-based.
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from structured_eval.models.nodes.array_node import ArrayNode
+
+
+def verdicts(node: ArrayNode, threshold: float) -> list[tuple[float, float, float]]:
+    """``(representative, threshold, weight=1.0)`` for each aligned item."""
+    return [(item.representative, threshold, 1.0) for item in node.items]
+
+
+def missing_spurious(node: ArrayNode) -> tuple[int, int]:
+    """``(n_missed, n_spurious)`` from the array's alignment result."""
+    mr = node.match_result
+    if mr is None:
+        return 0, 0
+    return len(mr.missed), len(mr.spurious)

structured_eval/metrics/utils/calculate.py

@@ -0,0 +1,72 @@
+"""Precision / recall / F1 arithmetic over resolved field/item verdicts.
+
+Each matched scalar field (or array item) is both a *predicted* and an
+*expected* entry; ``spurious`` entries add to predicted (FP), ``missing`` ones
+add to expected (FN). So a present-but-wrong entry lowers both precision and
+recall. Nested object/array children are graded at their own node and are not
+counted here.
+
+A ``verdicts`` argument is a list of ``(score, threshold, weight)`` from
+``structured_eval.metrics.utils.verdicts``. Each entry contributes its
+``weight`` (``1.0`` by default → plain counts) rather than a flat 1: in
+``GradingMode.HARD`` an entry is a TP when ``score >= threshold`` (counts its
+weight); in ``GradingMode.SOFT`` it contributes ``weight * score`` (threshold
+ignored). ``missing_weight`` / ``spurious_weight`` are the summed weights of the
+FN / FP entries (counts when uniform). How those weights are derived is the
+caller's choice (see ``WeightMode``).
+"""
+
+from __future__ import annotations
+
+from enum import StrEnum
+
+
+class GradingMode(StrEnum):
+    """How a verdict counts toward true positives."""
+
+    HARD = "hard"  # threshold gate: TP iff score >= threshold (counts its weight)
+    SOFT = "soft"  # graded: weight * score contributes, no threshold
+
+
+class WeightMode(StrEnum):
+    """How a node's children contribute to its weighted aggregate.
+
+    Extensible: more strategies (e.g. only first-level weights, or uniform per
+    level) can be added without touching the arithmetic below.
+    """
+
+    NONE = "none"  # ignore configured weights — every child counts 1.0
+    PROPORTIONAL = "proportional"  # weight each child by its configured ``weight``
+
+
+def prf_counts(
+    verdicts: list[tuple[float, float, float]],
+    missing_weight: float,
+    spurious_weight: float,
+    mode: GradingMode = GradingMode.HARD,
+) -> tuple[float, float, float]:
+    """Return weighted ``(tp, predicted, expected)``; uniform weights → counts."""
+    matched_weight = sum(weight for _, _, weight in verdicts)
+    predicted = matched_weight + spurious_weight
+    expected = matched_weight + missing_weight
+    if mode == GradingMode.SOFT:
+        tp = sum(weight * score for score, _, weight in verdicts)
+    else:
+        tp = sum(weight for score, threshold, weight in verdicts if score >= threshold)
+    return tp, predicted, expected
+
+
+def precision(tp: float, predicted: float, expected: float) -> float:
+    if predicted == 0:
+        return 1.0 if expected == 0 else 0.0  # empty object is vacuously precise
+    return tp / predicted
+
+
+def recall(tp: float, predicted: float, expected: float) -> float:
+    if expected == 0:
+        return 1.0 if predicted == 0 else 0.0
+    return tp / expected
+
+
+def f1(p: float, r: float) -> float:
+    return 2 * p * r / (p + r) if (p + r) else 0.0