aid-runtime-python 0.1.39__py3-none-any.whl

aid_runtime/evals/metrics.py

@@ -0,0 +1,335 @@
+"""Eval metrics.
+
+A metric turns ``(case, prediction)`` into a score in ``[0, 1]``. Each metric
+declares whether it is *deterministic*, and the gate uses that flag to refuse to
+be built on a metric that calls a model. This is the enforcement point for a
+Phase 0 decision: implement ``llm-judge``, never make CI depend on it.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+from collections.abc import Awaitable, Callable, Sequence
+from typing import Any, Protocol, runtime_checkable
+
+from pydantic import BaseModel, ConfigDict, Field
+
+from aid_runtime.errors import EvalError
+from aid_runtime.evals.dataset import EvalCase
+from aid_runtime.evals.predicate import evaluate_predicate
+from aid_runtime.structured import run_structured
+
+__all__ = [
+    "AssertMetric",
+    "ExactMatch",
+    "JsonSchemaValid",
+    "JudgeVerdict",
+    "LlmJudge",
+    "Metric",
+    "MetricResult",
+    "NormalizedContains",
+    "default_metrics",
+    "metric_names",
+    "normalize",
+]
+
+_WHITESPACE = re.compile(r"\s+")
+
+
+class MetricResult(BaseModel):
+    """One metric's verdict on one prediction."""
+
+    model_config = ConfigDict(frozen=True, extra="forbid")
+
+    metric: str
+    score: float = Field(ge=0.0, le=1.0)
+    passed: bool
+    detail: str = ""
+
+
+@runtime_checkable
+class Metric(Protocol):
+    """Scores a single prediction."""
+
+    @property
+    def name(self) -> str:
+        """Stable identifier used in baselines and thresholds."""
+        ...
+
+    @property
+    def deterministic(self) -> bool:
+        """Whether the same prediction always yields the same score.
+
+        A model-backed metric is not deterministic, and the gate refuses to
+        threshold on one: a CI signal that itself varies run to run cannot
+        distinguish a regression from its own noise.
+        """
+        ...
+
+    async def score(self, case: EvalCase, prediction: str) -> MetricResult: ...
+
+
+def normalize(text: str) -> str:
+    """Casefold and collapse whitespace, then strip surrounding punctuation."""
+    collapsed = _WHITESPACE.sub(" ", text.strip()).casefold()
+    return collapsed.strip(" .!?\"'")
+
+
+class ExactMatch:
+    """Normalized string equality against ``case.expected``."""
+
+    def __init__(self, *, name: str = "exact-match") -> None:
+        self._name = name
+
+    @property
+    def name(self) -> str:
+        return self._name
+
+    @property
+    def deterministic(self) -> bool:
+        return True
+
+    async def score(self, case: EvalCase, prediction: str) -> MetricResult:
+        if case.expected is None:
+            raise EvalError(f"case {case.id!r} has no expected value for {self.name}")
+        ok = normalize(prediction) == normalize(case.expected)
+        return MetricResult(
+            metric=self.name,
+            score=1.0 if ok else 0.0,
+            passed=ok,
+            detail="" if ok else f"expected {case.expected!r}",
+        )
+
+
+class NormalizedContains:
+    """Whether the normalized prediction contains the normalized expectation."""
+
+    def __init__(self, *, name: str = "contains") -> None:
+        self._name = name
+
+    @property
+    def name(self) -> str:
+        return self._name
+
+    @property
+    def deterministic(self) -> bool:
+        return True
+
+    async def score(self, case: EvalCase, prediction: str) -> MetricResult:
+        if case.expected is None:
+            raise EvalError(f"case {case.id!r} has no expected value for {self.name}")
+        ok = normalize(case.expected) in normalize(prediction)
+        return MetricResult(
+            metric=self.name,
+            score=1.0 if ok else 0.0,
+            passed=ok,
+            detail="" if ok else f"missing {case.expected!r}",
+        )
+
+
+class JsonSchemaValid:
+    """Whether the prediction parses as JSON and satisfies the case's schema.
+
+    The schema lives in the case's metadata under ``json_schema`` as a JSON
+    string, which keeps datasets in a single file format.
+    """
+
+    def __init__(self, *, name: str = "json-valid") -> None:
+        self._name = name
+
+    @property
+    def name(self) -> str:
+        return self._name
+
+    @property
+    def deterministic(self) -> bool:
+        return True
+
+    async def score(self, case: EvalCase, prediction: str) -> MetricResult:
+        schema = case.metadata.get("json_schema")
+        try:
+            parsed = json.loads(prediction)
+        except json.JSONDecodeError as exc:
+            return MetricResult(
+                metric=self.name, score=0.0, passed=False, detail=f"invalid JSON: {exc.msg}"
+            )
+        if schema is None:
+            return MetricResult(metric=self.name, score=1.0, passed=True, detail="parsed")
+
+        wanted = json.loads(schema)
+        problems = _schema_problems(wanted, parsed, path="$")
+        return MetricResult(
+            metric=self.name,
+            score=1.0 if not problems else 0.0,
+            passed=not problems,
+            detail="; ".join(problems),
+        )
+
+
+class AssertMetric:
+    """Whether a declared predicate holds for the case and its prediction.
+
+    The condition arrives as the IR's predicate AST -- the very tree the spec
+    layer parsed and checked -- so this metric needs no parser and cannot
+    disagree with the one that accepted the spec. A prediction that makes the
+    predicate false scores 0.0 and reports the spec's own ``message`` verbatim,
+    because the point of letting the author write the message is that a failure
+    quotes them.
+
+    A predicate that cannot be decided raises instead of scoring. See
+    ``aid_runtime.evals.predicate`` for why that distinction is load-bearing.
+    """
+
+    def __init__(self, *, expr: Any, message: str, name: str = "assert") -> None:
+        self._expr = expr
+        self._message = message
+        self._name = name
+
+    @property
+    def name(self) -> str:
+        return self._name
+
+    @property
+    def deterministic(self) -> bool:
+        return True
+
+    async def score(self, case: EvalCase, prediction: str) -> MetricResult:
+        ok = evaluate_predicate(self._expr, case=case, prediction=prediction)
+        return MetricResult(
+            metric=self.name,
+            score=1.0 if ok else 0.0,
+            passed=ok,
+            detail="" if ok else self._message,
+        )
+
+
+class JudgeVerdict(BaseModel):
+    """The structured output an LLM judge must return."""
+
+    model_config = ConfigDict(frozen=True, extra="forbid")
+
+    score: float = Field(ge=0.0, le=1.0)
+    reason: str = ""
+
+
+JudgeCall = Callable[[str], Awaitable[str]]
+
+
+class LlmJudge:
+    """Scores a prediction with a model, as a *metric only*.
+
+    ``deterministic`` is ``False`` and must stay that way. Using this metric for a
+    CI threshold would make the gate's own verdict probabilistic; the gate raises
+    rather than allow it. It is implemented because a judge is the right way to
+    explore quality locally and to produce a second opinion next to a
+    deterministic gate -- not because it can be trusted as the gate.
+    """
+
+    def __init__(
+        self,
+        call: JudgeCall,
+        *,
+        rubric: str,
+        name: str = "llm-judge",
+        retries: int = 2,
+    ) -> None:
+        self._call = call
+        self._rubric = rubric
+        self._name = name
+        self._retries = retries
+
+    @property
+    def name(self) -> str:
+        return self._name
+
+    @property
+    def deterministic(self) -> bool:
+        return False
+
+    async def score(self, case: EvalCase, prediction: str) -> MetricResult:
+        prompt = self._build_prompt(case, prediction)
+        try:
+            verdict = await run_structured(self._call, JudgeVerdict, prompt, retries=self._retries)
+        except Exception as exc:
+            return MetricResult(
+                metric=self.name, score=0.0, passed=False, detail=f"judge failed: {exc}"
+            )
+        return MetricResult(
+            metric=self.name,
+            score=verdict.score,
+            passed=verdict.score >= 0.5,
+            detail=verdict.reason,
+        )
+
+    def _build_prompt(self, case: EvalCase, prediction: str) -> str:
+        expected = case.expected if case.expected is not None else "<not provided>"
+        return (
+            f"{self._rubric}\n\n"
+            f"Input: {case.input}\n"
+            f"Reference: {expected}\n"
+            f"Candidate: {prediction}\n\n"
+            'Reply with JSON: {"score": <0.0-1.0>, "reason": "<short>"}'
+        )
+
+
+def default_metrics() -> tuple[Metric, ...]:
+    """The deterministic gate metrics: everything that needs no model call."""
+    return (ExactMatch(), NormalizedContains(), JsonSchemaValid())
+
+
+def metric_names(metrics: Sequence[Metric]) -> tuple[str, ...]:
+    return tuple(metric.name for metric in metrics)
+
+
+def _schema_problems(schema: Any, value: Any, *, path: str, prefix: str = "") -> list[str]:
+    """Check the JSON Schema subset a dataset realistically uses."""
+    problems: list[str] = []
+    if not isinstance(schema, dict):
+        return problems
+
+    declared = schema.get("type")
+    if isinstance(declared, str) and not _type_ok(value, declared):
+        problems.append(f"{prefix}{path}: expected {declared}, got {type(value).__name__}")
+
+    if isinstance(value, dict):
+        for name in schema.get("required") or []:
+            if name not in value:
+                problems.append(f"{prefix}{path}: missing required property {name!r}")
+        properties = schema.get("properties")
+        if isinstance(properties, dict):
+            for name, subschema in properties.items():
+                if name in value:
+                    problems.extend(
+                        _schema_problems(
+                            subschema, value[name], path=f"{path}.{name}", prefix=prefix
+                        )
+                    )
+
+    if isinstance(value, list):
+        items = schema.get("items")
+        if isinstance(items, dict):
+            for index, item in enumerate(value):
+                problems.extend(
+                    _schema_problems(items, item, path=f"{path}[{index}]", prefix=prefix)
+                )
+
+    return problems
+
+
+def _type_ok(value: Any, declared: str) -> bool:
+    if declared == "object":
+        return isinstance(value, dict)
+    if declared == "array":
+        return isinstance(value, list)
+    if declared == "string":
+        return isinstance(value, str)
+    if declared == "integer":
+        return isinstance(value, int) and not isinstance(value, bool)
+    if declared == "number":
+        return isinstance(value, (int, float)) and not isinstance(value, bool)
+    if declared == "boolean":
+        return isinstance(value, bool)
+    if declared == "null":
+        return value is None
+    return True

aid_runtime/evals/predicate.py

@@ -0,0 +1,211 @@
+"""The PredicateExpr evaluator an ``assert`` metric runs.
+
+The AST is data, not code: the IR parses and validates it, the generator emits it as
+a plain literal, and this module walks it. There is no ``eval`` and no ``exec``
+anywhere on that path, so a predicate can only ever do what this file implements --
+read the prediction, the case id and one metadata key, call four total string
+builtins, and compare. Anything else is a node kind this walker does not know, and
+an unknown node raises rather than passing.
+
+Raising matters more than it looks. A false assertion and an *undecidable* one are
+different outcomes: ``false`` is a legitimate result the gate must see, while a
+missing metadata key or a string where a number belongs means the eval itself is
+misconfigured. Returning ``False`` for either would report a broken eval as a
+failing candidate, and returning ``True`` would hide it, so both raise ``EvalError``.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from aid_runtime.errors import EvalError
+from aid_runtime.evals.dataset import EvalCase
+
+__all__ = ["evaluate_predicate", "type_name"]
+
+#: A value a predicate can carry. Numbers are normalized to ``float`` so that a JSON
+#: literal ``3`` and ``length(...)`` compare equal; booleans are kept apart from
+#: numbers on purpose, because ``1 == true`` is false in the other runtime and a
+#: grammar that answered differently in each language would be worse than no grammar.
+PredicateValue = float | str | bool
+
+_ORDERING = frozenset({"<", "<=", ">", ">="})
+
+
+def type_name(value: Any) -> str:
+    """The name a diagnostic uses for a value's type.
+
+    Booleans are checked before numbers because ``bool`` subclasses ``int`` in
+    Python and would otherwise be reported as a number.
+    """
+    if isinstance(value, bool):
+        return "boolean"
+    if isinstance(value, (int, float)):
+        return "number"
+    if isinstance(value, str):
+        return "string"
+    return type(value).__name__
+
+
+def evaluate_predicate(node: Any, *, case: EvalCase, prediction: str) -> bool:
+    """Evaluate ``node`` against one case and prediction, or raise if it cannot be.
+
+    A non-boolean result is an error rather than a truthiness test: the expression's
+    type is fixed at build time, so a non-boolean here means the AST and the walker
+    disagree, and guessing which one is right is exactly the kind of guess that makes
+    a gate untrustworthy.
+    """
+    value = _value(node, case=case, prediction=prediction)
+    if not isinstance(value, bool):
+        raise EvalError(f"predicate expected a boolean, got {type_name(value)}")
+    return value
+
+
+def _value(node: Any, *, case: EvalCase, prediction: str) -> PredicateValue:
+    if not isinstance(node, dict):
+        raise EvalError(f"predicate node must be an object, got {type(node).__name__}")
+    kind = node.get("kind")
+    match kind:
+        case "string":
+            return _literal(node, "string")
+        case "number":
+            return _number(node)
+        case "boolean":
+            return _boolean_literal(node)
+        case "invalid":
+            # Unreachable through a built metric -- the build drops an invalid
+            # expression -- but a hand-written AST must not become a silent pass.
+            raise EvalError(f"invalid predicate: {node.get('reason', 'unknown reason')}")
+        case "path":
+            return _path(node, case=case, prediction=prediction)
+        case "call":
+            return _call(node, case=case, prediction=prediction)
+        case "comparison":
+            return _compare(node, case=case, prediction=prediction)
+        case "not":
+            operand = _boolean(node, "operand", case=case, prediction=prediction)
+            return not operand
+        case "and":
+            left = _boolean(node, "left", case=case, prediction=prediction)
+            right = _boolean(node, "right", case=case, prediction=prediction)
+            # Both sides are always evaluated; see the module docstring of the
+            # TypeScript mirror for why short-circuiting would hide a misconfiguration.
+            return left and right
+        case "or":
+            left = _boolean(node, "left", case=case, prediction=prediction)
+            right = _boolean(node, "right", case=case, prediction=prediction)
+            return left or right
+    # Deliberately outside the match, so an unknown kind is a raise rather than a
+    # fall-through that would return None and read as falsy.
+    raise EvalError(f"unknown predicate node {kind!r}")
+
+
+def _literal(node: dict[str, Any], wanted: str) -> str:
+    value = node.get("value")
+    if not isinstance(value, str):
+        raise EvalError(f"predicate {wanted} literal must be a {wanted}, got {type_name(value)}")
+    return value
+
+
+def _number(node: dict[str, Any]) -> float:
+    value = node.get("value")
+    # ``bool`` is an ``int`` in Python, so an explicit exclusion is what keeps
+    # ``True`` from arriving as the number 1.
+    if isinstance(value, bool) or not isinstance(value, (int, float)):
+        raise EvalError(f"predicate number literal must be a number, got {type_name(value)}")
+    return float(value)
+
+
+def _boolean_literal(node: dict[str, Any]) -> bool:
+    value = node.get("value")
+    if not isinstance(value, bool):
+        raise EvalError(f"predicate boolean literal must be a boolean, got {type_name(value)}")
+    return value
+
+
+def _path(node: dict[str, Any], *, case: EvalCase, prediction: str) -> str:
+    path = node.get("path")
+    if path == "prediction":
+        return prediction
+    if path == "case.id":
+        return case.id
+    if path == "case.metadata":
+        name = node.get("name")
+        if not isinstance(name, str):
+            raise EvalError("case.metadata path must name a key")
+        if name not in case.metadata:
+            raise EvalError(f"unbound case.metadata key '{name}'")
+        return case.metadata[name]
+    raise EvalError(f"unknown predicate path {path!r}")
+
+
+def _call(node: dict[str, Any], *, case: EvalCase, prediction: str) -> str | float:
+    name = node.get("name")
+    args = node.get("args")
+    if not isinstance(args, list):
+        raise EvalError(f"predicate call {name!r} must carry an argument list")
+    values = [_value(arg, case=case, prediction=prediction) for arg in args]
+    if name == "length":
+        _arity(name, values, 1)
+        text = _string(name, values[0])
+        return float(len(text))
+    if name in {"contains", "startswith", "endswith"}:
+        _arity(name, values, 2)
+        text = _string(name, values[0])
+        needle = _string(name, values[1])
+        if name == "contains":
+            return needle in text
+        if name == "startswith":
+            return text.startswith(needle)
+        return text.endswith(needle)
+    raise EvalError(f"unknown predicate builtin {name!r}")
+
+
+def _arity(name: str, values: list[PredicateValue], wanted: int) -> None:
+    if len(values) != wanted:
+        raise EvalError(f"{name}() takes {wanted} argument(s), got {len(values)}")
+
+
+def _string(name: str, value: PredicateValue) -> str:
+    if not isinstance(value, str):
+        raise EvalError(f"{name}() requires strings, got {type_name(value)}")
+    return value
+
+
+def _compare(node: dict[str, Any], *, case: EvalCase, prediction: str) -> bool:
+    op = node.get("op")
+    left = _value(node.get("left"), case=case, prediction=prediction)
+    right = _value(node.get("right"), case=case, prediction=prediction)
+    if op in _ORDERING:
+        if (
+            isinstance(left, bool)
+            or isinstance(right, bool)
+            or not isinstance(left, (int, float))
+            or not isinstance(right, (int, float))
+        ):
+            raise EvalError(
+                f"ordering comparison {op!r} requires two numbers, "
+                f"got {type_name(left)} and {type_name(right)}"
+            )
+        if op == "<":
+            return float(left) < float(right)
+        if op == "<=":
+            return float(left) <= float(right)
+        if op == ">":
+            return float(left) > float(right)
+        return float(left) >= float(right)
+    if op not in {"==", "!="}:
+        raise EvalError(f"unknown predicate operator {op!r}")
+    if type_name(left) != type_name(right):
+        raise EvalError(
+            f"type mismatch: cannot apply {op!r} to {type_name(left)} and {type_name(right)}"
+        )
+    equal = left == right
+    return equal if op == "==" else not equal
+
+
+def _boolean(node: dict[str, Any], key: str, *, case: EvalCase, prediction: str) -> bool:
+    value = _value(node.get(key), case=case, prediction=prediction)
+    if not isinstance(value, bool):
+        raise EvalError(f"predicate expected a boolean, got {type_name(value)}")
+    return value