evalproof 0.2.0__py3-none-any.whl

evalproof/scorers/deterministic.py

@@ -0,0 +1,227 @@
+"""Scorers that need no model: string, pattern, schema and numeric checks."""
+
+from __future__ import annotations
+
+import json
+import math
+import re
+from typing import Any
+
+from evalproof.models import Case, Score, SystemOutput
+
+NUMBER = re.compile(r"[-+]?(?:\d{1,3}(?:,\d{3})+|\d+)(?:\.\d+)?")
+
+
+def normalize(text: str) -> str:
+    text = text.strip().lower()
+    text = re.sub(r"\s+", " ", text)
+    return text.strip(" .!?;:\"'`")
+
+
+def _case_list(case: Case, field: str | None) -> list[str]:
+    if not field:
+        return []
+    value = case.field(field)
+    if value is None:
+        return []
+    if isinstance(value, str):
+        return [value]
+    return [str(v) for v in value]
+
+
+def _binary(ok: bool, reason: str | None = None) -> Score:
+    return Score(value=1.0 if ok else 0.0, passed=ok, reason=reason)
+
+
+class ExactMatch:
+    def __init__(self, name: str, field: str = "reference", normalize: bool = True) -> None:
+        self.name, self.field, self.norm = name, field, normalize
+
+    def score(self, case: Case, out: SystemOutput) -> Score:
+        expected = case.field(self.field)
+        if expected is None:
+            return Score(value=None, reason=f"case has no {self.field!r}")
+        got, want = out.output, str(expected)
+        if self.norm:
+            got, want = normalize(got), normalize(want)
+        return _binary(got == want)
+
+
+class Contains:
+    """Substring checks. ``all_of`` must all appear; ``any_of`` needs at least one."""
+
+    def __init__(
+        self,
+        name: str,
+        any_of: list[str] | None = None,
+        all_of: list[str] | None = None,
+        any_of_field: str | None = None,
+        all_of_field: str | None = None,
+        case_sensitive: bool = False,
+    ) -> None:
+        self.name = name
+        self.any_of, self.all_of = any_of or [], all_of or []
+        self.any_of_field, self.all_of_field = any_of_field, all_of_field
+        self.case_sensitive = case_sensitive
+
+    def score(self, case: Case, out: SystemOutput) -> Score:
+        text = out.output if self.case_sensitive else out.output.lower()
+
+        def fold(s: str) -> str:
+            return s if self.case_sensitive else s.lower()
+
+        any_of = self.any_of + _case_list(case, self.any_of_field)
+        all_of = self.all_of + _case_list(case, self.all_of_field)
+        if not any_of and not all_of:
+            return Score(value=None, reason="nothing to look for")
+        missing = [s for s in all_of if fold(s) not in text]
+        any_ok = not any_of or any(fold(s) in text for s in any_of)
+        ok = not missing and any_ok
+        reason = None if ok else f"missing {missing}" if missing else "none of any_of found"
+        return _binary(ok, reason)
+
+
+class Regex:
+    """``pattern``/``any_of_field`` must match; ``none_of``/``none_of_field`` must not."""
+
+    def __init__(
+        self,
+        name: str,
+        pattern: str | None = None,
+        any_of_field: str | None = None,
+        none_of: list[str] | None = None,
+        none_of_field: str | None = None,
+        flags: str = "i",
+    ) -> None:
+        self.name = name
+        self.patterns = [pattern] if pattern else []
+        self.any_of_field = any_of_field
+        self.none_of = none_of or []
+        self.none_of_field = none_of_field
+        self.flags = (re.IGNORECASE if "i" in flags else 0) | (re.MULTILINE if "m" in flags else 0)
+        self.flags |= re.DOTALL if "s" in flags else 0
+
+    def score(self, case: Case, out: SystemOutput) -> Score:
+        accept = self.patterns + _case_list(case, self.any_of_field)
+        reject = self.none_of + _case_list(case, self.none_of_field)
+        if not accept:
+            return Score(value=None, reason="no accept pattern")
+        hit = next((p for p in accept if re.search(p, out.output, self.flags)), None)
+        bad = next((p for p in reject if re.search(p, out.output, self.flags)), None)
+        ok = hit is not None and bad is None
+        reason = None if ok else f"rejected by {bad!r}" if bad else "no accept pattern matched"
+        return _binary(ok, reason)
+
+
+def _check(instance: Any, schema: dict[str, Any], path: str = "$") -> list[str]:
+    """A deliberately small JSON Schema subset: type, enum, required, properties, items."""
+    errors: list[str] = []
+    types = {
+        "object": dict,
+        "array": list,
+        "string": str,
+        "boolean": bool,
+        "null": type(None),
+    }
+    expected = schema.get("type")
+    if expected == "number":
+        if not isinstance(instance, int | float) or isinstance(instance, bool):
+            return [f"{path}: expected number"]
+    elif expected == "integer":
+        if not isinstance(instance, int) or isinstance(instance, bool):
+            return [f"{path}: expected integer"]
+    elif expected in types and not isinstance(instance, types[expected]):
+        return [f"{path}: expected {expected}"]
+    if "enum" in schema and instance not in schema["enum"]:
+        errors.append(f"{path}: {instance!r} not in enum")
+    if isinstance(instance, dict):
+        for key in schema.get("required", []):
+            if key not in instance:
+                errors.append(f"{path}: missing required {key!r}")
+        for key, sub in schema.get("properties", {}).items():
+            if key in instance:
+                errors.extend(_check(instance[key], sub, f"{path}.{key}"))
+    if isinstance(instance, list) and "items" in schema:
+        for i, item in enumerate(instance):
+            errors.extend(_check(item, schema["items"], f"{path}[{i}]"))
+    return errors
+
+
+def extract_json(text: str) -> Any:
+    """Parse JSON from a model reply that may wrap it in prose or a code fence."""
+    text = re.sub(r"<think>.*?</think>", "", text, flags=re.DOTALL).strip()
+    fence = re.search(r"```(?:json)?\s*(.*?)```", text, re.DOTALL)
+    if fence:
+        text = fence.group(1).strip()
+    try:
+        return json.loads(text)
+    except json.JSONDecodeError:
+        pass
+    start = text.find("{")
+    while start != -1:
+        depth = 0
+        for i in range(start, len(text)):
+            if text[i] == "{":
+                depth += 1
+            elif text[i] == "}":
+                depth -= 1
+                if depth == 0:
+                    try:
+                        return json.loads(text[start : i + 1])
+                    except json.JSONDecodeError:
+                        break
+        start = text.find("{", start + 1)
+    raise ValueError("no JSON object found")
+
+
+class JsonSchema:
+    def __init__(
+        self, name: str, schema: dict[str, Any] | None = None, schema_field: str | None = None
+    ) -> None:
+        self.name, self.schema, self.schema_field = name, schema, schema_field
+
+    def score(self, case: Case, out: SystemOutput) -> Score:
+        schema = self.schema or (case.field(self.schema_field) if self.schema_field else None)
+        if not schema:
+            return Score(value=None, reason="no schema")
+        try:
+            data = extract_json(out.output)
+        except ValueError:
+            return _binary(False, "output is not JSON")
+        errors = _check(data, schema)
+        return _binary(not errors, "; ".join(errors[:3]) or None)
+
+
+class NumericTolerance:
+    def __init__(
+        self,
+        name: str,
+        field: str = "reference",
+        abs_tol: float = 0.0,
+        rel_tol: float = 0.0,
+        pick: str = "first",
+    ) -> None:
+        if pick not in {"first", "last", "any"}:
+            raise ValueError("pick must be first, last or any")
+        self.name, self.field, self.abs_tol, self.rel_tol, self.pick = (
+            name,
+            field,
+            abs_tol,
+            rel_tol,
+            pick,
+        )
+
+    def score(self, case: Case, out: SystemOutput) -> Score:
+        raw = case.field(self.field)
+        if raw is None:
+            return Score(value=None, reason=f"case has no {self.field!r}")
+        expected = float(str(raw).replace(",", ""))
+        found = [float(n.replace(",", "")) for n in NUMBER.findall(out.output)]
+        if not found:
+            return _binary(False, "no number in output")
+        candidates = {"first": found[:1], "last": found[-1:], "any": found}[self.pick]
+        ok = any(
+            math.isclose(x, expected, rel_tol=self.rel_tol, abs_tol=self.abs_tol)
+            for x in candidates
+        )
+        return _binary(ok, None if ok else f"expected {expected}, found {candidates}")

evalproof/scorers/judge.py

@@ -0,0 +1,187 @@
+"""LLM-as-judge scorers.
+
+A judge is a classifier, so it is treated like one: a strict JSON verdict, a parse
+retry instead of a guess, ``undecided`` instead of a silent pass, and — in
+``evalproof.calibration`` — measured error rates instead of trust.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Literal
+
+from evalproof.config import GenerationParams
+from evalproof.models import Case, Score, SystemOutput
+from evalproof.providers import Message, Provider, ProviderError
+from evalproof.scorers.deterministic import extract_json
+
+DEFAULT_RUBRIC = (
+    "Pass if the response answers the question correctly and is consistent with the "
+    "reference answer. Fail if it is wrong, contradicts the reference, invents facts the "
+    "reference does not support, or does not answer. Wording and length do not matter."
+)
+
+JUDGE_SYSTEM = (
+    "You are a strict, consistent evaluator. You grade one response at a time against a "
+    "rubric and a reference answer. Reply with a single JSON object and nothing else."
+)
+
+JUDGE_TEMPLATE = """Rubric:
+{rubric}
+
+Question:
+{question}
+
+Reference answer:
+{reference}
+
+Response to grade:
+{response}
+
+Reply with exactly this JSON shape:
+{{"reason": "<one sentence>", "pass": true or false, "score": <number from 0 to 1>}}"""
+
+RETRY_NUDGE = (
+    "That was not a valid JSON object of the requested shape. Reply with only the JSON "
+    'object: {"reason": "...", "pass": true or false, "score": 0 to 1}'
+)
+
+
+def _question(case: Case) -> str:
+    if isinstance(case.input, dict):
+        return str(case.input.get("question") or case.input)
+    return case.input
+
+
+def parse_verdict(text: str) -> tuple[bool, float, str]:
+    data = extract_json(text)
+    if not isinstance(data, dict) or "pass" not in data:
+        raise ValueError("verdict needs a 'pass' field")
+    passed = data["pass"]
+    if isinstance(passed, str):
+        passed = passed.strip().lower() in {"true", "yes", "pass"}
+    if not isinstance(passed, bool):
+        raise ValueError("'pass' must be a boolean")
+    raw_score = data.get("score", 1.0 if passed else 0.0)
+    try:
+        score = min(1.0, max(0.0, float(raw_score)))
+    except (TypeError, ValueError) as exc:
+        raise ValueError("'score' must be a number") from exc
+    return passed, score, str(data.get("reason", ""))[:500]
+
+
+class LLMJudge:
+    def __init__(
+        self,
+        name: str,
+        provider: Provider,
+        params: GenerationParams,
+        rubric: str = DEFAULT_RUBRIC,
+        max_retries: int = 1,
+    ) -> None:
+        self.name = name
+        self.provider = provider
+        self.params = params
+        self.rubric = rubric
+        self.max_retries = max_retries
+
+    def messages(self, case: Case, out: SystemOutput) -> list[Message]:
+        prompt = JUDGE_TEMPLATE.format(
+            rubric=self.rubric.strip(),
+            question=_question(case),
+            reference=case.reference or "(none provided)",
+            response=out.output or "(empty)",
+        )
+        return [Message(role="system", content=JUDGE_SYSTEM), Message(role="user", content=prompt)]
+
+    def score(self, case: Case, out: SystemOutput) -> Score:
+        if out.error:
+            return Score(value=0.0, passed=False, reason=f"system error: {out.error[:200]}")
+        messages = self.messages(case, out)
+        tokens = 0
+        last_text = ""
+        for _attempt in range(self.max_retries + 1):
+            try:
+                resp = self.provider.chat(messages, self.params)
+            except ProviderError as exc:
+                return Score(value=None, reason=f"judge error: {exc}", judge_tokens=tokens)
+            tokens += resp.tokens_in + resp.tokens_out
+            last_text = resp.text
+            try:
+                passed, value, reason = parse_verdict(resp.text)
+            except ValueError:
+                messages = [
+                    *messages,
+                    Message(role="assistant", content=resp.text),
+                    Message(role="user", content=RETRY_NUDGE),
+                ]
+                continue
+            return Score(value=value, passed=passed, reason=reason, judge_tokens=tokens)
+        return Score(
+            value=None,
+            reason=f"unparseable judge reply: {last_text[:200]!r}",
+            judge_tokens=tokens,
+        )
+
+
+PAIRWISE_TEMPLATE = """Question:
+{question}
+
+Reference answer:
+{reference}
+
+Response A:
+{a}
+
+Response B:
+{b}
+
+Which response answers the question better, judged against the reference? Reply with
+exactly this JSON shape: {{"reason": "<one sentence>", "winner": "A" or "B" or "tie"}}"""
+
+Winner = Literal["a", "b", "tie"]
+
+
+@dataclass(frozen=True)
+class PairwiseResult:
+    winner: Winner
+    consistent: bool
+    first: str  # verdict with (a, b) order, in a/b terms
+    second: str  # verdict with (b, a) order, mapped back to a/b terms
+
+
+def _pairwise_once(
+    provider: Provider, params: GenerationParams, case: Case, first: str, second: str
+) -> str:
+    prompt = PAIRWISE_TEMPLATE.format(
+        question=_question(case), reference=case.reference or "(none provided)", a=first, b=second
+    )
+    resp = provider.chat(
+        [Message(role="system", content=JUDGE_SYSTEM), Message(role="user", content=prompt)],
+        params,
+    )
+    data = extract_json(resp.text)
+    winner = str(data.get("winner", "")).strip().lower() if isinstance(data, dict) else ""
+    if winner not in {"a", "b", "tie"}:
+        raise ValueError(f"bad pairwise verdict: {resp.text[:120]!r}")
+    return winner
+
+
+def pairwise(
+    provider: Provider,
+    params: GenerationParams,
+    case: Case,
+    a: str,
+    b: str,
+    swap: bool = True,
+) -> PairwiseResult:
+    """Compare two outputs. With ``swap`` the judge sees both orders; a verdict that flips
+    with position is reported as a tie and flagged inconsistent (position bias)."""
+    first = _pairwise_once(provider, params, case, a, b)
+    if not swap:
+        return PairwiseResult(winner=first, consistent=True, first=first, second=first)  # type: ignore[arg-type]
+    raw = _pairwise_once(provider, params, case, b, a)
+    second = {"a": "b", "b": "a", "tie": "tie"}[raw]
+    consistent = first == second
+    winner: Winner = first if consistent else "tie"  # type: ignore[assignment]
+    return PairwiseResult(winner=winner, consistent=consistent, first=first, second=second)

evalproof/scorers/retrieval.py

@@ -0,0 +1,64 @@
+"""Retrieval metrics over ``SystemOutput.retrieved`` against a case's ``relevant`` ids."""
+
+from __future__ import annotations
+
+import math
+
+from evalproof.models import Case, Score, SystemOutput
+
+
+def _relevance(case: Case, field: str) -> dict[str, float]:
+    value = case.field(field)
+    if value is None:
+        return {}
+    if isinstance(value, dict):
+        return {str(k): float(v) for k, v in value.items()}
+    return {str(v): 1.0 for v in value}
+
+
+class _RetrievalScorer:
+    def __init__(
+        self, name: str, k: int = 10, field: str = "relevant", pass_threshold: float | None = None
+    ) -> None:
+        if k < 1:
+            raise ValueError("k must be >= 1")
+        self.name, self.k, self.field, self.pass_threshold = name, k, field, pass_threshold
+
+    def _finish(self, value: float) -> Score:
+        passed = None if self.pass_threshold is None else value >= self.pass_threshold
+        return Score(value=value, passed=passed)
+
+    def metric(self, rel: dict[str, float], ranked: list[str]) -> float:
+        raise NotImplementedError
+
+    def score(self, case: Case, out: SystemOutput) -> Score:
+        rel = _relevance(case, self.field)
+        if not rel:
+            return Score(value=None, reason=f"case has no {self.field!r}")
+        return self._finish(self.metric(rel, out.retrieved[: self.k]))
+
+
+class RecallAtK(_RetrievalScorer):
+    def metric(self, rel: dict[str, float], ranked: list[str]) -> float:
+        relevant = {d for d, g in rel.items() if g > 0}
+        return len(relevant & set(ranked)) / len(relevant) if relevant else 0.0
+
+
+class MRR(_RetrievalScorer):
+    def metric(self, rel: dict[str, float], ranked: list[str]) -> float:
+        for i, doc in enumerate(ranked, start=1):
+            if rel.get(doc, 0) > 0:
+                return 1.0 / i
+        return 0.0
+
+
+class NDCG(_RetrievalScorer):
+    """nDCG@k with exponential gain ``2^rel - 1``."""
+
+    def metric(self, rel: dict[str, float], ranked: list[str]) -> float:
+        def dcg(gains: list[float]) -> float:
+            return sum((2**g - 1) / math.log2(i + 2) for i, g in enumerate(gains))
+
+        actual = dcg([rel.get(doc, 0.0) for doc in ranked])
+        ideal = dcg(sorted(rel.values(), reverse=True)[: self.k])
+        return actual / ideal if ideal > 0 else 0.0

evalproof/stats.py

@@ -0,0 +1,145 @@
+"""Small, readable statistics. numpy and the standard library only — no SciPy.
+
+Every function here has a known-value test in ``tests/test_stats.py``.
+"""
+
+from __future__ import annotations
+
+import math
+from collections.abc import Callable, Sequence
+from dataclasses import dataclass
+from fractions import Fraction
+from statistics import NormalDist
+
+import numpy as np
+from numpy.typing import NDArray
+
+FloatArray = NDArray[np.float64]
+
+
+def z_for(alpha: float) -> float:
+    """Two-sided critical value: ``z_for(0.05)`` ≈ 1.959964."""
+    if not 0 < alpha < 1:
+        raise ValueError("alpha must be in (0, 1)")
+    return NormalDist().inv_cdf(1 - alpha / 2)
+
+
+def wilson(k: int, n: int, alpha: float = 0.05) -> tuple[float, float]:
+    """Wilson score interval for a binomial proportion. Behaves at 0/n and n/n."""
+    if n <= 0:
+        return (0.0, 1.0)
+    if not 0 <= k <= n:
+        raise ValueError("need 0 <= k <= n")
+    z = z_for(alpha)
+    p = k / n
+    denom = 1 + z * z / n
+    centre = (p + z * z / (2 * n)) / denom
+    half = z * math.sqrt(p * (1 - p) / n + z * z / (4 * n * n)) / denom
+    # At k=0 the lower bound is exactly 0 (centre == half), and at k=n the upper
+    # bound is exactly 1. Computing them by subtraction leaves platform-dependent
+    # rounding (2.8e-17 on x86_64 Linux, 0.0 on arm64 macOS), so pin them.
+    lo = 0.0 if k == 0 else max(0.0, centre - half)
+    hi = 1.0 if k == n else min(1.0, centre + half)
+    return (lo, hi)
+
+
+def bootstrap_ci(
+    values: Sequence[float] | FloatArray,
+    stat: Callable[[FloatArray], float] = lambda x: float(np.mean(x)),
+    *,
+    n_boot: int = 5000,
+    alpha: float = 0.05,
+    seed: int = 0,
+) -> tuple[float, float]:
+    """Percentile bootstrap interval, seeded so a report is reproducible."""
+    arr = np.asarray(values, dtype=np.float64)
+    if arr.size == 0:
+        raise ValueError("no values")
+    rng = np.random.default_rng(seed)
+    idx = rng.integers(0, arr.size, size=(n_boot, arr.size))
+    stats = np.array([stat(arr[row]) for row in idx])
+    lo, hi = np.quantile(stats, [alpha / 2, 1 - alpha / 2])
+    return (float(lo), float(hi))
+
+
+@dataclass(frozen=True)
+class PairedDelta:
+    """``delta = mean(candidate) - mean(baseline)`` over paired cases."""
+
+    delta: float
+    ci_low: float
+    ci_high: float
+    p_value: float
+    n: int
+
+
+def paired_bootstrap(
+    baseline: Sequence[float] | FloatArray,
+    candidate: Sequence[float] | FloatArray,
+    *,
+    n_boot: int = 5000,
+    alpha: float = 0.05,
+    seed: int = 0,
+) -> PairedDelta:
+    """Resample *case indices*, so each replicate keeps baseline and candidate paired.
+
+    The two-sided p-value is the percentile-bootstrap approximation
+    ``2 * min(P(d* <= 0), P(d* >= 0))`` — adequate for gating, and stated as such.
+    """
+    a = np.asarray(baseline, dtype=np.float64)
+    b = np.asarray(candidate, dtype=np.float64)
+    if a.shape != b.shape or a.size == 0:
+        raise ValueError("need two non-empty arrays of equal length")
+    diff = b - a
+    rng = np.random.default_rng(seed)
+    idx = rng.integers(0, diff.size, size=(n_boot, diff.size))
+    reps = diff[idx].mean(axis=1)
+    lo, hi = np.quantile(reps, [alpha / 2, 1 - alpha / 2])
+    if np.all(diff == 0):
+        p = 1.0
+    else:
+        p = float(min(1.0, 2 * min(np.mean(reps <= 0), np.mean(reps >= 0))))
+    return PairedDelta(float(diff.mean()), float(lo), float(hi), p, int(diff.size))
+
+
+def binom_cdf(k: int, n: int, p: float = 0.5) -> float:
+    """``P(X <= k)`` for ``X ~ Binomial(n, p)``, summed exactly and rounded once.
+
+    In floating point, ``comb(n, i)`` alone passes the largest float from about
+    n = 1,030, so a comparison with that many discordant pairs raised OverflowError
+    instead of returning a p-value. Integers and fractions do not overflow.
+    """
+    if k < 0:
+        return 0.0
+    if k >= n:
+        return 1.0
+    q = Fraction(p)
+    if q == Fraction(1, 2):
+        # The McNemar case: every outcome has probability 2**-n, so sum the binomial
+        # coefficients as integers and divide once (int / int is correctly rounded).
+        term = total = 1
+        for i in range(1, k + 1):
+            term = term * (n - i + 1) // i
+            total += term
+        return total / (1 << n)
+    return float(sum(math.comb(n, i) * q**i * (1 - q) ** (n - i) for i in range(k + 1)))
+
+
+def mcnemar_exact(b: int, c: int) -> float:
+    """Exact two-sided McNemar test on the discordant pairs.
+
+    ``b`` = baseline passed, candidate failed; ``c`` = baseline failed, candidate passed.
+    Only discordant pairs carry information about a difference between the two systems.
+    """
+    if b < 0 or c < 0:
+        raise ValueError("counts must be non-negative")
+    n = b + c
+    if n == 0:
+        return 1.0
+    return min(1.0, 2 * binom_cdf(min(b, c), n))
+
+
+def percentile(values: Sequence[float], q: float) -> float:
+    if not values:
+        return 0.0
+    return float(np.percentile(np.asarray(values, dtype=np.float64), q))