evalcore 0.1.0__py3-none-any.whl

evalkit/compare.py

@@ -0,0 +1,137 @@
+"""Comparison / regression engine - candidate vs baseline -> gate verdict.
+
+Generic across all consumers. Two ideas:
+
+* **Guardrails** - metrics that must hold regardless of the headline result
+  (e.g. ``false_negative_rate`` must stay under a ceiling and must not increase
+  vs. baseline). A guardrail breach is a hard ``fail``.
+* **Win metric** - the headline signal (``f1`` for ``/validation``; a pairwise
+  win-rate for judged suites). A regression beyond the band is a ``warn``.
+
+Verdict: any guardrail breach -> ``fail``; else win regressed -> ``warn``; else
+``pass``. The on-regression policy is configurable per suite.
+"""
+
+from evalkit import models
+
+_EPS = 1e-9
+
+
+def _metric(card: models.Scorecard, name: str) -> float | None:
+    found = card.metrics.get(name)
+    return found.value if found else None
+
+
+def _check_guardrail(
+    rule: dict, baseline: models.Scorecard, candidate: models.Scorecard
+) -> models.GuardrailResult:
+    metric = rule['metric']
+    cand = _metric(candidate, metric)
+    base = _metric(baseline, metric)
+    if cand is None:
+        return models.GuardrailResult(
+            metric=metric, passed=False, detail='metric absent on candidate'
+        )
+
+    problems: list[str] = []
+    if 'max' in rule and cand > rule['max'] + _EPS:
+        problems.append(f'{cand:.4f} > max {rule["max"]}')
+    if 'min' in rule and cand < rule['min'] - _EPS:
+        problems.append(f'{cand:.4f} < min {rule["min"]}')
+    if (
+        rule.get('must_not_increase')
+        and base is not None
+        and (cand > base + _EPS)
+    ):
+        problems.append(f'increased {base:.4f} -> {cand:.4f}')
+    if (
+        rule.get('must_not_decrease')
+        and base is not None
+        and (cand < base - _EPS)
+    ):
+        problems.append(f'decreased {base:.4f} -> {cand:.4f}')
+
+    if problems:
+        return models.GuardrailResult(
+            metric=metric, passed=False, detail='; '.join(problems)
+        )
+    return models.GuardrailResult(
+        metric=metric, passed=True, detail=f'{cand:.4f} ok'
+    )
+
+
+def _evaluate_win(
+    thresholds: dict, baseline: models.Scorecard, candidate: models.Scorecard
+) -> tuple[str | None, str]:
+    metric = thresholds.get('win_metric')
+    if not metric:
+        return None, 'neutral'
+    base = _metric(baseline, metric)
+    cand = _metric(candidate, metric)
+    if base is None or cand is None:
+        return metric, 'neutral'
+    higher_better = thresholds.get('win_higher_is_better', True)
+    min_delta = thresholds.get('win_min_delta', 0.0)
+    delta = cand - base if higher_better else base - cand
+    if delta > min_delta:
+        return metric, 'improved'
+    if delta < -min_delta:
+        return metric, 'regressed'
+    return metric, 'neutral'
+
+
+def compare(
+    baseline: models.Scorecard,
+    candidate: models.Scorecard,
+    thresholds: dict | None = None,
+) -> models.Comparison:
+    """Compare two scorecards and produce a gate verdict."""
+    thresholds = thresholds or {}
+
+    deltas: list[models.MetricDelta] = []
+    for metric in sorted(set(baseline.metrics) | set(candidate.metrics)):
+        base = _metric(baseline, metric)
+        cand = _metric(candidate, metric)
+        delta = cand - base if base is not None and cand is not None else None
+        deltas.append(
+            models.MetricDelta(
+                metric=metric, baseline=base, candidate=cand, delta=delta
+            )
+        )
+
+    guardrails = [
+        _check_guardrail(rule, baseline, candidate)
+        for rule in thresholds.get('guardrails', [])
+    ]
+    win_metric, win = _evaluate_win(thresholds, baseline, candidate)
+
+    breached = [g for g in guardrails if not g.passed]
+    on_regression = thresholds.get('on_regression', 'warn')
+    if breached:
+        verdict = 'fail'
+    elif win == 'regressed':
+        verdict = 'fail' if on_regression == 'fail' else 'warn'
+    else:
+        verdict = 'pass'
+
+    if breached:
+        summary = 'guardrail breach: ' + '; '.join(
+            f'{g.metric} ({g.detail})' for g in breached
+        )
+    elif win_metric:
+        summary = f'{win_metric} {win}'
+    else:
+        summary = 'no win metric configured'
+
+    return models.Comparison(
+        project=candidate.project,
+        suite=candidate.suite,
+        baseline_variant=baseline.variant.name,
+        candidate_variant=candidate.variant.name,
+        win_metric=win_metric,
+        win=win,
+        verdict=verdict,
+        deltas=deltas,
+        guardrails=guardrails,
+        summary=summary,
+    )

evalkit/graders/__init__.py

@@ -0,0 +1,18 @@
+"""Grader registry and built-in graders.
+
+Two extension shapes (see ``base``):
+
+* ``base.Grader`` - per-case: ``grade(case, output) -> [Score]`` (may be
+  async). Averaged by the runner. Used for deterministic checks (length,
+  format, regex) and LLM-judge rubric scoring.
+* ``base.AggregateGrader`` - whole-run: ``aggregate(results) -> [Score]``.
+  Used for metrics that only exist over a set (precision/recall/F1, win-rate).
+
+Importing this package registers the built-in grader types. Consumers register
+custom graders with ``base.register`` and load them via the CLI ``--plugins``
+flag, proving the generic/custom seam.
+"""
+
+from evalkit.graders import base, classification, deterministic, judge, numeric
+
+__all__ = ['base', 'classification', 'deterministic', 'judge', 'numeric']

evalkit/graders/base.py

@@ -0,0 +1,77 @@
+"""Grader protocols and the type registry.
+
+A grader spec is a plain dict from the suite config: ``{type, name, ...}``.
+``build_graders`` turns a list of specs into grader instances, split into the
+per-case and aggregate buckets the runner needs.
+"""
+
+import typing
+
+from evalkit import models
+
+
+@typing.runtime_checkable
+class Grader(typing.Protocol):
+    """Per-case grader. Scores are averaged across cases by the runner."""
+
+    name: str
+
+    def grade(
+        self, case: models.Case, output: models.Output
+    ) -> list[models.Score]: ...
+
+
+@typing.runtime_checkable
+class AggregateGrader(typing.Protocol):
+    """Whole-run grader for set-level metrics (P/R/F1, win-rate, ...)."""
+
+    name: str
+
+    def aggregate(
+        self, results: list[models.CaseResult]
+    ) -> list[models.Score]: ...
+
+
+_REGISTRY: dict[str, type] = {}
+
+
+def register(type_name: str) -> typing.Callable[[type], type]:
+    """Class decorator registering a grader under a suite-config ``type``."""
+
+    def _decorate(cls: type) -> type:
+        if type_name in _REGISTRY:
+            raise ValueError(f'grader type {type_name!r} already registered')
+        _REGISTRY[type_name] = cls
+        return cls
+
+    return _decorate
+
+
+def build_graders(
+    specs: list[dict],
+) -> tuple[list[Grader], list[AggregateGrader]]:
+    """Instantiate grader specs, partitioned into per-case and aggregate.
+
+    Each spec's ``type`` selects a registered class; remaining keys (minus
+    ``type``) are passed as keyword arguments to its constructor.
+    """
+    per_case: list[Grader] = []
+    aggregate: list[AggregateGrader] = []
+    for spec in specs:
+        spec = dict(spec)
+        type_name = spec.pop('type')
+        if type_name not in _REGISTRY:
+            raise ValueError(
+                f'unknown grader type {type_name!r}; '
+                f'known: {sorted(_REGISTRY)}'
+            )
+        grader = _REGISTRY[type_name](**spec)
+        if isinstance(grader, AggregateGrader):
+            aggregate.append(grader)
+        elif isinstance(grader, Grader):
+            per_case.append(grader)
+        else:  # pragma: no cover - defensive
+            raise TypeError(
+                f'{type_name!r} is neither Grader nor AggregateGrader'
+            )
+    return per_case, aggregate

evalkit/graders/classification.py

@@ -0,0 +1,107 @@
+"""Built-in classification (aggregate) grader.
+
+For structured-verdict suites - e.g. a moderation or intent classifier -
+where each case has a ground-truth label. Computes the set-level metrics that
+matter operationally: precision/recall/F1, and the two error rates a security
+gate lives or dies on - false-negative rate (a malicious item passed) and
+false-positive rate (a benign item blocked).
+
+The ``positive`` class is the one we care about catching (e.g. malicious +
+suspicious). Predicted and expected labels are mapped to positive/negative via
+the configured label sets, so the grader is reusable for any binary verdict.
+"""
+
+from evalkit import models, refs
+from evalkit.graders import base
+
+
+def _safe_div(numerator: float, denominator: float) -> float:
+    return numerator / denominator if denominator else 0.0
+
+
+@base.register('classification')
+class Classification:
+    """Binary precision/recall/F1 + FN/FP rates over a labeled dataset."""
+
+    def __init__(
+        self,
+        predicted_ref: str,
+        expected_ref: str,
+        positive_labels: list[str],
+        negative_labels: list[str] | None = None,
+        name: str = 'classification',
+    ):
+        self.name = name
+        self.predicted_ref = predicted_ref
+        self.expected_ref = expected_ref
+        self.positive = {label.lower() for label in positive_labels}
+        self.negative = {label.lower() for label in (negative_labels or [])}
+
+    def _is_positive(self, label) -> bool | None:
+        """Map a raw label to positive (True) / negative (False) / unknown."""
+        if label is None:
+            return None
+        token = str(label).lower()
+        if token in self.positive:
+            return True
+        if token in self.negative:
+            return False
+        # Unlisted labels default to negative so a stray verdict can't
+        # masquerade as a catch; counted via the ``errors`` metric below.
+        return False
+
+    def aggregate(
+        self, results: list[models.CaseResult]
+    ) -> list[models.Score]:
+        tp = fp = fn = tn = errors = 0
+        for result in results:
+            if result.output.error:
+                errors += 1
+                continue
+            context = {
+                'output': result.output.fields,
+                'case': result.case.model_dump(),
+                'expected': result.case.expected or {},
+                'input': result.case.input,
+            }
+            predicted = self._is_positive(
+                refs.resolve_ref(context, self.predicted_ref)
+            )
+            actual = self._is_positive(
+                refs.resolve_ref(context, self.expected_ref)
+            )
+            if predicted is None or actual is None:
+                errors += 1
+                continue
+            if actual and predicted:
+                tp += 1
+            elif actual and not predicted:
+                fn += 1
+            elif not actual and predicted:
+                fp += 1
+            else:
+                tn += 1
+
+        precision = _safe_div(tp, tp + fp)
+        recall = _safe_div(tp, tp + fn)
+        f1 = _safe_div(2 * precision * recall, precision + recall)
+        fnr = _safe_div(fn, fn + tp)
+        fpr = _safe_div(fp, fp + tn)
+        accuracy = _safe_div(tp + tn, tp + tn + fp + fn)
+
+        def agg(metric: str, value: float) -> models.Score:
+            return models.Score(
+                grader=self.name, metric=metric, value=value, kind='aggregate'
+            )
+
+        return [
+            agg('precision', precision),
+            agg('recall', recall),
+            agg('f1', f1),
+            agg('false_negative_rate', fnr),
+            agg('false_positive_rate', fpr),
+            agg('accuracy', accuracy),
+            agg('support_positive', float(tp + fn)),
+            agg('support_negative', float(tn + fp)),
+            agg('errors', float(errors)),
+        ]

evalkit/graders/deterministic.py

@@ -0,0 +1,127 @@
+"""Built-in deterministic (per-case) graders.
+
+Cheap, free, fully reproducible - the first tier. They read a field off the
+output via a ``$ref`` selector and emit a 1.0/0.0 score plus ``passed`` so the
+runner aggregates them into a pass-rate.
+"""
+
+import re
+
+from evalkit import models, refs
+from evalkit.graders import base
+
+
+def _context(case: models.Case, output: models.Output) -> dict:
+    return {
+        'input': case.input,
+        'expected': case.expected or {},
+        'output': output.fields,
+        'case': case.model_dump(),
+    }
+
+
+def _score(name: str, metric: str, case_id: str, ok: bool, detail: str):
+    return models.Score(
+        grader=name,
+        metric=metric,
+        value=1.0 if ok else 0.0,
+        passed=ok,
+        detail=detail,
+        case_id=case_id,
+        kind='per_case',
+    )
+
+
+@base.register('max_chars')
+class MaxChars:
+    """Assert a text field is at most ``maximum`` characters long."""
+
+    def __init__(self, field: str, maximum: int, name: str = 'max_chars'):
+        self.name = name
+        self.field = field
+        self.maximum = maximum
+
+    def grade(
+        self, case: models.Case, output: models.Output
+    ) -> list[models.Score]:
+        value = refs.resolve_ref(_context(case, output), self.field)
+        length = len(value) if isinstance(value, str) else 0
+        ok = length <= self.maximum
+        return [
+            _score(
+                self.name,
+                self.name,
+                case.id,
+                ok,
+                f'len={length} max={self.maximum}',
+            )
+        ]
+
+
+@base.register('regex_absent')
+class RegexAbsent:
+    """Assert a text field does NOT match ``pattern`` (e.g. no tokens)."""
+
+    def __init__(self, field: str, pattern: str, name: str = 'regex_absent'):
+        self.name = name
+        self.field = field
+        self.regex = re.compile(pattern)
+
+    def grade(
+        self, case: models.Case, output: models.Output
+    ) -> list[models.Score]:
+        value = refs.resolve_ref(_context(case, output), self.field)
+        text = value if isinstance(value, str) else ''
+        hit = self.regex.search(text)
+        ok = hit is None
+        detail = 'clean' if ok else f'matched {hit.group(0)!r}'
+        return [_score(self.name, self.name, case.id, ok, detail)]
+
+
+@base.register('regex_present')
+class RegexPresent:
+    """Assert a text field matches EVERY pattern in ``patterns`` (all-of).
+
+    The positive twin of :class:`RegexAbsent`: the cheap "did it include all
+    the required constructs" gate (e.g. a form's action URL, required hidden
+    fields). Patterns are compiled case-insensitively; a single ``pattern``
+    string is accepted as shorthand for a one-element list.
+    """
+
+    def __init__(
+        self,
+        patterns: list[str] | None = None,
+        field: str = 'html',
+        pattern: str | None = None,
+        name: str = 'regex_present',
+    ):
+        self.name = name
+        self.field = field
+        raw = list(patterns or ([] if pattern is None else [pattern]))
+        self.regexes = [(p, re.compile(p, re.IGNORECASE)) for p in raw]
+
+    def grade(
+        self, case: models.Case, output: models.Output
+    ) -> list[models.Score]:
+        value = refs.resolve_ref(_context(case, output), self.field)
+        text = value if isinstance(value, str) else ''
+        missing = [p for p, regex in self.regexes if not regex.search(text)]
+        ok = not missing
+        detail = 'all present' if ok else f'missing {missing}'
+        return [_score(self.name, self.name, case.id, ok, detail)]
+
+
+@base.register('non_empty')
+class NonEmpty:
+    """Assert a field resolves to a non-empty value."""
+
+    def __init__(self, field: str, name: str = 'non_empty'):
+        self.name = name
+        self.field = field
+
+    def grade(
+        self, case: models.Case, output: models.Output
+    ) -> list[models.Score]:
+        value = refs.resolve_ref(_context(case, output), self.field)
+        ok = bool(value)
+        return [_score(self.name, self.name, case.id, ok, f'value={value!r}')]