evalshift 0.3.0__py3-none-any.whl

evalshift/__init__.py

@@ -0,0 +1,12 @@
+"""EvalShift: a local-first CLI for safe LLM model migrations.
+
+EvalShift runs your prompts on two LLM model versions against a golden suite of
+inputs, scores the resulting outputs with structural, semantic, and LLM-as-judge
+evaluators, and produces a single-file HTML report with statistically rigorous
+regression analysis.
+"""
+
+from __future__ import annotations
+
+__version__ = "0.3.0"
+__all__ = ["__version__"]

evalshift/__main__.py

@@ -0,0 +1,17 @@
+"""Entry point for ``python -m evalshift``.
+
+Delegates to the Typer app defined in :mod:`evalshift.cli.main`.
+"""
+
+from __future__ import annotations
+
+from evalshift.cli.main import app
+
+
+def main() -> None:
+    """Invoke the EvalShift CLI."""
+    app()
+
+
+if __name__ == "__main__":
+    main()

evalshift/analysis/__init__.py

@@ -0,0 +1 @@
+"""Statistical analysis and slicing of evaluation scores."""

evalshift/analysis/slicing.py

@@ -0,0 +1,143 @@
+"""Group evaluation records into slices for analysis.
+
+A *slice* is a named subset of suite examples — typically defined by a
+tag in ``evalshift.yaml``'s ``slices:`` block. The implicit ``"all"``
+slice always exists and contains every example.
+
+The output of :func:`build_slices` is a mapping from slice name to a
+list of ``(prompt_id, evaluator_name, example_id, source_score,
+target_score, delta)`` tuples that the statistics layer can pair up and
+test.
+"""
+
+from __future__ import annotations
+
+from collections import defaultdict
+from dataclasses import dataclass
+
+from evalshift.evaluators.base import EvalRecord
+from evalshift.suite.models import Suite
+
+ALL_SLICE: str = "all"
+
+
+@dataclass(frozen=True, slots=True)
+class SlicedScore:
+    """One paired score belonging to a slice."""
+
+    prompt_id: str
+    evaluator_name: str
+    example_id: str
+    source_score: float
+    target_score: float
+    delta: float
+
+
+@dataclass(frozen=True, slots=True)
+class SliceAggregate:
+    """Per-slice aggregate stats (computed without significance testing)."""
+
+    name: str
+    n: int
+    source_avg_score: float
+    target_avg_score: float
+    delta_avg_score: float
+    delta_min_score: float
+    delta_max_score: float
+    source_score_stdev: float
+    target_score_stdev: float
+
+
+def build_slices(
+    *,
+    records: list[EvalRecord],
+    suite: Suite,
+    tag_to_slice: dict[str, str] | None = None,
+) -> dict[str, list[SlicedScore]]:
+    """Group evaluation records into slices keyed by slice name.
+
+    Args:
+        records: Every :class:`EvalRecord` from ``scores.jsonl``.
+        suite: The loaded suite, used to look up an example's ``tags``.
+        tag_to_slice: Mapping from a configured slice's tag (the value
+            of ``filter`` in MVP, simplified to a literal tag) to the
+            slice name surfaced in reports. ``None`` falls back to a
+            tag-name == slice-name identity mapping.
+
+    Returns:
+        ``{slice_name: [SlicedScore, ...]}`` always containing at
+        least the implicit ``"all"`` slice.
+    """
+    by_id = {ex.id: ex for ex in suite.examples}
+    out: dict[str, list[SlicedScore]] = defaultdict(list)
+
+    for rec in records:
+        # Skip records that errored at the evaluator layer — they don't
+        # carry meaningful scores.
+        if rec.error is not None:
+            continue
+        sliced = SlicedScore(
+            prompt_id=rec.prompt_id,
+            evaluator_name=rec.evaluator_name,
+            example_id=rec.example_id,
+            source_score=rec.source_score,
+            target_score=rec.target_score,
+            delta=rec.delta,
+        )
+        out[ALL_SLICE].append(sliced)
+        example = by_id.get(rec.example_id)
+        if example is None:
+            continue
+        for tag in example.tags:
+            slice_name = tag_to_slice.get(tag, tag) if tag_to_slice else tag
+            out[slice_name].append(sliced)
+
+    return dict(out)
+
+
+def aggregates(
+    sliced: list[SlicedScore],
+    name: str,
+) -> SliceAggregate:
+    """Compute simple aggregates over a list of paired scores."""
+    n = len(sliced)
+    if n == 0:
+        return SliceAggregate(
+            name=name,
+            n=0,
+            source_avg_score=0.0,
+            target_avg_score=0.0,
+            delta_avg_score=0.0,
+            delta_min_score=0.0,
+            delta_max_score=0.0,
+            source_score_stdev=0.0,
+            target_score_stdev=0.0,
+        )
+    source = [s.source_score for s in sliced]
+    target = [s.target_score for s in sliced]
+    deltas = [s.delta for s in sliced]
+    return SliceAggregate(
+        name=name,
+        n=n,
+        source_avg_score=_mean(source),
+        target_avg_score=_mean(target),
+        delta_avg_score=_mean(deltas),
+        delta_min_score=min(deltas),
+        delta_max_score=max(deltas),
+        source_score_stdev=_std(source),
+        target_score_stdev=_std(target),
+    )
+
+
+def _mean(xs: list[float]) -> float:
+    return float(sum(xs) / len(xs)) if xs else 0.0
+
+
+def _std(xs: list[float]) -> float:
+    if len(xs) < 2:
+        return 0.0
+    mu = _mean(xs)
+    return float((sum((x - mu) ** 2 for x in xs) / (len(xs) - 1)) ** 0.5)
+
+
+__all__ = ["ALL_SLICE", "SliceAggregate", "SlicedScore", "aggregates", "build_slices"]

evalshift/analysis/statistics.py

@@ -0,0 +1,382 @@
+"""Statistical analysis of paired source/target scores.
+
+Implements the contract from PDF §5.5 step-by-step:
+
+1. Pair scores per slice; require ``n>=5`` to test, flag ``n<20``.
+2. Test normality via Shapiro-Wilk on the deltas; route to a paired
+   t-test (normal) or Wilcoxon signed-rank (non-normal).
+3. Compute Cohen's d for paired samples (``mean(deltas)/std(deltas)``).
+4. Compute a 95% CI on the effect size — analytical for the t-test,
+   bootstrap for Wilcoxon.
+5. After every comparison in the run is collected, apply the
+   Benjamini-Hochberg correction (FDR=0.05) across all p-values to
+   control false-discovery in the multi-test setting.
+6. Classify severity from the *corrected* p-value + effect size.
+
+Why these choices:
+
+* **Paired tests** because each example is run on both models — the
+  measurements are inherently paired and ignoring that throws away
+  power.
+* **Shapiro-Wilk gate** because the t-test is sensitive to non-normal
+  deltas at small N; falling back to Wilcoxon (which only assumes
+  symmetric distribution) is the standard MVP-grade safety net.
+* **Cohen's d** as the effect size because it's interpretable in units
+  of standard deviation and the field has an intuition for it
+  (``|d|>0.8`` = "large").
+* **Benjamini-Hochberg** instead of Bonferroni because BH controls FDR
+  at the same level with substantially more power, especially when
+  some comparisons are real (which is exactly the case in a real
+  migration eval).
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Literal
+
+import numpy as np
+from scipy import stats
+
+from evalshift.analysis.slicing import SlicedScore
+
+Severity = Literal[
+    "critical",
+    "high",
+    "medium",
+    "low",
+    "improved",
+    "none",
+    "insufficient",
+]
+
+TestKind = Literal["paired_t", "wilcoxon", "skipped"]
+
+MIN_N_FOR_TEST: int = 5
+MIN_N_RELIABLE: int = 20
+NORMALITY_ALPHA: float = 0.05
+FDR_ALPHA: float = 0.05
+BOOTSTRAP_RESAMPLES: int = 2000
+DEFAULT_RNG_SEED: int = 0
+
+
+@dataclass(frozen=True, slots=True)
+class ComparisonResult:
+    """One (prompt_id, evaluator_name, slice_name) statistical verdict."""
+
+    prompt_id: str
+    evaluator_name: str
+    slice_name: str
+    n: int
+    test: TestKind
+    statistic: float
+    p_value: float
+    p_value_corrected: float
+    effect_size: float
+    effect_size_ci_low: float
+    effect_size_ci_high: float
+    delta_avg_score: float
+    severity: Severity
+    notes: list[str]
+
+
+def analyze(
+    *,
+    sliced_by_slice: dict[str, list[SlicedScore]],
+    rng: np.random.Generator | None = None,
+) -> list[ComparisonResult]:
+    """Run every comparison and apply BH correction across the whole set.
+
+    Returns a list of :class:`ComparisonResult` ordered by descending
+    severity then absolute effect size — i.e. the worst regressions
+    bubble to the top.
+    """
+    rng_inst = rng or np.random.default_rng(DEFAULT_RNG_SEED)
+
+    # Group within each slice by (prompt_id, evaluator_name) so we run
+    # one test per (prompt, evaluator, slice).
+    raw: list[ComparisonResult] = []
+    for slice_name, sliced in sliced_by_slice.items():
+        by_key: dict[tuple[str, str], list[SlicedScore]] = {}
+        for s in sliced:
+            by_key.setdefault((s.prompt_id, s.evaluator_name), []).append(s)
+        for (prompt_id, evaluator_name), pairs in by_key.items():
+            raw.append(
+                _one_comparison(
+                    prompt_id=prompt_id,
+                    evaluator_name=evaluator_name,
+                    slice_name=slice_name,
+                    pairs=pairs,
+                    rng=rng_inst,
+                ),
+            )
+
+    # Apply Benjamini-Hochberg across every comparison that actually
+    # ran a test (skipped comparisons keep p_value = 1.0 and don't
+    # participate in the correction).
+    testable_idx = [i for i, c in enumerate(raw) if c.test != "skipped"]
+    p_raw = [raw[i].p_value for i in testable_idx]
+    p_corrected_seq = _benjamini_hochberg(p_raw)
+    corrected_lookup: dict[int, float] = dict(
+        zip(testable_idx, p_corrected_seq, strict=True),
+    )
+
+    finalised: list[ComparisonResult] = []
+    for i, c in enumerate(raw):
+        if c.test == "skipped":
+            finalised.append(c)
+            continue
+        cp = corrected_lookup[i]
+        sev = _classify_severity(
+            corrected_p=cp, d=c.effect_size, mean_delta=c.delta_avg_score, n=c.n
+        )
+        finalised.append(
+            ComparisonResult(
+                prompt_id=c.prompt_id,
+                evaluator_name=c.evaluator_name,
+                slice_name=c.slice_name,
+                n=c.n,
+                test=c.test,
+                statistic=c.statistic,
+                p_value=c.p_value,
+                p_value_corrected=cp,
+                effect_size=c.effect_size,
+                effect_size_ci_low=c.effect_size_ci_low,
+                effect_size_ci_high=c.effect_size_ci_high,
+                delta_avg_score=c.delta_avg_score,
+                severity=sev,
+                notes=c.notes,
+            ),
+        )
+
+    finalised.sort(key=_severity_sort_key)
+    return finalised
+
+
+# ---------------------------------------------------------------------------
+# Helpers
+# ---------------------------------------------------------------------------
+
+
+def _one_comparison(
+    *,
+    prompt_id: str,
+    evaluator_name: str,
+    slice_name: str,
+    pairs: list[SlicedScore],
+    rng: np.random.Generator,
+) -> ComparisonResult:
+    """Compute one comparison, deferring severity until BH is done."""
+    n = len(pairs)
+    deltas = np.array([p.delta for p in pairs], dtype=float)
+    notes: list[str] = []
+
+    if n < MIN_N_FOR_TEST:
+        return ComparisonResult(
+            prompt_id=prompt_id,
+            evaluator_name=evaluator_name,
+            slice_name=slice_name,
+            n=n,
+            test="skipped",
+            statistic=0.0,
+            p_value=1.0,
+            p_value_corrected=1.0,
+            effect_size=0.0,
+            effect_size_ci_low=0.0,
+            effect_size_ci_high=0.0,
+            delta_avg_score=float(np.mean(deltas)) if n else 0.0,
+            severity="insufficient",
+            notes=[f"n={n} < {MIN_N_FOR_TEST}; no test run"],
+        )
+    if n < MIN_N_RELIABLE:
+        notes.append(f"n={n} < {MIN_N_RELIABLE}; results uncertain")
+
+    # Variance ≈ 0 (every delta identical, modulo float noise) → no
+    # inference possible. Use a small tolerance to dodge catastrophic
+    # cancellation that scipy already warns about.
+    if float(np.std(deltas, ddof=1)) < 1e-9:
+        return ComparisonResult(
+            prompt_id=prompt_id,
+            evaluator_name=evaluator_name,
+            slice_name=slice_name,
+            n=n,
+            test="skipped",
+            statistic=0.0,
+            p_value=1.0,
+            p_value_corrected=1.0,
+            effect_size=0.0,
+            effect_size_ci_low=0.0,
+            effect_size_ci_high=0.0,
+            delta_avg_score=float(np.mean(deltas)),
+            severity="none",
+            notes=[*notes, "zero variance — no test"],
+        )
+
+    # Pick the test based on a normality screen.
+    use_t = _is_normal(deltas)
+
+    if use_t:
+        result = stats.ttest_rel(
+            [p.target_score for p in pairs],
+            [p.source_score for p in pairs],
+        )
+        statistic = float(result.statistic)
+        p_value = float(result.pvalue)
+        d = _cohens_d_paired(deltas)
+        ci_low, ci_high = _t_test_d_ci(d=d, n=n)
+        test_kind: TestKind = "paired_t"
+    else:
+        result = stats.wilcoxon(deltas, zero_method="wilcox", correction=False)
+        statistic = float(result.statistic)
+        p_value = float(result.pvalue)
+        d = _cohens_d_paired(deltas)
+        ci_low, ci_high = _bootstrap_d_ci(deltas, rng=rng)
+        test_kind = "wilcoxon"
+        notes.append("non-normal deltas; used Wilcoxon")
+
+    return ComparisonResult(
+        prompt_id=prompt_id,
+        evaluator_name=evaluator_name,
+        slice_name=slice_name,
+        n=n,
+        test=test_kind,
+        statistic=statistic,
+        p_value=p_value,
+        p_value_corrected=p_value,  # placeholder; overwritten by BH
+        effect_size=d,
+        effect_size_ci_low=ci_low,
+        effect_size_ci_high=ci_high,
+        delta_avg_score=float(np.mean(deltas)),
+        severity="none",  # placeholder; classified after BH
+        notes=notes,
+    )
+
+
+def _is_normal(deltas: np.ndarray) -> bool:
+    """Shapiro-Wilk normality screen.
+
+    Skips for n>5000 where Shapiro is unreliable / overly powerful;
+    in that regime a t-test is robust enough by central-limit-theorem
+    grounds.
+    """
+    if deltas.size > 5000:
+        return True
+    try:
+        result = stats.shapiro(deltas)
+    except Exception:
+        return False
+    return float(result.pvalue) > NORMALITY_ALPHA
+
+
+def _cohens_d_paired(deltas: np.ndarray) -> float:
+    sd = float(np.std(deltas, ddof=1))
+    if sd < 1e-9:
+        return 0.0
+    return float(np.mean(deltas) / sd)
+
+
+def _t_test_d_ci(*, d: float, n: int) -> tuple[float, float]:
+    """Analytical 95% CI on Cohen's d (paired) using the SE approximation.
+
+    Standard error: sqrt(1/n + d^2/(2n)). Times 1.96 for 95%.
+    """
+    se = float(np.sqrt(1.0 / n + (d * d) / (2.0 * n)))
+    return d - 1.96 * se, d + 1.96 * se
+
+
+def _bootstrap_d_ci(
+    deltas: np.ndarray,
+    *,
+    rng: np.random.Generator,
+    resamples: int = BOOTSTRAP_RESAMPLES,
+) -> tuple[float, float]:
+    """Percentile-bootstrap 95% CI on Cohen's d (paired)."""
+    n = deltas.size
+    sample_ds = np.empty(resamples, dtype=float)
+    for i in range(resamples):
+        idx = rng.integers(0, n, size=n)
+        d = deltas[idx]
+        sd = float(np.std(d, ddof=1))
+        sample_ds[i] = float(np.mean(d)) / sd if sd != 0 else 0.0
+    return float(np.percentile(sample_ds, 2.5)), float(np.percentile(sample_ds, 97.5))
+
+
+def _benjamini_hochberg(p_values: list[float]) -> list[float]:
+    """Apply the BH FDR correction; returns adjusted p-values in input order.
+
+    Mirrors ``statsmodels.stats.multitest.multipletests(..., method='fdr_bh')``
+    semantics: each adjusted p-value is the smallest q such that the
+    BH cutoff would still call the corresponding raw p significant.
+    """
+    n = len(p_values)
+    if n == 0:
+        return []
+    arr = np.asarray(p_values, dtype=float)
+    order = np.argsort(arr)
+    ranked = arr[order]
+    adjusted_ranked = ranked * n / (np.arange(n) + 1)
+    # Enforce monotonicity (right-to-left running min) and clamp to 1.
+    adjusted_ranked = np.minimum.accumulate(adjusted_ranked[::-1])[::-1]
+    adjusted_ranked = np.clip(adjusted_ranked, 0.0, 1.0)
+    out = np.empty(n, dtype=float)
+    out[order] = adjusted_ranked
+    return [float(x) for x in out]
+
+
+def _classify_severity(
+    *,
+    corrected_p: float,
+    d: float,
+    mean_delta: float,
+    n: int,
+) -> Severity:
+    """Map (corrected p, effect size, direction, n) to severity per PDF §5.5."""
+    if n < MIN_N_FOR_TEST:
+        return "insufficient"
+    if corrected_p >= FDR_ALPHA:
+        return "none"
+    abs_d = abs(d)
+    if mean_delta > 0:
+        return "improved"
+    # mean_delta < 0 → regression
+    if corrected_p < 0.01 and abs_d > 0.8:
+        return "critical"
+    if abs_d > 0.5:
+        return "high"
+    if abs_d > 0.2:
+        return "medium"
+    return "low"
+
+
+_SEVERITY_RANK: dict[Severity, int] = {
+    "critical": 0,
+    "high": 1,
+    "medium": 2,
+    "low": 3,
+    "improved": 4,
+    "none": 5,
+    "insufficient": 6,
+}
+
+
+def _severity_sort_key(c: ComparisonResult) -> tuple[int, float, str, str, str]:
+    return (
+        _SEVERITY_RANK[c.severity],
+        -abs(c.effect_size),
+        c.prompt_id,
+        c.evaluator_name,
+        c.slice_name,
+    )
+
+
+__all__ = [
+    "BOOTSTRAP_RESAMPLES",
+    "FDR_ALPHA",
+    "MIN_N_FOR_TEST",
+    "MIN_N_RELIABLE",
+    "NORMALITY_ALPHA",
+    "ComparisonResult",
+    "Severity",
+    "TestKind",
+    "analyze",
+]

evalshift/cache/__init__.py

@@ -0,0 +1 @@
+"""SQLite cache for LLM responses and embeddings."""

evalshift/cache/schema.py

@@ -0,0 +1,102 @@
+"""SQLAlchemy schema for the local LLM-response cache.
+
+We use a single table — :class:`CachedCall` — keyed on the SHA-256 of
+(model + canonicalised prompt + canonicalised inputs + temperature +
+max_tokens). This keeps caching decisions transparent: identical inputs
+hash to the same key, period. No fuzzy matching, no embeddings, no
+similarity searches; the simplest thing that could possibly work, per
+the PDF's guidance for the MVP.
+
+The cache lives at ``~/.evalshift/cache.db`` by default. Tests pass an
+in-memory SQLite URL via the ``database_url`` parameter on the engine
+factory; never reach for the real DB in unit tests.
+"""
+
+from __future__ import annotations
+
+from datetime import UTC, datetime
+from pathlib import Path
+
+from sqlalchemy import DateTime, Float, Integer, String, Text
+from sqlalchemy.ext.asyncio import AsyncEngine, create_async_engine
+from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column
+
+DEFAULT_CACHE_PATH: Path = Path.home() / ".evalshift" / "cache.db"
+
+
+def _utcnow() -> datetime:
+    """UTC ``now()`` (named for monkeypatching in TTL tests)."""
+    return datetime.now(UTC)
+
+
+class Base(DeclarativeBase):
+    """Declarative base for the cache schema."""
+
+
+class CachedCall(Base):
+    """One row per cached LLM response.
+
+    Attributes:
+        cache_key: SHA-256 hex digest of the canonicalised request. The
+            primary key. See :func:`evalshift.cache.store.cache_key`.
+        model_id: The canonical model id used for the call (matches
+            :attr:`evalshift.models.registry.ModelMetadata.id`).
+        prompt_text: The fully-rendered prompt sent to the model. Stored
+            verbatim so we can audit cache hits.
+        inputs_json: Canonical-JSON-encoded ``inputs`` mapping.
+        response_text: The model's response text.
+        input_tokens / output_tokens: From the provider response.
+        cost_usd: Computed via ``litellm.completion_cost`` at write time.
+        latency_ms: Wall time of the original live call. Useful for
+            cost/perf reporting on cached results.
+        created_at: When the cache row was written. Driver of TTL.
+    """
+
+    __tablename__ = "cached_calls"
+
+    cache_key: Mapped[str] = mapped_column(String(64), primary_key=True)
+    model_id: Mapped[str] = mapped_column(String(128), nullable=False)
+    prompt_text: Mapped[str] = mapped_column(Text, nullable=False)
+    inputs_json: Mapped[str] = mapped_column(Text, nullable=False)
+    response_text: Mapped[str] = mapped_column(Text, nullable=False)
+    input_tokens: Mapped[int] = mapped_column(Integer, nullable=False)
+    output_tokens: Mapped[int] = mapped_column(Integer, nullable=False)
+    cost_usd: Mapped[float] = mapped_column(Float, nullable=False)
+    latency_ms: Mapped[int] = mapped_column(Integer, nullable=False)
+    created_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True),
+        nullable=False,
+        default=_utcnow,
+    )
+
+
+def default_database_url(path: Path | None = None) -> str:
+    """Return the SQLAlchemy URL for the on-disk cache DB.
+
+    Args:
+        path: Override the default location (mostly for tests). When
+            ``None``, uses :data:`DEFAULT_CACHE_PATH`.
+    """
+    target = path if path is not None else DEFAULT_CACHE_PATH
+    target.parent.mkdir(parents=True, exist_ok=True)
+    return f"sqlite+aiosqlite:///{target}"
+
+
+def create_engine(database_url: str | None = None) -> AsyncEngine:
+    """Create an async SQLAlchemy engine for the cache DB.
+
+    Args:
+        database_url: SQLAlchemy URL. Pass ``"sqlite+aiosqlite:///:memory:"``
+            for tests. When ``None``, defaults to the on-disk cache.
+    """
+    url = database_url if database_url is not None else default_database_url()
+    return create_async_engine(url, future=True)
+
+
+__all__ = [
+    "DEFAULT_CACHE_PATH",
+    "Base",
+    "CachedCall",
+    "create_engine",
+    "default_database_url",
+]