evalproof 0.2.0__py3-none-any.whl

evalproof/providers/anthropic.py

@@ -0,0 +1,84 @@
+"""Anthropic Messages API over plain HTTP. Optional: needs ANTHROPIC_API_KEY."""
+
+from __future__ import annotations
+
+import os
+import time
+from typing import Any
+
+import httpx
+
+from evalproof.config import GenerationParams
+from evalproof.providers.base import (
+    ChatResponse,
+    Message,
+    ProviderError,
+    split_system,
+    with_retries,
+)
+
+DEFAULT_BASE_URL = "https://api.anthropic.com"
+API_VERSION = "2023-06-01"
+
+
+class AnthropicProvider:
+    name = "anthropic"
+
+    def __init__(
+        self,
+        model: str,
+        *,
+        base_url: str | None = None,
+        timeout_s: float = 120.0,
+        client: httpx.Client | None = None,
+        api_key: str | None = None,
+    ) -> None:
+        self.model = model
+        self.base_url = (base_url or DEFAULT_BASE_URL).rstrip("/")
+        key = api_key or os.environ.get("ANTHROPIC_API_KEY")
+        if not key:
+            raise ProviderError("anthropic: set ANTHROPIC_API_KEY to use this provider")
+        self._headers = {
+            "x-api-key": key,
+            "anthropic-version": API_VERSION,
+            "content-type": "application/json",
+        }
+        self._client = client or httpx.Client(timeout=timeout_s)
+
+    def payload(self, messages: list[Message], params: GenerationParams) -> dict[str, Any]:
+        system, rest = split_system(messages)
+        body: dict[str, Any] = {
+            "model": self.model,
+            "max_tokens": params.max_tokens,
+            "temperature": params.temperature,
+            "messages": [m.model_dump() for m in rest],
+        }
+        if system:
+            body["system"] = system
+        body.update(params.extra)
+        return body
+
+    def chat(self, messages: list[Message], params: GenerationParams) -> ChatResponse:
+        body = self.payload(messages, params)
+
+        def call() -> httpx.Response:
+            resp = self._client.post(
+                f"{self.base_url}/v1/messages", json=body, headers=self._headers
+            )
+            resp.raise_for_status()
+            return resp
+
+        start = time.perf_counter()
+        resp = with_retries(call)
+        latency_ms = (time.perf_counter() - start) * 1000
+        data = resp.json()
+        blocks = data.get("content") or []
+        text = "".join(b.get("text", "") for b in blocks if b.get("type") == "text")
+        usage = data.get("usage") or {}
+        return ChatResponse(
+            text=text,
+            model=data.get("model", self.model),
+            tokens_in=int(usage.get("input_tokens") or 0),
+            tokens_out=int(usage.get("output_tokens") or 0),
+            latency_ms=latency_ms,
+        )

evalproof/providers/base.py

@@ -0,0 +1,75 @@
+"""The one thing evalproof needs from a model: send messages, get text and token counts back."""
+
+from __future__ import annotations
+
+import random
+import time
+from collections.abc import Callable
+from typing import Protocol, TypeVar
+
+import httpx
+from pydantic import BaseModel
+
+from evalproof.config import GenerationParams
+
+T = TypeVar("T")
+
+RETRY_STATUS = {408, 409, 425, 429, 500, 502, 503, 504, 529}
+
+
+class ProviderError(RuntimeError):
+    pass
+
+
+class Message(BaseModel):
+    role: str  # "system" | "user" | "assistant"
+    content: str
+
+
+class ChatResponse(BaseModel):
+    text: str
+    model: str
+    tokens_in: int = 0
+    tokens_out: int = 0
+    latency_ms: float = 0.0
+    cached: bool = False
+
+
+class Provider(Protocol):
+    name: str
+    model: str
+
+    def chat(self, messages: list[Message], params: GenerationParams) -> ChatResponse: ...
+
+
+def with_retries(
+    fn: Callable[[], T],
+    *,
+    attempts: int = 4,
+    base_delay: float = 0.5,
+    sleep: Callable[[float], None] = time.sleep,
+    rng: random.Random | None = None,
+) -> T:
+    """Retry transport errors and retryable HTTP statuses with jittered exponential backoff."""
+    rng = rng or random.Random(0)
+    last: Exception | None = None
+    for attempt in range(attempts):
+        try:
+            return fn()
+        except httpx.HTTPStatusError as exc:
+            if exc.response.status_code not in RETRY_STATUS:
+                raise ProviderError(
+                    f"HTTP {exc.response.status_code}: {exc.response.text[:300]}"
+                ) from exc
+            last = exc
+        except httpx.TransportError as exc:
+            last = exc
+        if attempt < attempts - 1:
+            sleep(base_delay * (2**attempt) * (0.5 + rng.random()))
+    raise ProviderError(f"giving up after {attempts} attempts: {last}") from last
+
+
+def split_system(messages: list[Message]) -> tuple[str | None, list[Message]]:
+    """Providers that take the system prompt as a separate field need it pulled out."""
+    system = "\n\n".join(m.content for m in messages if m.role == "system") or None
+    return system, [m for m in messages if m.role != "system"]

evalproof/providers/ollama.py

@@ -0,0 +1,70 @@
+"""Ollama, the default provider: local, free, and reproducible with a fixed seed."""
+
+from __future__ import annotations
+
+import time
+from typing import Any
+
+import httpx
+
+from evalproof.config import GenerationParams
+from evalproof.providers.base import ChatResponse, Message, ProviderError, with_retries
+
+DEFAULT_BASE_URL = "http://127.0.0.1:11434"
+
+
+class OllamaProvider:
+    name = "ollama"
+
+    def __init__(
+        self,
+        model: str,
+        *,
+        base_url: str | None = None,
+        timeout_s: float = 120.0,
+        client: httpx.Client | None = None,
+    ) -> None:
+        self.model = model
+        self.base_url = (base_url or DEFAULT_BASE_URL).rstrip("/")
+        self._client = client or httpx.Client(timeout=timeout_s)
+
+    def payload(self, messages: list[Message], params: GenerationParams) -> dict[str, Any]:
+        options: dict[str, Any] = {
+            "temperature": params.temperature,
+            "num_predict": params.max_tokens,
+        }
+        if params.seed is not None:
+            options["seed"] = params.seed
+        options.update(params.extra)
+        body: dict[str, Any] = {
+            "model": self.model,
+            "messages": [m.model_dump() for m in messages],
+            "stream": False,
+            "options": options,
+        }
+        if params.think is not None:
+            body["think"] = params.think
+        return body
+
+    def chat(self, messages: list[Message], params: GenerationParams) -> ChatResponse:
+        body = self.payload(messages, params)
+
+        def call() -> httpx.Response:
+            resp = self._client.post(f"{self.base_url}/api/chat", json=body)
+            resp.raise_for_status()
+            return resp
+
+        start = time.perf_counter()
+        resp = with_retries(call)
+        latency_ms = (time.perf_counter() - start) * 1000
+        data = resp.json()
+        message = data.get("message") or {}
+        if "content" not in message:
+            raise ProviderError(f"ollama: unexpected response shape: {str(data)[:300]}")
+        return ChatResponse(
+            text=message["content"],
+            model=data.get("model", self.model),
+            tokens_in=int(data.get("prompt_eval_count") or 0),
+            tokens_out=int(data.get("eval_count") or 0),
+            latency_ms=latency_ms,
+        )

evalproof/providers/openai.py

@@ -0,0 +1,76 @@
+"""OpenAI-compatible Chat Completions over plain HTTP. Optional: needs OPENAI_API_KEY.
+
+Also works against any server that speaks the same API, via ``base_url``.
+"""
+
+from __future__ import annotations
+
+import os
+import time
+from typing import Any
+
+import httpx
+
+from evalproof.config import GenerationParams
+from evalproof.providers.base import ChatResponse, Message, ProviderError, with_retries
+
+DEFAULT_BASE_URL = "https://api.openai.com/v1"
+
+
+class OpenAIProvider:
+    name = "openai"
+
+    def __init__(
+        self,
+        model: str,
+        *,
+        base_url: str | None = None,
+        timeout_s: float = 120.0,
+        client: httpx.Client | None = None,
+        api_key: str | None = None,
+    ) -> None:
+        self.model = model
+        self.base_url = (base_url or DEFAULT_BASE_URL).rstrip("/")
+        key = api_key or os.environ.get("OPENAI_API_KEY")
+        if not key:
+            raise ProviderError("openai: set OPENAI_API_KEY to use this provider")
+        self._headers = {"authorization": f"Bearer {key}", "content-type": "application/json"}
+        self._client = client or httpx.Client(timeout=timeout_s)
+
+    def payload(self, messages: list[Message], params: GenerationParams) -> dict[str, Any]:
+        body: dict[str, Any] = {
+            "model": self.model,
+            "messages": [m.model_dump() for m in messages],
+            "temperature": params.temperature,
+            "max_tokens": params.max_tokens,
+        }
+        if params.seed is not None:
+            body["seed"] = params.seed
+        body.update(params.extra)
+        return body
+
+    def chat(self, messages: list[Message], params: GenerationParams) -> ChatResponse:
+        body = self.payload(messages, params)
+
+        def call() -> httpx.Response:
+            resp = self._client.post(
+                f"{self.base_url}/chat/completions", json=body, headers=self._headers
+            )
+            resp.raise_for_status()
+            return resp
+
+        start = time.perf_counter()
+        resp = with_retries(call)
+        latency_ms = (time.perf_counter() - start) * 1000
+        data = resp.json()
+        choices = data.get("choices") or []
+        if not choices:
+            raise ProviderError(f"openai: no choices in response: {str(data)[:300]}")
+        usage = data.get("usage") or {}
+        return ChatResponse(
+            text=(choices[0].get("message") or {}).get("content") or "",
+            model=data.get("model", self.model),
+            tokens_in=int(usage.get("prompt_tokens") or 0),
+            tokens_out=int(usage.get("completion_tokens") or 0),
+            latency_ms=latency_ms,
+        )

evalproof/report.py

@@ -0,0 +1,149 @@
+"""Reports: Markdown for a PR comment or GitHub step summary, JSON for machines."""
+
+from __future__ import annotations
+
+import json
+from dataclasses import asdict
+from typing import Any
+
+from evalproof.calibration import CalibrationReport, Estimate
+from evalproof.compare import Comparison, MetricComparison
+from evalproof.gate import GateDecision
+from evalproof.models import Run
+from evalproof.stats import percentile, wilson
+
+STATUS_ICON = {"pass": "✅ pass", "warn": "⚠️ pass with warnings", "fail": "❌ fail"}
+
+
+def _pct(x: float | None) -> str:
+    return "—" if x is None else f"{100 * x:.1f}%"
+
+
+def _rate_or_mean(mc: MetricComparison, value: float) -> str:
+    return _pct(value) if mc.kind == "binary" else f"{value:.3f}"
+
+
+def _delta(mc: MetricComparison) -> str:
+    if mc.kind == "binary":
+        return f"{100 * mc.delta:+.1f} pts [{100 * mc.ci_low:+.1f}, {100 * mc.ci_high:+.1f}]"
+    return f"{mc.delta:+.3f} [{mc.ci_low:+.3f}, {mc.ci_high:+.3f}]"
+
+
+def comparison_markdown(
+    cmp: Comparison, decision: GateDecision | None = None, gate_metric: str | None = None
+) -> str:
+    lines = ["## evalproof", ""]
+    if decision is not None:
+        lines += [f"**Gate: {STATUS_ICON[decision.status]}** (exit {decision.exit_code})", ""]
+        lines += [f"- {r}" for r in decision.reasons]
+        lines += [f"- ⚠️ {w}" for w in decision.warnings]
+        if decision.reasons or decision.warnings:
+            lines.append("")
+    lines += [
+        f"Baseline `{cmp.baseline_label}` vs candidate `{cmp.candidate_label}` on "
+        f"{len(cmp.paired_ids)} paired cases.",
+        "",
+        "| Metric | Baseline | Candidate | Δ (95% CI) | Test | p |",
+        "|---|---:|---:|---:|---|---:|",
+    ]
+    for name, mc in cmp.metrics.items():
+        label = f"**{name}**" if name == gate_metric else name
+        lines.append(
+            f"| {label} | {_rate_or_mean(mc, mc.baseline)} | {_rate_or_mean(mc, mc.candidate)} | "
+            f"{_delta(mc)} | {mc.test} | {mc.p_value:.4f} |"
+        )
+    lines += [
+        "",
+        "| Run | Cases | Errors | p50 latency | p95 latency | Cost |",
+        "|---|---:|---:|---:|---:|---:|",
+    ]
+    for label, st in (("baseline", cmp.baseline), ("candidate", cmp.candidate)):
+        lines.append(
+            f"| {label} | {st.cases} | {st.errors} | {st.p50_latency_ms:.0f} ms | "
+            f"{st.p95_latency_ms:.0f} ms | ${st.cost_usd:.4f} |"
+        )
+    focus = cmp.metrics.get(gate_metric) if gate_metric else None
+    if focus and focus.regressed:
+        shown = ", ".join(f"`{c}`" for c in focus.regressed[:20])
+        more = f" and {len(focus.regressed) - 20} more" if len(focus.regressed) > 20 else ""
+        lines += ["", f"Regressed on **{focus.name}** ({len(focus.regressed)}): {shown}{more}"]
+    if focus and focus.improved:
+        lines.append(f"Improved on **{focus.name}**: {len(focus.improved)} case(s)")
+    return "\n".join(lines) + "\n"
+
+
+def run_markdown(run: Run) -> str:
+    lines = [
+        f"## {run.info.suite} — `{run.info.system}`",
+        "",
+        f"{len(run.results)} cases · wall {run.info.wall_time_s:.1f} s · "
+        f"dataset `{run.info.dataset_hash}`",
+        "",
+        "| Scorer | Decided | Pass rate (95% CI) | Mean |",
+        "|---|---:|---:|---:|",
+    ]
+    for name in run.scorer_names():
+        scores = [r.scores[name] for r in run.results if name in r.scores]
+        decided = [s for s in scores if s.value is not None]
+        binary = [s.passed for s in decided if s.passed is not None]
+        mean = (
+            sum(s.value for s in decided if s.value is not None) / len(decided) if decided else None
+        )
+        if binary:
+            k = sum(binary)
+            lo, hi = wilson(k, len(binary))
+            rate = f"{_pct(k / len(binary))} [{_pct(lo)}, {_pct(hi)}]"
+        else:
+            rate = "—"
+        lines.append(
+            f"| {name} | {len(decided)}/{len(scores)} | {rate} | "
+            f"{'—' if mean is None else f'{mean:.3f}'} |"
+        )
+    lat = [r.output.latency_ms for r in run.results if not r.output.error]
+    lines += [
+        "",
+        f"Latency p50 {percentile(lat, 50):.0f} ms · p95 {percentile(lat, 95):.0f} ms · "
+        f"errors {sum(1 for r in run.results if r.output.error)} · "
+        f"cached {sum(1 for r in run.results if r.output.cached)}",
+    ]
+    return "\n".join(lines) + "\n"
+
+
+def _est_row(e: Estimate) -> str:
+    ci = "—" if e.low is None or e.high is None else f"[{_pct(e.low)}, {_pct(e.high)}]"
+    note = f" ({e.note})" if e.note else ""
+    return f"| {e.method}{note} | {_pct(e.point)} | {ci} | {e.n} |"
+
+
+def calibration_markdown(rep: CalibrationReport) -> str:
+    cm = rep.confusion
+    lines = [
+        f"## Judge calibration — `{rep.judge_scorer}`",
+        "",
+        f"Gold source: {rep.gold_source} · {rep.n_gold} labelled of {rep.n_judged} judged "
+        f"· {rep.undecided_judge} undecided verdict(s) excluded",
+        "",
+        "| | Gold pass | Gold fail |",
+        "|---|---:|---:|",
+        f"| **Judge pass** | {cm.tp} | {cm.fp} |",
+        f"| **Judge fail** | {cm.fn} | {cm.tn} |",
+        "",
+        f"TPR {_pct(cm.tpr)} · FPR {_pct(cm.fpr)} · accuracy {_pct(cm.accuracy)} · "
+        f"Cohen's κ {'—' if cm.kappa is None else f'{cm.kappa:.3f}'}",
+        "",
+        "| Pass-rate estimate | Point | 95% CI | n |",
+        "|---|---:|---:|---:|",
+    ]
+    lines += [_est_row(e) for e in rep.estimates]
+    if rep.truth is not None:
+        lines.append(_est_row(rep.truth))
+    if rep.disagreements:
+        lines += [
+            "",
+            f"Judge disagreed with gold on: {', '.join(f'`{c}`' for c in rep.disagreements)}",
+        ]
+    return "\n".join(lines) + "\n"
+
+
+def to_json(obj: Any) -> str:
+    return json.dumps(asdict(obj), indent=2, default=str)

evalproof/runner.py

@@ -0,0 +1,135 @@
+"""Run a suite: call the system on every case, score every output, write one run artifact."""
+
+from __future__ import annotations
+
+import platform
+import sys
+import time
+from collections.abc import Callable
+from concurrent.futures import ThreadPoolExecutor
+from datetime import UTC, datetime
+from pathlib import Path
+
+import httpx
+
+from evalproof._version import __version__
+from evalproof.cache import CachedProvider, ResponseCache
+from evalproof.config import SuiteConfig, TargetConfig
+from evalproof.dataset import dataset_hash, load_dataset
+from evalproof.models import Case, CaseResult, Run, RunInfo
+from evalproof.providers import Provider, make_provider, parse_target
+from evalproof.scorers import Scorer, make_scorer
+from evalproof.systems import System, make_system
+
+Progress = Callable[[int, int, CaseResult], None]
+
+
+def _ollama_facts(cfg: TargetConfig, client: httpx.Client | None) -> dict[str, str]:
+    """Best effort: the server version and the model digest make a run reproducible."""
+    if cfg.kind != "ollama":
+        return {}
+    _, model = parse_target(cfg.target)
+    base = (cfg.base_url or "http://127.0.0.1:11434").rstrip("/")
+    http = client or httpx.Client(timeout=5)
+    facts: dict[str, str] = {}
+    try:
+        facts["ollama_version"] = http.get(f"{base}/api/version").json().get("version", "")
+        for m in http.get(f"{base}/api/tags").json().get("models", []):
+            if m.get("name") == model or m.get("model") == model:
+                facts[f"digest:{model}"] = str(m.get("digest", ""))[:12]
+    except (httpx.HTTPError, ValueError):
+        pass
+    return facts
+
+
+def environment(cfg: SuiteConfig, client: httpx.Client | None = None) -> dict[str, str]:
+    env = {
+        "python": sys.version.split()[0],
+        "platform": platform.platform(),
+        "machine": platform.machine(),
+        "processor": platform.processor() or "",
+    }
+    env.update(_ollama_facts(cfg.system, client))
+    if cfg.judge:
+        env.update(_ollama_facts(cfg.judge, client))
+    return env
+
+
+def score_case(case: Case, system: System, scorers: list[Scorer]) -> CaseResult:
+    out = system.run(case)
+    return CaseResult(
+        case_id=case.id,
+        tags=case.tags,
+        output=out,
+        scores={s.name: s.score(case, out) for s in scorers},
+    )
+
+
+def run_suite(
+    cfg: SuiteConfig,
+    *,
+    limit: int | None = None,
+    use_cache: bool = True,
+    client: httpx.Client | None = None,
+    progress: Progress | None = None,
+    system_override: TargetConfig | None = None,
+) -> Run:
+    cases = load_dataset(cfg.resolve(cfg.dataset))
+    if limit is not None:
+        cases = cases[:limit]
+    cache = ResponseCache(cfg.resolve(cfg.cache_dir)) if use_cache else None
+    system_cfg = system_override or cfg.system
+    system = make_system(
+        system_cfg, cache=cache, client=client, search_path=str(Path(cfg.base_dir).resolve())
+    )
+
+    judge: Provider | None = None
+    if cfg.judge is not None:
+        judge = make_provider(cfg.judge, client=client)
+        if cache is not None:
+            judge = CachedProvider(judge, cache, base_url=cfg.judge.base_url)
+    scorers = [make_scorer(s, cfg, judge) for s in cfg.scorers]
+
+    started = datetime.now(UTC)
+    t0 = time.perf_counter()
+    results: list[CaseResult] = []
+    if cfg.concurrency == 1:
+        for i, case in enumerate(cases, start=1):
+            res = score_case(case, system, scorers)
+            results.append(res)
+            if progress:
+                progress(i, len(cases), res)
+    else:
+        with ThreadPoolExecutor(max_workers=cfg.concurrency) as pool:
+            futures = [pool.submit(score_case, c, system, scorers) for c in cases]
+            for i, fut in enumerate(futures, start=1):
+                res = fut.result()
+                results.append(res)
+                if progress:
+                    progress(i, len(cases), res)
+    wall = time.perf_counter() - t0
+
+    info = RunInfo(
+        suite=cfg.name,
+        system=system.name,
+        dataset_path=str(cfg.dataset),
+        dataset_hash=dataset_hash(cases),
+        started_at=started.isoformat(timespec="seconds"),
+        finished_at=datetime.now(UTC).isoformat(timespec="seconds"),
+        wall_time_s=round(wall, 3),
+        evalproof_version=__version__,
+        seed=cfg.seed,
+        environment=environment(cfg, client),
+    )
+    return Run(info=info, results=results)
+
+
+def save_run(run: Run, path: str | Path) -> Path:
+    path = Path(path)
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(run.model_dump_json(indent=2), encoding="utf-8")
+    return path
+
+
+def load_run(path: str | Path) -> Run:
+    return Run.model_validate_json(Path(path).read_text(encoding="utf-8"))

evalproof/scorers/__init__.py

@@ -0,0 +1,80 @@
+"""Scorer registry: build scorers from suite config."""
+
+from __future__ import annotations
+
+from typing import Any, Protocol
+
+from evalproof.config import ConfigError, ScorerConfig, SuiteConfig
+from evalproof.models import Case, Score, SystemOutput
+from evalproof.providers import Provider
+from evalproof.scorers.deterministic import (
+    Contains,
+    ExactMatch,
+    JsonSchema,
+    NumericTolerance,
+    Regex,
+)
+from evalproof.scorers.judge import DEFAULT_RUBRIC, LLMJudge, PairwiseResult, pairwise
+from evalproof.scorers.retrieval import MRR, NDCG, RecallAtK
+
+__all__ = [
+    "MRR",
+    "NDCG",
+    "Contains",
+    "ExactMatch",
+    "JsonSchema",
+    "LLMJudge",
+    "NumericTolerance",
+    "PairwiseResult",
+    "RecallAtK",
+    "Regex",
+    "Scorer",
+    "make_scorer",
+    "pairwise",
+]
+
+
+class Scorer(Protocol):
+    name: str
+
+    def score(self, case: Case, out: SystemOutput) -> Score: ...
+
+
+_SIMPLE: dict[str, Any] = {
+    "exact_match": ExactMatch,
+    "contains": Contains,
+    "regex": Regex,
+    "json_schema": JsonSchema,
+    "numeric_tolerance": NumericTolerance,
+    "recall_at_k": RecallAtK,
+    "mrr": MRR,
+    "ndcg": NDCG,
+}
+
+
+def make_scorer(
+    cfg: ScorerConfig, suite: SuiteConfig, judge_provider: Provider | None = None
+) -> Scorer:
+    params = dict(cfg.params)
+    if cfg.type == "llm_judge":
+        if judge_provider is None or suite.judge is None:
+            raise ConfigError(f"scorer {cfg.name!r} needs a judge provider")
+        rubric = params.pop("rubric", None)
+        rubric_file = params.pop("rubric_file", None)
+        if rubric_file:
+            rubric = suite.resolve(rubric_file).read_text(encoding="utf-8")
+        max_retries = int(params.pop("max_retries", 1))
+        if params:
+            raise ConfigError(f"scorer {cfg.name!r}: unknown params {sorted(params)}")
+        return LLMJudge(
+            cfg.name,
+            judge_provider,
+            suite.judge.params,
+            rubric=rubric or DEFAULT_RUBRIC,
+            max_retries=max_retries,
+        )
+    try:
+        scorer: Scorer = _SIMPLE[cfg.type](cfg.name, **params)
+    except TypeError as exc:
+        raise ConfigError(f"scorer {cfg.name!r}: {exc}") from exc
+    return scorer