evalseal 0.1.0__py3-none-any.whl

evalseal/__init__.py

@@ -0,0 +1,3 @@
+"""EvalSeal: reproducibility and provenance receipts for LLM evaluations."""
+
+__version__ = "0.1.0"

evalseal/adapters/dataset.py

@@ -0,0 +1,36 @@
+from __future__ import annotations
+
+import hashlib
+import json
+from dataclasses import dataclass
+from pathlib import Path
+
+
+@dataclass
+class Case:
+    case_id: str
+    prompt: str
+    expected: str | None = None
+
+
+@dataclass
+class Dataset:
+    cases: list[Case]
+    hash: str
+
+    @classmethod
+    def from_jsonl(cls, path: str | Path) -> "Dataset":
+        raw = Path(path).read_text()
+        cases = []
+        seen: set[str] = set()
+        for lineno, line in enumerate(raw.splitlines(), start=1):
+            line = line.strip()
+            if not line:
+                continue
+            d = json.loads(line)
+            if d["case_id"] in seen:
+                raise ValueError(f"{path}:{lineno}: duplicate case_id {d['case_id']!r}")
+            seen.add(d["case_id"])
+            cases.append(Case(d["case_id"], d["prompt"], d.get("expected")))
+        h = "sha256:" + hashlib.sha256(raw.encode()).hexdigest()
+        return cls(cases, h)

evalseal/adapters/recording.py

@@ -0,0 +1,56 @@
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+from collections import Counter
+from pathlib import Path
+from typing import Callable
+
+
+def request_key(payload: dict) -> str:
+    """Stable hash of the *effective* request, so identical requests hit the cassette."""
+    blob = json.dumps(payload, sort_keys=True, separators=(",", ":"))
+    return hashlib.sha256(blob.encode()).hexdigest()
+
+
+class Cassette:
+    """JSON-backed record/replay store keyed by request hash.
+
+    Each key maps to an ordered *list* of responses. Repeating an identical request is
+    the whole point of an N-run eval, so the k-th call with a given key is served the
+    k-th recorded response. (A single response per key would replay repeat #1 N times
+    and hide every flip.)
+
+    Mode is controlled by env var EVALSEAL_RECORD (or the `record` argument):
+      unset/0 -> replay only (raises if a request is missing) — this is CI mode
+      1       -> record: already-recorded responses are reused, new calls are captured
+    """
+
+    def __init__(self, path: str | Path, record: bool | None = None):
+        self.path = Path(path)
+        if record is None:
+            record = os.environ.get("EVALSEAL_RECORD", "0") == "1"
+        self.record = record
+        self._data: dict[str, list[dict]] = {}
+        self._cursor: Counter[str] = Counter()
+        if self.path.exists():
+            self._data = json.loads(self.path.read_text())
+
+    def call(self, payload: dict, do_real_call: Callable[[], dict]) -> dict:
+        key = request_key(payload)
+        index = self._cursor[key]
+        self._cursor[key] += 1
+        recorded = self._data.get(key, [])
+        if index < len(recorded):
+            return recorded[index]
+        if not self.record:
+            raise RuntimeError(
+                f"No cassette entry #{index} for request {key[:12]} in {self.path} and "
+                f"recording is off. Run with EVALSEAL_RECORD=1 and an API key to capture it."
+            )
+        resp = do_real_call()          # real network call, record mode only
+        self._data.setdefault(key, []).append(resp)
+        self.path.parent.mkdir(parents=True, exist_ok=True)
+        self.path.write_text(json.dumps(self._data, indent=2, sort_keys=True))
+        return resp

evalseal/adapters/scorer.py

@@ -0,0 +1,74 @@
+from __future__ import annotations
+
+import hashlib
+import re
+from dataclasses import dataclass
+from typing import Optional, Protocol, runtime_checkable
+
+from .target import Target, TargetResponse
+
+_VERDICT_RE = re.compile(r"\b(PASS|FAIL)\b")
+
+
+@dataclass
+class ScoreResult:
+    score: float            # 0/1 for binary scorers
+    binary: bool
+    verdict: Optional[int]  # 0/1 or None
+    judge_response: Optional[TargetResponse] = None
+
+
+@runtime_checkable
+class Scorer(Protocol):
+    kind: str
+    def score(self, prompt: str, response_text: str, expected: Optional[str]) -> ScoreResult: ...
+
+
+@dataclass
+class ExactMatchScorer:
+    kind: str = "exact"
+
+    def score(self, prompt, response_text, expected):
+        ok = expected is not None and response_text.strip() == expected.strip()
+        return ScoreResult(float(ok), True, int(ok))
+
+
+@dataclass
+class RegexScorer:
+    pattern: str
+    kind: str = "regex"
+
+    def score(self, prompt, response_text, expected):
+        ok = re.search(self.pattern, response_text) is not None
+        return ScoreResult(float(ok), True, int(ok))
+
+
+def parse_verdict(text: str) -> int:
+    """First standalone PASS/FAIL token wins; anything unparseable counts as FAIL.
+    (A plain substring check would read "FAIL — this does not PASS" as a pass.)"""
+    m = _VERDICT_RE.search(text.upper())
+    return 1 if m and m.group(1) == "PASS" else 0
+
+
+@dataclass
+class LLMJudgeScorer:
+    """Grades a response pass/fail using another model. The judge is a Target, so it
+    gets its own recording + provenance. This is where flips are most visible."""
+    judge: Target
+    rubric: str
+    kind: str = "llm_judge"
+
+    @property
+    def rubric_hash(self) -> str:
+        return "sha256:" + hashlib.sha256(self.rubric.encode()).hexdigest()
+
+    def score(self, prompt, response_text, expected):
+        judge_prompt = (
+            f"{self.rubric}\n\n"
+            f"USER PROMPT:\n{prompt}\n\n"
+            f"RESPONSE TO GRADE:\n{response_text}\n\n"
+            f"Answer with exactly one word: PASS or FAIL."
+        )
+        jr = self.judge.generate(judge_prompt)
+        verdict = parse_verdict(jr.text)
+        return ScoreResult(float(verdict), True, verdict, judge_response=jr)

evalseal/adapters/target.py

@@ -0,0 +1,97 @@
+from __future__ import annotations
+
+import os
+from dataclasses import dataclass
+from typing import Any, Callable, Optional, Protocol, runtime_checkable
+
+import httpx
+
+from .recording import Cassette
+
+
+@dataclass
+class TargetResponse:
+    text: str
+    requested_model: str
+    served_model: Optional[str]
+    system_fingerprint: Optional[str]
+    base_url: str
+    effective_params: dict[str, Any]
+    params_source: str  # "explicit" | "provider_default"
+
+
+@runtime_checkable
+class Target(Protocol):
+    def generate(self, prompt: str) -> TargetResponse: ...
+
+
+@dataclass
+class LocalCallableTarget:
+    """Wrap any Python function as a target. Great for offline tests: the function
+    can return scripted-variance outputs so the executor is testable with no network."""
+    fn: Callable[[str], str]
+    name: str = "local"
+
+    def generate(self, prompt: str) -> TargetResponse:
+        return TargetResponse(
+            text=self.fn(prompt),
+            requested_model=self.name,
+            served_model=self.name,
+            system_fingerprint=None,
+            base_url="local://",
+            effective_params={},
+            params_source="explicit",
+        )
+
+
+@dataclass
+class OpenAICompatibleTarget:
+    """Calls any OpenAI-compatible /chat/completions endpoint, through the cassette."""
+    model: str
+    cassette: Cassette
+    base_url: str = "https://api.openai.com/v1"
+    temperature: Optional[float] = None      # None means "we did not set it"
+    seed: Optional[int] = None
+    api_key_env: str = "EVALSEAL_API_KEY"
+
+    def generate(self, prompt: str) -> TargetResponse:
+        # Record whether we explicitly set temperature or are inheriting the default.
+        params_source = "explicit" if self.temperature is not None else "provider_default"
+        body: dict[str, Any] = {
+            "model": self.model,
+            "messages": [{"role": "user", "content": prompt}],
+        }
+        if self.temperature is not None:
+            body["temperature"] = self.temperature
+        if self.seed is not None:
+            body["seed"] = self.seed
+
+        def do_real_call() -> dict:
+            key = os.environ.get(self.api_key_env)
+            if not key:
+                raise RuntimeError(f"Recording needs an API key in ${self.api_key_env}.")
+            with httpx.Client(timeout=60) as c:
+                r = c.post(
+                    f"{self.base_url.rstrip('/')}/chat/completions",
+                    headers={"Authorization": f"Bearer {key}"},
+                    json=body,
+                )
+                r.raise_for_status()
+                return r.json()
+
+        # Cassette key is the full effective body (never the API key).
+        raw = self.cassette.call({"url": self.base_url, "body": body}, do_real_call)
+
+        return TargetResponse(
+            text=raw["choices"][0]["message"]["content"] or "",
+            requested_model=self.model,
+            served_model=raw.get("model"),
+            system_fingerprint=raw.get("system_fingerprint"),
+            base_url=self.base_url,
+            effective_params={
+                "temperature": self.temperature,
+                "seed": self.seed,
+                "top_p": None,
+            },
+            params_source=params_source,
+        )

evalseal/analyze.py

@@ -0,0 +1,82 @@
+"""Variance analysis for repeated eval runs. Pure functions, no I/O, no network.
+
+A "verdict" is a binary pass(1)/fail(0). A "score" may be binary or a float in [0,1].
+We quantify reproducibility; we never claim determinism.
+"""
+from __future__ import annotations
+
+import random
+import statistics
+from collections import Counter
+from dataclasses import dataclass
+
+# Stability thresholds (constants so they're auditable, not magic numbers).
+BORDERLINE_MAX_FLIP = 0.20  # flip_rate in (0, 0.20]  -> BORDERLINE
+# flip_rate == 0 -> STABLE ; flip_rate > BORDERLINE_MAX_FLIP -> UNSTABLE
+
+_BOOTSTRAP_ITERS = 2000
+_RNG_SEED = 12345  # fixed so the CI computation itself is reproducible
+
+
+@dataclass(frozen=True)
+class CaseStats:
+    mean: float
+    ci95_low: float
+    ci95_high: float
+    flip_rate: float
+    stability: str  # "STABLE" | "BORDERLINE" | "UNSTABLE"
+    majority_verdict: int | None  # for binary; None for pure-float scores
+
+
+def _bootstrap_ci(scores: list[float], iters: int = _BOOTSTRAP_ITERS) -> tuple[float, float]:
+    """Percentile bootstrap 95% CI of the mean. Deterministic via fixed seed."""
+    n = len(scores)
+    if n == 0:
+        return (0.0, 0.0)
+    if n == 1:
+        return (scores[0], scores[0])
+    rng = random.Random(_RNG_SEED)
+    means = []
+    for _ in range(iters):
+        sample = [scores[rng.randrange(n)] for _ in range(n)]
+        means.append(sum(sample) / n)
+    means.sort()
+    lo = means[int(0.025 * iters)]
+    hi = means[int(0.975 * iters)]
+    return (lo, hi)
+
+
+def flip_rate(verdicts: list[int]) -> tuple[float, int]:
+    """Fraction of verdicts disagreeing with the majority. Returns (rate, majority)."""
+    if not verdicts:
+        return (0.0, 0)
+    counts = Counter(verdicts)
+    majority, majority_count = counts.most_common(1)[0]
+    disagree = len(verdicts) - majority_count
+    return (disagree / len(verdicts), majority)
+
+
+def classify_stability(rate: float) -> str:
+    if rate == 0.0:
+        return "STABLE"
+    if rate <= BORDERLINE_MAX_FLIP:
+        return "BORDERLINE"
+    return "UNSTABLE"
+
+
+def analyze_case(scores: list[float], *, binary: bool) -> CaseStats:
+    """Turn N per-run scores into a CaseStats. If binary, scores must be 0/1."""
+    if not scores:
+        raise ValueError("analyze_case requires at least one score")
+    mean = sum(scores) / len(scores)
+    lo, hi = _bootstrap_ci(scores)
+    if binary:
+        verdicts = [int(s) for s in scores]
+        rate, majority = flip_rate(verdicts)
+        return CaseStats(mean, lo, hi, rate, classify_stability(rate), majority)
+    # Float scores: define a "flip" as crossing the run-set's own median.
+    # This gives a variance signal without a fixed threshold assumption.
+    med = statistics.median(scores)
+    pseudo = [1 if s >= med else 0 for s in scores]
+    rate, _ = flip_rate(pseudo)
+    return CaseStats(mean, lo, hi, rate, classify_stability(rate), None)

evalseal/cli.py

@@ -0,0 +1,152 @@
+from __future__ import annotations
+
+import json
+import os
+from pathlib import Path
+
+import httpx
+import typer
+from rich.console import Console
+from rich.markdown import Markdown
+
+from .adapters.dataset import Dataset
+from .adapters.recording import Cassette
+from .adapters.scorer import ExactMatchScorer, LLMJudgeScorer, RegexScorer
+from .adapters.target import OpenAICompatibleTarget
+from .executor import run_eval
+from .ledger import LEDGER_PATH, last_hash, load_all, seal_and_append, verify_chain
+from .report import to_markdown, write_json, write_markdown
+
+# Distinct from 1 (uncaught error) and 2 (usage error) so CI can tell
+# "the eval is unstable" apart from "the tool broke".
+EXIT_UNSTABLE = 3
+
+app = typer.Typer(add_completion=False, help="Reproducibility receipts for LLM evals.")
+console = Console()
+
+LedgerOpt = typer.Option(LEDGER_PATH, "--ledger", help="Path to the ledger JSONL file.")
+
+
+def load_dotenv(path: Path = Path(".env")) -> None:
+    """Load KEY=VALUE lines from ./.env. Variables already in the environment win."""
+    if not path.is_file():
+        return
+    for line in path.read_text().splitlines():
+        line = line.strip()
+        if not line or line.startswith("#") or "=" not in line:
+            continue
+        key, _, value = line.partition("=")
+        key = key.strip().removeprefix("export ").strip()
+        value = value.strip().strip("\"'")
+        if key and value:
+            os.environ.setdefault(key, value)
+
+
+@app.callback()
+def _main() -> None:
+    load_dotenv()
+
+
+def _build_target(cfg: dict, cassette: Cassette) -> OpenAICompatibleTarget:
+    return OpenAICompatibleTarget(
+        model=cfg["model"],
+        cassette=cassette,
+        base_url=cfg.get("base_url", "https://api.openai.com/v1"),
+        temperature=cfg.get("temperature"),   # omit in config to surface the "unset" warning
+        seed=cfg.get("seed"),
+    )
+
+
+def _build_scorer(cfg: dict, cassette: Cassette):
+    if cfg["type"] == "exact":
+        return ExactMatchScorer()
+    if cfg["type"] == "regex":
+        return RegexScorer(pattern=cfg["pattern"])
+    if cfg["type"] == "llm_judge":
+        judge = OpenAICompatibleTarget(
+            model=cfg["judge_model"],
+            cassette=cassette,
+            base_url=cfg.get("base_url", "https://api.openai.com/v1"),
+            temperature=cfg.get("judge_temperature"),  # leave unset to demonstrate flips
+            seed=cfg.get("judge_seed"),
+        )
+        return LLMJudgeScorer(judge=judge, rubric=cfg["rubric"])
+    raise typer.BadParameter(f"unknown scorer type {cfg['type']!r}")
+
+
+@app.command()
+def run(
+    dataset: Path = typer.Option(..., exists=True, dir_okay=False),
+    target_config: Path = typer.Option(..., exists=True, dir_okay=False),
+    scorer_config: Path = typer.Option(..., exists=True, dir_okay=False),
+    n: int = typer.Option(5, min=1, help="Repeats per case (flip rate needs N>=5)."),
+    cassette: Path = typer.Option(Path("tests/cassettes/run.json")),
+    ledger: Path = LedgerOpt,
+):
+    """Run an eval N times, seal the result, emit report.json + report.md."""
+    ds = Dataset.from_jsonl(dataset)
+    cass = Cassette(cassette)
+    target = _build_target(json.loads(target_config.read_text()), cass)
+    scorer = _build_scorer(json.loads(scorer_config.read_text()), cass)
+
+    try:
+        record = run_eval(ds, target, scorer, n_repeats=n, prev_hash=last_hash(ledger))
+    except RuntimeError as e:
+        console.print(f"[red]{e}[/red]")
+        raise typer.Exit(code=1)
+    except httpx.HTTPStatusError as e:
+        resp = e.response
+        console.print(f"[red]HTTP {resp.status_code} from provider: {resp.text[:300]}[/red]")
+        if resp.status_code == 429:
+            console.print(
+                "[yellow]Rate limited. Responses recorded so far are saved in the cassette; "
+                "re-run the same command later to resume.[/yellow]"
+            )
+        raise typer.Exit(code=1)
+    record = seal_and_append(record, ledger)
+    write_json(record)
+    write_markdown(record)
+    console.print(Markdown(to_markdown(record)))
+
+    # CI gate: dedicated non-zero exit if any case is UNSTABLE.
+    if record.aggregate.n_unstable > 0:
+        console.print(f"[red]{record.aggregate.n_unstable} unstable case(s) — failing.[/red]")
+        raise typer.Exit(code=EXIT_UNSTABLE)
+
+
+@app.command()
+def verify(ledger: Path = LedgerOpt):
+    """Check the ledger chain integrity (tamper detection)."""
+    ok, msg = verify_chain(ledger)
+    color = "green" if ok else "red"
+    console.print(f"[{color}]{msg}[/{color}]")
+    raise typer.Exit(code=0 if ok else 1)
+
+
+@app.command()
+def diff(
+    a: int = typer.Argument(..., help="Ledger index of the baseline run (negative ok)."),
+    b: int = typer.Argument(..., help="Ledger index of the candidate run (negative ok)."),
+    ledger: Path = LedgerOpt,
+):
+    """Compare two runs; state whether a score change exceeds the noise floor."""
+    recs = load_all(ledger)
+    for i in (a, b):
+        if not -len(recs) <= i < len(recs):
+            raise typer.BadParameter(f"index {i} out of range; ledger has {len(recs)} record(s)")
+    ra, rb = recs[a], recs[b]
+    ma, mb = ra.aggregate.mean_score, rb.aggregate.mean_score
+
+    # Noise floor: widest per-case CI half-width across both runs. Deliberately
+    # conservative — "REAL CHANGE" is only claimed when no single case's noise explains it.
+    def halfwidth(r):
+        return max(((c.ci95[1] - c.ci95[0]) / 2 for c in r.results), default=0.0)
+
+    noise = max(halfwidth(ra), halfwidth(rb))
+    delta = mb - ma
+    verdict = "within noise" if abs(delta) <= noise else "REAL CHANGE"
+    console.print(
+        f"mean {ma:.3f} -> {mb:.3f}  (delta {delta:+.3f}, noise floor ±{noise:.3f}) => {verdict}"
+    )
+    if ra.manifest.dataset.hash != rb.manifest.dataset.hash:
+        console.print("[yellow]Warning: runs used different datasets.[/yellow]")

evalseal/executor.py

@@ -0,0 +1,142 @@
+from __future__ import annotations
+
+import re
+from urllib.parse import urlparse
+
+from .adapters.dataset import Dataset
+from .adapters.scorer import Scorer
+from .adapters.target import Target, TargetResponse
+from .analyze import analyze_case
+from .models import (
+    Aggregate, CaseResult, DatasetProvenance, EffectiveParams,
+    ProvenanceManifest, RunConfig, RunRecord, ScorerProvenance, TargetProvenance,
+)
+
+_CANONICAL_HOSTS = {"api.openai.com", "generativelanguage.googleapis.com"}
+_LOCAL_SCHEMES = {"local"}
+
+
+def _to_provenance(tr: TargetResponse) -> TargetProvenance:
+    return TargetProvenance(
+        requested_model=tr.requested_model,
+        served_model=tr.served_model,
+        system_fingerprint=tr.system_fingerprint,
+        base_url=tr.base_url,
+        effective_params=EffectiveParams(**tr.effective_params),
+        params_source=tr.params_source,
+    )
+
+
+def _is_same_model(requested: str, served: str) -> bool:
+    # Providers resolve aliases to dated snapshots (gpt-4o-mini -> gpt-4o-mini-2024-07-18,
+    # gpt-4 -> gpt-4-0613). That is the same model; the snapshot is still recorded in
+    # served_model. A bare prefix check is not enough: gpt-4o-mini is not gpt-4o.
+    snapshot = re.compile(rf"{re.escape(requested)}-(\d{{4}}-\d{{2}}-\d{{2}}|\d{{4}})")
+    return served == requested or snapshot.fullmatch(served) is not None
+
+
+def _provenance_warnings(tp: TargetProvenance, role: str = "TARGET") -> list[str]:
+    w = []
+    if tp.served_model and tp.requested_model and not _is_same_model(
+        tp.requested_model, tp.served_model
+    ):
+        w.append(
+            f"{role} SERVED MODEL MISMATCH: requested '{tp.requested_model}' but served "
+            f"'{tp.served_model}'. The report's model name may not be what ran."
+        )
+    parsed = urlparse(tp.base_url)
+    if parsed.scheme not in _LOCAL_SCHEMES and parsed.hostname not in _CANONICAL_HOSTS:
+        w.append(
+            f"{role} NON-CANONICAL ENDPOINT: {tp.base_url} — responses may be proxied/altered."
+        )
+    if tp.params_source == "provider_default" and tp.effective_params.temperature is None:
+        w.append(
+            f"{role} TEMPERATURE NOT SET: the provider default (often 1.0) was used silently; "
+            "verdicts near the decision boundary may not be reproducible."
+        )
+    return w
+
+
+def _drift_warnings(responses: list[TargetResponse], role: str) -> list[str]:
+    """The backend can change under you mid-run. Surface it instead of averaging over it."""
+    w = []
+    served = sorted({r.served_model for r in responses if r.served_model})
+    if len(served) > 1:
+        w.append(f"{role} SERVED MODEL CHANGED DURING RUN: {', '.join(served)}")
+    fps = sorted({r.system_fingerprint for r in responses if r.system_fingerprint})
+    if len(fps) > 1:
+        w.append(f"{role} SYSTEM FINGERPRINT CHANGED DURING RUN: {', '.join(fps)}")
+    return w
+
+
+def _dedupe(items: list[str]) -> list[str]:
+    return list(dict.fromkeys(items))
+
+
+def run_eval(
+    dataset: Dataset,
+    target: Target,
+    scorer: Scorer,
+    n_repeats: int = 5,
+    prev_hash: str = "GENESIS",
+) -> RunRecord:
+    if not dataset.cases:
+        raise ValueError("dataset has no cases")
+    if n_repeats < 1:
+        raise ValueError("n_repeats must be >= 1")
+
+    results: list[CaseResult] = []
+    target_resps: list[TargetResponse] = []
+    judge_resps: list[TargetResponse] = []
+
+    for case in dataset.cases:
+        scores: list[float] = []
+        binary = True
+        for _ in range(n_repeats):
+            tr = target.generate(case.prompt)
+            target_resps.append(tr)
+            sr = scorer.score(case.prompt, tr.text, case.expected)
+            if sr.judge_response is not None:
+                judge_resps.append(sr.judge_response)
+            scores.append(sr.score)
+            binary = binary and sr.binary
+        stats = analyze_case(scores, binary=binary)
+        results.append(CaseResult(
+            case_id=case.case_id,
+            scores=scores,
+            mean=stats.mean,
+            ci95=(stats.ci95_low, stats.ci95_high),
+            flip_rate=stats.flip_rate,
+            stability=stats.stability,
+            majority_verdict=stats.majority_verdict,
+        ))
+
+    # Every response is checked, not just the last one: a mismatch on any call matters.
+    warnings: list[str] = []
+    for tr in target_resps:
+        warnings += _provenance_warnings(_to_provenance(tr), "TARGET")
+    warnings += _drift_warnings(target_resps, "TARGET")
+    for jr in judge_resps:
+        warnings += _provenance_warnings(_to_provenance(jr), "JUDGE")
+    warnings += _drift_warnings(judge_resps, "JUDGE")
+
+    sp = ScorerProvenance(
+        type=scorer.kind,
+        judge=_to_provenance(judge_resps[0]) if judge_resps else None,
+        rubric_hash=getattr(scorer, "rubric_hash", None),
+    )
+    manifest = ProvenanceManifest(
+        target=_to_provenance(target_resps[0]),
+        scorer=sp,
+        dataset=DatasetProvenance(hash=dataset.hash, n_cases=len(dataset.cases)),
+        run_config=RunConfig(n_repeats=n_repeats),
+    )
+    agg = Aggregate(
+        n_cases=len(results),
+        mean_score=sum(r.mean for r in results) / len(results),
+        n_stable=sum(r.stability == "STABLE" for r in results),
+        n_borderline=sum(r.stability == "BORDERLINE" for r in results),
+        n_unstable=sum(r.stability == "UNSTABLE" for r in results),
+        warnings=_dedupe(warnings),
+    )
+    return RunRecord(manifest=manifest, results=results, aggregate=agg, prev_hash=prev_hash)

evalseal/ledger.py

@@ -0,0 +1,60 @@
+from __future__ import annotations
+
+import hashlib
+import json
+from pathlib import Path
+
+from .models import RunRecord
+
+LEDGER_PATH = Path(".evalseal/ledger.jsonl")
+GENESIS = "GENESIS"
+
+
+def _content_hash(record: RunRecord) -> str:
+    # Hash everything except the hash field itself.
+    payload = record.model_dump(mode="json")
+    payload.pop("hash", None)
+    blob = json.dumps(payload, sort_keys=True, separators=(",", ":"))
+    return "sha256:" + hashlib.sha256(blob.encode()).hexdigest()
+
+
+def load_all(path: Path = LEDGER_PATH) -> list[RunRecord]:
+    if not path.exists():
+        return []
+    return [
+        RunRecord.model_validate_json(line)
+        for line in path.read_text().splitlines()
+        if line.strip()
+    ]
+
+
+def last_hash(path: Path = LEDGER_PATH) -> str:
+    recs = load_all(path)
+    return recs[-1].hash if recs else GENESIS
+
+
+def seal_and_append(record: RunRecord, path: Path = LEDGER_PATH) -> RunRecord:
+    expected_prev = last_hash(path)
+    if record.prev_hash != expected_prev:
+        raise ValueError(
+            f"record.prev_hash {record.prev_hash[:20]} does not link to ledger head "
+            f"{expected_prev[:20]}; refusing to append a broken chain."
+        )
+    record.hash = _content_hash(record)
+    path.parent.mkdir(parents=True, exist_ok=True)
+    with path.open("a") as f:
+        f.write(record.model_dump_json() + "\n")
+    return record
+
+
+def verify_chain(path: Path = LEDGER_PATH) -> tuple[bool, str]:
+    """Returns (ok, message). Detects a tampered score or a broken prev-link."""
+    recs = load_all(path)
+    prev = GENESIS
+    for i, r in enumerate(recs):
+        if r.prev_hash != prev:
+            return (False, f"Broken chain at record {i}: prev_hash mismatch.")
+        if _content_hash(r) != r.hash:
+            return (False, f"TAMPER DETECTED at record {i}: content hash does not match.")
+        prev = r.hash
+    return (True, f"Chain intact: {len(recs)} record(s).")

evalseal/models.py

@@ -0,0 +1,78 @@
+from __future__ import annotations
+
+from datetime import datetime, timezone
+from typing import Literal, Optional
+
+from pydantic import BaseModel, Field
+
+
+def _now() -> str:
+    return datetime.now(timezone.utc).isoformat()
+
+
+class EffectiveParams(BaseModel):
+    temperature: Optional[float] = None
+    seed: Optional[int] = None
+    top_p: Optional[float] = None
+
+
+class TargetProvenance(BaseModel):
+    requested_model: str
+    served_model: Optional[str] = None        # from response; WARN if != requested
+    system_fingerprint: Optional[str] = None
+    base_url: str
+    effective_params: EffectiveParams
+    params_source: Literal["explicit", "provider_default"] = "explicit"
+
+
+class ScorerProvenance(BaseModel):
+    type: Literal["exact", "regex", "llm_judge"]
+    judge: Optional[TargetProvenance] = None  # the judge is a target too
+    rubric_hash: Optional[str] = None
+
+
+class DatasetProvenance(BaseModel):
+    hash: str
+    n_cases: int
+
+
+class RunConfig(BaseModel):
+    n_repeats: int = 5
+    harness_version: str = "evalseal/0.1.0"
+    started_at: str = Field(default_factory=_now)
+
+
+class ProvenanceManifest(BaseModel):
+    schema_version: str = "1.0"
+    target: TargetProvenance
+    scorer: ScorerProvenance
+    dataset: DatasetProvenance
+    run_config: RunConfig
+
+
+class CaseResult(BaseModel):
+    case_id: str
+    scores: list[float]
+    mean: float
+    ci95: tuple[float, float]
+    flip_rate: float
+    stability: str
+    majority_verdict: Optional[int] = None
+
+
+class Aggregate(BaseModel):
+    n_cases: int
+    mean_score: float
+    n_stable: int
+    n_borderline: int
+    n_unstable: int
+    warnings: list[str] = []
+
+
+class RunRecord(BaseModel):
+    manifest: ProvenanceManifest
+    results: list[CaseResult]
+    aggregate: Aggregate
+    prev_hash: str
+    hash: str = ""            # filled by the ledger at seal time
+    created_at: str = Field(default_factory=_now)

evalseal/report.py

@@ -0,0 +1,48 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+from .models import RunRecord
+
+
+def write_json(record: RunRecord, path: str | Path = "report.json") -> None:
+    Path(path).write_text(record.model_dump_json(indent=2))
+
+
+def to_markdown(record: RunRecord) -> str:
+    a = record.aggregate
+    t = record.manifest.target
+    judge = record.manifest.scorer.judge
+    lines = [
+        "# EvalSeal run\n",
+        f"- Model requested: `{t.requested_model}`",
+        f"- Served: `{t.served_model}`  |  fingerprint: `{t.system_fingerprint}`",
+        f"- Temperature: `{t.effective_params.temperature}` ({t.params_source})",
+        f"- Scorer: `{record.manifest.scorer.type}`"
+        + (f"  |  judge: `{judge.served_model or judge.requested_model}`"
+           f", temperature `{judge.effective_params.temperature}` ({judge.params_source})"
+           if judge else ""),
+        f"- N repeats: {record.manifest.run_config.n_repeats}",
+        f"- Dataset: `{record.manifest.dataset.hash[:20]}...` ({record.manifest.dataset.n_cases} cases)",
+        f"- Sealed hash: `{record.hash[:20]}...`\n",
+        f"**Aggregate:** {a.n_cases} cases · mean {a.mean_score:.2f} · "
+        f"{a.n_stable} stable / {a.n_borderline} borderline / {a.n_unstable} unstable\n",
+    ]
+    if a.warnings:
+        lines.append("## ⚠ Provenance warnings")
+        lines += [f"- {w}" for w in a.warnings]
+        lines.append("")
+    lines.append("## Per-case reproducibility\n")
+    lines.append("| case | verdicts | mean | 95% CI | flip rate | stability |")
+    lines.append("|---|---|---|---|---|---|")
+    for r in record.results:
+        verdicts = "".join("P" if s >= 0.5 else "F" for s in r.scores)
+        lines.append(
+            f"| {r.case_id} | `{verdicts}` | {r.mean:.2f} | [{r.ci95[0]:.2f}, {r.ci95[1]:.2f}] "
+            f"| {r.flip_rate:.0%} | {r.stability} |"
+        )
+    return "\n".join(lines) + "\n"
+
+
+def write_markdown(record: RunRecord, path: str | Path = "report.md") -> None:
+    Path(path).write_text(to_markdown(record))

evalseal-0.1.0.dist-info/METADATA

@@ -0,0 +1,116 @@
+Metadata-Version: 2.5
+Name: evalseal
+Version: 0.1.0
+Summary: Reproducibility and provenance receipts for LLM evaluations.
+Project-URL: Homepage, https://github.com/patibandlavenkatamanideep/evalseal
+Project-URL: Issues, https://github.com/patibandlavenkatamanideep/evalseal/issues
+Author: Venkata Manideep Patibandla
+License: MIT
+License-File: LICENSE
+Keywords: evaluation,llm,llm-as-judge,provenance,reproducibility
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Topic :: Software Development :: Testing
+Requires-Python: >=3.11
+Requires-Dist: httpx>=0.27
+Requires-Dist: pydantic>=2.6
+Requires-Dist: rich>=13.7
+Requires-Dist: typer>=0.12
+Provides-Extra: dev
+Requires-Dist: pytest-cov>=5.0; extra == 'dev'
+Requires-Dist: pytest>=8.0; extra == 'dev'
+Description-Content-Type: text/markdown
+
+# EvalSeal
+
+**Reproducibility receipts for LLM evals: run it N times, report the score with its noise, and seal what actually ran.**
+
+## The problem
+
+An eval score from a single run is one sample of a random process. Run the same eval again
+and borderline items quietly flip from PASS to FAIL, especially when an LLM judge grades
+them. Reports also name the model you *asked* for, not the one that *answered*. EvalSeal
+measures the flips, records the real provenance, and seals both into a tamper-evident ledger.
+
+## Quickstart
+
+```bash
+pip install evalseal
+
+# The recorded demo (examples + cassette) lives in the repo.
+git clone https://github.com/patibandlavenkatamanideep/evalseal && cd evalseal
+
+# Replays the committed cassette: no API key, no network.
+evalseal run \
+  --dataset examples/borderline_judge/dataset.jsonl \
+  --target-config examples/borderline_judge/target.json \
+  --scorer-config examples/borderline_judge/scorer.json \
+  --n 5
+```
+
+## Real flip rates
+
+Recorded 2026-09-15 and committed in `tests/cassettes/run.json`: `gemini-2.5-flash` as both
+the target and the judge, temperature left at the provider default, 20 arguable prompts,
+5 runs each. The quickstart above replays exactly this run.
+
+**Mean score: 0.92, but 5 of 20 cases did not get the same verdict every time.**
+
+| case | prompt | verdicts | mean | 95% CI | flip rate | stability |
+|---|---|---|---|---|---|---|
+| b01 | Is a hot dog a sandwich? | `FPPFP` | 0.60 | [0.20, 1.00] | 40% | UNSTABLE |
+| b10 | Blockchain for a child in exactly 20 words | `FPFPP` | 0.60 | [0.20, 1.00] | 40% | UNSTABLE |
+| b16 | "Do we only use 10% of our brains?" in a jokey tone | `PPFFP` | 0.60 | [0.20, 1.00] | 40% | UNSTABLE |
+| b05 | A borderline-polite refusal to a coworker | `PFPPP` | 0.80 | [0.40, 1.00] | 20% | BORDERLINE |
+| b19 | A technically accurate haiku about recursion | `PPFPP` | 0.80 | [0.40, 1.00] | 20% | BORDERLINE |
+| 15 others | | `PPPPP` | 1.00 | [1.00, 1.00] | 0% | STABLE |
+
+Treat the k-th repeat of every case as one ordinary single-run eval, and the five
+"single runs" of this identical eval scored **0.90, 0.95, 0.85, 0.90 and 1.00**. A single
+run can't tell you which of those numbers you got.
+
+The report also flagged `TEMPERATURE NOT SET` for both the target and the judge, which is
+the reason these borderline verdicts can come out differently from run to run.
+
+To re-record with your own key, copy `.env.example` to `.env`, add a free
+[Google AI Studio](https://aistudio.google.com/apikey) key, and run the quickstart with
+`EVALSEAL_RECORD=1`. If the free tier rate-limits you, run the same command again later;
+responses already recorded are kept.
+
+## Commands
+
+| command | what it does | exit code |
+|---|---|---|
+| `evalseal run` | Runs each case N times, analyzes variance, seals a record, writes `report.json` + `report.md`. | `0` all stable/borderline · `3` any case UNSTABLE · `1` error |
+| `evalseal verify` | Recomputes every hash in `.evalseal/ledger.jsonl` and checks the chain links. | `0` intact · `1` tampered or broken |
+| `evalseal diff A B` | Compares two ledger runs and says whether the mean moved beyond the noise floor. Use `--` for negative indices: `evalseal diff -- 0 -1`. | `0` |
+
+**Stability classes** are based on the flip rate, the share of a case's N verdicts that
+disagree with its majority: `STABLE` (0), `BORDERLINE` (≤ 20%), `UNSTABLE` (> 20%).
+
+**Provenance warnings** show up in the report when:
+- the served model differs from the requested one (for the target or the judge),
+- the endpoint isn't a canonical provider host,
+- temperature was left at the provider default,
+- the served model or system fingerprint changed partway through the run.
+
+## How it works
+
+The executor sends each prompt to the target N times and scores every response. An LLM
+judge is itself a target, so its own randomness is measured instead of assumed away.
+`analyze.py` computes the mean, a seeded bootstrap 95% CI, and the flip rate for each case.
+Every request goes through a cassette. In record mode, real responses are saved in call
+order; in replay mode, which is the default and what CI uses, they are served back, and a
+missing entry fails loudly. Each run is saved as a `RunRecord`: its manifest (requested vs.
+served model, fingerprint, parameters and whether they were set explicitly, rubric hash,
+dataset hash) plus its results. The record is hashed and linked to the previous record's
+hash in an append-only JSONL ledger, so editing any past score breaks `verify`.
+
+See [DESIGN.md](https://github.com/patibandlavenkatamanideep/evalseal/blob/main/DESIGN.md) for what this does and does not prove.

evalseal-0.1.0.dist-info/RECORD

@@ -0,0 +1,17 @@
+evalseal/__init__.py,sha256=prMy07gYwx9H-vSb9Qcm68fl8fslACzZLFSDiqCVocU,100
+evalseal/analyze.py,sha256=uzb-QL9tvwo9_qsLlvVyz2VY0l1zjY6fId5w7tyWihY,2894
+evalseal/cli.py,sha256=ZhabImU-yGbhCr1_2u-O8fTclynCQRERZiR9N8CfKaw,5882
+evalseal/executor.py,sha256=FdmNDAJVmUoEdeNFjgwTPfxmLoTKflwNqdZLUrA2Ye0,5523
+evalseal/ledger.py,sha256=SRh5Abs4-MkfCzsZ2SbbciN_WTRV-uzfVywQ4R6i3tQ,1946
+evalseal/models.py,sha256=Ct8TYa0VPfy6upI-Yg-9mGs9uYJLNsBcgDMk2dU0ZXI,1895
+evalseal/report.py,sha256=sb1XWsY2cyLhxq5lPC7K7x3WenQsg5YRM9UCJj2CPGE,2039
+evalseal/adapters/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+evalseal/adapters/dataset.py,sha256=JMaP5dMyGmoSTaDW59IKNhCl8n3RJBu5IKCpq7Z72Mg,950
+evalseal/adapters/recording.py,sha256=0xyaSD8cJliFvkEkAK3Wqjf37iplKUTAOjGYJqkHvZY,2278
+evalseal/adapters/scorer.py,sha256=W6ywL_cCdlZVkajDx_5Yzw05B6ROFtb7RBJAi_vsx1U,2220
+evalseal/adapters/target.py,sha256=zgpSk34KLjAqhmPc-STE5alRGrPDCkZO_B5QzXuly4c,3250
+evalseal-0.1.0.dist-info/METADATA,sha256=oo795y8bdeDFbuSpHiYfmTRH0l35UqP-9h9oJbbdKcQ,5844
+evalseal-0.1.0.dist-info/WHEEL,sha256=zOwg4jB6zX2kU910N-cMawjivD6tO8NEWvE12je1bVk,87
+evalseal-0.1.0.dist-info/entry_points.txt,sha256=j-GPjykIrYuVnOWt-2QaEaXQEpedrXNeA3vs-YwXHQY,46
+evalseal-0.1.0.dist-info/licenses/LICENSE,sha256=BUR4NJbROoLQlsxDQaqjIk8ThVqyIjc9AfXORDMlv9A,1084
+evalseal-0.1.0.dist-info/RECORD,,

evalseal-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.32.0
+Root-Is-Purelib: true
+Tag: py3-none-any

evalseal-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+evalseal = evalseal.cli:app

evalseal-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Venkata Manideep Patibandla
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.