evalpair 0.1.0__py3-none-any.whl

evalpair/cli.py

@@ -0,0 +1,164 @@
+"""Command line: evalpair suites | run | runs | show | compare | mcp."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import sys
+
+from . import __version__, analysis, store, suites
+from .backends import Backend, BackendError, Cache
+
+API_KEY_ENV = ("EVALPAIR_API_KEY", "OPENAI_API_KEY")
+
+
+def api_key() -> str | None:
+    return next((os.environ[k] for k in API_KEY_ENV if os.environ.get(k)), None)
+
+
+def make_backend(h: str, kind: str, model: str, base_url: str | None, temperature: float, seed: int | None,
+                 num_ctx: int | None, think: bool | None, cache_only: bool = False) -> Backend:
+    opts = {"temperature": temperature}
+    if seed is not None:
+        opts["seed"] = seed
+    if num_ctx and kind == "ollama":
+        opts["num_ctx"] = num_ctx
+    return Backend(kind, model, Cache(store.cache_path(h)), base_url, api_key(), opts, think, cache_only)
+
+
+def _print(obj, as_json: bool, text_fn) -> None:
+    if as_json:
+        print(json.dumps(obj, indent=1, ensure_ascii=False))
+    else:
+        print(text_fn(obj))
+
+
+def text_summary(s: dict) -> str:
+    lines = [f"{s['run']}  ({s['model']} on {s['suite']}, {s['items']} items)",
+             f"  score        {s['score']['text']}",
+             f"  no answer    {s['no_answer']}    truncated {s['truncated']}    "
+             f"mean output {s['mean_output_tokens']} tokens"]
+    nf = s.get("noise_floor")
+    if nf:
+        lines.append(f"  noise floor  {nf['answers_changed']} of {nf['items']} answers changed on a repeat "
+                     f"({nf['change_rate']['text']})")
+    for g, v in (s.get("by_group") or {}).items():
+        lines.append(f"    {g:<28} {100 * v['score']:5.1f}%  ({v['items']})")
+    if s["failures"]:
+        lines.append("  failures (unparsed and truncated first):")
+        for f in s["failures"]:
+            end = f["reply_end"].replace("\n", " ")[-160:]
+            lines.append(f"    {f['id']}: answer {f['answer']!r} via {f['how']}"
+                         f"{', truncated' if f['truncated'] else ''}  …{end}")
+    return "\n".join(lines)
+
+
+def text_compare(c: dict) -> str:
+    if not c.get("items"):
+        return "\n".join(c["notes"])
+    lines = [f"A  {c['a']['run']}: {c['a']['score']['text']}",
+             f"B  {c['b']['run']}: {c['b']['score']['text']}",
+             f"B − A        {c['difference']['text']}",
+             f"changed      {c['answers_changed']['text']} of {c['items']} items "
+             f"({c['worse_in_b']} worse in B, {c['better_in_b']} better)"]
+    for k, f in (c.get("noise_floor") or {}).items():
+        lines.append(f"floor ({k.upper()})    {f['change_rate']['text']}; excess {f['excess_over_floor']['text']}")
+    lines += ["", c["verdict"]] + [f"Note: {n}" for n in c["notes"]]
+    for kind in ("worse_in_b", "better_in_b"):
+        for e in c["examples"][kind]:
+            lines.append(f"  {kind.replace('_', ' ')}: {e['id']}  {e['a_answer']!r} → {e['b_answer']!r}")
+    return "\n".join(lines)
+
+
+def main(argv: list[str] | None = None) -> None:
+    p = argparse.ArgumentParser(prog="evalpair", description=__doc__)
+    p.add_argument("--version", action="version", version=f"evalpair {__version__}")
+    p.add_argument("--home", help="data folder (default $EVALPAIR_HOME or ~/.evalpair)")
+    sub = p.add_subparsers(dest="cmd", required=True)
+
+    sub.add_parser("suites", help="list the suites you can run").add_argument("--json", action="store_true")
+
+    r = sub.add_parser("run", help="run a suite against a model")
+    r.add_argument("suite", help="a suite name, or a path to a suite folder")
+    r.add_argument("--model", "-m", required=True)
+    r.add_argument("--backend", choices=["ollama", "openai"], default="ollama")
+    r.add_argument("--base-url", help="server URL (default: Ollama on localhost, or http://localhost:8000/v1)")
+    r.add_argument("--limit", "-n", type=int, help="only the first N items")
+    r.add_argument("--repeat", action="store_true", help="send every item twice to measure the noise floor")
+    r.add_argument("--workers", type=int, default=1)
+    r.add_argument("--label", help="run id (default: suite-model-timestamp)")
+    r.add_argument("--temperature", type=float, default=0.0)
+    r.add_argument("--seed", type=int, default=7)
+    r.add_argument("--num-ctx", type=int, help="Ollama context window")
+    r.add_argument("--think", choices=["on", "off"], help="Ollama thinking for models that support it")
+    r.add_argument("--json", action="store_true")
+
+    ls = sub.add_parser("runs", help="list saved runs, newest first")
+    ls.add_argument("--suite")
+    ls.add_argument("--json", action="store_true")
+
+    sh = sub.add_parser("show", help="summarise a run, with its failures")
+    sh.add_argument("run")
+    sh.add_argument("--failures", type=int, default=5)
+    sh.add_argument("--json", action="store_true")
+
+    c = sub.add_parser("compare", help="compare two runs item by item")
+    c.add_argument("a")
+    c.add_argument("b")
+    c.add_argument("--examples", type=int, default=5)
+    c.add_argument("--json", action="store_true")
+
+    sub.add_parser("mcp", help="run the MCP server on stdio")
+
+    args = p.parse_args(argv)
+    h = store.home(args.home)
+    try:
+        if args.cmd == "suites":
+            rows = [suites.find(n, h).summary() for n in suites.available(h)]
+            _print(rows, args.json, lambda rs: "\n".join(
+                f"{s['name']:<22} {s['items']:>5} items  {s['checker']:<12} {s['description']}" for s in rs))
+        elif args.cmd == "run":
+            suite = suites.find(args.suite, h)
+            think = None if args.think is None else args.think == "on"
+            be = make_backend(h, args.backend, args.model, args.base_url, args.temperature, args.seed,
+                              args.num_ctx, think)
+
+            tty = sys.stderr.isatty()
+
+            def progress(done, total):
+                if tty and (done == total or done % 10 == 0):
+                    print(f"\r{done}/{total}", end="", file=sys.stderr, flush=True)
+                elif not tty and (done == total or done % 50 == 0):  # a log file: whole lines, fewer of them
+                    print(f"{done}/{total}", file=sys.stderr, flush=True)
+            from .runner import run
+            meta = run(suite, be, h, args.limit, args.repeat, args.workers, args.label, progress)
+            if tty:
+                print(file=sys.stderr)
+            m, rows = store.load(h, meta["run"])
+            _print(analysis.summarize(m, rows), args.json, text_summary)
+        elif args.cmd == "runs":
+            rows = store.list_runs(h, args.suite)
+            _print(rows, args.json, lambda rs: "\n".join(
+                f"{m['run']:<50} {m['model']:<24} {m['items']:>5} items{'  +repeat' if m['repeat'] else ''}"
+                for m in rs) or "no runs yet")
+        elif args.cmd == "show":
+            m, rows = store.load(h, args.run)
+            _print(analysis.summarize(m, rows, args.failures), args.json, text_summary)
+        elif args.cmd == "compare":
+            ma, ra = store.load(h, args.a)
+            mb, rb = store.load(h, args.b)
+            _print(analysis.compare(ma, ra, mb, rb, args.examples), args.json, text_compare)
+        elif args.cmd == "mcp":
+            from .server import mcp
+            mcp.run()
+    except (suites.SuiteError, store.RunError, BackendError) as e:
+        print(f"evalpair: {e}", file=sys.stderr)
+        sys.exit(2)
+    except KeyboardInterrupt:
+        print("\ninterrupted; replies so far are cached, so the same command resumes", file=sys.stderr)
+        sys.exit(130)
+
+
+if __name__ == "__main__":
+    main()

evalpair/runner.py

@@ -0,0 +1,67 @@
+"""Run a suite against one model and save the run.
+
+Items are taken in file order (bundled suites are already a seeded random draw), so `limit=50` is the
+same 50 items every time and two limited runs still pair up. With `repeat`, every item is sent a second
+time as a separate call; how often the answer changes between the two passes is the run's noise floor.
+"""
+
+from __future__ import annotations
+
+import datetime as dt
+import threading
+import time
+from concurrent.futures import ThreadPoolExecutor
+from typing import Callable
+
+from . import __version__, store
+from .backends import Backend
+from .suites import Suite
+
+REPEAT_TAG = "repeat"
+
+
+def _row(suite: Suite, item: dict, pass_no: int, reply: dict) -> dict:
+    c = suite.check(reply["text"], item)
+    return {"id": item["id"], "pass": pass_no, "group": item.get(suite.group) if suite.group else None,
+            "answer": c.answer, "score": c.score, "how": c.how, "text": reply["text"],
+            "truncated": reply.get("done_reason") == "length", "prompt_tokens": reply.get("prompt_tokens", 0),
+            "output_tokens": reply.get("output_tokens", 0), "seconds": round(reply.get("seconds", 0.0), 3),
+            "cached": reply.get("cached", False)}
+
+
+def run(suite: Suite, backend: Backend, home: str, limit: int | None = None, repeat: bool = False,
+        workers: int = 1, label: str | None = None,
+        progress: Callable[[int, int], None] | None = None) -> dict:
+    items = suite.items[:limit] if limit else list(suite.items)
+    jobs = [(it, 1) for it in items] + ([(it, 2) for it in items] if repeat else [])
+    started = dt.datetime.now(dt.timezone.utc)
+    run_id = store.check_id(label) if label else store.slug(
+        f"{suite.name}-{backend.model}-{started.strftime('%Y%m%d-%H%M%S')}")
+    t0 = time.time()
+    done, lock = [0], threading.Lock()
+
+    def one(job):
+        it, p = job
+        reply = backend.chat(suite.render(it), suite.max_tokens, REPEAT_TAG if p == 2 else "")
+        with lock:
+            done[0] += 1
+            n = done[0]
+        if progress:
+            progress(n, len(jobs))
+        return _row(suite, it, p, reply)
+
+    if workers > 1:
+        with ThreadPoolExecutor(workers) as ex:
+            rows = list(ex.map(one, jobs))
+    else:
+        rows = [one(j) for j in jobs]
+    meta = {
+        "run": run_id, "suite": suite.name, "suite_hash": suite.hash, "checker": suite.checker,
+        **backend.describe(), "options": backend.options, "think": backend.think, "max_tokens": suite.max_tokens,
+        "items": len(items), "repeat": repeat, "calls": backend.stats["calls"], "cache_hits": backend.stats["hits"],
+        "started": started.isoformat(timespec="seconds"),
+        "finished": dt.datetime.now(dt.timezone.utc).isoformat(timespec="seconds"),
+        "wall_seconds": round(time.time() - t0, 1), "evalpair": __version__,
+    }
+    store.save(home, meta, rows)
+    return meta

evalpair/server.py

@@ -0,0 +1,116 @@
+"""MCP server front end (stdio). Four read-only tools over saved runs, and one that runs a suite."""
+
+from __future__ import annotations
+
+from typing import Annotated, Any, Literal
+
+import anyio
+import anyio.from_thread
+from mcp.server.mcpserver import Context, MCPServer
+from mcp.server.mcpserver.exceptions import ToolError
+from mcp.types import ToolAnnotations
+from pydantic import Field
+
+from . import __version__, analysis, store, suites
+from .backends import BackendError
+from .cli import make_backend
+
+INSTRUCTIONS = (
+    "evalpair runs an eval suite (questions with known answers, a prompt template and a deterministic checker) "
+    "against a model on Ollama or any OpenAI-compatible server, caches every call, and compares two runs item by "
+    "item. Scores come with 95% bootstrap intervals over items. A comparison says whether a score difference is "
+    "distinguishable from zero and whether answers changed more than the noise floor (the same model asked twice, "
+    "measured when a run is made with repeat=true). Prefer a comparison's verdict over reading two scores side by "
+    "side. Read a run's failures before trusting its score: unparsed or truncated replies are listed first."
+)
+
+mcp = MCPServer("evalpair", instructions=INSTRUCTIONS, version=__version__)
+READ_ONLY = ToolAnnotations(read_only_hint=True, destructive_hint=False, idempotent_hint=True, open_world_hint=False)
+RUNS = ToolAnnotations(read_only_hint=False, destructive_hint=False, idempotent_hint=False, open_world_hint=True)
+
+RunId = Annotated[str, Field(description="A run id from list_runs or run_eval.", min_length=1, max_length=120)]
+
+
+def home() -> str:
+    return store.home()
+
+
+def _guard(fn, *args, **kw):
+    try:
+        return fn(*args, **kw)
+    except (suites.SuiteError, store.RunError, BackendError) as e:
+        raise ToolError(str(e)) from None
+
+
+@mcp.tool(title="List eval suites", annotations=READ_ONLY)
+def list_suites() -> dict[str, Any]:
+    """Use to see which eval suites can be run: name, item count, checker and what each measures."""
+    h = home()
+    return {"suites": _guard(lambda: [suites.find(n, h).summary() for n in suites.available(h)]),
+            "your_suites_folder": f"{h}/suites"}
+
+
+@mcp.tool(title="List saved runs", annotations=READ_ONLY)
+def list_runs(
+    suite: Annotated[str | None, Field(description="Only runs of this suite.")] = None,
+    limit: Annotated[int, Field(description="How many, newest first.", ge=1, le=200)] = 20,
+) -> dict[str, Any]:
+    """Use to find earlier runs to show or compare: run id, suite, model, item count, whether it has a repeat pass."""
+    keep = ("run", "suite", "model", "backend", "items", "repeat", "finished", "digest")
+    return {"runs": [{k: m.get(k) for k in keep} for m in store.list_runs(home(), suite)[:limit]]}
+
+
+@mcp.tool(title="Run an eval suite", annotations=RUNS)
+async def run_eval(
+    suite: Annotated[str, Field(description="A suite name from list_suites, or a path to a suite folder.")],
+    model: Annotated[str, Field(description="Model name as the server knows it, e.g. 'llama3.1:8b' on Ollama.")],
+    ctx: Context,
+    backend: Annotated[Literal["ollama", "openai"], Field(
+        description="'ollama' for Ollama's API; 'openai' for any OpenAI-compatible server.")] = "ollama",
+    base_url: Annotated[str | None, Field(
+        description="Server URL. Defaults: http://localhost:11434 (ollama), http://localhost:8000/v1 (openai).")] = None,
+    limit: Annotated[int | None, Field(description="Only the first N items; start small, runs take a while.",
+                                       ge=1)] = None,
+    repeat: Annotated[bool, Field(description="Send every item twice to measure the noise floor.")] = False,
+    temperature: Annotated[float, Field(ge=0, le=2)] = 0.0,
+    seed: Annotated[int | None, Field(description="Sampling seed, where the server supports one.")] = 7,
+    think: Annotated[bool | None, Field(description="Ollama thinking on or off, for models that support it.")] = None,
+    label: Annotated[str | None, Field(description="Run id to save under (letters, digits, . _ -).")] = None,
+) -> dict[str, Any]:
+    """Use to run a suite against one model and save the run. Returns its score with an interval and its first failures. Calls the model; cached replies are reused."""
+    from .runner import run
+    h = home()
+    s = _guard(suites.find, suite, h)
+    be = _guard(make_backend, h, backend, model, base_url, temperature, seed, None, think)
+    n = min(limit or len(s.items), len(s.items)) * (2 if repeat else 1)
+
+    def progress(done, total):
+        if done == total or done % 5 == 0:
+            anyio.from_thread.run(ctx.report_progress, done, total, f"{done}/{total} calls")
+
+    meta = await anyio.to_thread.run_sync(lambda: _guard(run, s, be, h, limit, repeat, 1, label, progress))
+    m, rows = store.load(h, meta["run"])
+    return {**analysis.summarize(m, rows), "calls_made": meta["calls"], "calls_planned": n}
+
+
+@mcp.tool(title="Show a run", annotations=READ_ONLY)
+def show_run(
+    run: RunId,
+    failures: Annotated[int, Field(description="How many failed items to include.", ge=0, le=50)] = 5,
+) -> dict[str, Any]:
+    """Use to see a saved run's score with its interval, the noise floor if it has a repeat, and the end of failed replies."""
+    m, rows = _guard(store.load, home(), run)
+    return analysis.summarize(m, rows, failures)
+
+
+@mcp.tool(title="Compare two runs", annotations=READ_ONLY)
+def compare_runs(
+    a: Annotated[str, Field(description="The baseline run id.", min_length=1, max_length=120)],
+    b: Annotated[str, Field(description="The run to compare against it.", min_length=1, max_length=120)],
+    examples: Annotated[int, Field(description="Items to show that got worse and better.", ge=0, le=20)] = 5,
+) -> dict[str, Any]:
+    """Use to compare two runs of the same suite item by item: score difference with a paired interval, flips each way, and whether answers moved more than the noise floor."""
+    h = home()
+    ma, ra = _guard(store.load, h, a)
+    mb, rb = _guard(store.load, h, b)
+    return analysis.compare(ma, ra, mb, rb, examples)

evalpair/stats.py

@@ -0,0 +1,52 @@
+"""Percentile bootstrap over items, never over single calls.
+
+Every run of a suite answers the same items, so a comparison is paired: resample items, and an item's
+score under both runs moves together. With a `group` (e.g. MMLU subject), items are resampled within each
+group so every resample keeps the suite's mix. Pure Python, seeded, so the same inputs give the same interval.
+"""
+
+from __future__ import annotations
+
+import math
+import random
+from collections import defaultdict
+
+BOOT = 2000
+
+
+def mean(xs) -> float:
+    xs = list(xs)
+    return sum(xs) / len(xs) if xs else math.nan
+
+
+def _percentile(sorted_xs: list[float], q: float) -> float:
+    # linear interpolation, as numpy's default
+    pos = (len(sorted_xs) - 1) * q
+    lo, hi = math.floor(pos), math.ceil(pos)
+    return sorted_xs[lo] + (sorted_xs[hi] - sorted_xs[lo]) * (pos - lo)
+
+
+def mean_ci(values, groups=None, n_boot: int = BOOT, seed: int = 0, level: float = 0.95) -> dict:
+    values = [float(v) for v in values]
+    n = len(values)
+    if n == 0:
+        return {"mean": math.nan, "lo": math.nan, "hi": math.nan, "n": 0}
+    groups = list(groups) if groups is not None else [0] * n
+    by = defaultdict(list)
+    for v, g in zip(values, groups):
+        by[g].append(v)
+    strata = list(by.values())
+    rng = random.Random(seed)
+    boots = []
+    for _ in range(n_boot):
+        s = 0.0
+        for vs in strata:
+            s += sum(rng.choices(vs, k=len(vs)))
+        boots.append(s / n)
+    boots.sort()
+    a = (1 - level) / 2
+    return {"mean": sum(values) / n, "lo": _percentile(boots, a), "hi": _percentile(boots, 1 - a), "n": n}
+
+
+def excludes_zero(ci: dict) -> bool:
+    return ci["n"] > 0 and (ci["lo"] > 0 or ci["hi"] < 0)

evalpair/store.py

@@ -0,0 +1,86 @@
+"""Where evalpair keeps things: `$EVALPAIR_HOME`, or ~/.evalpair.
+
+    <home>/cache.sqlite          every model reply, keyed by request
+    <home>/runs/<run>/meta.json  what was run: suite and its hash, model, backend, options, counts
+    <home>/runs/<run>/rows.jsonl one line per item and pass: the reply, the parsed answer, the score
+    <home>/suites/<name>/        your own suites
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import re
+
+RUN_ID = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,119}$")
+
+
+class RunError(ValueError):
+    pass
+
+
+def home(path: str | None = None) -> str:
+    return os.path.abspath(os.path.expanduser(path or os.environ.get("EVALPAIR_HOME") or "~/.evalpair"))
+
+
+def cache_path(h: str) -> str:
+    return os.path.join(h, "cache.sqlite")
+
+
+def runs_dir(h: str) -> str:
+    return os.path.join(h, "runs")
+
+
+def slug(s: str) -> str:
+    return re.sub(r"[^A-Za-z0-9._-]+", "-", s).strip("-")[:60] or "run"
+
+
+def check_id(run_id: str) -> str:
+    if not RUN_ID.match(run_id or "") or ".." in run_id:
+        raise RunError(f"bad run id {run_id!r}: use letters, digits, '.', '_' and '-'")
+    return run_id
+
+
+def run_path(h: str, run_id: str) -> str:
+    return os.path.join(runs_dir(h), check_id(run_id))
+
+
+def save(h: str, meta: dict, rows: list[dict]) -> str:
+    d = run_path(h, meta["run"])
+    os.makedirs(d, exist_ok=True)
+    tmp = os.path.join(d, "rows.jsonl.tmp")
+    with open(tmp, "w", encoding="utf-8") as f:
+        for r in rows:
+            f.write(json.dumps(r, ensure_ascii=False) + "\n")
+    os.replace(tmp, os.path.join(d, "rows.jsonl"))
+    with open(os.path.join(d, "meta.json"), "w", encoding="utf-8") as f:
+        json.dump(meta, f, indent=1, ensure_ascii=False)
+    return d
+
+
+def load(h: str, run_id: str) -> tuple[dict, list[dict]]:
+    d = run_path(h, run_id)
+    if not os.path.isfile(os.path.join(d, "meta.json")):
+        known = [m["run"] for m in list_runs(h)][:10]
+        raise RunError(f"no run {run_id!r}" + (f"; recent: {', '.join(known)}" if known else ""))
+    with open(os.path.join(d, "meta.json"), encoding="utf-8") as f:
+        meta = json.load(f)
+    with open(os.path.join(d, "rows.jsonl"), encoding="utf-8") as f:
+        rows = [json.loads(line) for line in f if line.strip()]
+    return meta, rows
+
+
+def list_runs(h: str, suite: str | None = None) -> list[dict]:
+    """Newest first."""
+    out = []
+    d = runs_dir(h)
+    if not os.path.isdir(d):
+        return out
+    for name in os.listdir(d):
+        p = os.path.join(d, name, "meta.json")
+        if os.path.isfile(p):
+            with open(p, encoding="utf-8") as f:
+                m = json.load(f)
+            if suite is None or m.get("suite") == suite:
+                out.append(m)
+    return sorted(out, key=lambda m: m.get("finished", ""), reverse=True)