flakeval 0.1.0__py3-none-any.whl

flakeval/__init__.py

@@ -0,0 +1,5 @@
+from .io import load_jsonl, repeat, save_jsonl
+from .stats import Comparison, Scores, Summary, compare, summarize
+
+__all__ = ["Scores", "Summary", "Comparison", "summarize", "compare", "repeat", "load_jsonl", "save_jsonl"]
+__version__ = "0.1.0"

flakeval/cli.py

@@ -0,0 +1,99 @@
+"""Command line interface: `flakeval report`, `flakeval compare`, and `flakeval init` (copies the settings template)."""
+from __future__ import annotations
+
+import argparse
+import dataclasses
+import json
+import sys
+from importlib import resources
+from pathlib import Path
+
+from .io import load_jsonl
+from .stats import Comparison, Summary, compare, summarize
+
+
+def format_summary(name: str, s: Summary) -> str:
+    lines = [
+        f"== {name} ==",
+        f"items: {s.n_items}   runs per item: {s.n_runs}",
+        f"mean score: {s.mean:.4f}   95% CI: [{s.ci[0]:.4f}, {s.ci[1]:.4f}]",
+    ]
+    if s.n_runs > 1:
+        if s.run_noise_share != s.run_noise_share:  # NaN: the scores did not vary at all
+            lines.append("uncertainty from run-to-run noise: n/a (the scores did not vary at all)")
+        else:
+            lines.append(f"uncertainty from run-to-run noise: {s.run_noise_share:.0%} (rest: eval-set sampling)")
+        n_flaky = round(s.flaky_fraction * s.n_items)  # show the count: "0%" next to a listed unstable item looks like a bug
+        lines.append(f"flaky items (score changes between runs): {n_flaky} of {s.n_items} ({s.flaky_fraction:.1%})")
+        if s.flaky_items:
+            lines.append("most unstable items: " + ", ".join(f"{i} (sd {sd:.2f})" for i, sd in s.flaky_items[:5]))
+    lines += [f"WARNING: {w}" for w in s.warnings]
+    return "\n".join(lines)
+
+
+def format_comparison(c: Comparison) -> str:
+    lines = [
+        f"A: {c.mean_a:.4f} ({c.runs_a} runs)   B: {c.mean_b:.4f} ({c.runs_b} runs)   items: {c.n_items}",
+        f"delta (B - A): {c.delta:+.4f}   95% CI: [{c.ci[0]:+.4f}, {c.ci[1]:+.4f}]   p = {c.p_value:.4f}",
+        f"smallest reliably detectable difference: ~{c.mde:.4f}",
+        f"VERDICT: {c.verdict}",
+        c.advice,
+    ]
+    lines += [f"WARNING: {w}" for w in c.warnings]
+    return "\n".join(lines)
+
+
+def init_command(filename: str, force: bool = False, with_sample_data: bool = True) -> int:
+    """Copy the settings template (and a tiny sample dataset) out of the installed package."""
+    templates = resources.files("flakeval") / "templates"
+    dest = Path(filename)
+    if dest.exists() and not force:
+        print(f"{dest} already exists, so nothing was changed. Use --force to overwrite it "
+              f"(you would lose your edits), or choose another name: flakeval init other_name.py")
+        return 1
+    dest.parent.mkdir(parents=True, exist_ok=True)
+    dest.write_text((templates / "my_eval.py").read_text(encoding="utf-8"), encoding="utf-8")
+    print(f"created {dest}")
+    sample = dest.parent / "sample_data.csv"
+    if with_sample_data and not sample.exists():
+        sample.write_text((templates / "sample_data.csv").read_text(encoding="utf-8"), encoding="utf-8")
+        print(f"created {sample}  (12 easy questions, just to try things out)")
+    print(f"\nNext:\n  1. open {dest} and edit the settings at the top (data path, models, how to score)\n"
+          f"  2. python {dest} --dry-run     # tries 2 questions, saves nothing\n"
+          f"  3. python {dest}               # the real run")
+    return 0
+
+
+def main(argv: list[str] | None = None) -> int:
+    p = argparse.ArgumentParser(prog="flakeval", description="Is your eval improvement real, or just noise?")
+    sub = p.add_subparsers(dest="cmd", required=True)
+    r = sub.add_parser("report", help="noise report for one system")
+    r.add_argument("results")
+    c = sub.add_parser("compare", help="paired comparison of system A vs B")
+    c.add_argument("a")
+    c.add_argument("b")
+    i = sub.add_parser("init", help="create a settings file (my_eval.py) to test your own models and data")
+    i.add_argument("filename", nargs="?", default="my_eval.py")
+    i.add_argument("--force", action="store_true", help="overwrite the file if it exists")
+    i.add_argument("--no-sample-data", action="store_true", help="do not also create sample_data.csv")
+    for sp in (r, c):
+        sp.add_argument("--json", action="store_true", help="machine-readable output")
+        sp.add_argument("--seed", type=int, default=0)
+    args = p.parse_args(argv)
+
+    if args.cmd == "init":
+        return init_command(args.filename, args.force, not args.no_sample_data)
+    if args.cmd == "report":
+        s = summarize(load_jsonl(args.results), seed=args.seed)
+        print(json.dumps(dataclasses.asdict(s), indent=2) if args.json else format_summary(args.results, s))
+    else:
+        res = compare(load_jsonl(args.a), load_jsonl(args.b), seed=args.seed)
+        print(json.dumps(dataclasses.asdict(res), indent=2) if args.json else format_comparison(res))
+        # non-zero exit on regression => usable as a CI gate
+        if res.verdict == "REAL REGRESSION":
+            return 1
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

flakeval/io.py

@@ -0,0 +1,84 @@
+"""Reading/writing results and the `repeat` helper that runs an eval many times.
+
+Results format (JSONL, one line per item per run):
+    {"item_id": "q17", "run": 2, "score": 1.0}
+`correct` (bool) is accepted in place of `score`.
+"""
+from __future__ import annotations
+
+import json
+from collections import defaultdict
+from concurrent.futures import ThreadPoolExecutor
+from pathlib import Path
+from typing import Any, Callable, Iterable
+
+import numpy as np
+
+from .stats import Scores
+
+
+def load_jsonl(path: str | Path) -> Scores:
+    by_item: dict[str, dict[int, float]] = defaultdict(dict)
+    with open(path, encoding="utf-8") as f:
+        for line_no, line in enumerate(f, 1):
+            line = line.strip()
+            if not line:
+                continue
+            row = json.loads(line)
+            try:
+                item = str(row["item_id"])
+                run = int(row.get("run", len(by_item[item])))
+                score = float(row["score"] if "score" in row else row["correct"])
+            except KeyError as e:
+                raise ValueError(f"{path}:{line_no}: missing field {e}") from None
+            by_item[item][run] = score
+    if not by_item:
+        raise ValueError(f"{path}: no results found")
+    k = min(len(r) for r in by_item.values())
+    ids = sorted(by_item)
+    x = np.array([[by_item[i][r] for r in sorted(by_item[i])[:k]] for i in ids])
+    return Scores(ids, x)
+
+
+def save_jsonl(scores: Scores, path: str | Path) -> None:
+    Path(path).parent.mkdir(parents=True, exist_ok=True)
+    with open(path, "w", encoding="utf-8") as f:
+        for i, item in enumerate(scores.ids):
+            for r in range(scores.n_runs):
+                f.write(json.dumps({"item_id": item, "run": r, "score": float(scores.x[i, r])}) + "\n")
+
+
+def repeat(
+    fn: Callable[[Any], float],
+    items: Iterable[Any] | dict[str, Any],
+    runs: int = 5,
+    workers: int = 1,
+    save: str | Path | None = None,
+    with_run: bool = False,
+) -> Scores:
+    """Run `fn(item) -> score` `runs` times for every item.
+
+    `items` may be a dict {item_id: input} or a plain iterable (ids become "0", "1", ...).
+    Use workers > 1 for API-bound evals (threads).
+    With with_run=True, fn is called as fn(item, run_index), e.g. to derive a different
+    sampling seed for every run.
+    """
+    pairs = list(items.items()) if isinstance(items, dict) else [(str(i), x) for i, x in enumerate(items)]
+    jobs = [(i, r) for i in range(len(pairs)) for r in range(runs)]
+    out = np.zeros((len(pairs), runs))
+
+    def work(job: tuple[int, int]) -> tuple[int, int, float]:
+        i, r = job
+        return i, r, float(fn(pairs[i][1], r) if with_run else fn(pairs[i][1]))
+
+    if workers <= 1:
+        results = map(work, jobs)
+    else:
+        pool = ThreadPoolExecutor(max_workers=workers)
+        results = pool.map(work, jobs)
+    for i, r, v in results:
+        out[i, r] = v
+    scores = Scores([p[0] for p in pairs], out)
+    if save:
+        save_jsonl(scores, save)
+    return scores

flakeval/models.py

@@ -0,0 +1,68 @@
+"""Talking to a model: per-call random seeds, and a caller for any OpenAI-compatible chat endpoint.
+
+A system can instead supply its own Python function (key "call"), so any other SDK or local model works too.
+"""
+from __future__ import annotations
+
+import json
+import os
+import time
+import urllib.error
+import urllib.request
+import zlib
+
+
+def call_seed(base_seed: int, item_id: str, run: int) -> int:
+    """A different seed for every (question, run) so repeated runs genuinely differ, yet the study is reproducible."""
+    return zlib.crc32(f"{base_seed}|{item_id}|{run}".encode()) & 0x7FFFFFFF
+
+
+def make_caller(system: dict, max_tokens: int):
+    """Returns f(prompt, system_prompt, temperature, seed) -> answer text."""
+    if system.get("call"):
+        user_fn = system["call"]
+        return lambda prompt, system_prompt, temperature, seed: user_fn(
+            prompt=prompt, system_prompt=system_prompt, temperature=temperature, seed=seed, max_tokens=max_tokens)
+
+    for key in ("model", "base_url"):
+        if not system.get(key):
+            raise SystemExit(f"System {system.get('name')!r} needs {key!r} (or a 'call' function).")
+    api_key = None
+    if system.get("api_key_env"):
+        api_key = os.environ.get(system["api_key_env"])
+        if not api_key:
+            raise SystemExit(f"Environment variable {system['api_key_env']} is not set (needed for system "
+                             f"{system['name']!r}). Set it in your terminal before running.")
+    url = system["base_url"].rstrip("/") + "/chat/completions"
+    limit_param = system.get("token_limit_param", "max_tokens")
+
+    def caller(prompt, system_prompt, temperature, seed):
+        messages = ([{"role": "system", "content": system_prompt}] if system_prompt else []) + \
+                   [{"role": "user", "content": prompt}]
+        body = {"model": system["model"], "messages": messages, limit_param: max_tokens}
+        if temperature is not None:
+            body["temperature"] = temperature
+        if system.get("send_seed", True):
+            body["seed"] = seed
+        headers = {"Content-Type": "application/json"}
+        if api_key:
+            headers["Authorization"] = f"Bearer {api_key}"
+        req = urllib.request.Request(url, data=json.dumps(body).encode(), headers=headers)
+        last_error = None
+        for attempt in range(4):  # retry busy/overloaded servers; fail loudly on real errors
+            try:
+                with urllib.request.urlopen(req, timeout=180) as resp:
+                    return json.loads(resp.read())["choices"][0]["message"]["content"] or ""
+            except urllib.error.HTTPError as e:
+                if e.code in (429, 500, 502, 503, 504):
+                    last_error = f"HTTP {e.code}"
+                else:
+                    detail = e.read().decode("utf-8", "replace")[:300]
+                    raise RuntimeError(f"{system['name']}: server answered HTTP {e.code} ({detail}). Check model name, "
+                                       f"base_url and API key.") from None
+            except (urllib.error.URLError, TimeoutError, ConnectionError) as e:
+                last_error = str(e)
+            time.sleep(2 ** attempt)
+        raise RuntimeError(f"{system['name']}: gave up after 4 attempts ({last_error}). Is the server running?")
+
+    return caller

flakeval/progress.py

@@ -0,0 +1,101 @@
+"""The progress display shown while a study runs: one overall bar that counts model calls."""
+from __future__ import annotations
+
+import shutil
+import sys
+import threading
+import time
+
+
+def hms(seconds: float) -> str:
+    seconds = int(max(seconds, 0))
+    return f"{seconds // 3600}:{seconds % 3600 // 60:02d}:{seconds % 60:02d}"
+
+
+class Progress:
+    """One overall bar that counts model calls, with the current system shown next to it.
+
+    In a terminal it redraws a single line. When the output goes to a file or pipe (for example `| tee run.log`)
+    it prints one plain line every 10% instead, so logs do not fill up with redraw characters."""
+
+    def __init__(self, total: int, stream=None):
+        self.total, self.done = max(total, 1), 0
+        self.stream = stream or sys.stdout
+        self.tty = hasattr(self.stream, "isatty") and self.stream.isatty()
+        self.start = time.monotonic()
+        self.system, self.sys_index, self.sys_count, self.sys_calls, self.sys_done = "", 0, 0, 0, 0
+        self._last_draw, self._next_mark, self._drawn = 0.0, 10, False
+        self._lock = threading.RLock()
+
+    def start_system(self, name: str, index: int, count: int, calls: int) -> None:
+        with self._lock:
+            self.system, self.sys_index, self.sys_count, self.sys_calls, self.sys_done = name, index, count, calls, 0
+            self._update(force=True)
+
+    def advance(self) -> None:
+        with self._lock:
+            self.done += 1
+            self.sys_done += 1
+            self._update(force=self.done >= self.total)
+
+    def message(self, text: str) -> None:
+        """Print a normal line without leaving a half-drawn bar behind."""
+        with self._lock:
+            if self.tty and self._drawn:
+                self.stream.write("\r" + " " * (self._columns() - 1) + "\r")
+            self.stream.write(text + "\n")
+            self.stream.flush()
+            if self.tty:
+                self._draw()
+
+    def close(self) -> None:
+        with self._lock:
+            if self.tty and self._drawn:
+                self.stream.write("\n")
+                self.stream.flush()
+            self._drawn = False
+
+    # -- internals
+    def _columns(self) -> int:
+        return shutil.get_terminal_size((80, 20)).columns
+
+    def _eta(self) -> str:
+        if self.done < 3:
+            return "--"
+        elapsed = time.monotonic() - self.start
+        return "~" + hms(elapsed / self.done * (self.total - self.done))
+
+    def _variants(self) -> list[str]:
+        pct = 100 * self.done // self.total
+        filled = 20 * self.done // self.total
+        bar = "[" + "█" * filled + "░" * (20 - filled) + "]"
+        elapsed, eta = hms(time.monotonic() - self.start), self._eta()
+        where = f"{self.system} (system {self.sys_index}/{self.sys_count})" if self.system else ""
+        return [
+            f"{bar} {pct:3d}%  {self.done}/{self.total} calls | {where} {self.sys_done}/{self.sys_calls} | "
+            f"elapsed {elapsed} | ETA {eta}",
+            f"{bar} {pct:3d}%  {self.done}/{self.total} calls | {where} | ETA {eta}",
+            f"{bar} {pct:3d}%  {self.done}/{self.total} | ETA {eta}",
+            f"{pct}% ETA {eta}",
+        ]
+
+    def _draw(self) -> None:
+        cols = self._columns()
+        variants = self._variants()
+        line = next((v for v in variants if len(v) < cols), variants[-1][: cols - 1])
+        self.stream.write("\r" + line.ljust(cols - 1))
+        self.stream.flush()
+        self._drawn, self._last_draw = True, time.monotonic()
+
+    def _update(self, force: bool = False) -> None:
+        if self.tty:
+            if force or time.monotonic() - self._last_draw >= 0.2:
+                self._draw()
+            return
+        pct = 100 * self.done // self.total
+        if pct >= self._next_mark:  # plain log line every 10%
+            self._next_mark = (pct // 10 + 1) * 10
+            where = f" | {self.system} (system {self.sys_index}/{self.sys_count})" if self.system else ""
+            self.stream.write(f"progress: {pct}% ({self.done}/{self.total} calls){where} | "
+                              f"elapsed {hms(time.monotonic() - self.start)} | ETA {self._eta()}\n")
+            self.stream.flush()

flakeval/scoring.py

@@ -0,0 +1,78 @@
+"""Turning a model's answer into a score: the built-in scorers and a few text helpers."""
+from __future__ import annotations
+
+import hashlib
+import re
+import unicodedata
+
+NUMBER = re.compile(r"-?\d[\d,]*(?:\.\d+)?")
+ANSWER_LINE = re.compile(r"answer:[\s*$]*(-?[\d,]*\.?\d+)", re.IGNORECASE)      # "Answer: 18", "Answer: **$1,200**"
+BOXED = re.compile(r"\\boxed\{\s*\\?\$?\s*(-?[\d,]*\.?\d+)")                       # \boxed{13}
+CHOICE = re.compile(r"\b([A-H])\b")                                                # a standalone capital letter
+
+SCORERS = ("exact", "contains", "number", "choice_letter", "answer_line", "final_number", "custom")
+SCORER_HELP = """\
+  "exact"         the answer equals the expected text (ignores case, spaces and trailing punctuation)
+  "contains"      the expected text appears somewhere in the answer
+  "number"        the LAST number in the answer equals the expected number
+  "choice_letter" multiple choice: the FIRST standalone capital letter A-H in the answer equals the expected letter
+  "answer_line"   strict math: the number after the LAST "Answer:" in the reply equals the expected number
+  "final_number"  lenient math: like answer_line, but if there is no "Answer:" line it also accepts \\boxed{...}
+                  and finally the last number in the reply
+  "custom"        your own function called custom_score"""
+
+
+def fold_text(text) -> str:
+    """Treat look-alike characters as the same (for example the subscript in H₂O becomes H2O)."""
+    return unicodedata.normalize("NFKC", str(text))
+
+
+def question_hash(text: str) -> str:
+    """Short fingerprint of a question, saved with each answer so --rescore can tell if the question was edited."""
+    return hashlib.sha1(str(text).encode("utf-8")).hexdigest()[:10]
+
+
+def normalize_text(text) -> str:
+    return re.sub(r"\s+", " ", fold_text(text).strip().lower()).strip(" .!?\"'")
+
+
+def to_number(text: str):
+    try:
+        return float(fold_text(text).replace(",", ""))
+    except ValueError:
+        return None
+
+
+def _last_number(pattern: re.Pattern, text: str):
+    found = pattern.findall(text)
+    return to_number(found[-1]) if found else None
+
+
+def _matches_number(got, expected) -> float:
+    want = to_number(str(expected).replace("$", ""))
+    return float(got is not None and want is not None and abs(got - want) < 1e-6)
+
+
+def score_output(scorer: str, output: str, expected, row: dict, custom=None) -> float:
+    if scorer == "exact":
+        return float(normalize_text(output) == normalize_text(expected))
+    if scorer == "contains":
+        return float(normalize_text(expected) in normalize_text(output))
+    if scorer == "number":
+        found = NUMBER.findall(fold_text(output))
+        got, want = (to_number(found[-1]) if found else None), to_number(expected)
+        return float(got is not None and want is not None and abs(got - want) < 1e-6)
+    if scorer == "choice_letter":
+        match = CHOICE.search(fold_text(output).strip())
+        return float(bool(match) and match.group(1) == str(expected).strip().upper())
+    if scorer in ("answer_line", "final_number"):
+        text = fold_text(output)
+        got = _last_number(ANSWER_LINE, text)
+        if got is None and scorer == "final_number":
+            got = _last_number(BOXED, text)
+            if got is None:
+                got = _last_number(NUMBER, text)
+        return _matches_number(got, expected)
+    if scorer == "custom":
+        return float(custom(output, expected, row))
+    raise SystemExit(f"Unknown SCORER {scorer!r}. Choose one of:\n{SCORER_HELP}")