evalsize 0.1.0__py3-none-any.whl

evalsize/__init__.py

@@ -0,0 +1,36 @@
+"""How many eval examples do I need? Sample size planning for LLM evals."""
+
+from .comparison import (
+    Comparison,
+    compare_counts,
+    compare_results,
+    examples_to_compare,
+    power_to_compare,
+    smallest_detectable_improvement,
+)
+from .judge import CorrectedRate, JudgeAgreement, correct_judge_rate, judge_agreement, labels_to_check_judge
+from .precision import PassRate, examples_for_margin, margin_for_examples, pass_rate_interval
+from .reruns import effective_examples, question_share, questions_needed
+
+__all__ = [
+    "examples_for_margin",
+    "margin_for_examples",
+    "pass_rate_interval",
+    "examples_to_compare",
+    "power_to_compare",
+    "smallest_detectable_improvement",
+    "compare_counts",
+    "compare_results",
+    "effective_examples",
+    "questions_needed",
+    "question_share",
+    "judge_agreement",
+    "correct_judge_rate",
+    "labels_to_check_judge",
+    "PassRate",
+    "Comparison",
+    "JudgeAgreement",
+    "CorrectedRate",
+]
+
+__version__ = "0.1.0"

evalsize/_stats.py

@@ -0,0 +1,53 @@
+"""Small statistics helpers built on the standard library only."""
+
+from __future__ import annotations
+
+import math
+from statistics import NormalDist
+
+_NORMAL = NormalDist()
+
+
+def check_rate(value: float, name: str, allow_edges: bool = False) -> None:
+    low_ok = value >= 0 if allow_edges else value > 0
+    high_ok = value <= 1 if allow_edges else value < 1
+    if not (low_ok and high_ok):
+        bounds = "between 0 and 1" if allow_edges else "strictly between 0 and 1"
+        raise ValueError(f"{name} must be {bounds}, got {value!r}")
+
+
+def check_level(value: float, name: str) -> None:
+    if not 0 < value < 1:
+        raise ValueError(f"{name} must be strictly between 0 and 1, got {value!r}")
+
+
+def z_two_sided(alpha: float) -> float:
+    """Critical value for a two-sided test or interval at level alpha."""
+    return _NORMAL.inv_cdf(1 - alpha / 2)
+
+
+def z_power(power: float) -> float:
+    return _NORMAL.inv_cdf(power)
+
+
+def normal_cdf(x: float) -> float:
+    return _NORMAL.cdf(x)
+
+
+def wilson(successes: float, n: float, confidence: float) -> tuple:
+    """Wilson score interval for a proportion. Works well near 0 and 1."""
+    z = z_two_sided(1 - confidence)
+    p = successes / n
+    denom = 1 + z * z / n
+    centre = (p + z * z / (2 * n)) / denom
+    half = z / denom * math.sqrt(p * (1 - p) / n + z * z / (4 * n * n))
+    return max(0.0, centre - half), min(1.0, centre + half)
+
+
+def binom_two_sided_half(k: int, n: int) -> float:
+    """Exact two-sided p-value for k successes out of n when p = 0.5."""
+    if n == 0:
+        return 1.0
+    k = min(k, n - k)
+    tail = sum(math.comb(n, i) for i in range(k + 1)) / 2 ** n
+    return min(1.0, 2 * tail)

evalsize/cli.py

@@ -0,0 +1,151 @@
+"""Command line: ``evalsize compare``, ``evalsize margin``, ``evalsize detect`` and more."""
+
+from __future__ import annotations
+
+import argparse
+import sys
+from typing import List, Optional
+
+from . import (
+    compare_counts,
+    correct_judge_rate,
+    effective_examples,
+    examples_for_margin,
+    examples_to_compare,
+    labels_to_check_judge,
+    smallest_detectable_improvement,
+)
+
+
+def _pct(x: float) -> str:
+    return f"{x:.1%}"
+
+
+def _cmd_margin(args: argparse.Namespace) -> str:
+    n = examples_for_margin(args.margin, args.rate, args.confidence)
+    return (
+        f"{n} examples give a score of about {_pct(args.rate)} "
+        f"plus or minus {_pct(args.margin)} ({args.confidence:.0%} confidence)."
+    )
+
+
+def _cmd_compare(args: argparse.Namespace) -> str:
+    paired = not args.unpaired
+    n = examples_to_compare(args.baseline, args.improvement, paired, args.disagreement, args.alpha, args.power)
+    target = args.baseline + args.improvement
+    lines = [
+        f"To detect {_pct(args.baseline)} -> {_pct(target)} with {args.power:.0%} power "
+        f"(false alarm rate {args.alpha:g}):",
+        f"  {n} questions" + (" per model" if not paired else ", both models on the same questions"),
+    ]
+    if paired and args.disagreement is None:
+        lines.append(
+            "  This assumes the models make mistakes independently. Most model pairs agree more,\n"
+            "  so measure the disagreement rate on a pilot and pass --disagreement to get a smaller number."
+        )
+    return "\n".join(lines)
+
+
+def _cmd_detect(args: argparse.Namespace) -> str:
+    gain = smallest_detectable_improvement(
+        args.n, args.baseline, not args.unpaired, args.disagreement, args.alpha, args.power
+    )
+    if gain is None:
+        return f"With {args.n} questions no improvement over {_pct(args.baseline)} can be reliably detected."
+    return (
+        f"With {args.n} questions and a baseline of {_pct(args.baseline)}, the smallest improvement "
+        f"you can reliably detect is {gain * 100:.1f} percentage points."
+    )
+
+
+def _cmd_diff(args: argparse.Namespace) -> str:
+    return compare_counts(args.n, args.only_a, args.only_b, args.confidence).summary()
+
+
+def _cmd_reruns(args: argparse.Namespace) -> str:
+    lines = [f"{args.n} questions, question share {args.share:g}:"]
+    for k in (1, 2, 3, 5, 10):
+        lines.append(f"  {k:>2} run(s) each = worth {effective_examples(args.n, k, args.share):,.0f} single-run questions")
+    lines.append(f"  No number of reruns can beat {args.n / args.share:,.0f}." if args.share > 0 else "")
+    return "\n".join(line for line in lines if line)
+
+
+def _cmd_judge(args: argparse.Namespace) -> str:
+    if args.judge_rate is None:
+        n = labels_to_check_judge(args.margin, args.sensitivity, args.specificity, args.pass_rate)
+        return (
+            f"Label about {n} answers by hand to measure the judge's error rates "
+            f"to within {_pct(args.margin)}."
+        )
+    return correct_judge_rate(
+        args.judge_rate, args.n, args.sensitivity, args.specificity, args.human_passes, args.human_fails
+    ).summary()
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        prog="evalsize",
+        description="How many eval examples do I need? Sample size planning for LLM evals.",
+    )
+    sub = parser.add_subparsers(dest="command", required=True)
+
+    p = sub.add_parser("margin", help="examples needed to measure one score to within +/- margin")
+    p.add_argument("--rate", type=float, default=0.5, help="expected pass rate (default 0.5, the cautious case)")
+    p.add_argument("--margin", type=float, required=True, help="for example 0.03 for +/- 3 points")
+    p.add_argument("--confidence", type=float, default=0.95)
+    p.set_defaults(run=_cmd_margin)
+
+    def comparison_options(p: argparse.ArgumentParser) -> None:
+        p.add_argument("--baseline", type=float, required=True, help="model A's expected pass rate")
+        p.add_argument("--disagreement", type=float, help="share of questions where exactly one model is right")
+        p.add_argument("--unpaired", action="store_true", help="the two models answer different questions")
+        p.add_argument("--alpha", type=float, default=0.05)
+        p.add_argument("--power", type=float, default=0.8)
+
+    p = sub.add_parser("compare", help="questions needed to detect that B beats A")
+    comparison_options(p)
+    p.add_argument("--improvement", type=float, required=True, help="smallest gain worth detecting, e.g. 0.05")
+    p.set_defaults(run=_cmd_compare)
+
+    p = sub.add_parser("detect", help="smallest improvement a given number of questions can detect")
+    comparison_options(p)
+    p.add_argument("--n", type=int, required=True)
+    p.set_defaults(run=_cmd_detect)
+
+    p = sub.add_parser("diff", help="is B really better than A? (from disagreement counts)")
+    p.add_argument("--n", type=int, required=True, help="questions both models answered")
+    p.add_argument("--only-a", type=int, required=True, help="questions only A got right")
+    p.add_argument("--only-b", type=int, required=True, help="questions only B got right")
+    p.add_argument("--confidence", type=float, default=0.95)
+    p.set_defaults(run=_cmd_diff)
+
+    p = sub.add_parser("reruns", help="what running each question several times is worth")
+    p.add_argument("--n", type=int, required=True, help="number of questions")
+    p.add_argument("--share", type=float, required=True, help="question share of the variance (0 to 1)")
+    p.set_defaults(run=_cmd_reruns)
+
+    p = sub.add_parser("judge", help="human labels needed for an LLM judge, or correct its score")
+    p.add_argument("--sensitivity", type=float, default=0.9)
+    p.add_argument("--specificity", type=float, default=0.9)
+    p.add_argument("--pass-rate", type=float, default=0.7, help="expected pass rate, for planning labels")
+    p.add_argument("--margin", type=float, default=0.05, help="precision wanted for the error rates")
+    p.add_argument("--judge-rate", type=float, help="the judge's pass rate, to correct it")
+    p.add_argument("--n", type=int, default=1000, help="answers the judge graded")
+    p.add_argument("--human-passes", type=int, default=100)
+    p.add_argument("--human-fails", type=int, default=50)
+    p.set_defaults(run=_cmd_judge)
+    return parser
+
+
+def main(argv: Optional[List[str]] = None) -> int:
+    args = build_parser().parse_args(argv)
+    try:
+        print(args.run(args))
+    except ValueError as exc:
+        print(f"evalsize: {exc}", file=sys.stderr)
+        return 2
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

evalsize/comparison.py

@@ -0,0 +1,234 @@
+"""Comparing two models (or prompts) on pass/fail evals.
+
+"Paired" means both models answer the same questions, which is how most evals
+are run. Pairing usually needs far fewer questions than comparing two separate
+samples, because both models get the easy questions right and the hard ones
+wrong, and only the questions where they disagree carry information.
+
+The key planning input for a paired comparison is the **disagreement rate**:
+the share of questions where exactly one of the two models is right. Measure
+it on a small pilot if you can. If you leave it out, the functions assume the
+two models make mistakes independently, which is cautious: real models
+usually agree more than that, so you will often need fewer questions.
+"""
+
+from __future__ import annotations
+
+import math
+from dataclasses import dataclass
+from typing import Iterable, Optional
+
+from ._stats import (
+    binom_two_sided_half,
+    check_level,
+    check_rate,
+    normal_cdf,
+    z_power,
+    z_two_sided,
+)
+
+
+def _independent_disagreement(p_a: float, p_b: float) -> float:
+    return p_a * (1 - p_b) + p_b * (1 - p_a)
+
+
+def _check_disagreement(p_a: float, p_b: float, disagreement: float) -> None:
+    d = p_b - p_a
+    only_b = (disagreement + d) / 2
+    only_a = (disagreement - d) / 2
+    if only_a < -1e-12 or only_b < -1e-12:
+        raise ValueError(
+            f"disagreement {disagreement:g} is too low: going from {p_a:g} to {p_b:g} "
+            f"means the models must disagree on at least {abs(d):g} of questions"
+        )
+    if only_b > min(p_b, 1 - p_a) + 1e-12 or only_a > min(p_a, 1 - p_b) + 1e-12:
+        raise ValueError(
+            f"disagreement {disagreement:g} is too high for pass rates {p_a:g} and {p_b:g}"
+        )
+
+
+def _setup(baseline: float, improvement: float, disagreement: Optional[float]):
+    check_rate(baseline, "baseline")
+    if improvement == 0:
+        raise ValueError("improvement must not be zero")
+    target = baseline + improvement
+    check_rate(target, "baseline + improvement")
+    if disagreement is None:
+        disagreement = _independent_disagreement(baseline, target)
+    else:
+        check_rate(disagreement, "disagreement")
+        _check_disagreement(baseline, target, disagreement)
+    return target, disagreement
+
+
+def examples_to_compare(
+    baseline: float,
+    improvement: float,
+    paired: bool = True,
+    disagreement: Optional[float] = None,
+    alpha: float = 0.05,
+    power: float = 0.8,
+) -> int:
+    """Questions needed to reliably detect that model B beats model A.
+
+    Args:
+        baseline: model A's expected pass rate, for example 0.70.
+        improvement: the smallest gain worth detecting, for example 0.05 for
+            five percentage points.
+        paired: True if both models answer the same questions. If False, the
+            answer is the number of questions for each model.
+        disagreement: share of questions where exactly one model is right.
+            Only used when paired.
+        alpha: false alarm rate (two-sided).
+        power: chance of detecting the improvement if it is real.
+
+    Paired uses the McNemar sample size formula (Connor, 1987). Unpaired uses
+    the standard two-proportion formula.
+
+    >>> examples_to_compare(0.70, 0.05, disagreement=0.15)
+    469
+    """
+    check_level(alpha, "alpha")
+    check_level(power, "power")
+    target, psi = _setup(baseline, improvement, disagreement)
+    d = abs(improvement)
+    za, zb = z_two_sided(alpha), z_power(power)
+    if paired:
+        n = (za * math.sqrt(psi) + zb * math.sqrt(psi - d * d)) ** 2 / (d * d)
+    else:
+        p_bar = (baseline + target) / 2
+        spread = baseline * (1 - baseline) + target * (1 - target)
+        n = (za * math.sqrt(2 * p_bar * (1 - p_bar)) + zb * math.sqrt(spread)) ** 2 / (d * d)
+    return math.ceil(n - 1e-9)
+
+
+def power_to_compare(
+    n: int,
+    baseline: float,
+    improvement: float,
+    paired: bool = True,
+    disagreement: Optional[float] = None,
+    alpha: float = 0.05,
+) -> float:
+    """Chance that an eval with ``n`` questions detects the improvement."""
+    if n <= 0:
+        raise ValueError("n must be positive")
+    check_level(alpha, "alpha")
+    target, psi = _setup(baseline, improvement, disagreement)
+    d = abs(improvement)
+    za = z_two_sided(alpha)
+    if paired:
+        spread = math.sqrt(max(psi - d * d, 1e-15))
+        return normal_cdf((d * math.sqrt(n) - za * math.sqrt(psi)) / spread)
+    p_bar = (baseline + target) / 2
+    spread = math.sqrt(baseline * (1 - baseline) + target * (1 - target))
+    return normal_cdf((d * math.sqrt(n) - za * math.sqrt(2 * p_bar * (1 - p_bar))) / spread)
+
+
+def smallest_detectable_improvement(
+    n: int,
+    baseline: float,
+    paired: bool = True,
+    disagreement: Optional[float] = None,
+    alpha: float = 0.05,
+    power: float = 0.8,
+) -> Optional[float]:
+    """Smallest gain over ``baseline`` that ``n`` questions can reliably detect.
+
+    Returns None if no improvement up to a perfect score is detectable.
+
+    If you give a fixed ``disagreement``, the improvement can never exceed it,
+    so the answer is capped there.
+    """
+    check_level(power, "power")
+    check_rate(baseline, "baseline")
+    high = 1 - baseline - 1e-9
+    if disagreement is not None and paired:
+        high = min(high, disagreement)
+
+    def detectable(gain: float) -> bool:
+        try:
+            return power_to_compare(n, baseline, gain, paired, disagreement, alpha) >= power
+        except ValueError:
+            return False
+
+    if not detectable(high):
+        return None
+    low = 1e-6
+    for _ in range(60):
+        mid = (low + high) / 2
+        if detectable(mid):
+            high = mid
+        else:
+            low = mid
+    return high
+
+
+@dataclass(frozen=True)
+class Comparison:
+    n: int
+    only_a_right: int
+    only_b_right: int
+    difference: float
+    low: float
+    high: float
+    p_value: float
+    confidence: float
+
+    def summary(self) -> str:
+        direction = "better" if self.difference > 0 else "worse"
+        verdict = (
+            f"B looks {direction} than A"
+            if self.p_value < 1 - self.confidence
+            else "The difference could easily be noise"
+        )
+        return (
+            f"B minus A: {self.difference:+.1%} over {self.n} shared questions "
+            f"({self.confidence:.0%} interval {self.low:+.1%} to {self.high:+.1%}, p = {self.p_value:.3g}). "
+            f"{verdict}. The models disagreed on {self.only_a_right + self.only_b_right} questions."
+        )
+
+    def __str__(self) -> str:
+        return self.summary()
+
+
+def compare_counts(n: int, only_a_right: int, only_b_right: int, confidence: float = 0.95) -> Comparison:
+    """Is B really better than A? Uses only the questions where they disagree.
+
+    Args:
+        n: questions both models answered.
+        only_a_right: questions A got right and B got wrong.
+        only_b_right: questions B got right and A got wrong.
+
+    The p-value is the exact McNemar test. The interval for the difference
+    uses the Agresti and Min (2005) adjustment, which behaves well with small
+    counts.
+
+    >>> compare_counts(300, only_a_right=12, only_b_right=28).p_value < 0.05
+    True
+    """
+    if n <= 0:
+        raise ValueError("n must be positive")
+    if only_a_right < 0 or only_b_right < 0 or only_a_right + only_b_right > n:
+        raise ValueError("disagreement counts must be non-negative and add up to at most n")
+    check_level(confidence, "confidence")
+    diff = (only_b_right - only_a_right) / n
+    a_adj, b_adj, n_adj = only_a_right + 0.5, only_b_right + 0.5, n + 2
+    d_adj = (b_adj - a_adj) / n_adj
+    var = ((a_adj + b_adj) / n_adj - d_adj ** 2) / n_adj
+    half = z_two_sided(1 - confidence) * math.sqrt(max(var, 0.0))
+    p_value = binom_two_sided_half(only_b_right, only_a_right + only_b_right)
+    return Comparison(
+        n, only_a_right, only_b_right, diff,
+        max(-1.0, d_adj - half), min(1.0, d_adj + half), p_value, confidence,
+    )
+
+
+def compare_results(a: Iterable, b: Iterable, confidence: float = 0.95) -> Comparison:
+    """Same as ``compare_counts`` but from two lists of pass/fail results, in question order."""
+    a, b = [bool(x) for x in a], [bool(x) for x in b]
+    if len(a) != len(b):
+        raise ValueError("a and b must have one result per question, in the same order")
+    only_a = sum(1 for x, y in zip(a, b) if x and not y)
+    only_b = sum(1 for x, y in zip(a, b) if y and not x)
+    return compare_counts(len(a), only_a, only_b, confidence)

evalsize/judge.py

@@ -0,0 +1,139 @@
+"""When an LLM judge grades the eval, its mistakes shift the score.
+
+A judge that passes 10% of bad answers and fails 5% of good ones does not
+just add noise: it pushes the score in one direction. If you check the judge
+against human labels on a sample, you can correct the score and widen the
+interval to account for what you learned from only a sample.
+
+Terms used here, with humans as the reference:
+    sensitivity: share of answers humans pass that the judge also passes.
+    specificity: share of answers humans fail that the judge also fails.
+"""
+
+from __future__ import annotations
+
+import math
+from dataclasses import dataclass
+from typing import Iterable
+
+from ._stats import check_level, check_rate, z_two_sided
+from .precision import examples_for_margin
+
+
+@dataclass(frozen=True)
+class JudgeAgreement:
+    sensitivity: float
+    specificity: float
+    human_passes: int
+    human_fails: int
+    agreement: float
+
+    def summary(self) -> str:
+        return (
+            f"The judge agreed with humans on {self.agreement:.1%} of {self.human_passes + self.human_fails} labels. "
+            f"It passed {self.sensitivity:.1%} of answers humans passed "
+            f"and failed {self.specificity:.1%} of answers humans failed."
+        )
+
+    def __str__(self) -> str:
+        return self.summary()
+
+
+def judge_agreement(human: Iterable, judge: Iterable) -> JudgeAgreement:
+    """Measure the judge against human labels for the same answers (True means pass)."""
+    human, judge = [bool(x) for x in human], [bool(x) for x in judge]
+    if len(human) != len(judge):
+        raise ValueError("human and judge need one label per answer, in the same order")
+    passes = [j for h, j in zip(human, judge) if h]
+    fails = [j for h, j in zip(human, judge) if not h]
+    if not passes or not fails:
+        raise ValueError("the labelled sample needs at least one human pass and one human fail")
+    sensitivity = sum(passes) / len(passes)
+    specificity = sum(1 for j in fails if not j) / len(fails)
+    agreement = sum(1 for h, j in zip(human, judge) if h == j) / len(human)
+    return JudgeAgreement(sensitivity, specificity, len(passes), len(fails), agreement)
+
+
+@dataclass(frozen=True)
+class CorrectedRate:
+    judge_rate: float
+    corrected: float
+    low: float
+    high: float
+    confidence: float
+
+    def summary(self) -> str:
+        return (
+            f"The judge says {self.judge_rate:.1%}. Corrected for the judge's mistakes: "
+            f"{self.corrected:.1%} ({self.confidence:.0%} interval {self.low:.1%} to {self.high:.1%})."
+        )
+
+    def __str__(self) -> str:
+        return self.summary()
+
+
+def correct_judge_rate(
+    judge_rate: float,
+    n_judged: int,
+    sensitivity: float,
+    specificity: float,
+    human_passes: int,
+    human_fails: int,
+    confidence: float = 0.95,
+) -> CorrectedRate:
+    """Correct a judge-graded pass rate using the judge's measured error rates.
+
+    Args:
+        judge_rate: pass rate the judge gave over the whole eval.
+        n_judged: answers the judge graded.
+        sensitivity, specificity: from ``judge_agreement``.
+        human_passes, human_fails: how many human labels each of those was
+            measured on. Fewer labels means a wider interval.
+
+    Uses the Rogan and Gladen (1978) estimator with a delta method interval
+    that includes the uncertainty in all three measured rates.
+
+    >>> r = correct_judge_rate(0.82, 1000, 0.95, 0.70, 140, 60)
+    >>> round(r.corrected, 3)
+    0.8
+    """
+    check_rate(judge_rate, "judge_rate", allow_edges=True)
+    check_rate(sensitivity, "sensitivity", allow_edges=True)
+    check_rate(specificity, "specificity", allow_edges=True)
+    check_level(confidence, "confidence")
+    if n_judged <= 0 or human_passes <= 0 or human_fails <= 0:
+        raise ValueError("n_judged, human_passes and human_fails must be positive")
+    youden = sensitivity + specificity - 1
+    if youden <= 0.05:
+        raise ValueError(
+            "the judge is barely better than a coin flip against human labels "
+            "(sensitivity + specificity must be well above 1), so its score cannot be corrected"
+        )
+    raw = (judge_rate + specificity - 1) / youden
+    p = min(1.0, max(0.0, raw))
+    var = (
+        judge_rate * (1 - judge_rate) / n_judged
+        + p ** 2 * sensitivity * (1 - sensitivity) / human_passes
+        + (1 - p) ** 2 * specificity * (1 - specificity) / human_fails
+    ) / youden ** 2
+    half = z_two_sided(1 - confidence) * math.sqrt(var)
+    return CorrectedRate(judge_rate, p, max(0.0, p - half), min(1.0, p + half), confidence)
+
+
+def labels_to_check_judge(
+    margin: float = 0.05,
+    expected_sensitivity: float = 0.9,
+    expected_specificity: float = 0.9,
+    pass_rate: float = 0.7,
+    confidence: float = 0.95,
+) -> int:
+    """Human labels needed to measure the judge's error rates to within ``margin``.
+
+    Both error rates must be measured, and the rarer outcome (usually fails)
+    decides the total: if 30% of answers fail, you need to label about three
+    answers to find one fail.
+    """
+    check_rate(pass_rate, "pass_rate")
+    need_passes = examples_for_margin(margin, expected_sensitivity, confidence)
+    need_fails = examples_for_margin(margin, expected_specificity, confidence)
+    return math.ceil(max(need_passes / pass_rate, need_fails / (1 - pass_rate)) - 1e-9)

evalsize/precision.py

@@ -0,0 +1,78 @@
+"""How precise is one pass rate, and how many examples make it precise enough."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from ._stats import check_level, check_rate, wilson
+
+
+@dataclass(frozen=True)
+class PassRate:
+    passes: int
+    n: int
+    rate: float
+    low: float
+    high: float
+    confidence: float
+
+    def summary(self) -> str:
+        return (
+            f"Pass rate {self.rate:.1%} ({self.passes} of {self.n}). "
+            f"{self.confidence:.0%} confidence interval: {self.low:.1%} to {self.high:.1%}."
+        )
+
+    def __str__(self) -> str:
+        return self.summary()
+
+
+def pass_rate_interval(passes: int, n: int, confidence: float = 0.95) -> PassRate:
+    """Confidence interval for a pass rate, using the Wilson score method.
+
+    >>> pass_rate_interval(160, 200).summary()
+    'Pass rate 80.0% (160 of 200). 95% confidence interval: 73.9% to 85.0%.'
+    """
+    if n <= 0:
+        raise ValueError("n must be positive")
+    if not 0 <= passes <= n:
+        raise ValueError("passes must be between 0 and n")
+    check_level(confidence, "confidence")
+    low, high = wilson(passes, n, confidence)
+    return PassRate(passes, n, passes / n, low, high, confidence)
+
+
+def margin_for_examples(n: int, pass_rate: float = 0.5, confidence: float = 0.95) -> float:
+    """Half width of the confidence interval you can expect with ``n`` examples.
+
+    ``pass_rate`` is your best guess of the score. 0.5 gives the widest
+    (most cautious) answer.
+    """
+    if n <= 0:
+        raise ValueError("n must be positive")
+    check_rate(pass_rate, "pass_rate", allow_edges=True)
+    check_level(confidence, "confidence")
+    low, high = wilson(pass_rate * n, n, confidence)
+    return (high - low) / 2
+
+
+def examples_for_margin(margin: float, pass_rate: float = 0.5, confidence: float = 0.95) -> int:
+    """Smallest number of examples that gives a ``margin`` of plus or minus around the score.
+
+    >>> examples_for_margin(0.05, pass_rate=0.8)
+    245
+    """
+    check_level(margin, "margin")
+    check_rate(pass_rate, "pass_rate", allow_edges=True)
+    check_level(confidence, "confidence")
+    low, high = 1, 1
+    while margin_for_examples(high, pass_rate, confidence) > margin:
+        high *= 2
+        if high > 10**9:
+            raise ValueError("margin is too small to reach")
+    while low < high:
+        mid = (low + high) // 2
+        if margin_for_examples(mid, pass_rate, confidence) <= margin:
+            high = mid
+        else:
+            low = mid + 1
+    return low

evalsize/reruns.py

@@ -0,0 +1,74 @@
+"""More questions, or more runs per question?
+
+When a model is not deterministic, the same question can pass on one run and
+fail on the next. Running each question several times averages out that
+randomness, but it cannot fix the other source of noise: which questions you
+happened to pick. This module measures how much reruns are worth.
+
+The planning input is the **question share**: the part of the score's
+variance that comes from questions being easy or hard, as opposed to run to
+run randomness. A deterministic model has a question share of 1, and reruns
+add nothing. Estimate it from a pilot with ``question_share``.
+"""
+
+from __future__ import annotations
+
+import math
+from typing import Sequence
+
+from ._stats import check_rate
+
+
+def effective_examples(n_questions: int, runs_per_question: int, question_share: float) -> float:
+    """How many single-run questions your design is worth.
+
+    With ``n`` questions and ``k`` runs each, the score is as precise as
+    ``n / (s + (1 - s) / k)`` single-run questions, where ``s`` is the question
+    share. No number of reruns can do better than ``n / s``.
+
+    >>> round(effective_examples(200, 5, 0.6))
+    294
+    """
+    if n_questions <= 0 or runs_per_question <= 0:
+        raise ValueError("n_questions and runs_per_question must be positive")
+    check_rate(question_share, "question_share", allow_edges=True)
+    return n_questions / (question_share + (1 - question_share) / runs_per_question)
+
+
+def questions_needed(single_run_examples: int, runs_per_question: int, question_share: float) -> int:
+    """Questions needed, at ``runs_per_question`` runs each, to match a single-run plan.
+
+    Use it with ``examples_for_margin`` or ``examples_to_compare``, which assume
+    one run per question.
+    """
+    if single_run_examples <= 0 or runs_per_question <= 0:
+        raise ValueError("single_run_examples and runs_per_question must be positive")
+    check_rate(question_share, "question_share", allow_edges=True)
+    factor = question_share + (1 - question_share) / runs_per_question
+    return max(1, math.ceil(single_run_examples * factor - 1e-9))
+
+
+def question_share(results: Sequence[Sequence[float]]) -> float:
+    """Estimate the question share from a pilot where each question was run several times.
+
+    ``results`` has one row per question and one value per run (1 for pass,
+    0 for fail, or any score). Every question needs the same number of runs,
+    at least two. Uses the one-way analysis of variance estimator of the
+    intraclass correlation.
+    """
+    rows = [list(map(float, row)) for row in results]
+    if len(rows) < 2:
+        raise ValueError("need at least two questions")
+    k = len(rows[0])
+    if k < 2 or any(len(r) != k for r in rows):
+        raise ValueError("every question needs the same number of runs, at least two")
+    n = len(rows)
+    means = [sum(r) / k for r in rows]
+    grand = sum(means) / n
+    ms_between = k * sum((m - grand) ** 2 for m in means) / (n - 1)
+    ms_within = sum((x - m) ** 2 for r, m in zip(rows, means) for x in r) / (n * (k - 1))
+    between = max(0.0, (ms_between - ms_within) / k)
+    total = between + ms_within
+    if total == 0:
+        return 1.0
+    return between / total

evalsize-0.1.0.dist-info/METADATA

@@ -0,0 +1,173 @@
+Metadata-Version: 2.4
+Name: evalsize
+Version: 0.1.0
+Summary: How many eval examples do I need? Sample size and power planning for LLM evals
+Author: Meenal Asesh
+License-Expression: MIT
+Project-URL: Homepage, https://github.com/ai-with-md/evalsize
+Project-URL: Calculator, https://ai-with-md.github.io/evalsize/
+Keywords: llm,evaluation,evals,sample size,power analysis,statistics,llm-as-a-judge,a/b testing
+Classifier: Programming Language :: Python :: 3
+Classifier: Operating System :: OS Independent
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Provides-Extra: dev
+Requires-Dist: pytest; extra == "dev"
+Dynamic: license-file
+
+# evalsize
+
+**How many eval examples do I need?** Sample size and power planning for LLM evals, with a [calculator that runs in your browser](https://ai-with-md.github.io/evalsize/).
+
+[![tests](https://github.com/ai-with-md/evalsize/actions/workflows/tests.yml/badge.svg)](https://github.com/ai-with-md/evalsize/actions/workflows/tests.yml)
+![Python 3.9+](https://img.shields.io/badge/python-3.9%2B-blue)
+![License: MIT](https://img.shields.io/badge/license-MIT-green)
+
+Most teams pick an eval size by feel: 100 questions, maybe 500. Then a new prompt scores 2 points higher and nobody knows whether that is real. evalsize answers the planning questions before you spend the API budget:
+
+1. How many questions do I need to tell whether model B beats model A?
+2. How precise is the score I already have?
+3. Is the difference I measured real, or noise?
+4. Should I add more questions, or run each question several times?
+5. My LLM judge makes mistakes. How much does that move the score, and how many answers should I label by hand?
+
+## The number most people get wrong
+
+Comparing a 70% model with a 75% model, at the usual 80% power:
+
+| Setup | Questions needed |
+|---|---|
+| Each model on its own set of questions | 1,251 per model |
+| Same questions, models make mistakes independently | 1,254 |
+| Same questions, models disagree on 15% of them | **469** |
+
+When both models answer the same questions, only the questions where they **disagree** tell you which is better. Models often get the same questions right and the same ones wrong, so they usually disagree far less than chance would suggest, and you need far fewer questions. But only if you measure the disagreement rate. Run a small pilot, count the questions where exactly one model is right, and plan with that number.
+
+## Try it
+
+**In the browser:** [ai-with-md.github.io/evalsize](https://ai-with-md.github.io/evalsize/). Move the sliders, nothing to install.
+
+**In Python:**
+
+```bash
+pip install evalsize
+```
+
+No dependencies beyond the standard library. Python 3.9 or newer.
+
+## Use
+
+### 1. How many questions to compare two models?
+
+```python
+import evalsize as es
+
+es.examples_to_compare(baseline=0.70, improvement=0.05, disagreement=0.15)
+# 469
+
+es.smallest_detectable_improvement(n=300, baseline=0.70, disagreement=0.15)
+# 0.062  -> with 300 questions you can reliably detect about 6 points, not less
+```
+
+Leave out `disagreement` and evalsize assumes the models make mistakes independently, which is the cautious answer. Pass `paired=False` if the two models answer different questions.
+
+### 2. How precise is one score?
+
+```python
+es.pass_rate_interval(passes=160, n=200)
+# Pass rate 80.0% (160 of 200). 95% confidence interval: 73.9% to 85.0%.
+
+es.examples_for_margin(margin=0.03, pass_rate=0.80)
+# 682 questions for plus or minus 3 points
+```
+
+### 3. Is B really better than A?
+
+```python
+result = es.compare_results(a_passed, b_passed)  # two lists of True/False, same question order
+print(result)
+# B minus A: +5.3% over 300 shared questions (95% interval +1.2% to +9.4%, p = 0.0166).
+# B looks better than A. The models disagreed on 40 questions.
+```
+
+Or from counts: `es.compare_counts(n=300, only_a_right=12, only_b_right=28)`.
+
+### 4. More questions, or more runs per question?
+
+If a model gives different answers on different runs, rerunning each question averages out that randomness. It cannot fix the other source of noise: which questions you picked.
+
+```python
+share = es.question_share(pilot_runs)  # one row per question, one 0/1 per run
+es.effective_examples(n_questions=200, runs_per_question=5, question_share=share)
+# with share 0.6: worth 294 single-run questions, and no number of reruns can beat 333
+```
+
+A deterministic model has a question share of 1, and reruns add nothing. Per API call, adding questions always buys more precision than adding runs, so reruns make most sense when you have run out of good questions.
+
+### 5. Can I trust my LLM judge?
+
+A judge that passes some bad answers or fails some good ones does not just add noise. It moves the score in one direction. Label a random sample by hand, measure the judge, and correct:
+
+```python
+check = es.judge_agreement(human_labels, judge_labels)
+print(check)
+# The judge agreed with humans on 87.5% of 200 labels. It passed 95.0% of answers
+# humans passed and failed 70.0% of answers humans failed.
+
+es.correct_judge_rate(judge_rate=0.82, n_judged=1000,
+                      sensitivity=check.sensitivity, specificity=check.specificity,
+                      human_passes=check.human_passes, human_fails=check.human_fails)
+# The judge says 82.0%. Corrected for the judge's mistakes: 80.0% (95% interval 73.2% to 86.8%).
+
+es.labels_to_check_judge(margin=0.05, pass_rate=0.80)
+# how many answers to label by hand before you start
+```
+
+## Command line
+
+```bash
+evalsize compare --baseline 0.7 --improvement 0.05 --disagreement 0.15
+evalsize detect  --baseline 0.7 --n 300 --disagreement 0.15
+evalsize margin  --rate 0.8 --margin 0.03
+evalsize diff    --n 300 --only-a 12 --only-b 28
+evalsize reruns  --n 200 --share 0.6
+evalsize judge   --judge-rate 0.82 --sensitivity 0.95 --specificity 0.7 --human-passes 140 --human-fails 60
+```
+
+## How the numbers are checked
+
+Formulas are easy to get subtly wrong, so every one is tested against simulation. The test suite runs thousands of fake evals with known true pass rates and checks that the promises hold: the sample sizes deliver their stated power, the intervals cover the truth 95% of the time, the false alarm rate stays at 5%, and the rerun and judge corrections match what the simulations show. See [`tests/test_simulation.py`](https://github.com/ai-with-md/evalsize/blob/main/tests/test_simulation.py).
+
+## How it compares
+
+- **[evalstats](https://github.com/ianarawjo/evalstats)** analyzes results after you run an eval, with many tests and judge-bias corrections. evalsize is mostly about **planning before you run**: how many questions, how many runs, how many human labels. They work well together.
+- **General power calculators** (statsmodels, G*Power, online calculators) handle two separate groups well, but rarely the paired case with a disagreement rate, which is how most LLM evals are run, and they say nothing about reruns or imperfect judges.
+
+## Methods
+
+| Question | Method |
+|---|---|
+| Precision of one score | Wilson score interval |
+| Comparing two models, planning | McNemar sample size formula (Connor, 1987); two-proportion formula when unpaired |
+| Comparing two models, analysis | Exact McNemar test, Agresti and Min (2005) interval for the paired difference |
+| Reruns | Intraclass correlation (one-way analysis of variance estimator) and the design effect |
+| Imperfect judges | Rogan and Gladen (1978) estimator with a delta method interval that includes the uncertainty in the judge's measured error rates |
+
+Everything here is for pass/fail scores, which is how most evals are reported. Planning for scores on a scale (1 to 5, or 0 to 1) is a likely next step.
+
+## Development
+
+```bash
+python -m venv .venv
+source .venv/bin/activate
+pip install -e ".[dev]"
+pytest
+```
+
+The calculator is a [marimo](https://marimo.io) notebook in [`app/calculator.py`](https://github.com/ai-with-md/evalsize/blob/main/app/calculator.py). To run it locally: `pip install marimo matplotlib`, then `marimo run app/calculator.py`. It is published to GitHub Pages as a WebAssembly app on every push to `main`.
+
+## License
+
+MIT

evalsize-0.1.0.dist-info/RECORD

@@ -0,0 +1,13 @@
+evalsize/__init__.py,sha256=IctaPIANCZFVfnH_90yJ5h5LZDeUZKDXY0WKVoe_FQw,1002
+evalsize/_stats.py,sha256=dwR-G_PM__l4H_5BwYBkYR41MyahCuUx4QYxzdkx9wE,1678
+evalsize/cli.py,sha256=eiZz_i4tsMUXFw-ayFFOIJ1fD-Ai_DusMZZTQs55X7o,6637
+evalsize/comparison.py,sha256=ix4lUftZGg5Ww2HYApzvrn7NqxjqZeJ0vXbqsHUQpmM,8505
+evalsize/judge.py,sha256=_o4TQzpJVYY3uhSgVN5syOvyVNfolUA6RQzcM2e9eFE,5350
+evalsize/precision.py,sha256=vMiRi0mUbQKCwO8NsCIKp_lzhb1Ej7-P93mzQsbhOAg,2538
+evalsize/reruns.py,sha256=sMqPY9wIn_m_DxjCzHGDY7z96ib0g0hjnibu0iR5Xq0,3218
+evalsize-0.1.0.dist-info/licenses/LICENSE,sha256=HS_Pp1J0NNKF4niTKA1MdskkGwByEasM-oJb8NcpaGE,1069
+evalsize-0.1.0.dist-info/METADATA,sha256=IyorDk5WXXc8bWzlD1gUTRNQPvnSAi42FBsZ0jmgR5Y,8182
+evalsize-0.1.0.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
+evalsize-0.1.0.dist-info/entry_points.txt,sha256=u4TlRPvgiG9DXJbokb646Cxc1LkwU5VN42psVAho1Go,47
+evalsize-0.1.0.dist-info/top_level.txt,sha256=7YFrzKbaKFXWS7KSfFFHEYPzJfx4dyCBJ72hwD7IcUs,9
+evalsize-0.1.0.dist-info/RECORD,,

evalsize-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (84.2.1)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

evalsize-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+evalsize = evalsize.cli:main

evalsize-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Meenal Asesh
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

evalsize-0.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+evalsize