evalci 0.1.0__py3-none-any.whl

evalci/__init__.py

@@ -0,0 +1,25 @@
+from .stats import (
+    CIResult,
+    CompareResult,
+    PowerResult,
+    ci,
+    cluster_ci,
+    compare,
+    multi_compare,
+    power,
+)
+from .report import report
+from . import adapters
+
+__all__ = [
+    "ci",
+    "compare",
+    "power",
+    "multi_compare",
+    "cluster_ci",
+    "report",
+    "CIResult",
+    "CompareResult",
+    "PowerResult",
+    "adapters",
+]

evalci/_correction.py

@@ -0,0 +1,41 @@
+"""Multiple-comparison correction: Holm and Benjamini-Hochberg."""
+import numpy as np
+
+
+def holm(pvalues):
+    """Holm (1979) step-down Bonferroni correction. Returns adjusted p-values."""
+    p = np.asarray(pvalues, dtype=float)
+    n = len(p)
+    order = np.argsort(p)
+    adj = np.empty(n)
+    running_max = 0.0
+    for i, idx in enumerate(order):
+        running_max = max(running_max, (n - i) * p[idx])
+        adj[idx] = min(running_max, 1.0)
+    return adj
+
+
+def benjamini_hochberg(pvalues):
+    """Benjamini-Hochberg (1995) FDR correction. Returns adjusted (q) values."""
+    p = np.asarray(pvalues, dtype=float)
+    n = len(p)
+    order = np.argsort(p)[::-1]
+    adj = np.empty(n)
+    running_min = 1.0
+    for i, idx in enumerate(order):
+        rank = n - i
+        running_min = min(running_min, p[idx] * n / rank)
+        adj[idx] = running_min
+    return np.clip(adj, 0, 1)
+
+
+def correct_pvalues(pvalues, method="holm"):
+    method = method.lower()
+    if method in ("holm", "bonferroni-holm"):
+        return holm(pvalues)
+    if method in ("bh", "benjamini-hochberg", "fdr_bh"):
+        return benjamini_hochberg(pvalues)
+    if method == "bonferroni":
+        p = np.asarray(pvalues, dtype=float)
+        return np.clip(p * len(p), 0, 1)
+    raise ValueError(f"unknown correction method: {method!r}")

evalci/_intervals.py

@@ -0,0 +1,56 @@
+"""Interval estimators: Wilson, Clopper-Pearson, and bootstrap (percentile/BCa)."""
+import numpy as np
+from scipy import stats as _scipy_stats
+
+
+def wilson_interval(k, n, confidence=0.95):
+    """Wilson score interval for a binomial proportion."""
+    if n <= 0:
+        raise ValueError("n must be positive")
+    z = _scipy_stats.norm.ppf(0.5 + confidence / 2)
+    phat = k / n
+    z2 = z * z
+    denom = 1 + z2 / n
+    center = (phat + z2 / (2 * n)) / denom
+    half = (z * np.sqrt(phat * (1 - phat) / n + z2 / (4 * n * n))) / denom
+    return max(0.0, center - half), min(1.0, center + half)
+
+
+def clopper_pearson_interval(k, n, confidence=0.95):
+    """Exact (Clopper-Pearson) interval for a binomial proportion."""
+    if n <= 0:
+        raise ValueError("n must be positive")
+    alpha = 1 - confidence
+    lo = 0.0 if k == 0 else _scipy_stats.beta.ppf(alpha / 2, k, n - k + 1)
+    hi = 1.0 if k == n else _scipy_stats.beta.ppf(1 - alpha / 2, k + 1, n - k)
+    return float(lo), float(hi)
+
+
+def bootstrap_interval(
+    data,
+    statistic=np.mean,
+    method="BCa",
+    confidence=0.95,
+    n_resamples=9999,
+    random_state=None,
+    vectorized=True,
+):
+    """Bootstrap CI for an arbitrary statistic via scipy.stats.bootstrap."""
+    data = np.asarray(data, dtype=float)
+    scipy_method = {"bca": "BCa", "percentile": "percentile", "basic": "basic"}.get(
+        method.lower(), method
+    )
+    if np.all(data == data[0]):
+        # degenerate (zero-variance) sample: BCa's acceleration is undefined
+        point = float(statistic(data))
+        return point, point
+    res = _scipy_stats.bootstrap(
+        (data,),
+        statistic,
+        confidence_level=confidence,
+        method=scipy_method,
+        n_resamples=n_resamples,
+        random_state=random_state,
+        vectorized=vectorized,
+    )
+    return float(res.confidence_interval.low), float(res.confidence_interval.high)

evalci/_power.py

@@ -0,0 +1,65 @@
+"""Power / sample-size analysis: analytic (normal approximation) and simulation."""
+import numpy as np
+from scipy import stats as _scipy_stats
+
+
+def analytic_n(delta, target_power=0.8, baseline=0.5, alpha=0.05):
+    """Two-proportion normal-approximation sample size for a given effect size."""
+    p1, p2 = baseline, baseline + delta
+    z_alpha = _scipy_stats.norm.ppf(1 - alpha / 2)
+    z_beta = _scipy_stats.norm.ppf(target_power)
+    var = p1 * (1 - p1) + p2 * (1 - p2)
+    n = ((z_alpha + z_beta) ** 2) * var / (delta**2)
+    return int(np.ceil(n))
+
+
+def analytic_power(delta, n, baseline=0.5, alpha=0.05):
+    """Achieved power of a two-proportion test for given n and effect size."""
+    p1, p2 = baseline, baseline + delta
+    z_alpha = _scipy_stats.norm.ppf(1 - alpha / 2)
+    var = p1 * (1 - p1) + p2 * (1 - p2)
+    z_beta = np.sqrt(n * (delta**2) / var) - z_alpha
+    return float(_scipy_stats.norm.cdf(z_beta))
+
+
+def simulate_power(delta, n, baseline=0.5, alpha=0.05, rho=0.0, sims=5000, random_state=None):
+    """Monte Carlo power for a paired design with correlated item-level outcomes.
+
+    `rho` is the fraction of items whose correctness is shared between the two
+    models (a common-difficulty link) rather than drawn independently; it is a
+    simplified correlation knob, not an exact bivariate-Bernoulli correlation.
+    """
+    rng = np.random.default_rng(random_state)
+    p1, p2 = baseline, baseline + delta
+    shared = rng.random((sims, n)) < rho
+    shared_correct = rng.random((sims, n)) < (p1 + p2) / 2
+    a_ind = rng.random((sims, n)) < p1
+    b_ind = rng.random((sims, n)) < p2
+    a = np.where(shared, shared_correct, a_ind).astype(float)
+    b = np.where(shared, shared_correct, b_ind).astype(float)
+    diffs = a - b
+    means = diffs.mean(axis=1)
+    sds = diffs.std(axis=1, ddof=1)
+    sds[sds == 0] = 1e-12
+    z = means / (sds / np.sqrt(n))
+    pvals = 2 * (1 - _scipy_stats.norm.cdf(np.abs(z)))
+    return float(np.mean(pvals < alpha))
+
+
+def simulate_n(delta, target_power=0.8, baseline=0.5, alpha=0.05, rho=0.0, sims=3000, random_state=None):
+    """Bisection search for the n achieving target_power under simulate_power."""
+    lo, hi = 4, 64
+    max_n = 1_000_000
+    while simulate_power(delta, hi, baseline, alpha, rho, sims, random_state) < target_power:
+        lo = hi
+        hi *= 2
+        if hi > max_n:
+            return max_n
+    while hi - lo > 1:
+        mid = (lo + hi) // 2
+        p = simulate_power(delta, mid, baseline, alpha, rho, sims, random_state)
+        if p < target_power:
+            lo = mid
+        else:
+            hi = mid
+    return hi

evalci/_significance.py

@@ -0,0 +1,111 @@
+"""Significance-test internals: paired/unpaired permutation, bootstrap, McNemar."""
+import itertools
+
+import numpy as np
+from scipy import stats as _scipy_stats
+
+EXACT_ENUMERATION_THRESHOLD = 12
+
+
+def paired_bootstrap_ci(diffs, confidence=0.95, n_resamples=9999, rng=None):
+    rng = rng or np.random.default_rng()
+    n = len(diffs)
+    idx = rng.integers(0, n, size=(n_resamples, n))
+    resampled_means = diffs[idx].mean(axis=1)
+    alpha = 1 - confidence
+    lo, hi = np.percentile(resampled_means, [100 * alpha / 2, 100 * (1 - alpha / 2)])
+    return float(lo), float(hi)
+
+
+def unpaired_bootstrap_ci(a, b, confidence=0.95, n_resamples=9999, rng=None):
+    rng = rng or np.random.default_rng()
+    na, nb = len(a), len(b)
+    idx_a = rng.integers(0, na, size=(n_resamples, na))
+    idx_b = rng.integers(0, nb, size=(n_resamples, nb))
+    stats = a[idx_a].mean(axis=1) - b[idx_b].mean(axis=1)
+    alpha = 1 - confidence
+    lo, hi = np.percentile(stats, [100 * alpha / 2, 100 * (1 - alpha / 2)])
+    return float(lo), float(hi)
+
+
+def paired_permutation_p(diffs, n_resamples=9999, rng=None, exact_threshold=EXACT_ENUMERATION_THRESHOLD):
+    """Sign-flip permutation test on paired differences. Exact enumeration for small n."""
+    rng = rng or np.random.default_rng()
+    n = len(diffs)
+    obs = diffs.mean()
+    if n <= exact_threshold:
+        signs = np.array(list(itertools.product([-1, 1], repeat=n)), dtype=float)
+        resampled = (signs * diffs).mean(axis=1)
+        count = np.sum(np.abs(resampled) >= abs(obs) - 1e-9)
+        return float(count / len(signs))
+    signs = rng.choice([-1.0, 1.0], size=(n_resamples, n))
+    resampled = (signs * diffs).mean(axis=1)
+    count = np.sum(np.abs(resampled) >= abs(obs) - 1e-9)
+    return float((count + 1) / (n_resamples + 1))
+
+
+def unpaired_permutation_p(a, b, n_resamples=9999, rng=None):
+    """Label-shuffle permutation test for two independent samples."""
+    rng = rng or np.random.default_rng()
+    na = len(a)
+    combined = np.concatenate([a, b])
+    obs = a.mean() - b.mean()
+    tiled = np.broadcast_to(combined, (n_resamples, len(combined))).copy()
+    perms = rng.permuted(tiled, axis=1)
+    stat = perms[:, :na].mean(axis=1) - perms[:, na:].mean(axis=1)
+    count = np.sum(np.abs(stat) >= abs(obs) - 1e-9)
+    return float((count + 1) / (n_resamples + 1))
+
+
+def paired_bootstrap_p(diffs, n_resamples=9999, rng=None):
+    """Null-shifted bootstrap hypothesis test on paired differences."""
+    rng = rng or np.random.default_rng()
+    n = len(diffs)
+    obs = diffs.mean()
+    shifted = diffs - obs
+    idx = rng.integers(0, n, size=(n_resamples, n))
+    resampled = shifted[idx].mean(axis=1)
+    count = np.sum(np.abs(resampled) >= abs(obs) - 1e-9)
+    return float((count + 1) / (n_resamples + 1))
+
+
+def unpaired_bootstrap_p(a, b, n_resamples=9999, rng=None):
+    """Null-shifted bootstrap hypothesis test for two independent samples."""
+    rng = rng or np.random.default_rng()
+    obs = a.mean() - b.mean()
+    combined_mean = np.concatenate([a, b]).mean()
+    a_shift = a - a.mean() + combined_mean
+    b_shift = b - b.mean() + combined_mean
+    na, nb = len(a), len(b)
+    idx_a = rng.integers(0, na, size=(n_resamples, na))
+    idx_b = rng.integers(0, nb, size=(n_resamples, nb))
+    stats = a_shift[idx_a].mean(axis=1) - b_shift[idx_b].mean(axis=1)
+    count = np.sum(np.abs(stats) >= abs(obs) - 1e-9)
+    return float((count + 1) / (n_resamples + 1))
+
+
+def mcnemar_test(a, b, correction=True, exact=None):
+    """McNemar's test on paired binary (0/1 correctness) outcomes.
+
+    Returns (p_value, n01, n10, statistic) where n01 = b-correct-only,
+    n10 = a-correct-only, statistic is None when the exact binomial test is used.
+    """
+    a = np.asarray(a)
+    b = np.asarray(b)
+    n01 = int(np.sum((a == 0) & (b == 1)))
+    n10 = int(np.sum((a == 1) & (b == 0)))
+    n = n01 + n10
+    if exact is None:
+        exact = n < 25
+    if n == 0:
+        return 1.0, n01, n10, 0.0
+    if exact:
+        k = min(n01, n10)
+        p = _scipy_stats.binomtest(k, n, p=0.5, alternative="two-sided").pvalue
+        return float(p), n01, n10, None
+    if correction:
+        statistic = (abs(n01 - n10) - 1) ** 2 / n
+    else:
+        statistic = (n01 - n10) ** 2 / n
+    p = float(1 - _scipy_stats.chi2.cdf(statistic, df=1))
+    return p, n01, n10, float(statistic)

evalci/adapters/__init__.py

@@ -0,0 +1,7 @@
+"""Adapters that load external eval outputs into evalci's per-item DataFrame schema."""
+from . import csv, helm, lm_eval_harness
+from .csv import load as load_csv
+from .helm import load as load_helm
+from .lm_eval_harness import load as load_lm_eval_harness
+
+__all__ = ["load_lm_eval_harness", "load_helm", "load_csv", "lm_eval_harness", "helm", "csv"]

evalci/adapters/csv.py

@@ -0,0 +1,21 @@
+"""Load a plain per-item CSV (columns: item_id, score[, subset, sample_idx]) into the evalci schema."""
+from pathlib import Path
+
+import pandas as pd
+
+from ..schema import from_records
+
+
+def load(path, model=None):
+    path = Path(path)
+    model = model or path.stem
+    df = pd.read_csv(path)
+    if "item_id" not in df.columns or "score" not in df.columns:
+        raise ValueError(f"{path}: CSV must have 'item_id' and 'score' columns, got {list(df.columns)}")
+    return from_records(
+        df["item_id"],
+        model,
+        df["score"],
+        subsets=df["subset"] if "subset" in df.columns else None,
+        sample_idxs=df["sample_idx"] if "sample_idx" in df.columns else None,
+    )

evalci/adapters/helm.py

@@ -0,0 +1,59 @@
+"""Load per-item results from a HELM `per_instance_stats.json` file into the evalci schema.
+
+Expects the standard HELM per-instance-stats structure: a JSON list of records
+each with an "instance_id" and a "stats" list of {"name": {"name": <metric>},
+"mean": <value>, ...} entries. If your HELM run uses a different metric name,
+pass `metric_key` explicitly.
+"""
+import json
+from pathlib import Path
+
+from ..schema import from_records
+
+DEFAULT_METRIC_KEYS = ("exact_match", "quasi_exact_match", "f1_score")
+
+
+def _stat_value(stat):
+    if "mean" in stat and stat["mean"] is not None:
+        return float(stat["mean"])
+    if stat.get("count") == 1 and "sum" in stat:
+        return float(stat["sum"])
+    raise ValueError(f"could not extract a scalar value from stat record: {stat}")
+
+
+def _extract_score(record, metric_key):
+    stats = record.get("stats", [])
+    names = [s.get("name", {}).get("name") for s in stats]
+    if metric_key is not None:
+        for s, name in zip(stats, names):
+            if name == metric_key:
+                return _stat_value(s)
+        raise ValueError(f"metric_key={metric_key!r} not found; metrics present: {names}")
+    for key in DEFAULT_METRIC_KEYS:
+        for s, name in zip(stats, names):
+            if name == key:
+                return _stat_value(s)
+    raise ValueError(
+        f"could not find a metric in stats (tried {DEFAULT_METRIC_KEYS}); "
+        f"pass metric_key explicitly. metrics present: {names}"
+    )
+
+
+def load(path, model=None, subset=None, metric_key=None):
+    """Parse a HELM per_instance_stats.json file into a per-item DataFrame."""
+    path = Path(path)
+    model = model or path.stem
+    records = json.loads(path.read_text())
+    if not isinstance(records, list):
+        raise ValueError(f"{path}: expected a JSON list of per-instance stat records")
+    if not records:
+        raise ValueError(f"{path}: no records found")
+
+    item_ids, scores = [], []
+    for r in records:
+        item_id = r.get("instance_id", r.get("instance", {}).get("id"))
+        item_ids.append(item_id)
+        scores.append(_extract_score(r, metric_key))
+
+    subsets = [subset] * len(item_ids)
+    return from_records(item_ids, model, scores, subsets=subsets)

evalci/adapters/lm_eval_harness.py

@@ -0,0 +1,67 @@
+"""Load per-item results from lm-evaluation-harness output into the evalci schema.
+
+Supports both the `--log_samples` JSONL format (`samples_<task>_*.jsonl`, one
+JSON record per line) and the older single-JSON format with a top-level
+"samples" field. If your harness version uses different metric key names, pass
+`metric_key` explicitly.
+"""
+import json
+from pathlib import Path
+
+from ..schema import from_records
+
+DEFAULT_METRIC_KEYS = ("acc", "exact_match", "acc_norm", "score")
+
+
+def _extract_score(record, metric_key):
+    if metric_key is not None:
+        if metric_key not in record:
+            raise ValueError(f"metric_key={metric_key!r} not found in sample record: {list(record.keys())}")
+        return float(record[metric_key])
+    for key in DEFAULT_METRIC_KEYS:
+        if key in record:
+            return float(record[key])
+    raise ValueError(
+        f"could not find a metric in sample record (tried {DEFAULT_METRIC_KEYS}); "
+        f"pass metric_key explicitly. keys present: {list(record.keys())}"
+    )
+
+
+def load(path, model=None, task=None, metric_key=None):
+    """Parse an lm-evaluation-harness results file into a per-item DataFrame."""
+    path = Path(path)
+    model = model or path.stem
+    records = []
+
+    if path.suffix == ".jsonl":
+        task_name = task or path.stem.replace("samples_", "").split("_2")[0]
+        for line in path.read_text().splitlines():
+            line = line.strip()
+            if line:
+                records.append((task_name, json.loads(line)))
+    else:
+        data = json.loads(path.read_text())
+        samples = data.get("samples")
+        if samples is None:
+            raise ValueError(f"{path}: no 'samples' field found; is this an lm-eval-harness output file?")
+        if isinstance(samples, dict):
+            for task_name, items in samples.items():
+                if task is not None and task_name != task:
+                    continue
+                records.extend((task_name, r) for r in items)
+        elif isinstance(samples, list):
+            records.extend((task or "task", r) for r in samples)
+        else:
+            raise ValueError(f"{path}: unrecognized 'samples' structure: {type(samples)}")
+
+    if not records:
+        raise ValueError(f"{path}: no sample records found (task filter={task!r})")
+
+    item_ids, subsets, sample_idxs, scores = [], [], [], []
+    for task_name, r in records:
+        item_ids.append(r.get("doc_id", r.get("idx", r.get("doc_hash"))))
+        subsets.append(task_name)
+        sample_idxs.append(r.get("repeat_idx", 0))
+        scores.append(_extract_score(r, metric_key))
+
+    return from_records(item_ids, model, scores, subsets=subsets, sample_idxs=sample_idxs)

evalci/cli.py

@@ -0,0 +1,96 @@
+"""`evalci compare results_a.json results_b.json` — CLI entry point."""
+import argparse
+import json
+import sys
+from pathlib import Path
+
+from . import adapters
+from .schema import to_paired_arrays
+from .stats import compare
+from .report import report
+
+
+def _sniff_format(path):
+    path = Path(path)
+    if path.suffix == ".csv":
+        return "csv"
+    if path.suffix == ".jsonl":
+        return "lm-eval-harness"
+    if path.suffix == ".json":
+        try:
+            data = json.loads(path.read_text())
+        except json.JSONDecodeError:
+            return "csv"
+        if isinstance(data, dict) and "samples" in data:
+            return "lm-eval-harness"
+        if isinstance(data, list):
+            return "helm"
+    raise ValueError(f"could not auto-detect format for {path}; pass --format explicitly")
+
+
+def _load(path, fmt, model, metric_key):
+    fmt = fmt or _sniff_format(path)
+    if fmt == "lm-eval-harness":
+        return adapters.load_lm_eval_harness(path, model=model, metric_key=metric_key)
+    if fmt == "helm":
+        return adapters.load_helm(path, model=model, metric_key=metric_key)
+    if fmt == "csv":
+        return adapters.load_csv(path, model=model)
+    raise ValueError(f"unknown format: {fmt!r}")
+
+
+def build_parser():
+    parser = argparse.ArgumentParser(prog="evalci")
+    sub = parser.add_subparsers(dest="command", required=True)
+
+    cmp_p = sub.add_parser("compare", help="Compare two models' per-item results")
+    cmp_p.add_argument("results_a")
+    cmp_p.add_argument("results_b")
+    cmp_p.add_argument("--format", choices=["lm-eval-harness", "helm", "csv"], default=None)
+    cmp_p.add_argument("--method", choices=["permutation", "bootstrap", "mcnemar"], default="permutation")
+    cmp_p.add_argument("--unpaired", action="store_true", help="treat samples as independent, not item-paired")
+    cmp_p.add_argument("--metric-key", default=None)
+    cmp_p.add_argument("--confidence", type=float, default=0.95)
+    cmp_p.add_argument("--n-resamples", type=int, default=9999)
+    cmp_p.add_argument("--output-format", choices=["markdown", "latex"], default="markdown")
+    cmp_p.add_argument("--model-a", default=None, help="label for results_a (default: filename)")
+    cmp_p.add_argument("--model-b", default=None, help="label for results_b (default: filename)")
+
+    return parser
+
+
+def main(argv=None):
+    parser = build_parser()
+    args = parser.parse_args(argv)
+
+    if args.command == "compare":
+        df_a = _load(args.results_a, args.format, args.model_a, args.metric_key)
+        df_b = _load(args.results_b, args.format, args.model_b, args.metric_key)
+        model_a, model_b = df_a["model"].iloc[0], df_b["model"].iloc[0]
+        if model_a == model_b:
+            print(
+                f"error: both inputs resolved to the same model label {model_a!r}; "
+                "pass --model-a/--model-b to disambiguate",
+                file=sys.stderr,
+            )
+            return 1
+        import pandas as pd
+
+        merged = pd.concat([df_a, df_b], ignore_index=True)
+        a, b = to_paired_arrays(merged, model_a, model_b)
+        if len(a) == 0:
+            print(f"error: no overlapping item_ids between {args.results_a} and {args.results_b}", file=sys.stderr)
+            return 1
+        result = compare(
+            a, b, paired=not args.unpaired, method=args.method,
+            confidence=args.confidence, n_resamples=args.n_resamples,
+        )
+        print(report(result, format=args.output_format))
+        return 0
+
+    parser.error(f"unknown command: {args.command}")
+    return 2
+
+
+if __name__ == "__main__":
+    sys.exit(main())

evalci/report.py

@@ -0,0 +1,86 @@
+"""Render ci()/compare()/multi_compare() results as Markdown or LaTeX."""
+import pandas as pd
+
+from .stats import CIResult, CompareResult
+
+
+def _stars(p):
+    if p < 0.001:
+        return "***"
+    if p < 0.01:
+        return "**"
+    if p < 0.05:
+        return "*"
+    return ""
+
+
+def _fmt(x, precision):
+    return f"{x:.{precision}f}"
+
+
+def _report_ci(result, precision):
+    return (
+        f"{_fmt(result.estimate, precision)} "
+        f"[{int(result.confidence * 100)}% CI {_fmt(result.lower, precision)}, "
+        f"{_fmt(result.upper, precision)}], n={result.n}"
+    )
+
+
+def _report_compare(result, stars, precision):
+    star = f"{_stars(result.p_value)}" if stars else ""
+    kind = "paired" if result.paired else "unpaired"
+    return (
+        f"Δ={_fmt(result.delta, precision)}, "
+        f"{int(result.confidence * 100)}% CI [{_fmt(result.ci[0], precision)}, {_fmt(result.ci[1], precision)}], "
+        f"{kind} {result.method} p={result.p_value:.4g}{star}, n={result.n}"
+    )
+
+
+def _markdown_table(headers, rows):
+    lines = ["| " + " | ".join(headers) + " |", "| " + " | ".join(["---"] * len(headers)) + " |"]
+    for row in rows:
+        lines.append("| " + " | ".join(str(c) for c in row) + " |")
+    return "\n".join(lines)
+
+
+def _latex_table(headers, rows):
+    lines = [r"\begin{tabular}{" + "l" * len(headers) + "}", r"\toprule",
+              " & ".join(headers) + r" \\", r"\midrule"]
+    for row in rows:
+        lines.append(" & ".join(str(c) for c in row) + r" \\")
+    lines += [r"\bottomrule", r"\end{tabular}"]
+    return "\n".join(lines)
+
+
+def _report_table(df, format, stars, precision):
+    has_subset = "subset" in df.columns
+    headers = (["subset"] if has_subset else []) + ["model_a", "model_b", "delta", "ci", "p_adj"]
+    if stars:
+        headers.append("sig")
+    rows = []
+    for _, r in df.iterrows():
+        row = []
+        if has_subset:
+            row.append(r["subset"])
+        row.append(r["model_a"])
+        row.append(r["model_b"])
+        row.append(_fmt(r["delta"], precision))
+        row.append(f"[{_fmt(r['ci_low'], precision)}, {_fmt(r['ci_high'], precision)}]")
+        row.append(f"{r['p_adj']:.4g}")
+        if stars:
+            row.append(_stars(r["p_adj"]))
+        rows.append(row)
+    if format == "latex":
+        return _latex_table(headers, rows)
+    return _markdown_table(headers, rows)
+
+
+def report(result, format="markdown", stars=True, precision=3):
+    """Render a CIResult, CompareResult, or multi_compare() DataFrame."""
+    if isinstance(result, pd.DataFrame):
+        return _report_table(result, format, stars, precision)
+    if isinstance(result, CompareResult):
+        return _report_compare(result, stars, precision)
+    if isinstance(result, CIResult):
+        return _report_ci(result, precision)
+    raise TypeError(f"don't know how to report a {type(result)!r}")

evalci/schema.py

@@ -0,0 +1,45 @@
+"""The per-item results schema shared by multi_compare(), cluster_ci(), and adapters."""
+import numpy as np
+import pandas as pd
+
+REQUIRED_COLUMNS = ("item_id", "model", "score")
+OPTIONAL_COLUMNS = ("subset", "sample_idx")
+PER_ITEM_COLUMNS = REQUIRED_COLUMNS + OPTIONAL_COLUMNS
+
+
+def validate_schema(df):
+    if not isinstance(df, pd.DataFrame):
+        raise TypeError("expected a pandas DataFrame with columns "
+                         f"{REQUIRED_COLUMNS} (+ optional {OPTIONAL_COLUMNS})")
+    missing = [c for c in REQUIRED_COLUMNS if c not in df.columns]
+    if missing:
+        raise ValueError(f"per-item DataFrame is missing required column(s): {missing}")
+
+
+def to_paired_arrays(df, model_a, model_b, subset=None, subset_col="subset"):
+    """Pivot a per-item DataFrame to two aligned score arrays for model_a/model_b.
+
+    Multiple samples per item (`sample_idx`) are averaged into a single per-item
+    score before pairing.
+    """
+    validate_schema(df)
+    sub = df if subset is None or subset_col not in df.columns else df[df[subset_col] == subset]
+    pivot = sub.pivot_table(index="item_id", columns="model", values="score", aggfunc="mean")
+    missing = [m for m in (model_a, model_b) if m not in pivot.columns]
+    if missing:
+        raise ValueError(f"model(s) not found in DataFrame: {missing}")
+    paired = pivot[[model_a, model_b]].dropna()
+    return paired[model_a].to_numpy(dtype=float), paired[model_b].to_numpy(dtype=float)
+
+
+def from_records(item_ids, models, scores, subsets=None, sample_idxs=None):
+    """Build a per-item DataFrame from parallel arrays (used by adapters)."""
+    n = len(item_ids)
+    data = {
+        "item_id": list(item_ids),
+        "model": list(models) if not isinstance(models, str) else [models] * n,
+        "score": np.asarray(scores, dtype=float),
+    }
+    data["subset"] = list(subsets) if subsets is not None else [None] * n
+    data["sample_idx"] = list(sample_idxs) if sample_idxs is not None else [0] * n
+    return pd.DataFrame(data)

evalci/stats.py

@@ -0,0 +1,278 @@
+"""Public statistics API: ci, compare, power, multi_compare, cluster_ci."""
+import itertools
+from dataclasses import dataclass, field
+from typing import Optional, Tuple
+
+import numpy as np
+import pandas as pd
+
+from . import _power
+from ._correction import correct_pvalues
+from ._intervals import bootstrap_interval, clopper_pearson_interval, wilson_interval
+from ._significance import (
+    mcnemar_test,
+    paired_bootstrap_ci,
+    paired_bootstrap_p,
+    paired_permutation_p,
+    unpaired_bootstrap_ci,
+    unpaired_bootstrap_p,
+    unpaired_permutation_p,
+)
+from .schema import to_paired_arrays, validate_schema
+
+_BINARY_CI_METHODS = {"wilson", "clopper-pearson", "exact"}
+
+
+@dataclass
+class CIResult:
+    estimate: float
+    lower: float
+    upper: float
+    method: str
+    n: int
+    confidence: float = 0.95
+
+    def __iter__(self):
+        return iter((self.lower, self.upper))
+
+    def __repr__(self):
+        return (
+            f"CIResult(estimate={self.estimate:.4f}, "
+            f"ci=[{self.lower:.4f}, {self.upper:.4f}], "
+            f"method={self.method!r}, n={self.n})"
+        )
+
+
+@dataclass
+class CompareResult:
+    delta: float
+    ci: Tuple[float, float]
+    p_value: float
+    method: str
+    paired: bool
+    n: int
+    confidence: float = 0.95
+    extra: dict = field(default_factory=dict)
+
+    def __repr__(self):
+        return (
+            f"CompareResult(delta={self.delta:.4f}, "
+            f"ci=[{self.ci[0]:.4f}, {self.ci[1]:.4f}], "
+            f"p_value={self.p_value:.4g}, method={self.method!r}, "
+            f"paired={self.paired}, n={self.n})"
+        )
+
+
+@dataclass
+class PowerResult:
+    delta: float
+    n: int
+    power: float
+    alpha: float
+    method: str
+
+
+def ci(scores, method="wilson", confidence=0.95, n_resamples=9999, random_state=None):
+    """Confidence interval on a single model's score.
+
+    method: "wilson" or "clopper-pearson"/"exact" for binary (0/1) scores,
+    or "bootstrap" (percentile/BCa on the mean) for continuous scores.
+    """
+    scores = np.asarray(scores, dtype=float)
+    n = len(scores)
+    if n == 0:
+        raise ValueError("scores must be non-empty")
+    method_l = method.lower()
+    if method_l in _BINARY_CI_METHODS:
+        if not np.all(np.isin(scores, [0.0, 1.0])):
+            raise ValueError(
+                f"method={method!r} requires binary 0/1 scores; use method='bootstrap' "
+                "for continuous scores"
+            )
+        k = int(scores.sum())
+        if method_l == "wilson":
+            lo, hi = wilson_interval(k, n, confidence)
+        else:
+            lo, hi = clopper_pearson_interval(k, n, confidence)
+        return CIResult(estimate=k / n, lower=lo, upper=hi, method=method_l, n=n, confidence=confidence)
+    if method_l == "bootstrap":
+        est = float(scores.mean())
+        lo, hi = bootstrap_interval(
+            scores, np.mean, confidence=confidence, n_resamples=n_resamples, random_state=random_state
+        )
+        return CIResult(estimate=est, lower=lo, upper=hi, method="bootstrap", n=n, confidence=confidence)
+    raise ValueError(f"unknown method: {method!r}")
+
+
+def compare(
+    a,
+    b,
+    paired=True,
+    method="permutation",
+    confidence=0.95,
+    n_resamples=9999,
+    correction=True,
+    random_state=None,
+):
+    """Model-vs-model comparison. method: "permutation", "bootstrap", or "mcnemar" (paired only)."""
+    a = np.asarray(a, dtype=float)
+    b = np.asarray(b, dtype=float)
+    rng = np.random.default_rng(random_state)
+    method_l = method.lower()
+
+    if paired:
+        if len(a) != len(b):
+            raise ValueError("paired comparison requires equal-length arrays")
+        diffs = a - b
+        n = len(diffs)
+        delta = float(diffs.mean())
+        lo, hi = paired_bootstrap_ci(diffs, confidence=confidence, n_resamples=n_resamples, rng=rng)
+        extra = {}
+        if method_l == "permutation":
+            p = paired_permutation_p(diffs, n_resamples=n_resamples, rng=rng)
+        elif method_l == "bootstrap":
+            p = paired_bootstrap_p(diffs, n_resamples=n_resamples, rng=rng)
+        elif method_l == "mcnemar":
+            p, n01, n10, statistic = mcnemar_test(a, b, correction=correction)
+            extra = {"n01": n01, "n10": n10, "statistic": statistic}
+        else:
+            raise ValueError(f"unknown method: {method!r}")
+    else:
+        if method_l == "mcnemar":
+            raise ValueError("mcnemar requires paired=True")
+        n = len(a) + len(b)
+        delta = float(a.mean() - b.mean())
+        lo, hi = unpaired_bootstrap_ci(a, b, confidence=confidence, n_resamples=n_resamples, rng=rng)
+        extra = {}
+        if method_l == "permutation":
+            p = unpaired_permutation_p(a, b, n_resamples=n_resamples, rng=rng)
+        elif method_l == "bootstrap":
+            p = unpaired_bootstrap_p(a, b, n_resamples=n_resamples, rng=rng)
+        else:
+            raise ValueError(f"unknown method: {method!r}")
+
+    return CompareResult(
+        delta=delta,
+        ci=(lo, hi),
+        p_value=p,
+        method=method_l,
+        paired=paired,
+        n=n,
+        confidence=confidence,
+        extra=extra,
+    )
+
+
+def power(
+    delta,
+    n=None,
+    power=0.8,
+    baseline=0.5,
+    alpha=0.05,
+    method="analytic",
+    rho=0.0,
+    sims=5000,
+    random_state=None,
+):
+    """Sample-size (n is None) or achieved-power (n given) calculator.
+
+    method="analytic" uses a two-proportion normal approximation (fast, ignores
+    pairing correlation). method="simulation" simulates a paired design with a
+    `rho` correlation knob between the two models' item-level correctness.
+    """
+    method_l = method.lower()
+    if method_l == "analytic":
+        if n is None:
+            solved_n = _power.analytic_n(delta, target_power=power, baseline=baseline, alpha=alpha)
+            return PowerResult(delta=delta, n=solved_n, power=power, alpha=alpha, method="analytic")
+        achieved = _power.analytic_power(delta, n, baseline=baseline, alpha=alpha)
+        return PowerResult(delta=delta, n=n, power=achieved, alpha=alpha, method="analytic")
+    if method_l == "simulation":
+        if n is None:
+            solved_n = _power.simulate_n(
+                delta, target_power=power, baseline=baseline, alpha=alpha, rho=rho,
+                sims=sims, random_state=random_state,
+            )
+            return PowerResult(delta=delta, n=solved_n, power=power, alpha=alpha, method="simulation")
+        achieved = _power.simulate_power(
+            delta, n, baseline=baseline, alpha=alpha, rho=rho, sims=sims, random_state=random_state
+        )
+        return PowerResult(delta=delta, n=n, power=achieved, alpha=alpha, method="simulation")
+    raise ValueError(f"unknown method: {method!r}")
+
+
+def multi_compare(
+    df,
+    correction="holm",
+    method="permutation",
+    paired=True,
+    alpha=0.05,
+    n_resamples=9999,
+    random_state=None,
+    subset_col="subset",
+):
+    """Pairwise model comparisons across (optionally subset-stratified) benchmarks,
+    with multiple-comparison correction across all resulting p-values."""
+    validate_schema(df)
+    models = sorted(df["model"].unique())
+    if len(models) < 2:
+        raise ValueError("need at least 2 models to compare")
+    has_subset = subset_col in df.columns and df[subset_col].notna().any()
+    subsets = sorted(df[subset_col].dropna().unique()) if has_subset else [None]
+
+    rows = []
+    for subset in subsets:
+        for model_a, model_b in itertools.combinations(models, 2):
+            a, b = to_paired_arrays(df, model_a, model_b, subset=subset, subset_col=subset_col)
+            if len(a) == 0:
+                continue
+            res = compare(
+                a, b, paired=paired, method=method, confidence=1 - alpha,
+                n_resamples=n_resamples, random_state=random_state,
+            )
+            rows.append(
+                {
+                    "subset": subset,
+                    "model_a": model_a,
+                    "model_b": model_b,
+                    "delta": res.delta,
+                    "ci_low": res.ci[0],
+                    "ci_high": res.ci[1],
+                    "p_value": res.p_value,
+                    "n": res.n,
+                }
+            )
+    result = pd.DataFrame(rows)
+    if result.empty:
+        return result
+    result["p_adj"] = correct_pvalues(result["p_value"].to_numpy(), method=correction)
+    result["significant"] = result["p_adj"] < alpha
+    if not has_subset:
+        result = result.drop(columns=["subset"])
+    return result
+
+
+def cluster_ci(scores, clusters, statistic=np.mean, confidence=0.95, n_resamples=9999, random_state=None):
+    """Clustered bootstrap CI (resamples whole clusters, e.g. repeated decodes or
+    grouped questions, rather than individual items)."""
+    scores = np.asarray(scores, dtype=float)
+    clusters = np.asarray(clusters)
+    if len(scores) != len(clusters):
+        raise ValueError("scores and clusters must be the same length")
+    rng = np.random.default_rng(random_state)
+    unique_clusters = np.unique(clusters)
+    n_clusters = len(unique_clusters)
+    grouped = [scores[clusters == c] for c in unique_clusters]
+    overall = float(statistic(scores))
+
+    boot_stats = np.empty(n_resamples)
+    for i in range(n_resamples):
+        chosen = rng.integers(0, n_clusters, size=n_clusters)
+        resampled = np.concatenate([grouped[c] for c in chosen])
+        boot_stats[i] = statistic(resampled)
+    alpha = 1 - confidence
+    lo, hi = np.percentile(boot_stats, [100 * alpha / 2, 100 * (1 - alpha / 2)])
+    return CIResult(
+        estimate=overall, lower=float(lo), upper=float(hi),
+        method="cluster_bootstrap", n=len(scores), confidence=confidence,
+    )

evalci-0.1.0.dist-info/METADATA

@@ -0,0 +1,159 @@
+Metadata-Version: 2.4
+Name: evalci
+Version: 0.1.0
+Summary: Statistical significance & confidence intervals for LLM evals
+Author: Shreyas
+License-Expression: MIT
+Project-URL: Homepage, https://github.com/Shreyaskc/evalci
+Project-URL: Repository, https://github.com/Shreyaskc/evalci
+Project-URL: Issues, https://github.com/Shreyaskc/evalci/issues
+Keywords: llm,evaluation,statistics,confidence-interval,significance-testing,benchmarking
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Science/Research
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Topic :: Scientific/Engineering
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: numpy
+Requires-Dist: scipy
+Requires-Dist: pandas
+Provides-Extra: test
+Requires-Dist: pytest; extra == "test"
+Requires-Dist: statsmodels; extra == "test"
+Dynamic: license-file
+
+# evalci
+
+Statistically sound comparisons between LLMs on benchmarks: confidence
+intervals on accuracy, paired significance tests, power analysis, clustered
+standard errors for multi-sample decoding, and multiple-comparison correction
+across many models/benchmarks — all validated against `statsmodels`/exact
+enumeration fixtures.
+
+```python
+>>> import evalci
+>>> result = evalci.compare(model_a_scores, model_b_scores, method="permutation")
+>>> evalci.report(result)
+'Δ=0.034, 95% CI [0.005, 0.063], paired permutation p=0.025*, n=1319'  # exact numbers depend on your data
+```
+
+## Status
+
+Core library (statistics, eval-shaped workflows, adapters, CLI) is implemented
+and tested. Not yet released to PyPI; no arXiv paper or DOI yet.
+
+## Install
+
+Not yet on PyPI. Install from source:
+
+```bash
+git clone https://github.com/Shreyaskc/evalci.git
+cd evalci
+pip install -e ".[test]"   # add [test] to also get pytest/statsmodels for running the test suite
+```
+
+Requires Python ≥3.9. Runtime dependencies are numpy, scipy, and pandas only.
+
+## Usage
+
+### Confidence interval on a single model
+
+```python
+import evalci
+
+# binary (0/1) per-item correctness
+evalci.ci(scores, method="wilson")           # Wilson score interval
+evalci.ci(scores, method="clopper-pearson")  # exact interval
+
+# continuous scores (e.g. a similarity metric)
+evalci.ci(scores, method="bootstrap")        # percentile/BCa bootstrap on the mean
+```
+
+### Comparing two models on the same items
+
+```python
+result = evalci.compare(model_a_scores, model_b_scores, paired=True, method="permutation")
+# result.delta, result.ci, result.p_value, result.n
+
+evalci.compare(a, b, method="bootstrap")   # null-shifted bootstrap hypothesis test
+evalci.compare(a, b, method="mcnemar")     # McNemar's test for paired binary outcomes
+evalci.compare(a, b, paired=False, method="permutation")  # independent samples
+```
+
+### Sample-size / power calculator
+
+```python
+evalci.power(delta=0.03, power=0.8)          # required n to detect a 3-point gap at 80% power
+evalci.power(delta=0.03, n=1500)             # achieved power at n=1500
+evalci.power(delta=0.03, power=0.8, method="simulation", rho=0.3)  # correlated-items simulation
+```
+
+### Many models × many benchmarks, with correction
+
+```python
+import pandas as pd
+
+# per-item schema: item_id, model, score, [subset], [sample_idx]
+df = pd.DataFrame(...)
+table = evalci.multi_compare(df, correction="holm")
+print(evalci.report(table, format="markdown"))
+```
+
+### Clustered standard errors (repeated decoding, grouped questions)
+
+```python
+# clusters groups multiple samples of the same underlying item
+evalci.cluster_ci(scores, clusters)
+```
+
+### Loading results from eval harnesses
+
+```python
+from evalci.adapters import load_lm_eval_harness, load_helm, load_csv
+
+df_a = load_lm_eval_harness("results_a.json", model="model-a")
+df_b = load_helm("per_instance_stats.json", model="model-b")
+```
+
+### CLI
+
+```bash
+evalci compare results_a.json results_b.json --method permutation
+evalci compare results_a.json results_b.json --format helm --method mcnemar
+```
+
+Auto-detects lm-evaluation-harness / HELM / CSV format from the file
+extension/content; pass `--format` to override, and `--model-a`/`--model-b` to
+label the two runs explicitly.
+
+## What's validated, and how
+
+Statistical correctness is the point of this library, so the test suite
+cross-checks every routine against an independent reference rather than just
+re-testing its own math:
+
+- Wilson and Clopper-Pearson intervals against `statsmodels.stats.proportion.proportion_confint`
+- McNemar's test (exact and asymptotic) against `statsmodels.stats.contingency_tables.mcnemar`
+- Holm and Benjamini-Hochberg correction against `statsmodels.stats.multitest.multipletests`
+- The paired permutation test against brute-force exact enumeration of all sign flips (small n)
+- Bootstrap CIs via a coverage simulation (nominal 95% CIs should contain the true parameter ~95% of the time)
+
+`statsmodels` is a test-only dependency (`pip install -e ".[test]"`), not a
+runtime dependency.
+
+```bash
+pytest tests/
+```
+
+## API surface
+
+`evalci.ci`, `evalci.compare`, `evalci.power`, `evalci.multi_compare`,
+`evalci.cluster_ci`, `evalci.report`, `evalci.adapters.{load_lm_eval_harness,
+load_helm, load_csv}`.
+
+## License
+
+MIT — see [LICENSE](LICENSE).

evalci-0.1.0.dist-info/RECORD

@@ -0,0 +1,19 @@
+evalci/__init__.py,sha256=BIGglYzvKq-4pxZLaHH6aKC6StQorG8XxoleIvPDHpw,368
+evalci/_correction.py,sha256=R-meWXZ37Dq8Wc5XZ-BIr6wN1FRn_-B7ZdBD9iN_tz0,1318
+evalci/_intervals.py,sha256=86Bn6fM3PBixRMgmJ_D-Vp_o6GckW70E0B_ePWjMQZI,1902
+evalci/_power.py,sha256=zj6aMc4Ac792Ji8YbyMQCdtxZV1LnQ5nCTnMCcP9j7A,2616
+evalci/_significance.py,sha256=qPppQSUut_iG-lmRnAyziE_HMWn2gCtp5svRDujO5UM,4337
+evalci/cli.py,sha256=7ahTjHMO9P1U6AISv39b-QWvxUCjEak-f2Szpa1XYT4,3620
+evalci/report.py,sha256=BC9HDdF3Ij8sWaQp8Sv7VFM0OQvm5GrHgIj7KzJlhK4,2864
+evalci/schema.py,sha256=HaUQ0d7dvIqJTzHrqeCmokBFWQau4_3enxyRUIqI_KQ,2004
+evalci/stats.py,sha256=QrQZ4hnw1yEkqduskVtE0qDT-En1zplMYJ_ch9Jchyk,9697
+evalci/adapters/__init__.py,sha256=qjTGbejXDzF8hSuY9lqNSnH0f7PR-d1Ms_UZyQYBkD0,352
+evalci/adapters/csv.py,sha256=oOt26sAt8dVV5aD7xognQrtBMn3mflzUP76jNh6qGs0,699
+evalci/adapters/helm.py,sha256=nPstY8VawTmI9Mc-I4o-swqHd7lwhq1wEOHQknP9yCw,2221
+evalci/adapters/lm_eval_harness.py,sha256=CvfYLej08y6LhKlZHJDLACdlCKLUNpY9Is00ERqCnyY,2694
+evalci-0.1.0.dist-info/licenses/LICENSE,sha256=E5J5RZEY_2ybAJCvq79TWdtnQy4nO-IDeeyQ2LMF2l4,1064
+evalci-0.1.0.dist-info/METADATA,sha256=O6K9eOMn790pLQ-F-lYzCdbccqkLt0qtgIWGFZe2fPA,5327
+evalci-0.1.0.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
+evalci-0.1.0.dist-info/entry_points.txt,sha256=yL6bWF7XTvcJba_eK0dzi9bXIORTvw7ahqL3ioKqWbY,43
+evalci-0.1.0.dist-info/top_level.txt,sha256=HAQ7N_S3eZUMnC33FI6oywwl66lJ2Mx9kduKb6ukPEs,7
+evalci-0.1.0.dist-info/RECORD,,

evalci-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (82.0.1)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

evalci-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+evalci = evalci.cli:main

evalci-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Shreyas
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

evalci-0.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+evalci