evalci 0.1.0__tar.gz

evalci-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Shreyas
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

evalci-0.1.0/PKG-INFO

@@ -0,0 +1,159 @@
+Metadata-Version: 2.4
+Name: evalci
+Version: 0.1.0
+Summary: Statistical significance & confidence intervals for LLM evals
+Author: Shreyas
+License-Expression: MIT
+Project-URL: Homepage, https://github.com/Shreyaskc/evalci
+Project-URL: Repository, https://github.com/Shreyaskc/evalci
+Project-URL: Issues, https://github.com/Shreyaskc/evalci/issues
+Keywords: llm,evaluation,statistics,confidence-interval,significance-testing,benchmarking
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Science/Research
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Topic :: Scientific/Engineering
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: numpy
+Requires-Dist: scipy
+Requires-Dist: pandas
+Provides-Extra: test
+Requires-Dist: pytest; extra == "test"
+Requires-Dist: statsmodels; extra == "test"
+Dynamic: license-file
+
+# evalci
+
+Statistically sound comparisons between LLMs on benchmarks: confidence
+intervals on accuracy, paired significance tests, power analysis, clustered
+standard errors for multi-sample decoding, and multiple-comparison correction
+across many models/benchmarks — all validated against `statsmodels`/exact
+enumeration fixtures.
+
+```python
+>>> import evalci
+>>> result = evalci.compare(model_a_scores, model_b_scores, method="permutation")
+>>> evalci.report(result)
+'Δ=0.034, 95% CI [0.005, 0.063], paired permutation p=0.025*, n=1319'  # exact numbers depend on your data
+```
+
+## Status
+
+Core library (statistics, eval-shaped workflows, adapters, CLI) is implemented
+and tested. Not yet released to PyPI; no arXiv paper or DOI yet.
+
+## Install
+
+Not yet on PyPI. Install from source:
+
+```bash
+git clone https://github.com/Shreyaskc/evalci.git
+cd evalci
+pip install -e ".[test]"   # add [test] to also get pytest/statsmodels for running the test suite
+```
+
+Requires Python ≥3.9. Runtime dependencies are numpy, scipy, and pandas only.
+
+## Usage
+
+### Confidence interval on a single model
+
+```python
+import evalci
+
+# binary (0/1) per-item correctness
+evalci.ci(scores, method="wilson")           # Wilson score interval
+evalci.ci(scores, method="clopper-pearson")  # exact interval
+
+# continuous scores (e.g. a similarity metric)
+evalci.ci(scores, method="bootstrap")        # percentile/BCa bootstrap on the mean
+```
+
+### Comparing two models on the same items
+
+```python
+result = evalci.compare(model_a_scores, model_b_scores, paired=True, method="permutation")
+# result.delta, result.ci, result.p_value, result.n
+
+evalci.compare(a, b, method="bootstrap")   # null-shifted bootstrap hypothesis test
+evalci.compare(a, b, method="mcnemar")     # McNemar's test for paired binary outcomes
+evalci.compare(a, b, paired=False, method="permutation")  # independent samples
+```
+
+### Sample-size / power calculator
+
+```python
+evalci.power(delta=0.03, power=0.8)          # required n to detect a 3-point gap at 80% power
+evalci.power(delta=0.03, n=1500)             # achieved power at n=1500
+evalci.power(delta=0.03, power=0.8, method="simulation", rho=0.3)  # correlated-items simulation
+```
+
+### Many models × many benchmarks, with correction
+
+```python
+import pandas as pd
+
+# per-item schema: item_id, model, score, [subset], [sample_idx]
+df = pd.DataFrame(...)
+table = evalci.multi_compare(df, correction="holm")
+print(evalci.report(table, format="markdown"))
+```
+
+### Clustered standard errors (repeated decoding, grouped questions)
+
+```python
+# clusters groups multiple samples of the same underlying item
+evalci.cluster_ci(scores, clusters)
+```
+
+### Loading results from eval harnesses
+
+```python
+from evalci.adapters import load_lm_eval_harness, load_helm, load_csv
+
+df_a = load_lm_eval_harness("results_a.json", model="model-a")
+df_b = load_helm("per_instance_stats.json", model="model-b")
+```
+
+### CLI
+
+```bash
+evalci compare results_a.json results_b.json --method permutation
+evalci compare results_a.json results_b.json --format helm --method mcnemar
+```
+
+Auto-detects lm-evaluation-harness / HELM / CSV format from the file
+extension/content; pass `--format` to override, and `--model-a`/`--model-b` to
+label the two runs explicitly.
+
+## What's validated, and how
+
+Statistical correctness is the point of this library, so the test suite
+cross-checks every routine against an independent reference rather than just
+re-testing its own math:
+
+- Wilson and Clopper-Pearson intervals against `statsmodels.stats.proportion.proportion_confint`
+- McNemar's test (exact and asymptotic) against `statsmodels.stats.contingency_tables.mcnemar`
+- Holm and Benjamini-Hochberg correction against `statsmodels.stats.multitest.multipletests`
+- The paired permutation test against brute-force exact enumeration of all sign flips (small n)
+- Bootstrap CIs via a coverage simulation (nominal 95% CIs should contain the true parameter ~95% of the time)
+
+`statsmodels` is a test-only dependency (`pip install -e ".[test]"`), not a
+runtime dependency.
+
+```bash
+pytest tests/
+```
+
+## API surface
+
+`evalci.ci`, `evalci.compare`, `evalci.power`, `evalci.multi_compare`,
+`evalci.cluster_ci`, `evalci.report`, `evalci.adapters.{load_lm_eval_harness,
+load_helm, load_csv}`.
+
+## License
+
+MIT — see [LICENSE](LICENSE).

evalci-0.1.0/README.md

@@ -0,0 +1,132 @@
+# evalci
+
+Statistically sound comparisons between LLMs on benchmarks: confidence
+intervals on accuracy, paired significance tests, power analysis, clustered
+standard errors for multi-sample decoding, and multiple-comparison correction
+across many models/benchmarks — all validated against `statsmodels`/exact
+enumeration fixtures.
+
+```python
+>>> import evalci
+>>> result = evalci.compare(model_a_scores, model_b_scores, method="permutation")
+>>> evalci.report(result)
+'Δ=0.034, 95% CI [0.005, 0.063], paired permutation p=0.025*, n=1319'  # exact numbers depend on your data
+```
+
+## Status
+
+Core library (statistics, eval-shaped workflows, adapters, CLI) is implemented
+and tested. Not yet released to PyPI; no arXiv paper or DOI yet.
+
+## Install
+
+Not yet on PyPI. Install from source:
+
+```bash
+git clone https://github.com/Shreyaskc/evalci.git
+cd evalci
+pip install -e ".[test]"   # add [test] to also get pytest/statsmodels for running the test suite
+```
+
+Requires Python ≥3.9. Runtime dependencies are numpy, scipy, and pandas only.
+
+## Usage
+
+### Confidence interval on a single model
+
+```python
+import evalci
+
+# binary (0/1) per-item correctness
+evalci.ci(scores, method="wilson")           # Wilson score interval
+evalci.ci(scores, method="clopper-pearson")  # exact interval
+
+# continuous scores (e.g. a similarity metric)
+evalci.ci(scores, method="bootstrap")        # percentile/BCa bootstrap on the mean
+```
+
+### Comparing two models on the same items
+
+```python
+result = evalci.compare(model_a_scores, model_b_scores, paired=True, method="permutation")
+# result.delta, result.ci, result.p_value, result.n
+
+evalci.compare(a, b, method="bootstrap")   # null-shifted bootstrap hypothesis test
+evalci.compare(a, b, method="mcnemar")     # McNemar's test for paired binary outcomes
+evalci.compare(a, b, paired=False, method="permutation")  # independent samples
+```
+
+### Sample-size / power calculator
+
+```python
+evalci.power(delta=0.03, power=0.8)          # required n to detect a 3-point gap at 80% power
+evalci.power(delta=0.03, n=1500)             # achieved power at n=1500
+evalci.power(delta=0.03, power=0.8, method="simulation", rho=0.3)  # correlated-items simulation
+```
+
+### Many models × many benchmarks, with correction
+
+```python
+import pandas as pd
+
+# per-item schema: item_id, model, score, [subset], [sample_idx]
+df = pd.DataFrame(...)
+table = evalci.multi_compare(df, correction="holm")
+print(evalci.report(table, format="markdown"))
+```
+
+### Clustered standard errors (repeated decoding, grouped questions)
+
+```python
+# clusters groups multiple samples of the same underlying item
+evalci.cluster_ci(scores, clusters)
+```
+
+### Loading results from eval harnesses
+
+```python
+from evalci.adapters import load_lm_eval_harness, load_helm, load_csv
+
+df_a = load_lm_eval_harness("results_a.json", model="model-a")
+df_b = load_helm("per_instance_stats.json", model="model-b")
+```
+
+### CLI
+
+```bash
+evalci compare results_a.json results_b.json --method permutation
+evalci compare results_a.json results_b.json --format helm --method mcnemar
+```
+
+Auto-detects lm-evaluation-harness / HELM / CSV format from the file
+extension/content; pass `--format` to override, and `--model-a`/`--model-b` to
+label the two runs explicitly.
+
+## What's validated, and how
+
+Statistical correctness is the point of this library, so the test suite
+cross-checks every routine against an independent reference rather than just
+re-testing its own math:
+
+- Wilson and Clopper-Pearson intervals against `statsmodels.stats.proportion.proportion_confint`
+- McNemar's test (exact and asymptotic) against `statsmodels.stats.contingency_tables.mcnemar`
+- Holm and Benjamini-Hochberg correction against `statsmodels.stats.multitest.multipletests`
+- The paired permutation test against brute-force exact enumeration of all sign flips (small n)
+- Bootstrap CIs via a coverage simulation (nominal 95% CIs should contain the true parameter ~95% of the time)
+
+`statsmodels` is a test-only dependency (`pip install -e ".[test]"`), not a
+runtime dependency.
+
+```bash
+pytest tests/
+```
+
+## API surface
+
+`evalci.ci`, `evalci.compare`, `evalci.power`, `evalci.multi_compare`,
+`evalci.cluster_ci`, `evalci.report`, `evalci.adapters.{load_lm_eval_harness,
+load_helm, load_csv}`.
+
+## License
+
+MIT — see [LICENSE](LICENSE).

evalci-0.1.0/pyproject.toml

@@ -0,0 +1,42 @@
+[build-system]
+requires = ["setuptools>=61.0"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "evalci"
+version = "0.1.0"
+description = "Statistical significance & confidence intervals for LLM evals"
+readme = "README.md"
+license = "MIT"
+requires-python = ">=3.9"
+authors = [
+    { name = "Shreyas" },
+]
+keywords = ["llm", "evaluation", "statistics", "confidence-interval", "significance-testing", "benchmarking"]
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "Intended Audience :: Science/Research",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3 :: Only",
+    "Topic :: Scientific/Engineering",
+    "Topic :: Scientific/Engineering :: Artificial Intelligence",
+]
+dependencies = [
+    "numpy",
+    "scipy",
+    "pandas",
+]
+
+[project.urls]
+Homepage = "https://github.com/Shreyaskc/evalci"
+Repository = "https://github.com/Shreyaskc/evalci"
+Issues = "https://github.com/Shreyaskc/evalci/issues"
+
+[project.optional-dependencies]
+test = ["pytest", "statsmodels"]
+
+[project.scripts]
+evalci = "evalci.cli:main"
+
+[tool.setuptools.packages.find]
+where = ["src"]

evalci-0.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

evalci-0.1.0/src/evalci/__init__.py

@@ -0,0 +1,25 @@
+from .stats import (
+    CIResult,
+    CompareResult,
+    PowerResult,
+    ci,
+    cluster_ci,
+    compare,
+    multi_compare,
+    power,
+)
+from .report import report
+from . import adapters
+
+__all__ = [
+    "ci",
+    "compare",
+    "power",
+    "multi_compare",
+    "cluster_ci",
+    "report",
+    "CIResult",
+    "CompareResult",
+    "PowerResult",
+    "adapters",
+]

evalci-0.1.0/src/evalci/_correction.py

@@ -0,0 +1,41 @@
+"""Multiple-comparison correction: Holm and Benjamini-Hochberg."""
+import numpy as np
+
+
+def holm(pvalues):
+    """Holm (1979) step-down Bonferroni correction. Returns adjusted p-values."""
+    p = np.asarray(pvalues, dtype=float)
+    n = len(p)
+    order = np.argsort(p)
+    adj = np.empty(n)
+    running_max = 0.0
+    for i, idx in enumerate(order):
+        running_max = max(running_max, (n - i) * p[idx])
+        adj[idx] = min(running_max, 1.0)
+    return adj
+
+
+def benjamini_hochberg(pvalues):
+    """Benjamini-Hochberg (1995) FDR correction. Returns adjusted (q) values."""
+    p = np.asarray(pvalues, dtype=float)
+    n = len(p)
+    order = np.argsort(p)[::-1]
+    adj = np.empty(n)
+    running_min = 1.0
+    for i, idx in enumerate(order):
+        rank = n - i
+        running_min = min(running_min, p[idx] * n / rank)
+        adj[idx] = running_min
+    return np.clip(adj, 0, 1)
+
+
+def correct_pvalues(pvalues, method="holm"):
+    method = method.lower()
+    if method in ("holm", "bonferroni-holm"):
+        return holm(pvalues)
+    if method in ("bh", "benjamini-hochberg", "fdr_bh"):
+        return benjamini_hochberg(pvalues)
+    if method == "bonferroni":
+        p = np.asarray(pvalues, dtype=float)
+        return np.clip(p * len(p), 0, 1)
+    raise ValueError(f"unknown correction method: {method!r}")

evalci-0.1.0/src/evalci/_intervals.py

@@ -0,0 +1,56 @@
+"""Interval estimators: Wilson, Clopper-Pearson, and bootstrap (percentile/BCa)."""
+import numpy as np
+from scipy import stats as _scipy_stats
+
+
+def wilson_interval(k, n, confidence=0.95):
+    """Wilson score interval for a binomial proportion."""
+    if n <= 0:
+        raise ValueError("n must be positive")
+    z = _scipy_stats.norm.ppf(0.5 + confidence / 2)
+    phat = k / n
+    z2 = z * z
+    denom = 1 + z2 / n
+    center = (phat + z2 / (2 * n)) / denom
+    half = (z * np.sqrt(phat * (1 - phat) / n + z2 / (4 * n * n))) / denom
+    return max(0.0, center - half), min(1.0, center + half)
+
+
+def clopper_pearson_interval(k, n, confidence=0.95):
+    """Exact (Clopper-Pearson) interval for a binomial proportion."""
+    if n <= 0:
+        raise ValueError("n must be positive")
+    alpha = 1 - confidence
+    lo = 0.0 if k == 0 else _scipy_stats.beta.ppf(alpha / 2, k, n - k + 1)
+    hi = 1.0 if k == n else _scipy_stats.beta.ppf(1 - alpha / 2, k + 1, n - k)
+    return float(lo), float(hi)
+
+
+def bootstrap_interval(
+    data,
+    statistic=np.mean,
+    method="BCa",
+    confidence=0.95,
+    n_resamples=9999,
+    random_state=None,
+    vectorized=True,
+):
+    """Bootstrap CI for an arbitrary statistic via scipy.stats.bootstrap."""
+    data = np.asarray(data, dtype=float)
+    scipy_method = {"bca": "BCa", "percentile": "percentile", "basic": "basic"}.get(
+        method.lower(), method
+    )
+    if np.all(data == data[0]):
+        # degenerate (zero-variance) sample: BCa's acceleration is undefined
+        point = float(statistic(data))
+        return point, point
+    res = _scipy_stats.bootstrap(
+        (data,),
+        statistic,
+        confidence_level=confidence,
+        method=scipy_method,
+        n_resamples=n_resamples,
+        random_state=random_state,
+        vectorized=vectorized,
+    )
+    return float(res.confidence_interval.low), float(res.confidence_interval.high)

evalci-0.1.0/src/evalci/_power.py

@@ -0,0 +1,65 @@
+"""Power / sample-size analysis: analytic (normal approximation) and simulation."""
+import numpy as np
+from scipy import stats as _scipy_stats
+
+
+def analytic_n(delta, target_power=0.8, baseline=0.5, alpha=0.05):
+    """Two-proportion normal-approximation sample size for a given effect size."""
+    p1, p2 = baseline, baseline + delta
+    z_alpha = _scipy_stats.norm.ppf(1 - alpha / 2)
+    z_beta = _scipy_stats.norm.ppf(target_power)
+    var = p1 * (1 - p1) + p2 * (1 - p2)
+    n = ((z_alpha + z_beta) ** 2) * var / (delta**2)
+    return int(np.ceil(n))
+
+
+def analytic_power(delta, n, baseline=0.5, alpha=0.05):
+    """Achieved power of a two-proportion test for given n and effect size."""
+    p1, p2 = baseline, baseline + delta
+    z_alpha = _scipy_stats.norm.ppf(1 - alpha / 2)
+    var = p1 * (1 - p1) + p2 * (1 - p2)
+    z_beta = np.sqrt(n * (delta**2) / var) - z_alpha
+    return float(_scipy_stats.norm.cdf(z_beta))
+
+
+def simulate_power(delta, n, baseline=0.5, alpha=0.05, rho=0.0, sims=5000, random_state=None):
+    """Monte Carlo power for a paired design with correlated item-level outcomes.
+
+    `rho` is the fraction of items whose correctness is shared between the two
+    models (a common-difficulty link) rather than drawn independently; it is a
+    simplified correlation knob, not an exact bivariate-Bernoulli correlation.
+    """
+    rng = np.random.default_rng(random_state)
+    p1, p2 = baseline, baseline + delta
+    shared = rng.random((sims, n)) < rho
+    shared_correct = rng.random((sims, n)) < (p1 + p2) / 2
+    a_ind = rng.random((sims, n)) < p1
+    b_ind = rng.random((sims, n)) < p2
+    a = np.where(shared, shared_correct, a_ind).astype(float)
+    b = np.where(shared, shared_correct, b_ind).astype(float)
+    diffs = a - b
+    means = diffs.mean(axis=1)
+    sds = diffs.std(axis=1, ddof=1)
+    sds[sds == 0] = 1e-12
+    z = means / (sds / np.sqrt(n))
+    pvals = 2 * (1 - _scipy_stats.norm.cdf(np.abs(z)))
+    return float(np.mean(pvals < alpha))
+
+
+def simulate_n(delta, target_power=0.8, baseline=0.5, alpha=0.05, rho=0.0, sims=3000, random_state=None):
+    """Bisection search for the n achieving target_power under simulate_power."""
+    lo, hi = 4, 64
+    max_n = 1_000_000
+    while simulate_power(delta, hi, baseline, alpha, rho, sims, random_state) < target_power:
+        lo = hi
+        hi *= 2
+        if hi > max_n:
+            return max_n
+    while hi - lo > 1:
+        mid = (lo + hi) // 2
+        p = simulate_power(delta, mid, baseline, alpha, rho, sims, random_state)
+        if p < target_power:
+            lo = mid
+        else:
+            hi = mid
+    return hi

evalci-0.1.0/src/evalci/_significance.py

@@ -0,0 +1,111 @@
+"""Significance-test internals: paired/unpaired permutation, bootstrap, McNemar."""
+import itertools
+
+import numpy as np
+from scipy import stats as _scipy_stats
+
+EXACT_ENUMERATION_THRESHOLD = 12
+
+
+def paired_bootstrap_ci(diffs, confidence=0.95, n_resamples=9999, rng=None):
+    rng = rng or np.random.default_rng()
+    n = len(diffs)
+    idx = rng.integers(0, n, size=(n_resamples, n))
+    resampled_means = diffs[idx].mean(axis=1)
+    alpha = 1 - confidence
+    lo, hi = np.percentile(resampled_means, [100 * alpha / 2, 100 * (1 - alpha / 2)])
+    return float(lo), float(hi)
+
+
+def unpaired_bootstrap_ci(a, b, confidence=0.95, n_resamples=9999, rng=None):
+    rng = rng or np.random.default_rng()
+    na, nb = len(a), len(b)
+    idx_a = rng.integers(0, na, size=(n_resamples, na))
+    idx_b = rng.integers(0, nb, size=(n_resamples, nb))
+    stats = a[idx_a].mean(axis=1) - b[idx_b].mean(axis=1)
+    alpha = 1 - confidence
+    lo, hi = np.percentile(stats, [100 * alpha / 2, 100 * (1 - alpha / 2)])
+    return float(lo), float(hi)
+
+
+def paired_permutation_p(diffs, n_resamples=9999, rng=None, exact_threshold=EXACT_ENUMERATION_THRESHOLD):
+    """Sign-flip permutation test on paired differences. Exact enumeration for small n."""
+    rng = rng or np.random.default_rng()
+    n = len(diffs)
+    obs = diffs.mean()
+    if n <= exact_threshold:
+        signs = np.array(list(itertools.product([-1, 1], repeat=n)), dtype=float)
+        resampled = (signs * diffs).mean(axis=1)
+        count = np.sum(np.abs(resampled) >= abs(obs) - 1e-9)
+        return float(count / len(signs))
+    signs = rng.choice([-1.0, 1.0], size=(n_resamples, n))
+    resampled = (signs * diffs).mean(axis=1)
+    count = np.sum(np.abs(resampled) >= abs(obs) - 1e-9)
+    return float((count + 1) / (n_resamples + 1))
+
+
+def unpaired_permutation_p(a, b, n_resamples=9999, rng=None):
+    """Label-shuffle permutation test for two independent samples."""
+    rng = rng or np.random.default_rng()
+    na = len(a)
+    combined = np.concatenate([a, b])
+    obs = a.mean() - b.mean()
+    tiled = np.broadcast_to(combined, (n_resamples, len(combined))).copy()
+    perms = rng.permuted(tiled, axis=1)
+    stat = perms[:, :na].mean(axis=1) - perms[:, na:].mean(axis=1)
+    count = np.sum(np.abs(stat) >= abs(obs) - 1e-9)
+    return float((count + 1) / (n_resamples + 1))
+
+
+def paired_bootstrap_p(diffs, n_resamples=9999, rng=None):
+    """Null-shifted bootstrap hypothesis test on paired differences."""
+    rng = rng or np.random.default_rng()
+    n = len(diffs)
+    obs = diffs.mean()
+    shifted = diffs - obs
+    idx = rng.integers(0, n, size=(n_resamples, n))
+    resampled = shifted[idx].mean(axis=1)
+    count = np.sum(np.abs(resampled) >= abs(obs) - 1e-9)
+    return float((count + 1) / (n_resamples + 1))
+
+
+def unpaired_bootstrap_p(a, b, n_resamples=9999, rng=None):
+    """Null-shifted bootstrap hypothesis test for two independent samples."""
+    rng = rng or np.random.default_rng()
+    obs = a.mean() - b.mean()
+    combined_mean = np.concatenate([a, b]).mean()
+    a_shift = a - a.mean() + combined_mean
+    b_shift = b - b.mean() + combined_mean
+    na, nb = len(a), len(b)
+    idx_a = rng.integers(0, na, size=(n_resamples, na))
+    idx_b = rng.integers(0, nb, size=(n_resamples, nb))
+    stats = a_shift[idx_a].mean(axis=1) - b_shift[idx_b].mean(axis=1)
+    count = np.sum(np.abs(stats) >= abs(obs) - 1e-9)
+    return float((count + 1) / (n_resamples + 1))
+
+
+def mcnemar_test(a, b, correction=True, exact=None):
+    """McNemar's test on paired binary (0/1 correctness) outcomes.
+
+    Returns (p_value, n01, n10, statistic) where n01 = b-correct-only,
+    n10 = a-correct-only, statistic is None when the exact binomial test is used.
+    """
+    a = np.asarray(a)
+    b = np.asarray(b)
+    n01 = int(np.sum((a == 0) & (b == 1)))
+    n10 = int(np.sum((a == 1) & (b == 0)))
+    n = n01 + n10
+    if exact is None:
+        exact = n < 25
+    if n == 0:
+        return 1.0, n01, n10, 0.0
+    if exact:
+        k = min(n01, n10)
+        p = _scipy_stats.binomtest(k, n, p=0.5, alternative="two-sided").pvalue
+        return float(p), n01, n10, None
+    if correction:
+        statistic = (abs(n01 - n10) - 1) ** 2 / n
+    else:
+        statistic = (n01 - n10) ** 2 / n
+    p = float(1 - _scipy_stats.chi2.cdf(statistic, df=1))
+    return p, n01, n10, float(statistic)

evalci-0.1.0/src/evalci/adapters/__init__.py

@@ -0,0 +1,7 @@
+"""Adapters that load external eval outputs into evalci's per-item DataFrame schema."""
+from . import csv, helm, lm_eval_harness
+from .csv import load as load_csv
+from .helm import load as load_helm
+from .lm_eval_harness import load as load_lm_eval_harness
+
+__all__ = ["load_lm_eval_harness", "load_helm", "load_csv", "lm_eval_harness", "helm", "csv"]