abeval 0.1.0__tar.gz

abeval-0.1.0/.github/FUNDING.yml

@@ -0,0 +1 @@
+github: mohammadi-hadi

abeval-0.1.0/.github/dependabot.yml

@@ -0,0 +1,10 @@
+version: 2
+updates:
+  - package-ecosystem: github-actions
+    directory: /
+    schedule:
+      interval: monthly
+  - package-ecosystem: pip
+    directory: /
+    schedule:
+      interval: monthly

abeval-0.1.0/.github/workflows/test.yml

@@ -0,0 +1,29 @@
+name: Tests
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.10", "3.14"]
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install
+        run: pip install -e ".[dev]"
+      - name: Lint
+        run: ruff check .
+      - name: Test
+        run: pytest -q
+      - name: CLI smoke test
+        run: |
+          abeval --version
+          abeval compare examples/run_a.jsonl examples/run_b.jsonl
+          abeval power --baseline 0.75 --delta 0.03 --json

abeval-0.1.0/CITATION.cff

@@ -0,0 +1,18 @@
+cff-version: 1.2.0
+message: "If abeval is useful in your work, please cite this repository."
+title: "abeval: A/B-test statistics for LLM evals"
+type: software
+authors:
+  - family-names: Mohammadi
+    given-names: Hadi
+version: 0.1.0
+date-released: 2026-08-05
+license: MIT
+repository-code: "https://github.com/mohammadi-hadi/abeval"
+url: "https://mohammadi.cv/abeval/"
+keywords:
+  - llm
+  - evaluation
+  - statistics
+  - ab-testing
+  - significance

abeval-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Hadi Mohammadi
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

abeval-0.1.0/Makefile

@@ -0,0 +1,33 @@
+.PHONY: install lint test demo serve build
+
+install:
+	python3 -m venv .venv && .venv/bin/pip install -e ".[dev]"
+
+lint:
+	.venv/bin/ruff check .
+
+test:
+	.venv/bin/python -m pytest -q
+
+demo:
+	.venv/bin/python -m abeval compare examples/run_a.jsonl examples/run_b.jsonl
+	.venv/bin/python -m abeval power --baseline 0.75 --delta 0.03
+
+serve:
+	.venv/bin/python -m http.server 8765 -d docs
+
+# Build sdist + wheel from a clean `git archive` export in a temp directory
+# OUTSIDE the working tree (hatchling walks upward and would otherwise pick up
+# local ignore files), so only committed files can reach a distribution.
+# The artifacts are audited afterwards.
+build:
+	rm -rf dist
+	@export_dir=$$(mktemp -d) && \
+	git archive HEAD | tar -x -C "$$export_dir" && \
+	.venv/bin/python -m build --outdir dist "$$export_dir" && \
+	rm -rf "$$export_dir"
+	@if tar -tzf dist/*.tar.gz | grep -qiE 'claude|gitignore'; then \
+		echo "TAINTED SDIST"; exit 1; fi
+	@if unzip -l dist/*.whl | grep -qiE 'claude|gitignore'; then \
+		echo "TAINTED WHEEL"; exit 1; fi
+	@echo "artifacts clean:" && ls dist

abeval-0.1.0/PKG-INFO

@@ -0,0 +1,201 @@
+Metadata-Version: 2.4
+Name: abeval
+Version: 0.1.0
+Summary: A/B-test statistics for LLM evals: error bars, paired comparisons, and sample-size planning
+Project-URL: Homepage, https://github.com/mohammadi-hadi/abeval
+Project-URL: Calculator, https://mohammadi.cv/abeval/
+Author: Hadi Mohammadi
+License: MIT
+License-File: LICENSE
+Keywords: ab-testing,evaluation,llm,sample-size,significance,statistics
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Python: >=3.10
+Provides-Extra: dev
+Requires-Dist: build; extra == 'dev'
+Requires-Dist: pytest; extra == 'dev'
+Requires-Dist: ruff; extra == 'dev'
+Description-Content-Type: text/markdown
+
+# abeval
+
+A/B-test statistics for LLM evals.
+
+Eval scores are sample estimates, but they're routinely reported as exact
+numbers: "our prompt scores 76.5%, the old one 63.5%, ship it." On 200 items
+much smaller gaps than that are pure noise. abeval treats an eval like an A/B
+test — confidence intervals for a single run, paired significance tests for
+comparing two runs on the same items, and sample-size planning *before* you
+spend on inference. Standard library only, no dependencies.
+
+**Sample-size calculator:** https://mohammadi.cv/abeval/
+
+## Install
+
+```
+pip install abeval
+```
+
+Or from source: `git clone https://github.com/mohammadi-hadi/abeval && cd abeval && make install`.
+
+## Is my new prompt actually better?
+
+Point `abeval compare` at two JSONL result files that share item ids:
+
+```
+$ abeval compare examples/run_a.jsonl examples/run_b.jsonl
+A: 63.5%   B: 76.5%   (n=200 paired items)
+B - A: +13.0%  [4.5%, 21.5%]  (95% CI)
+p (sign-flip permutation): 0.0041   p (paired t): 0.0034
+discordant items: B-only wins 53, A-only wins 27   p (exact McNemar): 0.0049
+verdict: significant at the chosen level
+```
+
+The comparison is **paired**: both runs are scored on the same items, so
+item difficulty cancels out and you detect much smaller differences than two
+independent runs would allow. Items present in only one run are dropped (and
+reported).
+
+Every command takes `--json` for machine-readable output, and `--seed` makes
+the resampling reproducible.
+
+## Error bars for one run
+
+```
+$ abeval ci examples/run_a.jsonl
+score: 63.5%  [56.6%, 69.9%]  (n=200, 95% CI, wilson)
+```
+
+Binary metrics get a Wilson score interval, continuous metrics a t interval
+(or `--bootstrap`). If your items aren't independent — 5 questions generated
+from each source document, say — pass `--cluster-key document` and the
+interval widens to account for within-cluster correlation.
+
+## How many items do I need?
+
+Run this *before* the eval, not after:
+
+```
+$ abeval power --baseline 0.75 --delta 0.03
+to detect 3.0% difference from a 75.0% baseline (corr=0.5):
+  n = 1568 paired items  (80% power, 95% confidence)
+  with  100 items you can detect >= 11.9%
+  with  200 items you can detect >= 8.4%
+  with  500 items you can detect >= 5.3%
+  with 1000 items you can detect >= 3.8%
+```
+
+The lesson generalizes: a 200-item eval cannot see a 3-point improvement.
+`--corr` is the item-level correlation between the two runs' outcomes (0.5 is
+a reasonable default for two variants of the same system; higher correlation
+means fewer items needed). There's an interactive version at
+https://mohammadi.cv/abeval/.
+
+## How noisy is my judge?
+
+If an LLM judge scores the same item differently on repeat calls, that noise
+eats your statistical power. Feed repeated judgments to `reliability`:
+
+```
+$ abeval reliability examples/judge_repeats.jsonl
+40 items, 120 judgments (3.0 per item)
+ICC (signal share of variance): 0.764
+between-item sd: 1.708   judge noise sd: 0.9487
+exact agreement across repeats: 15.0%
+repeats per item to push judge noise under 10%: 3
+```
+
+ICC is the share of score variance that is real item signal rather than judge
+noise, from a one-way random-effects decomposition. The last line answers the
+practical question: how many repeat judgments to average per item.
+
+## Data format
+
+JSONL, one item per line, any extra fields ignored:
+
+```json
+{"id": "item-001", "score": 1}
+{"id": "item-002", "score": 0, "category": "reasoning"}
+```
+
+`--metric` and `--id-key` rename the fields; booleans are accepted as 0/1.
+The files in `examples/` are generated by `examples/make_fixtures.py`
+(seeded, so they're reproducible).
+
+## Python API
+
+Everything the CLI does is a plain function:
+
+```python
+from abeval import paired_compare, proportion_ci, sample_size, sd_diff_from_rates
+
+proportion_ci(127, 200)                  # Wilson interval, level=0.95
+result = paired_compare(scores_a, scores_b, seed=0)
+result.diff, result.ci_lo, result.ci_hi, result.p_permutation
+
+sd = sd_diff_from_rates(0.75, 0.78, corr=0.5)
+sample_size(0.03, sd)                    # -> 1568 paired items
+```
+
+## What's inside
+
+| Question | Method |
+|---|---|
+| CI for a pass rate | Wilson score interval |
+| CI for a mean score | Student t, percentile bootstrap, or cluster-robust t |
+| Is B better than A? | Sign-flip permutation test on paired differences (primary), paired t, exact McNemar on discordant pairs (binary) |
+| How many items? | Normal-approximation power for the paired design, with a Monte-Carlo check (`power_simulated`) |
+| Judge noise | One-way random-effects ICC with unbalanced-design correction |
+
+The permutation p-value is the primary test: it's exact under item
+exchangeability and makes no normality assumption. The paired t and McNemar
+values are printed alongside because reviewers ask for them.
+
+The approach follows Miller,
+["Adding Error Bars to Evals"](https://arxiv.org/abs/2411.00640) (2024) —
+report standard errors, use paired designs, plan sample sizes — packaged as a
+tool you can point at result files.
+
+## Honest limitations
+
+- The power formulas use the normal approximation; for very small samples or
+  rates near 0/1 trust `power_simulated` over the closed form.
+- Cluster-robust intervals are approximate with fewer than ~20 clusters.
+- Multiple-comparison correction is on you: if you compare ten prompts and
+  ship the best p-value, it's inflated (a `sweep` command with Bonferroni
+  correction is on the roadmap).
+- This is not a stats library. If you need regressions or GLMs, use
+  statsmodels; abeval covers the eval loop with zero dependencies.
+
+## Sponsoring
+
+abeval is MIT-licensed and dependency-free, and it stays that way. Sponsoring
+funds the roadmap below and the maintenance time to keep the statistics
+trustworthy. Sponsors are credited in release notes and vote on what lands
+next: [GitHub Sponsors](https://github.com/sponsors/mohammadi-hadi).
+
+## Roadmap
+
+- `sweep`: compare N runs against a baseline with multiple-comparison
+  correction.
+- Confidence intervals for pairwise win rates (Bradley-Terry) for
+  head-to-head judge outputs.
+- Variance-reduction guidance: how much power you buy by averaging k judge
+  repeats, given your measured ICC.
+
+## Related projects
+
+- [judgewatch](https://github.com/mohammadi-hadi/judgewatch) — monthly bias
+  audits of LLM judges; abeval tells you whether this month's change is real.
+- [judgekit](https://github.com/mohammadi-hadi/judgekit) — audit your own
+  judge pipelines.
+- [trajectory-judge](https://github.com/mohammadi-hadi/trajectory-judge) —
+  evaluating agent trajectories with LLM judges.
+
+## License
+
+MIT — see [LICENSE](LICENSE).

abeval-0.1.0/README.md

@@ -0,0 +1,178 @@
+# abeval
+
+A/B-test statistics for LLM evals.
+
+Eval scores are sample estimates, but they're routinely reported as exact
+numbers: "our prompt scores 76.5%, the old one 63.5%, ship it." On 200 items
+much smaller gaps than that are pure noise. abeval treats an eval like an A/B
+test — confidence intervals for a single run, paired significance tests for
+comparing two runs on the same items, and sample-size planning *before* you
+spend on inference. Standard library only, no dependencies.
+
+**Sample-size calculator:** https://mohammadi.cv/abeval/
+
+## Install
+
+```
+pip install abeval
+```
+
+Or from source: `git clone https://github.com/mohammadi-hadi/abeval && cd abeval && make install`.
+
+## Is my new prompt actually better?
+
+Point `abeval compare` at two JSONL result files that share item ids:
+
+```
+$ abeval compare examples/run_a.jsonl examples/run_b.jsonl
+A: 63.5%   B: 76.5%   (n=200 paired items)
+B - A: +13.0%  [4.5%, 21.5%]  (95% CI)
+p (sign-flip permutation): 0.0041   p (paired t): 0.0034
+discordant items: B-only wins 53, A-only wins 27   p (exact McNemar): 0.0049
+verdict: significant at the chosen level
+```
+
+The comparison is **paired**: both runs are scored on the same items, so
+item difficulty cancels out and you detect much smaller differences than two
+independent runs would allow. Items present in only one run are dropped (and
+reported).
+
+Every command takes `--json` for machine-readable output, and `--seed` makes
+the resampling reproducible.
+
+## Error bars for one run
+
+```
+$ abeval ci examples/run_a.jsonl
+score: 63.5%  [56.6%, 69.9%]  (n=200, 95% CI, wilson)
+```
+
+Binary metrics get a Wilson score interval, continuous metrics a t interval
+(or `--bootstrap`). If your items aren't independent — 5 questions generated
+from each source document, say — pass `--cluster-key document` and the
+interval widens to account for within-cluster correlation.
+
+## How many items do I need?
+
+Run this *before* the eval, not after:
+
+```
+$ abeval power --baseline 0.75 --delta 0.03
+to detect 3.0% difference from a 75.0% baseline (corr=0.5):
+  n = 1568 paired items  (80% power, 95% confidence)
+  with  100 items you can detect >= 11.9%
+  with  200 items you can detect >= 8.4%
+  with  500 items you can detect >= 5.3%
+  with 1000 items you can detect >= 3.8%
+```
+
+The lesson generalizes: a 200-item eval cannot see a 3-point improvement.
+`--corr` is the item-level correlation between the two runs' outcomes (0.5 is
+a reasonable default for two variants of the same system; higher correlation
+means fewer items needed). There's an interactive version at
+https://mohammadi.cv/abeval/.
+
+## How noisy is my judge?
+
+If an LLM judge scores the same item differently on repeat calls, that noise
+eats your statistical power. Feed repeated judgments to `reliability`:
+
+```
+$ abeval reliability examples/judge_repeats.jsonl
+40 items, 120 judgments (3.0 per item)
+ICC (signal share of variance): 0.764
+between-item sd: 1.708   judge noise sd: 0.9487
+exact agreement across repeats: 15.0%
+repeats per item to push judge noise under 10%: 3
+```
+
+ICC is the share of score variance that is real item signal rather than judge
+noise, from a one-way random-effects decomposition. The last line answers the
+practical question: how many repeat judgments to average per item.
+
+## Data format
+
+JSONL, one item per line, any extra fields ignored:
+
+```json
+{"id": "item-001", "score": 1}
+{"id": "item-002", "score": 0, "category": "reasoning"}
+```
+
+`--metric` and `--id-key` rename the fields; booleans are accepted as 0/1.
+The files in `examples/` are generated by `examples/make_fixtures.py`
+(seeded, so they're reproducible).
+
+## Python API
+
+Everything the CLI does is a plain function:
+
+```python
+from abeval import paired_compare, proportion_ci, sample_size, sd_diff_from_rates
+
+proportion_ci(127, 200)                  # Wilson interval, level=0.95
+result = paired_compare(scores_a, scores_b, seed=0)
+result.diff, result.ci_lo, result.ci_hi, result.p_permutation
+
+sd = sd_diff_from_rates(0.75, 0.78, corr=0.5)
+sample_size(0.03, sd)                    # -> 1568 paired items
+```
+
+## What's inside
+
+| Question | Method |
+|---|---|
+| CI for a pass rate | Wilson score interval |
+| CI for a mean score | Student t, percentile bootstrap, or cluster-robust t |
+| Is B better than A? | Sign-flip permutation test on paired differences (primary), paired t, exact McNemar on discordant pairs (binary) |
+| How many items? | Normal-approximation power for the paired design, with a Monte-Carlo check (`power_simulated`) |
+| Judge noise | One-way random-effects ICC with unbalanced-design correction |
+
+The permutation p-value is the primary test: it's exact under item
+exchangeability and makes no normality assumption. The paired t and McNemar
+values are printed alongside because reviewers ask for them.
+
+The approach follows Miller,
+["Adding Error Bars to Evals"](https://arxiv.org/abs/2411.00640) (2024) —
+report standard errors, use paired designs, plan sample sizes — packaged as a
+tool you can point at result files.
+
+## Honest limitations
+
+- The power formulas use the normal approximation; for very small samples or
+  rates near 0/1 trust `power_simulated` over the closed form.
+- Cluster-robust intervals are approximate with fewer than ~20 clusters.
+- Multiple-comparison correction is on you: if you compare ten prompts and
+  ship the best p-value, it's inflated (a `sweep` command with Bonferroni
+  correction is on the roadmap).
+- This is not a stats library. If you need regressions or GLMs, use
+  statsmodels; abeval covers the eval loop with zero dependencies.
+
+## Sponsoring
+
+abeval is MIT-licensed and dependency-free, and it stays that way. Sponsoring
+funds the roadmap below and the maintenance time to keep the statistics
+trustworthy. Sponsors are credited in release notes and vote on what lands
+next: [GitHub Sponsors](https://github.com/sponsors/mohammadi-hadi).
+
+## Roadmap
+
+- `sweep`: compare N runs against a baseline with multiple-comparison
+  correction.
+- Confidence intervals for pairwise win rates (Bradley-Terry) for
+  head-to-head judge outputs.
+- Variance-reduction guidance: how much power you buy by averaging k judge
+  repeats, given your measured ICC.
+
+## Related projects
+
+- [judgewatch](https://github.com/mohammadi-hadi/judgewatch) — monthly bias
+  audits of LLM judges; abeval tells you whether this month's change is real.
+- [judgekit](https://github.com/mohammadi-hadi/judgekit) — audit your own
+  judge pipelines.
+- [trajectory-judge](https://github.com/mohammadi-hadi/trajectory-judge) —
+  evaluating agent trajectories with LLM judges.
+
+## License
+
+MIT — see [LICENSE](LICENSE).

abeval-0.1.0/abeval/__init__.py

@@ -0,0 +1,34 @@
+"""abeval: A/B-test statistics for LLM evals.
+
+Error bars for a single run, paired significance tests for comparing two
+runs, sample-size planning before you spend on inference, and judge-noise
+reliability. Standard library only.
+"""
+
+from .ci import Interval, clustered_mean_ci, mean_ci, proportion_ci
+from .compare import Comparison, mcnemar_exact, paired_compare
+from .data import extract, pair, read_jsonl
+from .power import mde, power_simulated, sample_size, sd_diff_from_rates
+from .reliability import Reliability, judge_reliability
+
+__version__ = "0.1.0"
+
+__all__ = [
+    "Comparison",
+    "Interval",
+    "Reliability",
+    "__version__",
+    "clustered_mean_ci",
+    "extract",
+    "judge_reliability",
+    "mcnemar_exact",
+    "mde",
+    "mean_ci",
+    "pair",
+    "paired_compare",
+    "power_simulated",
+    "proportion_ci",
+    "read_jsonl",
+    "sample_size",
+    "sd_diff_from_rates",
+]