noisefloor-evals 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (72) hide show
  1. noisefloor_evals-0.1.0/.github/workflows/ci.yml +102 -0
  2. noisefloor_evals-0.1.0/.github/workflows/release.yml +67 -0
  3. noisefloor_evals-0.1.0/.gitignore +13 -0
  4. noisefloor_evals-0.1.0/CHANGELOG.md +27 -0
  5. noisefloor_evals-0.1.0/CITATION.cff +25 -0
  6. noisefloor_evals-0.1.0/CONTRIBUTING.md +46 -0
  7. noisefloor_evals-0.1.0/LICENSE +21 -0
  8. noisefloor_evals-0.1.0/PKG-INFO +333 -0
  9. noisefloor_evals-0.1.0/README.md +304 -0
  10. noisefloor_evals-0.1.0/action.yml +161 -0
  11. noisefloor_evals-0.1.0/docs/github-action.md +75 -0
  12. noisefloor_evals-0.1.0/docs/methodology.md +182 -0
  13. noisefloor_evals-0.1.0/examples/README.md +21 -0
  14. noisefloor_evals-0.1.0/examples/data/leaderboard.csv +7 -0
  15. noisefloor_evals-0.1.0/examples/data/rag_baseline.jsonl +600 -0
  16. noisefloor_evals-0.1.0/examples/data/rag_candidate.jsonl +600 -0
  17. noisefloor_evals-0.1.0/examples/data/rag_candidate_k5.jsonl +3000 -0
  18. noisefloor_evals-0.1.0/examples/data/suite/main/arithmetic.jsonl +400 -0
  19. noisefloor_evals-0.1.0/examples/data/suite/main/code_repair.jsonl +300 -0
  20. noisefloor_evals-0.1.0/examples/data/suite/main/commonsense.jsonl +500 -0
  21. noisefloor_evals-0.1.0/examples/data/suite/main/data_extraction.jsonl +350 -0
  22. noisefloor_evals-0.1.0/examples/data/suite/main/instruction_following.jsonl +500 -0
  23. noisefloor_evals-0.1.0/examples/data/suite/main/long_context.jsonl +600 -0
  24. noisefloor_evals-0.1.0/examples/data/suite/main/multilingual_qa.jsonl +600 -0
  25. noisefloor_evals-0.1.0/examples/data/suite/main/planning.jsonl +300 -0
  26. noisefloor_evals-0.1.0/examples/data/suite/main/reading_comprehension.jsonl +450 -0
  27. noisefloor_evals-0.1.0/examples/data/suite/main/safety_refusals.jsonl +350 -0
  28. noisefloor_evals-0.1.0/examples/data/suite/main/sql_generation.jsonl +400 -0
  29. noisefloor_evals-0.1.0/examples/data/suite/main/summarization_judge.jsonl +400 -0
  30. noisefloor_evals-0.1.0/examples/data/suite/main/tool_use.jsonl +600 -0
  31. noisefloor_evals-0.1.0/examples/data/suite/main/translation.jsonl +500 -0
  32. noisefloor_evals-0.1.0/examples/data/suite/pr/arithmetic.jsonl +400 -0
  33. noisefloor_evals-0.1.0/examples/data/suite/pr/code_repair.jsonl +300 -0
  34. noisefloor_evals-0.1.0/examples/data/suite/pr/commonsense.jsonl +500 -0
  35. noisefloor_evals-0.1.0/examples/data/suite/pr/data_extraction.jsonl +350 -0
  36. noisefloor_evals-0.1.0/examples/data/suite/pr/instruction_following.jsonl +500 -0
  37. noisefloor_evals-0.1.0/examples/data/suite/pr/long_context.jsonl +600 -0
  38. noisefloor_evals-0.1.0/examples/data/suite/pr/multilingual_qa.jsonl +600 -0
  39. noisefloor_evals-0.1.0/examples/data/suite/pr/planning.jsonl +300 -0
  40. noisefloor_evals-0.1.0/examples/data/suite/pr/reading_comprehension.jsonl +450 -0
  41. noisefloor_evals-0.1.0/examples/data/suite/pr/safety_refusals.jsonl +350 -0
  42. noisefloor_evals-0.1.0/examples/data/suite/pr/sql_generation.jsonl +400 -0
  43. noisefloor_evals-0.1.0/examples/data/suite/pr/summarization_judge.jsonl +400 -0
  44. noisefloor_evals-0.1.0/examples/data/suite/pr/tool_use.jsonl +600 -0
  45. noisefloor_evals-0.1.0/examples/data/suite/pr/translation.jsonl +500 -0
  46. noisefloor_evals-0.1.0/examples/generate.py +125 -0
  47. noisefloor_evals-0.1.0/examples/validate.py +197 -0
  48. noisefloor_evals-0.1.0/pyproject.toml +62 -0
  49. noisefloor_evals-0.1.0/src/noisefloor/__init__.py +70 -0
  50. noisefloor_evals-0.1.0/src/noisefloor/__main__.py +5 -0
  51. noisefloor_evals-0.1.0/src/noisefloor/_dist.py +152 -0
  52. noisefloor_evals-0.1.0/src/noisefloor/bootstrap.py +147 -0
  53. noisefloor_evals-0.1.0/src/noisefloor/cli.py +539 -0
  54. noisefloor_evals-0.1.0/src/noisefloor/data.py +365 -0
  55. noisefloor_evals-0.1.0/src/noisefloor/gate.py +80 -0
  56. noisefloor_evals-0.1.0/src/noisefloor/leaderboard.py +269 -0
  57. noisefloor_evals-0.1.0/src/noisefloor/multiple.py +76 -0
  58. noisefloor_evals-0.1.0/src/noisefloor/power.py +203 -0
  59. noisefloor_evals-0.1.0/src/noisefloor/py.typed +0 -0
  60. noisefloor_evals-0.1.0/src/noisefloor/report.py +610 -0
  61. noisefloor_evals-0.1.0/src/noisefloor/stats.py +361 -0
  62. noisefloor_evals-0.1.0/src/noisefloor/suite.py +102 -0
  63. noisefloor_evals-0.1.0/tests/test_bootstrap.py +68 -0
  64. noisefloor_evals-0.1.0/tests/test_cli.py +215 -0
  65. noisefloor_evals-0.1.0/tests/test_data.py +119 -0
  66. noisefloor_evals-0.1.0/tests/test_dist.py +77 -0
  67. noisefloor_evals-0.1.0/tests/test_gate.py +48 -0
  68. noisefloor_evals-0.1.0/tests/test_leaderboard.py +81 -0
  69. noisefloor_evals-0.1.0/tests/test_multiple.py +83 -0
  70. noisefloor_evals-0.1.0/tests/test_power.py +95 -0
  71. noisefloor_evals-0.1.0/tests/test_stats.py +183 -0
  72. noisefloor_evals-0.1.0/tests/test_validation.py +81 -0
@@ -0,0 +1,102 @@
1
+ name: CI
2
+
3
+ on:
4
+ push:
5
+ branches: [main]
6
+ pull_request:
7
+
8
+ permissions:
9
+ contents: read
10
+
11
+ concurrency:
12
+ group: ${{ github.workflow }}-${{ github.ref }}
13
+ cancel-in-progress: true
14
+
15
+ jobs:
16
+ lint:
17
+ runs-on: ubuntu-latest
18
+ steps:
19
+ - uses: actions/checkout@v4
20
+ - uses: actions/setup-python@v5
21
+ with:
22
+ python-version: "3.12"
23
+ - run: python -m pip install -e ".[dev]"
24
+ - run: ruff format --check .
25
+ - run: ruff check .
26
+ - run: mypy
27
+
28
+ test:
29
+ runs-on: ubuntu-latest
30
+ strategy:
31
+ fail-fast: false
32
+ matrix:
33
+ python-version: ["3.10", "3.11", "3.12", "3.13"]
34
+ steps:
35
+ - uses: actions/checkout@v4
36
+ - uses: actions/setup-python@v5
37
+ with:
38
+ python-version: ${{ matrix.python-version }}
39
+ - run: python -m pip install -e ".[dev]"
40
+ - run: pytest -m "not slow"
41
+
42
+ validation:
43
+ name: statistical validation (Monte Carlo)
44
+ runs-on: ubuntu-latest
45
+ steps:
46
+ - uses: actions/checkout@v4
47
+ - uses: actions/setup-python@v5
48
+ with:
49
+ python-version: "3.12"
50
+ - run: python -m pip install -e ".[dev]"
51
+ - run: pytest -m slow
52
+
53
+ package:
54
+ runs-on: ubuntu-latest
55
+ steps:
56
+ - uses: actions/checkout@v4
57
+ - uses: actions/setup-python@v5
58
+ with:
59
+ python-version: "3.12"
60
+ - run: python -m pip install build twine
61
+ - run: python -m build
62
+ - run: twine check --strict dist/*
63
+ - name: Smoke-test the built wheel
64
+ run: |
65
+ python -m venv /tmp/wheel-env
66
+ /tmp/wheel-env/bin/pip install dist/*.whl
67
+ /tmp/wheel-env/bin/noisefloor --version
68
+ /tmp/wheel-env/bin/noisefloor compare examples/data/rag_baseline.jsonl examples/data/rag_candidate.jsonl
69
+
70
+ action:
71
+ name: dogfood the GitHub Action
72
+ runs-on: ubuntu-latest
73
+ steps:
74
+ - uses: actions/checkout@v4
75
+ - name: Noisy improvement should pass the default gate
76
+ id: noisy
77
+ uses: ./
78
+ with:
79
+ baseline: examples/data/rag_baseline.jsonl
80
+ candidate: examples/data/rag_candidate.jsonl
81
+ comment: "false"
82
+ - name: Real regression should fail the gate
83
+ id: regression
84
+ continue-on-error: true
85
+ uses: ./
86
+ with:
87
+ baseline: examples/data/suite/main/long_context.jsonl
88
+ candidate: examples/data/suite/pr/long_context.jsonl
89
+ comment: "false"
90
+ - name: Check outcomes
91
+ env:
92
+ NOISY_VERDICT: ${{ steps.noisy.outputs.verdict }}
93
+ NOISY_PASSED: ${{ steps.noisy.outputs.passed }}
94
+ REGRESSION_OUTCOME: ${{ steps.regression.outcome }}
95
+ REGRESSION_VERDICT: ${{ steps.regression.outputs.verdict }}
96
+ run: |
97
+ set -eu
98
+ test "$NOISY_VERDICT" = "inconclusive"
99
+ test "$NOISY_PASSED" = "true"
100
+ test "$REGRESSION_OUTCOME" = "failure"
101
+ test "$REGRESSION_VERDICT" = "regression"
102
+ echo "action behaves as specified"
@@ -0,0 +1,67 @@
1
+ name: Release
2
+
3
+ # Publishes to PyPI, using PyPI Trusted Publishing (OIDC): no API token is
4
+ # stored anywhere. Trigger it either way:
5
+ # - push a version tag: git tag v0.2.0 && git push origin v0.2.0
6
+ # - or publish a GitHub release for tag v0.2.0 from the web UI
7
+ # If both fire for the same tag, runs are serialized and the second skips
8
+ # files PyPI already has, so a version is never uploaded twice.
9
+ on:
10
+ push:
11
+ tags: ["v*"]
12
+ release:
13
+ types: [published]
14
+
15
+ concurrency:
16
+ group: release-${{ github.event.release.tag_name || github.ref_name }}
17
+ cancel-in-progress: false
18
+
19
+ permissions:
20
+ contents: read
21
+
22
+ jobs:
23
+ build:
24
+ runs-on: ubuntu-latest
25
+ steps:
26
+ - uses: actions/checkout@v4
27
+ - uses: actions/setup-python@v5
28
+ with:
29
+ python-version: "3.12"
30
+ - name: Check the tag matches the package version
31
+ env:
32
+ TAG: ${{ github.event.release.tag_name || github.ref_name }}
33
+ run: |
34
+ version=$(python -c "import tomllib; print(tomllib.load(open('pyproject.toml','rb'))['project']['version'])")
35
+ if [ "${TAG#v}" != "$version" ]; then
36
+ echo "::error::release tag $TAG does not match pyproject version $version"
37
+ exit 1
38
+ fi
39
+ - run: python -m pip install build twine
40
+ - run: python -m build
41
+ - run: twine check --strict dist/*
42
+ - name: Smoke-test the wheel
43
+ run: |
44
+ python -m venv /tmp/env
45
+ /tmp/env/bin/pip install dist/*.whl
46
+ /tmp/env/bin/noisefloor --version
47
+ - uses: actions/upload-artifact@v4
48
+ with:
49
+ name: dist
50
+ path: dist/
51
+
52
+ publish:
53
+ needs: build
54
+ runs-on: ubuntu-latest
55
+ environment:
56
+ name: pypi
57
+ url: https://pypi.org/project/noisefloor-evals/
58
+ permissions:
59
+ id-token: write
60
+ steps:
61
+ - uses: actions/download-artifact@v4
62
+ with:
63
+ name: dist
64
+ path: dist/
65
+ - uses: pypa/gh-action-pypi-publish@release/v1
66
+ with:
67
+ skip-existing: true
@@ -0,0 +1,13 @@
1
+ __pycache__/
2
+ *.py[cod]
3
+ .venv/
4
+ venv/
5
+ dist/
6
+ build/
7
+ *.egg-info/
8
+ .pytest_cache/
9
+ .mypy_cache/
10
+ .ruff_cache/
11
+ .coverage
12
+ htmlcov/
13
+ .DS_Store
@@ -0,0 +1,27 @@
1
+ # Changelog
2
+
3
+ All notable changes to this project are documented here. The format follows
4
+ [Keep a Changelog](https://keepachangelog.com/en/1.1.0/) and the project uses
5
+ [Semantic Versioning](https://semver.org/).
6
+
7
+ ## [0.1.0] - 2026-11-21
8
+
9
+ ### Added
10
+ - `summary`: t-based confidence intervals, CR1 cluster-robust standard errors,
11
+ Wilson intervals, design effect, and within/between variance decomposition
12
+ for repeated samples, with projections for K = 1 to 16.
13
+ - `compare`: paired comparisons with p-values, wins/losses/ties, variance
14
+ removed by pairing, minimum detectable effect, and the sample size needed to
15
+ confirm the observed effect.
16
+ - `gate`: CI gate with no-regression, non-inferiority (`--max-regression`), and
17
+ require-improvement policies; Markdown output for PR comments.
18
+ - `suite`: per-benchmark comparisons with Holm, Benjamini–Hochberg, or
19
+ Bonferroni correction.
20
+ - `leaderboard`: joint-bootstrap rank confidence intervals and statistical
21
+ tiers, from per-question runs or from summary scores.
22
+ - `power`: sample size, minimum detectable effect, achieved power, margin
23
+ sizing, design effects, and planning from a pilot run.
24
+ - Vectorized (cluster) bootstrap for means and paired differences.
25
+ - Dependency-free Student-t distribution.
26
+ - Composite GitHub Action with job summary, PR comment, and outputs.
27
+ - Monte Carlo validation suite and synthetic example datasets.
@@ -0,0 +1,25 @@
1
+ cff-version: 1.2.0
2
+ message: "If you use noisefloor, please cite it as below."
3
+ title: "noisefloor: statistical rigor for LLM evaluations"
4
+ authors:
5
+ - family-names: Ibrahim
6
+ given-names: Abualgasim
7
+ version: 0.1.0
8
+ date-released: 2026-11-21
9
+ license: MIT
10
+ repository-code: "https://github.com/Gasooom/noisefloor"
11
+ keywords:
12
+ - llm evaluation
13
+ - statistics
14
+ - confidence intervals
15
+ - power analysis
16
+ references:
17
+ - type: article
18
+ title: "Adding Error Bars to Evals: A Statistical Approach to Language Model Evaluations"
19
+ authors:
20
+ - family-names: Miller
21
+ given-names: Evan
22
+ year: 2024
23
+ identifiers:
24
+ - type: other
25
+ value: "arXiv:2411.00640"
@@ -0,0 +1,46 @@
1
+ # Contributing to noisefloor
2
+
3
+ Thanks for helping make evals more honest.
4
+
5
+ ## Development setup
6
+
7
+ ```bash
8
+ git clone https://github.com/Gasooom/noisefloor
9
+ cd noisefloor
10
+ python -m venv .venv && source .venv/bin/activate
11
+ pip install -e ".[dev]"
12
+ ```
13
+
14
+ ## Before opening a pull request
15
+
16
+ ```bash
17
+ ruff format .
18
+ ruff check .
19
+ mypy # strict mode
20
+ pytest -m "not slow" # fast unit tests (~1s)
21
+ pytest -m slow # Monte Carlo validation (~30s)
22
+ ```
23
+
24
+ CI runs all of the above on Python 3.10 to 3.13.
25
+
26
+ ## Ground rules
27
+
28
+ - **Statistical changes need a simulation.** A unit test shows the code does
29
+ what you meant; a Monte Carlo check in `tests/test_validation.py` shows that
30
+ what you meant is true (coverage, error rates, power). Add one for any new
31
+ estimator, interval, or test.
32
+ - **Cite your method.** Add the formula and a reference to
33
+ `docs/methodology.md`.
34
+ - **Keep the dependency footprint at numpy.** If you need a special function,
35
+ implement it in `_dist.py` and test it against published values.
36
+ - **Every number gets an interval.** Reports must never show a bare point
37
+ estimate.
38
+ - **Errors should say what to do.** Prefer `"could not find a score column
39
+ (tried ...); pass --score-field"` over a stack trace.
40
+
41
+ ## Regenerating the example data
42
+
43
+ ```bash
44
+ python examples/generate.py
45
+ python examples/validate.py --sims 4000 # refresh the README validation table
46
+ ```
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Abualgasim Ibrahim
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,333 @@
1
+ Metadata-Version: 2.5
2
+ Name: noisefloor-evals
3
+ Version: 0.1.0
4
+ Summary: Is your eval improvement real, or just noise? Statistical rigor for LLM evaluations.
5
+ Project-URL: Homepage, https://github.com/Gasooom/noisefloor
6
+ Project-URL: Issues, https://github.com/Gasooom/noisefloor/issues
7
+ Project-URL: Documentation, https://github.com/Gasooom/noisefloor/blob/main/docs/methodology.md
8
+ Project-URL: Changelog, https://github.com/Gasooom/noisefloor/blob/main/CHANGELOG.md
9
+ Author: Abualgasim Ibrahim
10
+ License-Expression: MIT
11
+ License-File: LICENSE
12
+ Keywords: ci,confidence-intervals,evals,evaluation,llm,power-analysis,statistics
13
+ Classifier: Development Status :: 4 - Beta
14
+ Classifier: Intended Audience :: Developers
15
+ Classifier: Intended Audience :: Science/Research
16
+ Classifier: Programming Language :: Python :: 3
17
+ Classifier: Programming Language :: Python :: 3 :: Only
18
+ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
19
+ Classifier: Topic :: Scientific/Engineering :: Mathematics
20
+ Classifier: Typing :: Typed
21
+ Requires-Python: >=3.10
22
+ Requires-Dist: numpy>=1.24
23
+ Provides-Extra: dev
24
+ Requires-Dist: hypothesis>=6; extra == 'dev'
25
+ Requires-Dist: mypy>=1.10; extra == 'dev'
26
+ Requires-Dist: pytest>=8; extra == 'dev'
27
+ Requires-Dist: ruff>=0.5; extra == 'dev'
28
+ Description-Content-Type: text/markdown
29
+
30
+ <div align="center">
31
+
32
+ # noisefloor
33
+
34
+ **Is your eval improvement real, or just noise?**
35
+
36
+ Statistical rigor for LLM evaluations: honest error bars, paired comparisons,
37
+ cluster-robust standard errors, power analysis, multiple-comparison control,
38
+ leaderboard tiers, and a CI gate that blocks real regressions, not noise.
39
+
40
+ [![CI](https://github.com/Gasooom/noisefloor/actions/workflows/ci.yml/badge.svg)](https://github.com/Gasooom/noisefloor/actions/workflows/ci.yml)
41
+ ![Python](https://img.shields.io/badge/python-3.10%E2%80%933.13-blue)
42
+ ![Dependencies](https://img.shields.io/badge/dependencies-numpy%20only-brightgreen)
43
+ ![Typed](https://img.shields.io/badge/mypy-strict-blue)
44
+ ![License](https://img.shields.io/badge/license-MIT-green)
45
+ [![PyPI](https://img.shields.io/pypi/v/noisefloor-evals)](https://pypi.org/project/noisefloor-evals/)
46
+
47
+ </div>
48
+
49
+ ---
50
+
51
+ ## The problem
52
+
53
+ A team changes a prompt, swaps a model, or tweaks an agent. They rerun their
54
+ eval of a few hundred questions and see **72.0% → 76.2%**. They ship it.
55
+
56
+ Here is what noisefloor says about that exact result:
57
+
58
+ ```console
59
+ $ noisefloor compare rag_baseline.jsonl rag_candidate.jsonl
60
+
61
+ noisefloor · compare rag_baseline → rag_candidate (600 paired questions, 120 clusters)
62
+
63
+ score 95% CI
64
+ rag_baseline 72.0% [67.3, 76.7]
65
+ rag_candidate 76.2% [71.9, 80.5]
66
+
67
+ difference +4.2pp [-0.4, +8.8] p = 0.076
68
+
69
+ verdict INCONCLUSIVE the difference is within the noise floor
70
+
71
+ · pairing removed 47% of the variance (per-question correlation 0.26)
72
+ · wins / losses / ties: 98 / 73 / 429
73
+ · smallest effect this eval can detect (80% power): ±6.6pp
74
+ · to reliably detect a +4.2pp effect you need ~1,494 questions (you have 600)
75
+ ```
76
+
77
+ This dataset is simulated, so we know the truth: the candidate really is
78
+ better, by **+1.4pp**. The observed +4.2pp is **3× the real effect**. An eval
79
+ this size cannot tell a real 1.4pp gain apart from luck, and noisefloor says so
80
+ instead of letting a lucky draw become a launch decision.
81
+
82
+ Most of the benchmarks people quote are smaller than their decisions assume:
83
+
84
+ | Benchmark (public size) | Questions | 95% error bar on one score (at 50%) | Smallest real gain you can reliably detect* |
85
+ |---|---:|---:|---:|
86
+ | AIME 2024 | 30 | ±18.7pp | 24.1pp |
87
+ | HumanEval | 164 | ±7.7pp | 10.8pp |
88
+ | GPQA Diamond | 198 | ±7.0pp | 9.9pp |
89
+ | SWE-bench Verified | 500 | ±4.4pp | 6.2pp |
90
+ | MMLU | 14,042 | ±0.8pp | 1.2pp |
91
+
92
+ <sub>*Paired comparison, per-question correlation 0.5, 80% power, α = 0.05. Computed with `noisefloor power`.</sub>
93
+
94
+ The statistics to handle this exist. Miller (2024), [*Adding Error Bars to
95
+ Evals*](https://arxiv.org/abs/2411.00640), lays them out for LLM evaluation.
96
+ What has been missing is a tool that brings them into the workflow engineers
97
+ actually use: a terminal, a pull request, a CI check. **noisefloor is that
98
+ tool.**
99
+
100
+ ## What it does
101
+
102
+ | Command | Answers |
103
+ |---|---|
104
+ | `noisefloor summary` | What is my score, *with an honest error bar*? Is clustering inflating my confidence? Should I buy more questions or more samples per question? |
105
+ | `noisefloor compare` | Is B really better than A on the same questions? (paired test, cluster-robust) |
106
+ | `noisefloor gate` | Should this PR be allowed to merge? Exits `1` on policy failure and writes a PR comment. |
107
+ | `noisefloor suite` | Across 14 benchmarks, which changes survive multiple-comparison correction? |
108
+ | `noisefloor power` | How many questions do I need *before* I spend compute? |
109
+ | `noisefloor leaderboard` | Which models are actually distinguishable? Rank confidence intervals and statistical tiers. |
110
+
111
+ Also included: a **GitHub Action** ([docs](docs/github-action.md)), a typed
112
+ **Python API**, and a **Monte Carlo validation suite** that verifies every
113
+ guarantee against data with known ground truth.
114
+
115
+ ## Install
116
+
117
+ ```bash
118
+ pip install noisefloor-evals
119
+ ```
120
+
121
+ The package is published as `noisefloor-evals`; you import it as `noisefloor`, and
122
+ the command is `noisefloor`.
123
+
124
+ Requires Python ≥ 3.10. The only runtime dependency is numpy; the Student-t
125
+ distribution is implemented from scratch, so there is no SciPy.
126
+
127
+ ## Quick start
128
+
129
+ Your eval results just need one row per question (or per sample) with an id
130
+ and a score:
131
+
132
+ ```jsonl
133
+ {"id": "doc017-q2", "doc_id": "doc017", "correct": true}
134
+ {"id": "doc017-q3", "doc_id": "doc017", "correct": false}
135
+ ```
136
+
137
+ Common field names (`id`/`question_id`/`task_id`, `score`/`correct`/`passed`,
138
+ `cluster`/`doc_id`/`group`) are detected automatically. Booleans, `"pass"`/`"fail"`,
139
+ and continuous scores all work. Repeated ids are treated as repeated samples
140
+ of the same question. Use `--id-field`, `--score-field`, and `--cluster-field`
141
+ (dotted paths like `metrics.acc` work) for anything else.
142
+
143
+ Every example below runs on the files in [`examples/data`](examples/data).
144
+
145
+ ### A score with an honest error bar
146
+
147
+ ```console
148
+ $ noisefloor summary rag_candidate_k5.jsonl
149
+
150
+ noisefloor · summary rag_candidate_k5 (600 questions, 120 clusters)
151
+
152
+ score 72.6% ± 3.5pp 95% CI [69.1, 76.1] SE 1.8pp, clustered
153
+ clustering inflates variance 2.38×; a naive i.i.d. analysis would claim ± 2.3pp
154
+
155
+ resampling 5.0 samples/question; 76% of single-sample variance is sampling noise
156
+ K=1: ±5.5pp K=2: ±4.3pp K=4: ±3.6pp K=8: ±3.2pp K=16: ±3.0pp K=∞: ±2.7pp
157
+ doubling samples/question would cut the error bar by only 10%, doubling questions by 29%. Buy more questions, not more samples.
158
+ ```
159
+
160
+ The questions come in groups of five per document, so a naive analysis would
161
+ report an error bar about 35% too narrow. The variance decomposition then
162
+ answers a real budgeting question: with 5 samples per question already,
163
+ more resampling buys almost nothing.
164
+
165
+ ### How many questions do I need?
166
+
167
+ ```console
168
+ $ noisefloor power --effect 2pp --baseline-rate 70%
169
+
170
+ noisefloor · power questions to detect +2.0pp (baseline 70.0%, 80% power, α = 0.05)
171
+
172
+ correlation questions
173
+ 0.0 8,077
174
+ 0.3 5,655
175
+ 0.5 4,040
176
+ 0.7 2,425
177
+ ```
178
+
179
+ Run `--n 198` to get the smallest effect a 198-question eval can detect,
180
+ `--n` with `--effect` for achieved power, `--margin 2pp` to size a single
181
+ score, `--cluster-size`/`--icc` for clustered designs, or `--pilot base.jsonl
182
+ cand.jsonl` to plan from runs you already have.
183
+
184
+ ### Many benchmarks without fooling yourself
185
+
186
+ ```console
187
+ $ noisefloor suite runs/main/ runs/pr/ --correction bh
188
+ ...
189
+ 4 of 14 benchmarks look significant at α = 0.05 before correction;
190
+ 2 survive Benjamini-Hochberg correction (1 better, 1 worse).
191
+ ```
192
+
193
+ In this simulated suite exactly two benchmarks truly changed. Before
194
+ correction, four *look* significant, which is enough for a misleading release
195
+ note. After correction, exactly the two real effects remain.
196
+
197
+ ### Leaderboards with rank uncertainty
198
+
199
+ ```console
200
+ $ noisefloor leaderboard --summary leaderboard.csv
201
+
202
+ tier rank rank CI model score 95% CI
203
+ 1 1 1–4 aurora-xl 64.6% [57.9, 71.3]
204
+ 1 2 1–4 basalt-pro 63.1% [56.4, 69.8]
205
+ 1 3 1–4 cirrus-2 62.6% [55.9, 69.3]
206
+ 1 4 1–5 delta-large 58.1% [51.2, 65.0]
207
+ ─────────────────────────────────────────────────────────
208
+ 2 5 4–6 ember-mini 51.5% [44.5, 58.5]
209
+ 2 6 5–6 fjord-7b 41.9% [35.0, 48.8]
210
+ ```
211
+
212
+ On a 198-question benchmark, the "#1 model" could plausibly be #4. Pass
213
+ per-question result files instead of `--summary` to get a paired analysis
214
+ with joint-bootstrap rank intervals.
215
+
216
+ ## In CI: block real regressions, not noise
217
+
218
+ ```yaml
219
+ - uses: Gasooom/noisefloor@main
220
+ with:
221
+ baseline: results/main.jsonl
222
+ candidate: results/pr.jsonl
223
+ max-regression: 1pp # optional non-inferiority margin
224
+ ```
225
+
226
+ The action prints the comparison, writes a job summary, keeps **one**
227
+ up-to-date comment on the PR, and fails only when the change fails your
228
+ policy:
229
+
230
+ | Policy | Fails when |
231
+ |---|---|
232
+ | default | the candidate is **significantly worse** |
233
+ | `max-regression: 1pp` | a regression > 1pp **cannot be ruled out** |
234
+ | `require-improvement: true` | the candidate is **not significantly better** |
235
+
236
+ See [docs/github-action.md](docs/github-action.md). This repository's own CI
237
+ dogfoods the action: it asserts that the noisy +4.2pp example passes and a real
238
+ regression fails.
239
+
240
+ ## Python API
241
+
242
+ ```python
243
+ import noisefloor as nf
244
+
245
+ base = nf.load("baseline.jsonl")
246
+ cand = nf.load("candidate.jsonl")
247
+
248
+ c = nf.compare(base, cand)
249
+ c.verdict # 'improvement' | 'regression' | 'inconclusive'
250
+ c.diff.mean, c.diff.ci_low, c.diff.ci_high, c.p_value
251
+ c.mde # smallest effect this eval can detect
252
+ c.n_required # questions needed to confirm the observed effect
253
+
254
+ nf.sample_size(0.02, baseline_rate=0.7, correlation=0.5) # -> 4040
255
+ nf.evaluate_gate(c, nf.GatePolicy(max_regression=0.01)).passed
256
+
257
+ runs = {name: (nf.load(f"main/{name}.jsonl"), nf.load(f"pr/{name}.jsonl")) for name in names}
258
+ nf.compare_suite(runs, correction="holm").regressions
259
+ ```
260
+
261
+ Every function is typed (`mypy --strict`) and returns frozen dataclasses.
262
+
263
+ ## Does it actually work? (validation)
264
+
265
+ Every guarantee noisefloor makes is a frequency claim, so each one is checked
266
+ by simulation against data with known ground truth
267
+ ([`examples/validate.py`](examples/validate.py), 4,000 simulations per row; also
268
+ run in CI):
269
+
270
+ | Check | Target | noisefloor | Common shortcut |
271
+ |---|:-:|:-:|:-:|
272
+ | 95% CI coverage, i.i.d. questions (n=100) | 95% | **94.9%** | n/a |
273
+ | 95% CI coverage, tiny judge-scored eval (n=12) | 95% | **94.5%** | 92.4% (z-interval) |
274
+ | 95% CI coverage, clustered eval (60 passages × 5 questions) | 95% | **95.2%** | 88.1% (ignore clusters) |
275
+ | False alarm rate, equally good models (clustered eval) | 5% | **5.1%** | 9.0% (ignore clusters) |
276
+ | Power at planned n = 1,468 for a +5pp gain | 80% | **79.1%** | n/a |
277
+ | ≥1 false "win" across 10 unchanged benchmarks | ≤ 5% | **5.1%** | 40.5% (no correction) |
278
+
279
+ The Student-t implementation is checked against published tables to 7
280
+ significant figures, and the estimators against hand-derived values. Run the
281
+ full suite with `pytest`, or only the fast tests with `pytest -m "not slow"`.
282
+
283
+ ## How it works
284
+
285
+ The full derivations are in [docs/methodology.md](docs/methodology.md). In short:
286
+
287
+ - **Unit of analysis = question.** Repeated samples are averaged per question,
288
+ and their spread is kept as within-question variance.
289
+ - **Cluster-robust (CR1) standard errors** with G−1 degrees of freedom when
290
+ questions share a passage, repository, or conversation.
291
+ - **Paired differences** whenever two systems answer the same questions. Shared
292
+ difficulty cancels, which often removes half the variance.
293
+ - **Student-t intervals** everywhere, plus Wilson intervals for small binary
294
+ evals and a vectorized cluster bootstrap as a cross-check.
295
+ - **Power analysis** with correlation and design effects, including planning
296
+ from a pilot run.
297
+ - **Holm / Benjamini–Hochberg** corrections across benchmarks.
298
+ - **Joint-bootstrap rank intervals** and paired-test tiers for leaderboards.
299
+
300
+ ## Design principles
301
+
302
+ 1. **Never print a bare number.** Every estimate comes with its interval.
303
+ 2. **Say what to do next.** "Inconclusive" alone is useless. "You need ~1,500
304
+ questions" or "buy questions, not samples" is actionable.
305
+ 3. **Validate the guarantees, not just the code.** Unit tests show the code
306
+ does what it says; Monte Carlo tests show that what it says is true.
307
+ 4. **Boring to install.** Pure Python plus numpy, no services, no API keys, works
308
+ offline on any eval harness's output.
309
+
310
+ ## Roadmap
311
+
312
+ - [ ] Native readers for popular harness outputs (lm-evaluation-harness, Inspect, OpenAI Evals, promptfoo)
313
+ - [ ] Sequential testing (always-valid p-values) for evals that stream in
314
+ - [ ] LLM-as-judge noise: judge-resampling variance and inter-judge agreement
315
+ - [ ] Stratified / weighted estimates for evals that mix difficulty buckets
316
+ - [ ] HTML report with forest plots
317
+
318
+ ## Contributing
319
+
320
+ Issues and pull requests are welcome; see [CONTRIBUTING.md](CONTRIBUTING.md).
321
+ New statistical features should come with a Monte Carlo check in
322
+ `tests/test_validation.py`.
323
+
324
+ ## Citation
325
+
326
+ If noisefloor helps your research, please cite it (see [CITATION.cff](CITATION.cff))
327
+ and the paper that motivated it:
328
+
329
+ > Miller, E. (2024). *Adding Error Bars to Evals: A Statistical Approach to Language Model Evaluations.* arXiv:2411.00640.
330
+
331
+ ## License
332
+
333
+ MIT © Abualgasim Ibrahim