noisefloor-evals 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- noisefloor_evals-0.1.0/.github/workflows/ci.yml +102 -0
- noisefloor_evals-0.1.0/.github/workflows/release.yml +67 -0
- noisefloor_evals-0.1.0/.gitignore +13 -0
- noisefloor_evals-0.1.0/CHANGELOG.md +27 -0
- noisefloor_evals-0.1.0/CITATION.cff +25 -0
- noisefloor_evals-0.1.0/CONTRIBUTING.md +46 -0
- noisefloor_evals-0.1.0/LICENSE +21 -0
- noisefloor_evals-0.1.0/PKG-INFO +333 -0
- noisefloor_evals-0.1.0/README.md +304 -0
- noisefloor_evals-0.1.0/action.yml +161 -0
- noisefloor_evals-0.1.0/docs/github-action.md +75 -0
- noisefloor_evals-0.1.0/docs/methodology.md +182 -0
- noisefloor_evals-0.1.0/examples/README.md +21 -0
- noisefloor_evals-0.1.0/examples/data/leaderboard.csv +7 -0
- noisefloor_evals-0.1.0/examples/data/rag_baseline.jsonl +600 -0
- noisefloor_evals-0.1.0/examples/data/rag_candidate.jsonl +600 -0
- noisefloor_evals-0.1.0/examples/data/rag_candidate_k5.jsonl +3000 -0
- noisefloor_evals-0.1.0/examples/data/suite/main/arithmetic.jsonl +400 -0
- noisefloor_evals-0.1.0/examples/data/suite/main/code_repair.jsonl +300 -0
- noisefloor_evals-0.1.0/examples/data/suite/main/commonsense.jsonl +500 -0
- noisefloor_evals-0.1.0/examples/data/suite/main/data_extraction.jsonl +350 -0
- noisefloor_evals-0.1.0/examples/data/suite/main/instruction_following.jsonl +500 -0
- noisefloor_evals-0.1.0/examples/data/suite/main/long_context.jsonl +600 -0
- noisefloor_evals-0.1.0/examples/data/suite/main/multilingual_qa.jsonl +600 -0
- noisefloor_evals-0.1.0/examples/data/suite/main/planning.jsonl +300 -0
- noisefloor_evals-0.1.0/examples/data/suite/main/reading_comprehension.jsonl +450 -0
- noisefloor_evals-0.1.0/examples/data/suite/main/safety_refusals.jsonl +350 -0
- noisefloor_evals-0.1.0/examples/data/suite/main/sql_generation.jsonl +400 -0
- noisefloor_evals-0.1.0/examples/data/suite/main/summarization_judge.jsonl +400 -0
- noisefloor_evals-0.1.0/examples/data/suite/main/tool_use.jsonl +600 -0
- noisefloor_evals-0.1.0/examples/data/suite/main/translation.jsonl +500 -0
- noisefloor_evals-0.1.0/examples/data/suite/pr/arithmetic.jsonl +400 -0
- noisefloor_evals-0.1.0/examples/data/suite/pr/code_repair.jsonl +300 -0
- noisefloor_evals-0.1.0/examples/data/suite/pr/commonsense.jsonl +500 -0
- noisefloor_evals-0.1.0/examples/data/suite/pr/data_extraction.jsonl +350 -0
- noisefloor_evals-0.1.0/examples/data/suite/pr/instruction_following.jsonl +500 -0
- noisefloor_evals-0.1.0/examples/data/suite/pr/long_context.jsonl +600 -0
- noisefloor_evals-0.1.0/examples/data/suite/pr/multilingual_qa.jsonl +600 -0
- noisefloor_evals-0.1.0/examples/data/suite/pr/planning.jsonl +300 -0
- noisefloor_evals-0.1.0/examples/data/suite/pr/reading_comprehension.jsonl +450 -0
- noisefloor_evals-0.1.0/examples/data/suite/pr/safety_refusals.jsonl +350 -0
- noisefloor_evals-0.1.0/examples/data/suite/pr/sql_generation.jsonl +400 -0
- noisefloor_evals-0.1.0/examples/data/suite/pr/summarization_judge.jsonl +400 -0
- noisefloor_evals-0.1.0/examples/data/suite/pr/tool_use.jsonl +600 -0
- noisefloor_evals-0.1.0/examples/data/suite/pr/translation.jsonl +500 -0
- noisefloor_evals-0.1.0/examples/generate.py +125 -0
- noisefloor_evals-0.1.0/examples/validate.py +197 -0
- noisefloor_evals-0.1.0/pyproject.toml +62 -0
- noisefloor_evals-0.1.0/src/noisefloor/__init__.py +70 -0
- noisefloor_evals-0.1.0/src/noisefloor/__main__.py +5 -0
- noisefloor_evals-0.1.0/src/noisefloor/_dist.py +152 -0
- noisefloor_evals-0.1.0/src/noisefloor/bootstrap.py +147 -0
- noisefloor_evals-0.1.0/src/noisefloor/cli.py +539 -0
- noisefloor_evals-0.1.0/src/noisefloor/data.py +365 -0
- noisefloor_evals-0.1.0/src/noisefloor/gate.py +80 -0
- noisefloor_evals-0.1.0/src/noisefloor/leaderboard.py +269 -0
- noisefloor_evals-0.1.0/src/noisefloor/multiple.py +76 -0
- noisefloor_evals-0.1.0/src/noisefloor/power.py +203 -0
- noisefloor_evals-0.1.0/src/noisefloor/py.typed +0 -0
- noisefloor_evals-0.1.0/src/noisefloor/report.py +610 -0
- noisefloor_evals-0.1.0/src/noisefloor/stats.py +361 -0
- noisefloor_evals-0.1.0/src/noisefloor/suite.py +102 -0
- noisefloor_evals-0.1.0/tests/test_bootstrap.py +68 -0
- noisefloor_evals-0.1.0/tests/test_cli.py +215 -0
- noisefloor_evals-0.1.0/tests/test_data.py +119 -0
- noisefloor_evals-0.1.0/tests/test_dist.py +77 -0
- noisefloor_evals-0.1.0/tests/test_gate.py +48 -0
- noisefloor_evals-0.1.0/tests/test_leaderboard.py +81 -0
- noisefloor_evals-0.1.0/tests/test_multiple.py +83 -0
- noisefloor_evals-0.1.0/tests/test_power.py +95 -0
- noisefloor_evals-0.1.0/tests/test_stats.py +183 -0
- noisefloor_evals-0.1.0/tests/test_validation.py +81 -0
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
name: CI
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
branches: [main]
|
|
6
|
+
pull_request:
|
|
7
|
+
|
|
8
|
+
permissions:
|
|
9
|
+
contents: read
|
|
10
|
+
|
|
11
|
+
concurrency:
|
|
12
|
+
group: ${{ github.workflow }}-${{ github.ref }}
|
|
13
|
+
cancel-in-progress: true
|
|
14
|
+
|
|
15
|
+
jobs:
|
|
16
|
+
lint:
|
|
17
|
+
runs-on: ubuntu-latest
|
|
18
|
+
steps:
|
|
19
|
+
- uses: actions/checkout@v4
|
|
20
|
+
- uses: actions/setup-python@v5
|
|
21
|
+
with:
|
|
22
|
+
python-version: "3.12"
|
|
23
|
+
- run: python -m pip install -e ".[dev]"
|
|
24
|
+
- run: ruff format --check .
|
|
25
|
+
- run: ruff check .
|
|
26
|
+
- run: mypy
|
|
27
|
+
|
|
28
|
+
test:
|
|
29
|
+
runs-on: ubuntu-latest
|
|
30
|
+
strategy:
|
|
31
|
+
fail-fast: false
|
|
32
|
+
matrix:
|
|
33
|
+
python-version: ["3.10", "3.11", "3.12", "3.13"]
|
|
34
|
+
steps:
|
|
35
|
+
- uses: actions/checkout@v4
|
|
36
|
+
- uses: actions/setup-python@v5
|
|
37
|
+
with:
|
|
38
|
+
python-version: ${{ matrix.python-version }}
|
|
39
|
+
- run: python -m pip install -e ".[dev]"
|
|
40
|
+
- run: pytest -m "not slow"
|
|
41
|
+
|
|
42
|
+
validation:
|
|
43
|
+
name: statistical validation (Monte Carlo)
|
|
44
|
+
runs-on: ubuntu-latest
|
|
45
|
+
steps:
|
|
46
|
+
- uses: actions/checkout@v4
|
|
47
|
+
- uses: actions/setup-python@v5
|
|
48
|
+
with:
|
|
49
|
+
python-version: "3.12"
|
|
50
|
+
- run: python -m pip install -e ".[dev]"
|
|
51
|
+
- run: pytest -m slow
|
|
52
|
+
|
|
53
|
+
package:
|
|
54
|
+
runs-on: ubuntu-latest
|
|
55
|
+
steps:
|
|
56
|
+
- uses: actions/checkout@v4
|
|
57
|
+
- uses: actions/setup-python@v5
|
|
58
|
+
with:
|
|
59
|
+
python-version: "3.12"
|
|
60
|
+
- run: python -m pip install build twine
|
|
61
|
+
- run: python -m build
|
|
62
|
+
- run: twine check --strict dist/*
|
|
63
|
+
- name: Smoke-test the built wheel
|
|
64
|
+
run: |
|
|
65
|
+
python -m venv /tmp/wheel-env
|
|
66
|
+
/tmp/wheel-env/bin/pip install dist/*.whl
|
|
67
|
+
/tmp/wheel-env/bin/noisefloor --version
|
|
68
|
+
/tmp/wheel-env/bin/noisefloor compare examples/data/rag_baseline.jsonl examples/data/rag_candidate.jsonl
|
|
69
|
+
|
|
70
|
+
action:
|
|
71
|
+
name: dogfood the GitHub Action
|
|
72
|
+
runs-on: ubuntu-latest
|
|
73
|
+
steps:
|
|
74
|
+
- uses: actions/checkout@v4
|
|
75
|
+
- name: Noisy improvement should pass the default gate
|
|
76
|
+
id: noisy
|
|
77
|
+
uses: ./
|
|
78
|
+
with:
|
|
79
|
+
baseline: examples/data/rag_baseline.jsonl
|
|
80
|
+
candidate: examples/data/rag_candidate.jsonl
|
|
81
|
+
comment: "false"
|
|
82
|
+
- name: Real regression should fail the gate
|
|
83
|
+
id: regression
|
|
84
|
+
continue-on-error: true
|
|
85
|
+
uses: ./
|
|
86
|
+
with:
|
|
87
|
+
baseline: examples/data/suite/main/long_context.jsonl
|
|
88
|
+
candidate: examples/data/suite/pr/long_context.jsonl
|
|
89
|
+
comment: "false"
|
|
90
|
+
- name: Check outcomes
|
|
91
|
+
env:
|
|
92
|
+
NOISY_VERDICT: ${{ steps.noisy.outputs.verdict }}
|
|
93
|
+
NOISY_PASSED: ${{ steps.noisy.outputs.passed }}
|
|
94
|
+
REGRESSION_OUTCOME: ${{ steps.regression.outcome }}
|
|
95
|
+
REGRESSION_VERDICT: ${{ steps.regression.outputs.verdict }}
|
|
96
|
+
run: |
|
|
97
|
+
set -eu
|
|
98
|
+
test "$NOISY_VERDICT" = "inconclusive"
|
|
99
|
+
test "$NOISY_PASSED" = "true"
|
|
100
|
+
test "$REGRESSION_OUTCOME" = "failure"
|
|
101
|
+
test "$REGRESSION_VERDICT" = "regression"
|
|
102
|
+
echo "action behaves as specified"
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
name: Release
|
|
2
|
+
|
|
3
|
+
# Publishes to PyPI, using PyPI Trusted Publishing (OIDC): no API token is
|
|
4
|
+
# stored anywhere. Trigger it either way:
|
|
5
|
+
# - push a version tag: git tag v0.2.0 && git push origin v0.2.0
|
|
6
|
+
# - or publish a GitHub release for tag v0.2.0 from the web UI
|
|
7
|
+
# If both fire for the same tag, runs are serialized and the second skips
|
|
8
|
+
# files PyPI already has, so a version is never uploaded twice.
|
|
9
|
+
on:
|
|
10
|
+
push:
|
|
11
|
+
tags: ["v*"]
|
|
12
|
+
release:
|
|
13
|
+
types: [published]
|
|
14
|
+
|
|
15
|
+
concurrency:
|
|
16
|
+
group: release-${{ github.event.release.tag_name || github.ref_name }}
|
|
17
|
+
cancel-in-progress: false
|
|
18
|
+
|
|
19
|
+
permissions:
|
|
20
|
+
contents: read
|
|
21
|
+
|
|
22
|
+
jobs:
|
|
23
|
+
build:
|
|
24
|
+
runs-on: ubuntu-latest
|
|
25
|
+
steps:
|
|
26
|
+
- uses: actions/checkout@v4
|
|
27
|
+
- uses: actions/setup-python@v5
|
|
28
|
+
with:
|
|
29
|
+
python-version: "3.12"
|
|
30
|
+
- name: Check the tag matches the package version
|
|
31
|
+
env:
|
|
32
|
+
TAG: ${{ github.event.release.tag_name || github.ref_name }}
|
|
33
|
+
run: |
|
|
34
|
+
version=$(python -c "import tomllib; print(tomllib.load(open('pyproject.toml','rb'))['project']['version'])")
|
|
35
|
+
if [ "${TAG#v}" != "$version" ]; then
|
|
36
|
+
echo "::error::release tag $TAG does not match pyproject version $version"
|
|
37
|
+
exit 1
|
|
38
|
+
fi
|
|
39
|
+
- run: python -m pip install build twine
|
|
40
|
+
- run: python -m build
|
|
41
|
+
- run: twine check --strict dist/*
|
|
42
|
+
- name: Smoke-test the wheel
|
|
43
|
+
run: |
|
|
44
|
+
python -m venv /tmp/env
|
|
45
|
+
/tmp/env/bin/pip install dist/*.whl
|
|
46
|
+
/tmp/env/bin/noisefloor --version
|
|
47
|
+
- uses: actions/upload-artifact@v4
|
|
48
|
+
with:
|
|
49
|
+
name: dist
|
|
50
|
+
path: dist/
|
|
51
|
+
|
|
52
|
+
publish:
|
|
53
|
+
needs: build
|
|
54
|
+
runs-on: ubuntu-latest
|
|
55
|
+
environment:
|
|
56
|
+
name: pypi
|
|
57
|
+
url: https://pypi.org/project/noisefloor-evals/
|
|
58
|
+
permissions:
|
|
59
|
+
id-token: write
|
|
60
|
+
steps:
|
|
61
|
+
- uses: actions/download-artifact@v4
|
|
62
|
+
with:
|
|
63
|
+
name: dist
|
|
64
|
+
path: dist/
|
|
65
|
+
- uses: pypa/gh-action-pypi-publish@release/v1
|
|
66
|
+
with:
|
|
67
|
+
skip-existing: true
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
All notable changes to this project are documented here. The format follows
|
|
4
|
+
[Keep a Changelog](https://keepachangelog.com/en/1.1.0/) and the project uses
|
|
5
|
+
[Semantic Versioning](https://semver.org/).
|
|
6
|
+
|
|
7
|
+
## [0.1.0] - 2026-11-21
|
|
8
|
+
|
|
9
|
+
### Added
|
|
10
|
+
- `summary`: t-based confidence intervals, CR1 cluster-robust standard errors,
|
|
11
|
+
Wilson intervals, design effect, and within/between variance decomposition
|
|
12
|
+
for repeated samples, with projections for K = 1 to 16.
|
|
13
|
+
- `compare`: paired comparisons with p-values, wins/losses/ties, variance
|
|
14
|
+
removed by pairing, minimum detectable effect, and the sample size needed to
|
|
15
|
+
confirm the observed effect.
|
|
16
|
+
- `gate`: CI gate with no-regression, non-inferiority (`--max-regression`), and
|
|
17
|
+
require-improvement policies; Markdown output for PR comments.
|
|
18
|
+
- `suite`: per-benchmark comparisons with Holm, Benjamini–Hochberg, or
|
|
19
|
+
Bonferroni correction.
|
|
20
|
+
- `leaderboard`: joint-bootstrap rank confidence intervals and statistical
|
|
21
|
+
tiers, from per-question runs or from summary scores.
|
|
22
|
+
- `power`: sample size, minimum detectable effect, achieved power, margin
|
|
23
|
+
sizing, design effects, and planning from a pilot run.
|
|
24
|
+
- Vectorized (cluster) bootstrap for means and paired differences.
|
|
25
|
+
- Dependency-free Student-t distribution.
|
|
26
|
+
- Composite GitHub Action with job summary, PR comment, and outputs.
|
|
27
|
+
- Monte Carlo validation suite and synthetic example datasets.
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
cff-version: 1.2.0
|
|
2
|
+
message: "If you use noisefloor, please cite it as below."
|
|
3
|
+
title: "noisefloor: statistical rigor for LLM evaluations"
|
|
4
|
+
authors:
|
|
5
|
+
- family-names: Ibrahim
|
|
6
|
+
given-names: Abualgasim
|
|
7
|
+
version: 0.1.0
|
|
8
|
+
date-released: 2026-11-21
|
|
9
|
+
license: MIT
|
|
10
|
+
repository-code: "https://github.com/Gasooom/noisefloor"
|
|
11
|
+
keywords:
|
|
12
|
+
- llm evaluation
|
|
13
|
+
- statistics
|
|
14
|
+
- confidence intervals
|
|
15
|
+
- power analysis
|
|
16
|
+
references:
|
|
17
|
+
- type: article
|
|
18
|
+
title: "Adding Error Bars to Evals: A Statistical Approach to Language Model Evaluations"
|
|
19
|
+
authors:
|
|
20
|
+
- family-names: Miller
|
|
21
|
+
given-names: Evan
|
|
22
|
+
year: 2024
|
|
23
|
+
identifiers:
|
|
24
|
+
- type: other
|
|
25
|
+
value: "arXiv:2411.00640"
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
# Contributing to noisefloor
|
|
2
|
+
|
|
3
|
+
Thanks for helping make evals more honest.
|
|
4
|
+
|
|
5
|
+
## Development setup
|
|
6
|
+
|
|
7
|
+
```bash
|
|
8
|
+
git clone https://github.com/Gasooom/noisefloor
|
|
9
|
+
cd noisefloor
|
|
10
|
+
python -m venv .venv && source .venv/bin/activate
|
|
11
|
+
pip install -e ".[dev]"
|
|
12
|
+
```
|
|
13
|
+
|
|
14
|
+
## Before opening a pull request
|
|
15
|
+
|
|
16
|
+
```bash
|
|
17
|
+
ruff format .
|
|
18
|
+
ruff check .
|
|
19
|
+
mypy # strict mode
|
|
20
|
+
pytest -m "not slow" # fast unit tests (~1s)
|
|
21
|
+
pytest -m slow # Monte Carlo validation (~30s)
|
|
22
|
+
```
|
|
23
|
+
|
|
24
|
+
CI runs all of the above on Python 3.10 to 3.13.
|
|
25
|
+
|
|
26
|
+
## Ground rules
|
|
27
|
+
|
|
28
|
+
- **Statistical changes need a simulation.** A unit test shows the code does
|
|
29
|
+
what you meant; a Monte Carlo check in `tests/test_validation.py` shows that
|
|
30
|
+
what you meant is true (coverage, error rates, power). Add one for any new
|
|
31
|
+
estimator, interval, or test.
|
|
32
|
+
- **Cite your method.** Add the formula and a reference to
|
|
33
|
+
`docs/methodology.md`.
|
|
34
|
+
- **Keep the dependency footprint at numpy.** If you need a special function,
|
|
35
|
+
implement it in `_dist.py` and test it against published values.
|
|
36
|
+
- **Every number gets an interval.** Reports must never show a bare point
|
|
37
|
+
estimate.
|
|
38
|
+
- **Errors should say what to do.** Prefer `"could not find a score column
|
|
39
|
+
(tried ...); pass --score-field"` over a stack trace.
|
|
40
|
+
|
|
41
|
+
## Regenerating the example data
|
|
42
|
+
|
|
43
|
+
```bash
|
|
44
|
+
python examples/generate.py
|
|
45
|
+
python examples/validate.py --sims 4000 # refresh the README validation table
|
|
46
|
+
```
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Abualgasim Ibrahim
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,333 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: noisefloor-evals
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Is your eval improvement real, or just noise? Statistical rigor for LLM evaluations.
|
|
5
|
+
Project-URL: Homepage, https://github.com/Gasooom/noisefloor
|
|
6
|
+
Project-URL: Issues, https://github.com/Gasooom/noisefloor/issues
|
|
7
|
+
Project-URL: Documentation, https://github.com/Gasooom/noisefloor/blob/main/docs/methodology.md
|
|
8
|
+
Project-URL: Changelog, https://github.com/Gasooom/noisefloor/blob/main/CHANGELOG.md
|
|
9
|
+
Author: Abualgasim Ibrahim
|
|
10
|
+
License-Expression: MIT
|
|
11
|
+
License-File: LICENSE
|
|
12
|
+
Keywords: ci,confidence-intervals,evals,evaluation,llm,power-analysis,statistics
|
|
13
|
+
Classifier: Development Status :: 4 - Beta
|
|
14
|
+
Classifier: Intended Audience :: Developers
|
|
15
|
+
Classifier: Intended Audience :: Science/Research
|
|
16
|
+
Classifier: Programming Language :: Python :: 3
|
|
17
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
18
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
19
|
+
Classifier: Topic :: Scientific/Engineering :: Mathematics
|
|
20
|
+
Classifier: Typing :: Typed
|
|
21
|
+
Requires-Python: >=3.10
|
|
22
|
+
Requires-Dist: numpy>=1.24
|
|
23
|
+
Provides-Extra: dev
|
|
24
|
+
Requires-Dist: hypothesis>=6; extra == 'dev'
|
|
25
|
+
Requires-Dist: mypy>=1.10; extra == 'dev'
|
|
26
|
+
Requires-Dist: pytest>=8; extra == 'dev'
|
|
27
|
+
Requires-Dist: ruff>=0.5; extra == 'dev'
|
|
28
|
+
Description-Content-Type: text/markdown
|
|
29
|
+
|
|
30
|
+
<div align="center">
|
|
31
|
+
|
|
32
|
+
# noisefloor
|
|
33
|
+
|
|
34
|
+
**Is your eval improvement real, or just noise?**
|
|
35
|
+
|
|
36
|
+
Statistical rigor for LLM evaluations: honest error bars, paired comparisons,
|
|
37
|
+
cluster-robust standard errors, power analysis, multiple-comparison control,
|
|
38
|
+
leaderboard tiers, and a CI gate that blocks real regressions, not noise.
|
|
39
|
+
|
|
40
|
+
[](https://github.com/Gasooom/noisefloor/actions/workflows/ci.yml)
|
|
41
|
+

|
|
42
|
+

|
|
43
|
+

|
|
44
|
+

|
|
45
|
+
[](https://pypi.org/project/noisefloor-evals/)
|
|
46
|
+
|
|
47
|
+
</div>
|
|
48
|
+
|
|
49
|
+
---
|
|
50
|
+
|
|
51
|
+
## The problem
|
|
52
|
+
|
|
53
|
+
A team changes a prompt, swaps a model, or tweaks an agent. They rerun their
|
|
54
|
+
eval of a few hundred questions and see **72.0% → 76.2%**. They ship it.
|
|
55
|
+
|
|
56
|
+
Here is what noisefloor says about that exact result:
|
|
57
|
+
|
|
58
|
+
```console
|
|
59
|
+
$ noisefloor compare rag_baseline.jsonl rag_candidate.jsonl
|
|
60
|
+
|
|
61
|
+
noisefloor · compare rag_baseline → rag_candidate (600 paired questions, 120 clusters)
|
|
62
|
+
|
|
63
|
+
score 95% CI
|
|
64
|
+
rag_baseline 72.0% [67.3, 76.7]
|
|
65
|
+
rag_candidate 76.2% [71.9, 80.5]
|
|
66
|
+
|
|
67
|
+
difference +4.2pp [-0.4, +8.8] p = 0.076
|
|
68
|
+
|
|
69
|
+
verdict INCONCLUSIVE the difference is within the noise floor
|
|
70
|
+
|
|
71
|
+
· pairing removed 47% of the variance (per-question correlation 0.26)
|
|
72
|
+
· wins / losses / ties: 98 / 73 / 429
|
|
73
|
+
· smallest effect this eval can detect (80% power): ±6.6pp
|
|
74
|
+
· to reliably detect a +4.2pp effect you need ~1,494 questions (you have 600)
|
|
75
|
+
```
|
|
76
|
+
|
|
77
|
+
This dataset is simulated, so we know the truth: the candidate really is
|
|
78
|
+
better, by **+1.4pp**. The observed +4.2pp is **3× the real effect**. An eval
|
|
79
|
+
this size cannot tell a real 1.4pp gain apart from luck, and noisefloor says so
|
|
80
|
+
instead of letting a lucky draw become a launch decision.
|
|
81
|
+
|
|
82
|
+
Most of the benchmarks people quote are smaller than their decisions assume:
|
|
83
|
+
|
|
84
|
+
| Benchmark (public size) | Questions | 95% error bar on one score (at 50%) | Smallest real gain you can reliably detect* |
|
|
85
|
+
|---|---:|---:|---:|
|
|
86
|
+
| AIME 2024 | 30 | ±18.7pp | 24.1pp |
|
|
87
|
+
| HumanEval | 164 | ±7.7pp | 10.8pp |
|
|
88
|
+
| GPQA Diamond | 198 | ±7.0pp | 9.9pp |
|
|
89
|
+
| SWE-bench Verified | 500 | ±4.4pp | 6.2pp |
|
|
90
|
+
| MMLU | 14,042 | ±0.8pp | 1.2pp |
|
|
91
|
+
|
|
92
|
+
<sub>*Paired comparison, per-question correlation 0.5, 80% power, α = 0.05. Computed with `noisefloor power`.</sub>
|
|
93
|
+
|
|
94
|
+
The statistics to handle this exist. Miller (2024), [*Adding Error Bars to
|
|
95
|
+
Evals*](https://arxiv.org/abs/2411.00640), lays them out for LLM evaluation.
|
|
96
|
+
What has been missing is a tool that brings them into the workflow engineers
|
|
97
|
+
actually use: a terminal, a pull request, a CI check. **noisefloor is that
|
|
98
|
+
tool.**
|
|
99
|
+
|
|
100
|
+
## What it does
|
|
101
|
+
|
|
102
|
+
| Command | Answers |
|
|
103
|
+
|---|---|
|
|
104
|
+
| `noisefloor summary` | What is my score, *with an honest error bar*? Is clustering inflating my confidence? Should I buy more questions or more samples per question? |
|
|
105
|
+
| `noisefloor compare` | Is B really better than A on the same questions? (paired test, cluster-robust) |
|
|
106
|
+
| `noisefloor gate` | Should this PR be allowed to merge? Exits `1` on policy failure and writes a PR comment. |
|
|
107
|
+
| `noisefloor suite` | Across 14 benchmarks, which changes survive multiple-comparison correction? |
|
|
108
|
+
| `noisefloor power` | How many questions do I need *before* I spend compute? |
|
|
109
|
+
| `noisefloor leaderboard` | Which models are actually distinguishable? Rank confidence intervals and statistical tiers. |
|
|
110
|
+
|
|
111
|
+
Also included: a **GitHub Action** ([docs](docs/github-action.md)), a typed
|
|
112
|
+
**Python API**, and a **Monte Carlo validation suite** that verifies every
|
|
113
|
+
guarantee against data with known ground truth.
|
|
114
|
+
|
|
115
|
+
## Install
|
|
116
|
+
|
|
117
|
+
```bash
|
|
118
|
+
pip install noisefloor-evals
|
|
119
|
+
```
|
|
120
|
+
|
|
121
|
+
The package is published as `noisefloor-evals`; you import it as `noisefloor`, and
|
|
122
|
+
the command is `noisefloor`.
|
|
123
|
+
|
|
124
|
+
Requires Python ≥ 3.10. The only runtime dependency is numpy; the Student-t
|
|
125
|
+
distribution is implemented from scratch, so there is no SciPy.
|
|
126
|
+
|
|
127
|
+
## Quick start
|
|
128
|
+
|
|
129
|
+
Your eval results just need one row per question (or per sample) with an id
|
|
130
|
+
and a score:
|
|
131
|
+
|
|
132
|
+
```jsonl
|
|
133
|
+
{"id": "doc017-q2", "doc_id": "doc017", "correct": true}
|
|
134
|
+
{"id": "doc017-q3", "doc_id": "doc017", "correct": false}
|
|
135
|
+
```
|
|
136
|
+
|
|
137
|
+
Common field names (`id`/`question_id`/`task_id`, `score`/`correct`/`passed`,
|
|
138
|
+
`cluster`/`doc_id`/`group`) are detected automatically. Booleans, `"pass"`/`"fail"`,
|
|
139
|
+
and continuous scores all work. Repeated ids are treated as repeated samples
|
|
140
|
+
of the same question. Use `--id-field`, `--score-field`, and `--cluster-field`
|
|
141
|
+
(dotted paths like `metrics.acc` work) for anything else.
|
|
142
|
+
|
|
143
|
+
Every example below runs on the files in [`examples/data`](examples/data).
|
|
144
|
+
|
|
145
|
+
### A score with an honest error bar
|
|
146
|
+
|
|
147
|
+
```console
|
|
148
|
+
$ noisefloor summary rag_candidate_k5.jsonl
|
|
149
|
+
|
|
150
|
+
noisefloor · summary rag_candidate_k5 (600 questions, 120 clusters)
|
|
151
|
+
|
|
152
|
+
score 72.6% ± 3.5pp 95% CI [69.1, 76.1] SE 1.8pp, clustered
|
|
153
|
+
clustering inflates variance 2.38×; a naive i.i.d. analysis would claim ± 2.3pp
|
|
154
|
+
|
|
155
|
+
resampling 5.0 samples/question; 76% of single-sample variance is sampling noise
|
|
156
|
+
K=1: ±5.5pp K=2: ±4.3pp K=4: ±3.6pp K=8: ±3.2pp K=16: ±3.0pp K=∞: ±2.7pp
|
|
157
|
+
doubling samples/question would cut the error bar by only 10%, doubling questions by 29%. Buy more questions, not more samples.
|
|
158
|
+
```
|
|
159
|
+
|
|
160
|
+
The questions come in groups of five per document, so a naive analysis would
|
|
161
|
+
report an error bar about 35% too narrow. The variance decomposition then
|
|
162
|
+
answers a real budgeting question: with 5 samples per question already,
|
|
163
|
+
more resampling buys almost nothing.
|
|
164
|
+
|
|
165
|
+
### How many questions do I need?
|
|
166
|
+
|
|
167
|
+
```console
|
|
168
|
+
$ noisefloor power --effect 2pp --baseline-rate 70%
|
|
169
|
+
|
|
170
|
+
noisefloor · power questions to detect +2.0pp (baseline 70.0%, 80% power, α = 0.05)
|
|
171
|
+
|
|
172
|
+
correlation questions
|
|
173
|
+
0.0 8,077
|
|
174
|
+
0.3 5,655
|
|
175
|
+
0.5 4,040
|
|
176
|
+
0.7 2,425
|
|
177
|
+
```
|
|
178
|
+
|
|
179
|
+
Run `--n 198` to get the smallest effect a 198-question eval can detect,
|
|
180
|
+
`--n` with `--effect` for achieved power, `--margin 2pp` to size a single
|
|
181
|
+
score, `--cluster-size`/`--icc` for clustered designs, or `--pilot base.jsonl
|
|
182
|
+
cand.jsonl` to plan from runs you already have.
|
|
183
|
+
|
|
184
|
+
### Many benchmarks without fooling yourself
|
|
185
|
+
|
|
186
|
+
```console
|
|
187
|
+
$ noisefloor suite runs/main/ runs/pr/ --correction bh
|
|
188
|
+
...
|
|
189
|
+
4 of 14 benchmarks look significant at α = 0.05 before correction;
|
|
190
|
+
2 survive Benjamini-Hochberg correction (1 better, 1 worse).
|
|
191
|
+
```
|
|
192
|
+
|
|
193
|
+
In this simulated suite exactly two benchmarks truly changed. Before
|
|
194
|
+
correction, four *look* significant, which is enough for a misleading release
|
|
195
|
+
note. After correction, exactly the two real effects remain.
|
|
196
|
+
|
|
197
|
+
### Leaderboards with rank uncertainty
|
|
198
|
+
|
|
199
|
+
```console
|
|
200
|
+
$ noisefloor leaderboard --summary leaderboard.csv
|
|
201
|
+
|
|
202
|
+
tier rank rank CI model score 95% CI
|
|
203
|
+
1 1 1–4 aurora-xl 64.6% [57.9, 71.3]
|
|
204
|
+
1 2 1–4 basalt-pro 63.1% [56.4, 69.8]
|
|
205
|
+
1 3 1–4 cirrus-2 62.6% [55.9, 69.3]
|
|
206
|
+
1 4 1–5 delta-large 58.1% [51.2, 65.0]
|
|
207
|
+
─────────────────────────────────────────────────────────
|
|
208
|
+
2 5 4–6 ember-mini 51.5% [44.5, 58.5]
|
|
209
|
+
2 6 5–6 fjord-7b 41.9% [35.0, 48.8]
|
|
210
|
+
```
|
|
211
|
+
|
|
212
|
+
On a 198-question benchmark, the "#1 model" could plausibly be #4. Pass
|
|
213
|
+
per-question result files instead of `--summary` to get a paired analysis
|
|
214
|
+
with joint-bootstrap rank intervals.
|
|
215
|
+
|
|
216
|
+
## In CI: block real regressions, not noise
|
|
217
|
+
|
|
218
|
+
```yaml
|
|
219
|
+
- uses: Gasooom/noisefloor@main
|
|
220
|
+
with:
|
|
221
|
+
baseline: results/main.jsonl
|
|
222
|
+
candidate: results/pr.jsonl
|
|
223
|
+
max-regression: 1pp # optional non-inferiority margin
|
|
224
|
+
```
|
|
225
|
+
|
|
226
|
+
The action prints the comparison, writes a job summary, keeps **one**
|
|
227
|
+
up-to-date comment on the PR, and fails only when the change fails your
|
|
228
|
+
policy:
|
|
229
|
+
|
|
230
|
+
| Policy | Fails when |
|
|
231
|
+
|---|---|
|
|
232
|
+
| default | the candidate is **significantly worse** |
|
|
233
|
+
| `max-regression: 1pp` | a regression > 1pp **cannot be ruled out** |
|
|
234
|
+
| `require-improvement: true` | the candidate is **not significantly better** |
|
|
235
|
+
|
|
236
|
+
See [docs/github-action.md](docs/github-action.md). This repository's own CI
|
|
237
|
+
dogfoods the action: it asserts that the noisy +4.2pp example passes and a real
|
|
238
|
+
regression fails.
|
|
239
|
+
|
|
240
|
+
## Python API
|
|
241
|
+
|
|
242
|
+
```python
|
|
243
|
+
import noisefloor as nf
|
|
244
|
+
|
|
245
|
+
base = nf.load("baseline.jsonl")
|
|
246
|
+
cand = nf.load("candidate.jsonl")
|
|
247
|
+
|
|
248
|
+
c = nf.compare(base, cand)
|
|
249
|
+
c.verdict # 'improvement' | 'regression' | 'inconclusive'
|
|
250
|
+
c.diff.mean, c.diff.ci_low, c.diff.ci_high, c.p_value
|
|
251
|
+
c.mde # smallest effect this eval can detect
|
|
252
|
+
c.n_required # questions needed to confirm the observed effect
|
|
253
|
+
|
|
254
|
+
nf.sample_size(0.02, baseline_rate=0.7, correlation=0.5) # -> 4040
|
|
255
|
+
nf.evaluate_gate(c, nf.GatePolicy(max_regression=0.01)).passed
|
|
256
|
+
|
|
257
|
+
runs = {name: (nf.load(f"main/{name}.jsonl"), nf.load(f"pr/{name}.jsonl")) for name in names}
|
|
258
|
+
nf.compare_suite(runs, correction="holm").regressions
|
|
259
|
+
```
|
|
260
|
+
|
|
261
|
+
Every function is typed (`mypy --strict`) and returns frozen dataclasses.
|
|
262
|
+
|
|
263
|
+
## Does it actually work? (validation)
|
|
264
|
+
|
|
265
|
+
Every guarantee noisefloor makes is a frequency claim, so each one is checked
|
|
266
|
+
by simulation against data with known ground truth
|
|
267
|
+
([`examples/validate.py`](examples/validate.py), 4,000 simulations per row; also
|
|
268
|
+
run in CI):
|
|
269
|
+
|
|
270
|
+
| Check | Target | noisefloor | Common shortcut |
|
|
271
|
+
|---|:-:|:-:|:-:|
|
|
272
|
+
| 95% CI coverage, i.i.d. questions (n=100) | 95% | **94.9%** | n/a |
|
|
273
|
+
| 95% CI coverage, tiny judge-scored eval (n=12) | 95% | **94.5%** | 92.4% (z-interval) |
|
|
274
|
+
| 95% CI coverage, clustered eval (60 passages × 5 questions) | 95% | **95.2%** | 88.1% (ignore clusters) |
|
|
275
|
+
| False alarm rate, equally good models (clustered eval) | 5% | **5.1%** | 9.0% (ignore clusters) |
|
|
276
|
+
| Power at planned n = 1,468 for a +5pp gain | 80% | **79.1%** | n/a |
|
|
277
|
+
| ≥1 false "win" across 10 unchanged benchmarks | ≤ 5% | **5.1%** | 40.5% (no correction) |
|
|
278
|
+
|
|
279
|
+
The Student-t implementation is checked against published tables to 7
|
|
280
|
+
significant figures, and the estimators against hand-derived values. Run the
|
|
281
|
+
full suite with `pytest`, or only the fast tests with `pytest -m "not slow"`.
|
|
282
|
+
|
|
283
|
+
## How it works
|
|
284
|
+
|
|
285
|
+
The full derivations are in [docs/methodology.md](docs/methodology.md). In short:
|
|
286
|
+
|
|
287
|
+
- **Unit of analysis = question.** Repeated samples are averaged per question,
|
|
288
|
+
and their spread is kept as within-question variance.
|
|
289
|
+
- **Cluster-robust (CR1) standard errors** with G−1 degrees of freedom when
|
|
290
|
+
questions share a passage, repository, or conversation.
|
|
291
|
+
- **Paired differences** whenever two systems answer the same questions. Shared
|
|
292
|
+
difficulty cancels, which often removes half the variance.
|
|
293
|
+
- **Student-t intervals** everywhere, plus Wilson intervals for small binary
|
|
294
|
+
evals and a vectorized cluster bootstrap as a cross-check.
|
|
295
|
+
- **Power analysis** with correlation and design effects, including planning
|
|
296
|
+
from a pilot run.
|
|
297
|
+
- **Holm / Benjamini–Hochberg** corrections across benchmarks.
|
|
298
|
+
- **Joint-bootstrap rank intervals** and paired-test tiers for leaderboards.
|
|
299
|
+
|
|
300
|
+
## Design principles
|
|
301
|
+
|
|
302
|
+
1. **Never print a bare number.** Every estimate comes with its interval.
|
|
303
|
+
2. **Say what to do next.** "Inconclusive" alone is useless. "You need ~1,500
|
|
304
|
+
questions" or "buy questions, not samples" is actionable.
|
|
305
|
+
3. **Validate the guarantees, not just the code.** Unit tests show the code
|
|
306
|
+
does what it says; Monte Carlo tests show that what it says is true.
|
|
307
|
+
4. **Boring to install.** Pure Python plus numpy, no services, no API keys, works
|
|
308
|
+
offline on any eval harness's output.
|
|
309
|
+
|
|
310
|
+
## Roadmap
|
|
311
|
+
|
|
312
|
+
- [ ] Native readers for popular harness outputs (lm-evaluation-harness, Inspect, OpenAI Evals, promptfoo)
|
|
313
|
+
- [ ] Sequential testing (always-valid p-values) for evals that stream in
|
|
314
|
+
- [ ] LLM-as-judge noise: judge-resampling variance and inter-judge agreement
|
|
315
|
+
- [ ] Stratified / weighted estimates for evals that mix difficulty buckets
|
|
316
|
+
- [ ] HTML report with forest plots
|
|
317
|
+
|
|
318
|
+
## Contributing
|
|
319
|
+
|
|
320
|
+
Issues and pull requests are welcome; see [CONTRIBUTING.md](CONTRIBUTING.md).
|
|
321
|
+
New statistical features should come with a Monte Carlo check in
|
|
322
|
+
`tests/test_validation.py`.
|
|
323
|
+
|
|
324
|
+
## Citation
|
|
325
|
+
|
|
326
|
+
If noisefloor helps your research, please cite it (see [CITATION.cff](CITATION.cff))
|
|
327
|
+
and the paper that motivated it:
|
|
328
|
+
|
|
329
|
+
> Miller, E. (2024). *Adding Error Bars to Evals: A Statistical Approach to Language Model Evaluations.* arXiv:2411.00640.
|
|
330
|
+
|
|
331
|
+
## License
|
|
332
|
+
|
|
333
|
+
MIT © Abualgasim Ibrahim
|