evalnoise 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Meenal Asesh
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,292 @@
1
+ Metadata-Version: 2.4
2
+ Name: evalnoise
3
+ Version: 0.1.0
4
+ Summary: Where does the noise in an eval score come from? Split it across questions, judges, prompt wordings and runs, and plan where to spend the next API calls
5
+ Author: Meenal Asesh
6
+ License-Expression: MIT
7
+ Project-URL: Homepage, https://github.com/ai-with-md/evalnoise
8
+ Project-URL: Calculator, https://ai-with-md.github.io/evalnoise/
9
+ Project-URL: Issues, https://github.com/ai-with-md/evalnoise/issues
10
+ Keywords: llm,evaluation,evals,llm-as-a-judge,error bars,generalizability theory,variance components,measurement error,statistics
11
+ Classifier: Programming Language :: Python :: 3
12
+ Classifier: Operating System :: OS Independent
13
+ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
14
+ Requires-Python: >=3.9
15
+ Description-Content-Type: text/markdown
16
+ License-File: LICENSE
17
+ Provides-Extra: dev
18
+ Requires-Dist: pytest; extra == "dev"
19
+ Dynamic: license-file
20
+
21
+ # evalnoise
22
+
23
+ **Where does the noise in my eval score come from?** Split it across questions, judges, prompt wordings and reruns, get an error bar that counts all of them, and see where your next API calls do the most good. With a [calculator that runs in your browser](https://ai-with-md.github.io/evalnoise/).
24
+
25
+ [![PyPI](https://img.shields.io/pypi/v/evalnoise)](https://pypi.org/project/evalnoise/)
26
+ [![tests](https://github.com/ai-with-md/evalnoise/actions/workflows/tests.yml/badge.svg)](https://github.com/ai-with-md/evalnoise/actions/workflows/tests.yml)
27
+ ![Python 3.9+](https://img.shields.io/badge/python-3.9%2B-blue)
28
+ ![License: MIT](https://img.shields.io/badge/license-MIT-green)
29
+
30
+ [![evalnoise showing that two thirds of the noise in an eval score comes from which judge was used](https://raw.githubusercontent.com/ai-with-md/evalnoise/main/docs/calculator.png)](https://ai-with-md.github.io/evalnoise/)
31
+
32
+ An eval score moves when you change the questions. It also moves when you swap the judge, reword the prompt, or simply run it again. The usual error bar, even the careful kind that groups scores by question, only counts the first of these. So a "+3 points" win can turn out to be one judge's taste, and a score can shift by more than its error bar when someone reruns the eval with a different grader.
33
+
34
+ evalnoise takes a small pilot, where every question is scored under every judge, wording and run, and works out how much each source adds. Then it answers the practical question: with the next $50 of API calls, should you add questions, judges, wordings or reruns?
35
+
36
+ ## The number most people get wrong
37
+
38
+ Simulated evals where the true score is known: 50 to 80 questions, each answered under 2 or 3 prompt wordings, 2 runs each, and graded by 3 judges. Judges and wordings differ a little, as they do in practice. 1,000 simulated evals per row.
39
+
40
+ | | Usual interval (grouped by question) | evalnoise |
41
+ |---|---|---|
42
+ | Covers the true score (95% promised) | 74% | 94% |
43
+ | The same, for pass/fail verdicts | 72% | 96% |
44
+ | Two equally good models, judges that lean one way or the other: calls the difference real (5% promised) | 37% | 4.9% |
45
+ | Two models 6 points apart, no leaning judges: finds the difference | 99% | 96% |
46
+ | No judge or wording noise at all: covers the true score | 95% | 95%, about the same width |
47
+
48
+ When judges and wordings don't matter, evalnoise gives the same answer as the usual interval. When they do, it says so, and says by how much.
49
+
50
+ ## Try it
51
+
52
+ **In the browser:** [ai-with-md.github.io/evalnoise](https://ai-with-md.github.io/evalnoise/). Upload a pilot or try the examples, nothing to install.
53
+
54
+ **In Python:**
55
+
56
+ ```bash
57
+ pip install evalnoise
58
+ ```
59
+
60
+ No dependencies. Python 3.9 or later.
61
+
62
+ ## Use
63
+
64
+ ### 1. Run a small pilot
65
+
66
+ Score every question under every combination you want to learn about: every judge, every prompt wording, every run. For example 60 questions x 3 judges x 3 wordings x 2 runs. Keep one row per score:
67
+
68
+ ```text
69
+ question_id,judge,prompt,run,verdict
70
+ q001,judge-a,wording-1,1,pass
71
+ q001,judge-b,wording-1,1,fail
72
+ ...
73
+ ```
74
+
75
+ Scores can be numbers (0 to 1, 1 to 5, anything), true/false, or words like pass and fail. Use at least 3 judges and 3 wordings if you can: with only 2, how much they differ is little more than a guess (see [Limits](#limits)).
76
+
77
+ ### 2. Where does the noise come from?
78
+
79
+ ```python
80
+ import evalnoise as en
81
+
82
+ pilot = en.read_scores("examples/pilot.csv")
83
+ print(pilot.report())
84
+ # Read 1,080 rows: question from 'question_id', score from 'verdict', judge from 'judge', prompt from 'prompt', run from 'run'.
85
+
86
+ result = pilot.study()
87
+ print(result.report())
88
+ ```
89
+
90
+ ```text
91
+ Score 0.713 from 60 questions x 3 judges x 3 prompts x 2 runs (1,080 scores).
92
+ Interval counting every source of noise (95%): 0.574 to 0.852 (+-0.139).
93
+ The usual interval, counting only which questions were picked: 0.658 to 0.768 (+-0.0545).
94
+ The noise estimate itself rests on few judges, wordings or runs (about 4 degrees of freedom), which widens the interval. More of them in the pilot would firm it up.
95
+ Where the noise in this score comes from:
96
+ judge 67% some judges are stricter than others across the board
97
+ question 22% some questions are harder than others
98
+ question x judge x prompt x run 5% everything else, including one-off noise in single scores
99
+ prompt 3% some prompt wordings score higher across the board
100
+ question x judge x prompt 2% an interaction between question, judge and prompt
101
+ question x prompt x run 1% answers change from run to run
102
+ By judge: judge-a 0.731, judge-b 0.633, judge-c 0.775
103
+ ```
104
+
105
+ Two thirds of the uncertainty is which judge you happened to pick: judge-b passes 63% of answers, judge-c passes 78%. Adding questions will not fix that.
106
+
107
+ Everything in the report is also available as attributes: `result.mean`, `result.se`, `result.naive_se`, `result.interval()`, and `result.components`, each with its `variance` and `share`.
108
+
109
+ ### 3. What would another design give?
110
+
111
+ ```python
112
+ print(result.project(items=500, judges=1))
113
+ # 500 questions x 1 judge x 3 prompts x 2 runs: the score would be good to about +-0.142 (95%).
114
+ print(result.project(items=200, judges=3, prompts=3, runs=1))
115
+ # 200 questions x 3 judges x 3 prompts x 1 run: the score would be good to about +-0.0884 (95%).
116
+ ```
117
+
118
+ 500 questions with one judge are no better than the pilot. 200 questions with three judges and a single run cost less and do much better.
119
+
120
+ ### 4. Where should the next calls go?
121
+
122
+ Give the price of a call to the model you are testing and a call to the judge, and either a budget or a target:
123
+
124
+ ```python
125
+ print(result.plan(budget=40, generation_cost=0.01, judge_cost=0.004, limits={"judge": 3}))
126
+ ```
127
+
128
+ ```text
129
+ Best design for $40.00: 454 questions, 3 judges, 4 prompts, 1 run.
130
+ 1,816 model calls and 5,448 judge calls, costing $39.95.
131
+ The score would be good to about +-0.0843 (95%).
132
+ On the same basis, the pilot's own design ($7.92 at these prices) gives about +-0.0985.
133
+ ```
134
+
135
+ ```python
136
+ print(result.plan(target=0.1, generation_cost=0.01, judge_cost=0.004, limits={"judge": 3}))
137
+ ```
138
+
139
+ ```text
140
+ Best design for +-0.1: 140 questions, 3 judges, 1 prompt, 1 run.
141
+ 140 model calls and 420 judge calls, costing $3.08.
142
+ The score would be good to about +-0.0999 (95%).
143
+ On the same basis, the pilot's own design ($7.92 at these prices) gives about +-0.0985.
144
+ Careful: the pilot had only 3 prompts, so how much prompts differ is a rough estimate; dropping to 1 is a bet that it stays small.
145
+ ```
146
+
147
+ `limits` caps how many judges, wordings or runs to consider (only three judges are available here); `max_items` caps the questions when your eval set has a fixed size.
148
+
149
+ ### 5. Comparing two models or prompts
150
+
151
+ Score both on the same questions, judges, wordings and runs, with a column naming the model:
152
+
153
+ ```python
154
+ two = en.read_scores("examples/two_models.csv")
155
+ print(two.compare().report()) # or compare(new="candidate", old="baseline")
156
+ ```
157
+
158
+ ```text
159
+ Difference, candidate minus baseline: +0.18 from 80 questions x 4 judges x 2 runs (640 paired scores).
160
+ Interval counting every source of noise (95%): -0.118 to 0.477 (+-0.297).
161
+ The usual interval, counting only which questions were picked: 0.0472 to 0.312 (+-0.132).
162
+ The usual interval would call this a real difference, but with the judge, wording and run noise counted it could be noise.
163
+ The noise estimate itself rests on few judges, wordings or runs (about 5 degrees of freedom), which widens the interval. More of them in the pilot would firm it up.
164
+ Where the noise in this difference comes from:
165
+ judge 68% judges disagree about which version is better overall (for example, a judge that favours one model's style)
166
+ question 17% the new version is better on some questions and worse on others
167
+ question x run 8% answers change from run to run
168
+ question x judge x run 6% everything else, including one-off noise in single scores
169
+ By judge: judge-a +0.0438, judge-b +0.0688, judge-c +0.481, judge-d +0.125
170
+ ```
171
+
172
+ Most of the "win" comes from judge-c. Three of the four judges see a gap of 0.04 to 0.13 points on a 1 to 5 scale.
173
+
174
+ Comparisons are kinder than single scores: a judge that is strict with both models cancels out. What is left is noise in the gap itself, like a judge that prefers one model's style. Runs are paired by their label, so run 1 of one model is compared with run 1 of the other.
175
+
176
+ ### Judges you will always use
177
+
178
+ The analysis treats judges and wordings as a sample from all the ones you might have used, so the score generalizes beyond them. If you will always grade with exactly these judges, mark them fixed and their own leniency becomes part of what you are measuring:
179
+
180
+ ```python
181
+ print(pilot.study(fixed=["judge"]).report())
182
+ # Interval counting every source of noise (95%): 0.653 to 0.773 (+-0.0598).
183
+ ```
184
+
185
+ ### Runs are nested
186
+
187
+ Run 1 of one question has nothing to do with run 1 of another, so runs are analysed as nested within each question and wording. Columns named run, seed, sample, attempt, trial and similar are treated this way. If run 1 really is the same for every question (all run-1 answers came from one batch on one day), pass `nested=[]`.
188
+
189
+ ### Rows in memory
190
+
191
+ ```python
192
+ rows = [
193
+ {"item": "q1", "judge": "a", "prompt": "p1", "score": 1},
194
+ {"item": "q1", "judge": "b", "prompt": "p1", "score": 0},
195
+ ...
196
+ ]
197
+ result = en.study(rows) # every column other than item and score is a facet
198
+ result = en.study(df, item="question", score="correct") # a pandas DataFrame works too
199
+ diff = en.compare(rows, system="model", new="v2", old="v1")
200
+ ```
201
+
202
+ ### Reading your files
203
+
204
+ `read_scores` reads CSV, JSON (a list of objects) and JSON Lines, with one row per score. It finds the question, score and system columns by common names (`question_id`, `item`, `id`; `score`, `verdict`, `passed`, `correct`; `model`, `system`, `version`) and treats columns named like judges, graders, prompts, templates, runs and seeds as facets. Nested JSON fields are named with dots, like `metadata.judge`. Columns that look like text or ids are left out and reported. Check `table.report()` the first time, and name columns yourself when the guess is wrong:
205
+
206
+ ```python
207
+ table = en.read_scores("results.jsonl", item="example_id", score="judge_score", facets=["grader", "seed"])
208
+ ```
209
+
210
+ Questions with any missing score are left out of the analysis and listed in the report.
211
+
212
+ ## Command line
213
+
214
+ ```bash
215
+ evalnoise check examples/pilot.csv
216
+ evalnoise check examples/two_models.csv # two systems in the file: compares them
217
+ evalnoise check examples/two_models.csv --only baseline # or analyse one of them
218
+ evalnoise check results.jsonl --fixed judge -v
219
+ evalnoise project examples/pilot.csv items=500 judges=1
220
+ evalnoise plan examples/pilot.csv --budget 40 --model-cost 0.01 --judge-cost 0.004 --limit judge=3
221
+ evalnoise plan examples/pilot.csv --target 0.05 --model-cost 0.01 --judge-cost 0.004 --max-items 800
222
+ ```
223
+
224
+ Use `--item`, `--score`, `--facets`, `--system`, `--new` and `--old` when the automatic guess is wrong, and `-v` to see which columns were used.
225
+
226
+ ## How the numbers are checked
227
+
228
+ The test suite simulates evals from a known model, where questions, judges, wordings and runs each add their own noise, and checks each promise:
229
+
230
+ | Promise | Held in simulation |
231
+ |---|---|
232
+ | The interval covers the true score (95% promised) | 94% (usual interval: 74%) |
233
+ | The same for pass/fail verdicts | 96% |
234
+ | Comparing equal models with leaning judges: false "real difference" (5% promised) | 4.9% (usual interval: 37%) |
235
+ | Comparing models 6 points apart: finds the difference | 96% |
236
+ | With no judge or wording noise, the interval is no wider than needed | 95% coverage, about the same width as the usual interval |
237
+ | The judge spread is estimated without bias | yes, on average |
238
+ | A projection predicts the spread of scores at the new design | within 10% |
239
+ | The plan from a pilot beats spending the same money on the pilot's design | yes |
240
+
241
+ The two-way analysis is also checked against the textbook formulas, and run labels can be shuffled within each question without changing anything. See [`tests/test_simulation.py`](https://github.com/ai-with-md/evalnoise/blob/main/tests/test_simulation.py).
242
+
243
+ ## How it compares
244
+
245
+ - **Error bars grouped by question**, as in [Adding Error Bars to Evals](https://arxiv.org/abs/2411.00640) and Inspect's clustered standard errors, are what evalnoise calls the usual interval. They are right when the judge and wording are fixed for good. evalnoise adds the noise from judges, wordings and reruns, and tells you which of them matters.
246
+ - **[Total Evaluation Error](https://arxiv.org/abs/2604.11581)** (Messing, NeurIPS 2026) makes the same argument for LLM pipelines and finds usual error bars 53 to 70% too narrow. evalnoise is a small Python implementation of the idea for balanced pilots.
247
+ - **[aiEvalR](https://cran.r-project.org/package=aiEvalR)** brings generalizability theory to AI evaluation in R. evalnoise does it in Python with no dependencies, plus budget planning with API prices.
248
+ - **Judge bias correction** with human labels ([judgy](https://github.com/ai-evals-course/judgy), [evalstats](https://github.com/ianarawjo/evalstats)) answers a different question: whether the judge agrees with people. evalnoise asks how much the score depends on which judge, wording and run you used.
249
+ - **[evalsize](https://github.com/ai-with-md/evalsize)** tells you how many questions you need for a pass rate. evalnoise tells you whether more questions are even the right fix.
250
+
251
+ ## Limits
252
+
253
+ - **Balanced pilots only.** Every question must be scored under every combination of judges, wordings and runs. Questions with gaps are left out. Unbalanced designs need a mixed model fitted by REML, which this package does not do.
254
+ - **Few judges or wordings give rough answers.** The spread between judges is estimated from the judges you used. With 2, it is close to a guess: in simulation, when judges differ a lot, the interval covered 86% with 2 judges and 91% with 4, against 95% promised. The interval widens to reflect this, and the report says so, but use 3 or more of each when you can.
255
+ - **The plan treats the pilot's estimates as exact.** When a pilot had only a few judges or wordings, a plan that drops to fewer of them comes with a warning.
256
+ - **Judges and wordings are treated as a sample** from all the ones you might have used, unless you mark them fixed. Decide which one you mean: "this model with this judge" or "this model, judged fairly".
257
+ - **Effects are assumed to add up.** For pass/fail verdicts this works well for the average and the interval (see the simulations above), but the components describe pass rates, not anything deeper.
258
+ - **Projections and plans** use a normal approximation for the interval, so they are a little optimistic for pilots with very few judges or wordings.
259
+
260
+ ## Methods
261
+
262
+ evalnoise runs a generalizability study (G-study) and a decision study (D-study), the standard tools of generalizability theory:
263
+
264
+ - Variance components come from the expected mean squares of a balanced random-effects analysis of variance, with questions crossed with every facet and runs nested within question and wording. Negative estimates are set to zero.
265
+ - The error variance of the overall score adds each component divided by the number of levels it is averaged over. Components of fixed facets alone drop out.
266
+ - The interval uses a t distribution with Satterthwaite degrees of freedom, so it widens when the estimate rests on few judges or wordings.
267
+ - Projections and plans reuse the same formula at other sample sizes. Plans search every allowed combination of judges, wordings and runs and pick the number of questions that fits the budget or the target.
268
+
269
+ References: Cronbach, Gleser, Nanda and Rajaratnam, *The Dependability of Behavioral Measurements* (1972); Brennan, *Generalizability Theory* (2001); Satterthwaite, "An approximate distribution of estimates of variance components" (1946); Messing, "Hidden Measurement Error in LLM Pipelines" ([arXiv 2604.11581](https://arxiv.org/abs/2604.11581)).
270
+
271
+ ## More tools
272
+
273
+ Small, free tools for building and checking AI systems:
274
+
275
+ - **[evalsize](https://github.com/ai-with-md/evalsize)**: how many eval examples you need before a difference between prompts or models means anything.
276
+ - **[costplan](https://github.com/ai-with-md/costplan)**: what an AI agent will cost and what budget cap to set, from a handful of pilot runs.
277
+ - **[sturdytools](https://github.com/ai-with-md/sturdytools)**: retries, timeouts, rate limits and circuit breakers for the tools an AI agent calls.
278
+
279
+ ## Development
280
+
281
+ ```bash
282
+ python -m venv .venv
283
+ source .venv/bin/activate
284
+ pip install -e ".[dev]"
285
+ pytest
286
+ ```
287
+
288
+ The calculator is a [marimo](https://marimo.io) notebook in [`app/calculator.py`](https://github.com/ai-with-md/evalnoise/blob/main/app/calculator.py), published to GitHub Pages as a WebAssembly app on every push to `main`.
289
+
290
+ ## License
291
+
292
+ MIT
@@ -0,0 +1,272 @@
1
+ # evalnoise
2
+
3
+ **Where does the noise in my eval score come from?** Split it across questions, judges, prompt wordings and reruns, get an error bar that counts all of them, and see where your next API calls do the most good. With a [calculator that runs in your browser](https://ai-with-md.github.io/evalnoise/).
4
+
5
+ [![PyPI](https://img.shields.io/pypi/v/evalnoise)](https://pypi.org/project/evalnoise/)
6
+ [![tests](https://github.com/ai-with-md/evalnoise/actions/workflows/tests.yml/badge.svg)](https://github.com/ai-with-md/evalnoise/actions/workflows/tests.yml)
7
+ ![Python 3.9+](https://img.shields.io/badge/python-3.9%2B-blue)
8
+ ![License: MIT](https://img.shields.io/badge/license-MIT-green)
9
+
10
+ [![evalnoise showing that two thirds of the noise in an eval score comes from which judge was used](https://raw.githubusercontent.com/ai-with-md/evalnoise/main/docs/calculator.png)](https://ai-with-md.github.io/evalnoise/)
11
+
12
+ An eval score moves when you change the questions. It also moves when you swap the judge, reword the prompt, or simply run it again. The usual error bar, even the careful kind that groups scores by question, only counts the first of these. So a "+3 points" win can turn out to be one judge's taste, and a score can shift by more than its error bar when someone reruns the eval with a different grader.
13
+
14
+ evalnoise takes a small pilot, where every question is scored under every judge, wording and run, and works out how much each source adds. Then it answers the practical question: with the next $50 of API calls, should you add questions, judges, wordings or reruns?
15
+
16
+ ## The number most people get wrong
17
+
18
+ Simulated evals where the true score is known: 50 to 80 questions, each answered under 2 or 3 prompt wordings, 2 runs each, and graded by 3 judges. Judges and wordings differ a little, as they do in practice. 1,000 simulated evals per row.
19
+
20
+ | | Usual interval (grouped by question) | evalnoise |
21
+ |---|---|---|
22
+ | Covers the true score (95% promised) | 74% | 94% |
23
+ | The same, for pass/fail verdicts | 72% | 96% |
24
+ | Two equally good models, judges that lean one way or the other: calls the difference real (5% promised) | 37% | 4.9% |
25
+ | Two models 6 points apart, no leaning judges: finds the difference | 99% | 96% |
26
+ | No judge or wording noise at all: covers the true score | 95% | 95%, about the same width |
27
+
28
+ When judges and wordings don't matter, evalnoise gives the same answer as the usual interval. When they do, it says so, and says by how much.
29
+
30
+ ## Try it
31
+
32
+ **In the browser:** [ai-with-md.github.io/evalnoise](https://ai-with-md.github.io/evalnoise/). Upload a pilot or try the examples, nothing to install.
33
+
34
+ **In Python:**
35
+
36
+ ```bash
37
+ pip install evalnoise
38
+ ```
39
+
40
+ No dependencies. Python 3.9 or later.
41
+
42
+ ## Use
43
+
44
+ ### 1. Run a small pilot
45
+
46
+ Score every question under every combination you want to learn about: every judge, every prompt wording, every run. For example 60 questions x 3 judges x 3 wordings x 2 runs. Keep one row per score:
47
+
48
+ ```text
49
+ question_id,judge,prompt,run,verdict
50
+ q001,judge-a,wording-1,1,pass
51
+ q001,judge-b,wording-1,1,fail
52
+ ...
53
+ ```
54
+
55
+ Scores can be numbers (0 to 1, 1 to 5, anything), true/false, or words like pass and fail. Use at least 3 judges and 3 wordings if you can: with only 2, how much they differ is little more than a guess (see [Limits](#limits)).
56
+
57
+ ### 2. Where does the noise come from?
58
+
59
+ ```python
60
+ import evalnoise as en
61
+
62
+ pilot = en.read_scores("examples/pilot.csv")
63
+ print(pilot.report())
64
+ # Read 1,080 rows: question from 'question_id', score from 'verdict', judge from 'judge', prompt from 'prompt', run from 'run'.
65
+
66
+ result = pilot.study()
67
+ print(result.report())
68
+ ```
69
+
70
+ ```text
71
+ Score 0.713 from 60 questions x 3 judges x 3 prompts x 2 runs (1,080 scores).
72
+ Interval counting every source of noise (95%): 0.574 to 0.852 (+-0.139).
73
+ The usual interval, counting only which questions were picked: 0.658 to 0.768 (+-0.0545).
74
+ The noise estimate itself rests on few judges, wordings or runs (about 4 degrees of freedom), which widens the interval. More of them in the pilot would firm it up.
75
+ Where the noise in this score comes from:
76
+ judge 67% some judges are stricter than others across the board
77
+ question 22% some questions are harder than others
78
+ question x judge x prompt x run 5% everything else, including one-off noise in single scores
79
+ prompt 3% some prompt wordings score higher across the board
80
+ question x judge x prompt 2% an interaction between question, judge and prompt
81
+ question x prompt x run 1% answers change from run to run
82
+ By judge: judge-a 0.731, judge-b 0.633, judge-c 0.775
83
+ ```
84
+
85
+ Two thirds of the uncertainty is which judge you happened to pick: judge-b passes 63% of answers, judge-c passes 78%. Adding questions will not fix that.
86
+
87
+ Everything in the report is also available as attributes: `result.mean`, `result.se`, `result.naive_se`, `result.interval()`, and `result.components`, each with its `variance` and `share`.
88
+
89
+ ### 3. What would another design give?
90
+
91
+ ```python
92
+ print(result.project(items=500, judges=1))
93
+ # 500 questions x 1 judge x 3 prompts x 2 runs: the score would be good to about +-0.142 (95%).
94
+ print(result.project(items=200, judges=3, prompts=3, runs=1))
95
+ # 200 questions x 3 judges x 3 prompts x 1 run: the score would be good to about +-0.0884 (95%).
96
+ ```
97
+
98
+ 500 questions with one judge are no better than the pilot. 200 questions with three judges and a single run cost less and do much better.
99
+
100
+ ### 4. Where should the next calls go?
101
+
102
+ Give the price of a call to the model you are testing and a call to the judge, and either a budget or a target:
103
+
104
+ ```python
105
+ print(result.plan(budget=40, generation_cost=0.01, judge_cost=0.004, limits={"judge": 3}))
106
+ ```
107
+
108
+ ```text
109
+ Best design for $40.00: 454 questions, 3 judges, 4 prompts, 1 run.
110
+ 1,816 model calls and 5,448 judge calls, costing $39.95.
111
+ The score would be good to about +-0.0843 (95%).
112
+ On the same basis, the pilot's own design ($7.92 at these prices) gives about +-0.0985.
113
+ ```
114
+
115
+ ```python
116
+ print(result.plan(target=0.1, generation_cost=0.01, judge_cost=0.004, limits={"judge": 3}))
117
+ ```
118
+
119
+ ```text
120
+ Best design for +-0.1: 140 questions, 3 judges, 1 prompt, 1 run.
121
+ 140 model calls and 420 judge calls, costing $3.08.
122
+ The score would be good to about +-0.0999 (95%).
123
+ On the same basis, the pilot's own design ($7.92 at these prices) gives about +-0.0985.
124
+ Careful: the pilot had only 3 prompts, so how much prompts differ is a rough estimate; dropping to 1 is a bet that it stays small.
125
+ ```
126
+
127
+ `limits` caps how many judges, wordings or runs to consider (only three judges are available here); `max_items` caps the questions when your eval set has a fixed size.
128
+
129
+ ### 5. Comparing two models or prompts
130
+
131
+ Score both on the same questions, judges, wordings and runs, with a column naming the model:
132
+
133
+ ```python
134
+ two = en.read_scores("examples/two_models.csv")
135
+ print(two.compare().report()) # or compare(new="candidate", old="baseline")
136
+ ```
137
+
138
+ ```text
139
+ Difference, candidate minus baseline: +0.18 from 80 questions x 4 judges x 2 runs (640 paired scores).
140
+ Interval counting every source of noise (95%): -0.118 to 0.477 (+-0.297).
141
+ The usual interval, counting only which questions were picked: 0.0472 to 0.312 (+-0.132).
142
+ The usual interval would call this a real difference, but with the judge, wording and run noise counted it could be noise.
143
+ The noise estimate itself rests on few judges, wordings or runs (about 5 degrees of freedom), which widens the interval. More of them in the pilot would firm it up.
144
+ Where the noise in this difference comes from:
145
+ judge 68% judges disagree about which version is better overall (for example, a judge that favours one model's style)
146
+ question 17% the new version is better on some questions and worse on others
147
+ question x run 8% answers change from run to run
148
+ question x judge x run 6% everything else, including one-off noise in single scores
149
+ By judge: judge-a +0.0438, judge-b +0.0688, judge-c +0.481, judge-d +0.125
150
+ ```
151
+
152
+ Most of the "win" comes from judge-c. Three of the four judges see a gap of 0.04 to 0.13 points on a 1 to 5 scale.
153
+
154
+ Comparisons are kinder than single scores: a judge that is strict with both models cancels out. What is left is noise in the gap itself, like a judge that prefers one model's style. Runs are paired by their label, so run 1 of one model is compared with run 1 of the other.
155
+
156
+ ### Judges you will always use
157
+
158
+ The analysis treats judges and wordings as a sample from all the ones you might have used, so the score generalizes beyond them. If you will always grade with exactly these judges, mark them fixed and their own leniency becomes part of what you are measuring:
159
+
160
+ ```python
161
+ print(pilot.study(fixed=["judge"]).report())
162
+ # Interval counting every source of noise (95%): 0.653 to 0.773 (+-0.0598).
163
+ ```
164
+
165
+ ### Runs are nested
166
+
167
+ Run 1 of one question has nothing to do with run 1 of another, so runs are analysed as nested within each question and wording. Columns named run, seed, sample, attempt, trial and similar are treated this way. If run 1 really is the same for every question (all run-1 answers came from one batch on one day), pass `nested=[]`.
168
+
169
+ ### Rows in memory
170
+
171
+ ```python
172
+ rows = [
173
+ {"item": "q1", "judge": "a", "prompt": "p1", "score": 1},
174
+ {"item": "q1", "judge": "b", "prompt": "p1", "score": 0},
175
+ ...
176
+ ]
177
+ result = en.study(rows) # every column other than item and score is a facet
178
+ result = en.study(df, item="question", score="correct") # a pandas DataFrame works too
179
+ diff = en.compare(rows, system="model", new="v2", old="v1")
180
+ ```
181
+
182
+ ### Reading your files
183
+
184
+ `read_scores` reads CSV, JSON (a list of objects) and JSON Lines, with one row per score. It finds the question, score and system columns by common names (`question_id`, `item`, `id`; `score`, `verdict`, `passed`, `correct`; `model`, `system`, `version`) and treats columns named like judges, graders, prompts, templates, runs and seeds as facets. Nested JSON fields are named with dots, like `metadata.judge`. Columns that look like text or ids are left out and reported. Check `table.report()` the first time, and name columns yourself when the guess is wrong:
185
+
186
+ ```python
187
+ table = en.read_scores("results.jsonl", item="example_id", score="judge_score", facets=["grader", "seed"])
188
+ ```
189
+
190
+ Questions with any missing score are left out of the analysis and listed in the report.
191
+
192
+ ## Command line
193
+
194
+ ```bash
195
+ evalnoise check examples/pilot.csv
196
+ evalnoise check examples/two_models.csv # two systems in the file: compares them
197
+ evalnoise check examples/two_models.csv --only baseline # or analyse one of them
198
+ evalnoise check results.jsonl --fixed judge -v
199
+ evalnoise project examples/pilot.csv items=500 judges=1
200
+ evalnoise plan examples/pilot.csv --budget 40 --model-cost 0.01 --judge-cost 0.004 --limit judge=3
201
+ evalnoise plan examples/pilot.csv --target 0.05 --model-cost 0.01 --judge-cost 0.004 --max-items 800
202
+ ```
203
+
204
+ Use `--item`, `--score`, `--facets`, `--system`, `--new` and `--old` when the automatic guess is wrong, and `-v` to see which columns were used.
205
+
206
+ ## How the numbers are checked
207
+
208
+ The test suite simulates evals from a known model, where questions, judges, wordings and runs each add their own noise, and checks each promise:
209
+
210
+ | Promise | Held in simulation |
211
+ |---|---|
212
+ | The interval covers the true score (95% promised) | 94% (usual interval: 74%) |
213
+ | The same for pass/fail verdicts | 96% |
214
+ | Comparing equal models with leaning judges: false "real difference" (5% promised) | 4.9% (usual interval: 37%) |
215
+ | Comparing models 6 points apart: finds the difference | 96% |
216
+ | With no judge or wording noise, the interval is no wider than needed | 95% coverage, about the same width as the usual interval |
217
+ | The judge spread is estimated without bias | yes, on average |
218
+ | A projection predicts the spread of scores at the new design | within 10% |
219
+ | The plan from a pilot beats spending the same money on the pilot's design | yes |
220
+
221
+ The two-way analysis is also checked against the textbook formulas, and run labels can be shuffled within each question without changing anything. See [`tests/test_simulation.py`](https://github.com/ai-with-md/evalnoise/blob/main/tests/test_simulation.py).
222
+
223
+ ## How it compares
224
+
225
+ - **Error bars grouped by question**, as in [Adding Error Bars to Evals](https://arxiv.org/abs/2411.00640) and Inspect's clustered standard errors, are what evalnoise calls the usual interval. They are right when the judge and wording are fixed for good. evalnoise adds the noise from judges, wordings and reruns, and tells you which of them matters.
226
+ - **[Total Evaluation Error](https://arxiv.org/abs/2604.11581)** (Messing, NeurIPS 2026) makes the same argument for LLM pipelines and finds usual error bars 53 to 70% too narrow. evalnoise is a small Python implementation of the idea for balanced pilots.
227
+ - **[aiEvalR](https://cran.r-project.org/package=aiEvalR)** brings generalizability theory to AI evaluation in R. evalnoise does it in Python with no dependencies, plus budget planning with API prices.
228
+ - **Judge bias correction** with human labels ([judgy](https://github.com/ai-evals-course/judgy), [evalstats](https://github.com/ianarawjo/evalstats)) answers a different question: whether the judge agrees with people. evalnoise asks how much the score depends on which judge, wording and run you used.
229
+ - **[evalsize](https://github.com/ai-with-md/evalsize)** tells you how many questions you need for a pass rate. evalnoise tells you whether more questions are even the right fix.
230
+
231
+ ## Limits
232
+
233
+ - **Balanced pilots only.** Every question must be scored under every combination of judges, wordings and runs. Questions with gaps are left out. Unbalanced designs need a mixed model fitted by REML, which this package does not do.
234
+ - **Few judges or wordings give rough answers.** The spread between judges is estimated from the judges you used. With 2, it is close to a guess: in simulation, when judges differ a lot, the interval covered 86% with 2 judges and 91% with 4, against 95% promised. The interval widens to reflect this, and the report says so, but use 3 or more of each when you can.
235
+ - **The plan treats the pilot's estimates as exact.** When a pilot had only a few judges or wordings, a plan that drops to fewer of them comes with a warning.
236
+ - **Judges and wordings are treated as a sample** from all the ones you might have used, unless you mark them fixed. Decide which one you mean: "this model with this judge" or "this model, judged fairly".
237
+ - **Effects are assumed to add up.** For pass/fail verdicts this works well for the average and the interval (see the simulations above), but the components describe pass rates, not anything deeper.
238
+ - **Projections and plans** use a normal approximation for the interval, so they are a little optimistic for pilots with very few judges or wordings.
239
+
240
+ ## Methods
241
+
242
+ evalnoise runs a generalizability study (G-study) and a decision study (D-study), the standard tools of generalizability theory:
243
+
244
+ - Variance components come from the expected mean squares of a balanced random-effects analysis of variance, with questions crossed with every facet and runs nested within question and wording. Negative estimates are set to zero.
245
+ - The error variance of the overall score adds each component divided by the number of levels it is averaged over. Components of fixed facets alone drop out.
246
+ - The interval uses a t distribution with Satterthwaite degrees of freedom, so it widens when the estimate rests on few judges or wordings.
247
+ - Projections and plans reuse the same formula at other sample sizes. Plans search every allowed combination of judges, wordings and runs and pick the number of questions that fits the budget or the target.
248
+
249
+ References: Cronbach, Gleser, Nanda and Rajaratnam, *The Dependability of Behavioral Measurements* (1972); Brennan, *Generalizability Theory* (2001); Satterthwaite, "An approximate distribution of estimates of variance components" (1946); Messing, "Hidden Measurement Error in LLM Pipelines" ([arXiv 2604.11581](https://arxiv.org/abs/2604.11581)).
250
+
251
+ ## More tools
252
+
253
+ Small, free tools for building and checking AI systems:
254
+
255
+ - **[evalsize](https://github.com/ai-with-md/evalsize)**: how many eval examples you need before a difference between prompts or models means anything.
256
+ - **[costplan](https://github.com/ai-with-md/costplan)**: what an AI agent will cost and what budget cap to set, from a handful of pilot runs.
257
+ - **[sturdytools](https://github.com/ai-with-md/sturdytools)**: retries, timeouts, rate limits and circuit breakers for the tools an AI agent calls.
258
+
259
+ ## Development
260
+
261
+ ```bash
262
+ python -m venv .venv
263
+ source .venv/bin/activate
264
+ pip install -e ".[dev]"
265
+ pytest
266
+ ```
267
+
268
+ The calculator is a [marimo](https://marimo.io) notebook in [`app/calculator.py`](https://github.com/ai-with-md/evalnoise/blob/main/app/calculator.py), published to GitHub Pages as a WebAssembly app on every push to `main`.
269
+
270
+ ## License
271
+
272
+ MIT
@@ -0,0 +1,37 @@
1
+ [build-system]
2
+ requires = ["setuptools>=77"]
3
+ build-backend = "setuptools.build_meta"
4
+
5
+ [project]
6
+ name = "evalnoise"
7
+ version = "0.1.0"
8
+ description = "Where does the noise in an eval score come from? Split it across questions, judges, prompt wordings and runs, and plan where to spend the next API calls"
9
+ readme = "README.md"
10
+ license = "MIT"
11
+ authors = [{name = "Meenal Asesh"}]
12
+ requires-python = ">=3.9"
13
+ dependencies = []
14
+ keywords = ["llm", "evaluation", "evals", "llm-as-a-judge", "error bars", "generalizability theory", "variance components", "measurement error", "statistics"]
15
+ classifiers = [
16
+ "Programming Language :: Python :: 3",
17
+ "Operating System :: OS Independent",
18
+ "Topic :: Scientific/Engineering :: Artificial Intelligence",
19
+ ]
20
+
21
+ [project.optional-dependencies]
22
+ dev = ["pytest"]
23
+
24
+ [project.scripts]
25
+ evalnoise = "evalnoise.cli:main"
26
+
27
+ [project.urls]
28
+ Homepage = "https://github.com/ai-with-md/evalnoise"
29
+ Calculator = "https://ai-with-md.github.io/evalnoise/"
30
+ Issues = "https://github.com/ai-with-md/evalnoise/issues"
31
+
32
+ [tool.setuptools.packages.find]
33
+ where = ["src"]
34
+
35
+ [tool.pytest.ini_options]
36
+ testpaths = ["tests", "src"]
37
+ addopts = "--doctest-modules"
@@ -0,0 +1,4 @@
1
+ [egg_info]
2
+ tag_build =
3
+ tag_date = 0
4
+
@@ -0,0 +1,8 @@
1
+ """evalnoise: where does the noise in an eval score come from, and where should the next calls go?"""
2
+
3
+ from .files import ScoreTable, read_scores
4
+ from .study import Component, Plan, Projection, Study, compare, study, to_score
5
+
6
+ __version__ = "0.1.0"
7
+
8
+ __all__ = ["Component", "Plan", "Projection", "Study", "ScoreTable", "compare", "read_scores", "study", "to_score", "__version__"]