raterkit 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (35) hide show
  1. raterkit-0.1.0/.github/FUNDING.yml +1 -0
  2. raterkit-0.1.0/.github/workflows/ci.yml +47 -0
  3. raterkit-0.1.0/.github/workflows/release.yml +51 -0
  4. raterkit-0.1.0/CITATION.cff +11 -0
  5. raterkit-0.1.0/LICENSE +21 -0
  6. raterkit-0.1.0/Makefile +27 -0
  7. raterkit-0.1.0/PKG-INFO +269 -0
  8. raterkit-0.1.0/README.md +239 -0
  9. raterkit-0.1.0/examples/goemotions/audit_goemotions.py +329 -0
  10. raterkit-0.1.0/examples/goemotions/fetch_data.py +32 -0
  11. raterkit-0.1.0/examples/goemotions/results/figures/label_alpha.png +0 -0
  12. raterkit-0.1.0/examples/goemotions/results/goemotions.json +1014 -0
  13. raterkit-0.1.0/examples/goemotions/results/goemotions.md +15 -0
  14. raterkit-0.1.0/pyproject.toml +69 -0
  15. raterkit-0.1.0/requirements-results.txt +6 -0
  16. raterkit-0.1.0/results/demo.json +949 -0
  17. raterkit-0.1.0/results/figures/depth_curve.png +0 -0
  18. raterkit-0.1.0/results/figures/rater_forest.png +0 -0
  19. raterkit-0.1.0/results/report.md +242 -0
  20. raterkit-0.1.0/results/table.md +10 -0
  21. raterkit-0.1.0/src/raterkit/__init__.py +21 -0
  22. raterkit-0.1.0/src/raterkit/cli.py +74 -0
  23. raterkit-0.1.0/src/raterkit/dataset.py +117 -0
  24. raterkit-0.1.0/src/raterkit/demo.py +107 -0
  25. raterkit-0.1.0/src/raterkit/probes.py +398 -0
  26. raterkit-0.1.0/src/raterkit/py.typed +0 -0
  27. raterkit-0.1.0/src/raterkit/report.py +225 -0
  28. raterkit-0.1.0/src/raterkit/result.py +84 -0
  29. raterkit-0.1.0/src/raterkit/schema.py +104 -0
  30. raterkit-0.1.0/src/raterkit/stats.py +330 -0
  31. raterkit-0.1.0/src/raterkit/synthetic.py +118 -0
  32. raterkit-0.1.0/tests/test_cli.py +85 -0
  33. raterkit-0.1.0/tests/test_probes.py +187 -0
  34. raterkit-0.1.0/tests/test_schema_result.py +78 -0
  35. raterkit-0.1.0/tests/test_stats.py +152 -0
@@ -0,0 +1 @@
1
+ github: [mohammadi-hadi]
@@ -0,0 +1,47 @@
1
+ name: ci
2
+
3
+ on:
4
+ push:
5
+ branches: [main]
6
+ pull_request:
7
+
8
+ jobs:
9
+ test:
10
+ runs-on: ubuntu-latest
11
+ strategy:
12
+ matrix:
13
+ python-version: ["3.11", "3.12", "3.13"]
14
+ steps:
15
+ - uses: actions/checkout@v4
16
+ - uses: actions/setup-python@v5
17
+ with:
18
+ python-version: ${{ matrix.python-version }}
19
+ - name: Install
20
+ run: pip install -e ".[dev]"
21
+ - name: Lint
22
+ run: |
23
+ ruff check src tests
24
+ ruff format --check src tests
25
+ - name: Type check
26
+ if: matrix.python-version == '3.12'
27
+ run: mypy src
28
+ - name: Test
29
+ run: pytest -q
30
+
31
+ results-drift:
32
+ runs-on: ubuntu-latest
33
+ steps:
34
+ - uses: actions/checkout@v4
35
+ - uses: actions/setup-python@v5
36
+ with:
37
+ python-version: "3.12"
38
+ - name: Install pinned results environment
39
+ run: |
40
+ pip install -r requirements-results.txt
41
+ pip install -e . --no-deps
42
+ - name: Regenerate demo results and README table
43
+ run: |
44
+ python -m raterkit.cli demo --out results
45
+ python -m raterkit.cli inject-readme README.md --results results
46
+ - name: Fail if committed results drift
47
+ run: git diff --exit-code -- results/report.md results/table.md results/demo.json README.md
@@ -0,0 +1,51 @@
1
+ name: release
2
+
3
+ on:
4
+ push:
5
+ tags: ["v*"]
6
+
7
+ jobs:
8
+ build:
9
+ runs-on: ubuntu-latest
10
+ steps:
11
+ - uses: actions/checkout@v4
12
+ - uses: actions/setup-python@v5
13
+ with:
14
+ python-version: "3.12"
15
+ - name: Build distributions
16
+ run: |
17
+ pip install build twine
18
+ python -m build
19
+ twine check dist/*
20
+ - uses: actions/upload-artifact@v4
21
+ with:
22
+ name: dist
23
+ path: dist/
24
+
25
+ publish:
26
+ needs: build
27
+ runs-on: ubuntu-latest
28
+ environment: pypi
29
+ permissions:
30
+ id-token: write
31
+ steps:
32
+ - uses: actions/download-artifact@v4
33
+ with:
34
+ name: dist
35
+ path: dist/
36
+ - uses: pypa/gh-action-pypi-publish@release/v1
37
+
38
+ github-release:
39
+ needs: publish
40
+ runs-on: ubuntu-latest
41
+ permissions:
42
+ contents: write
43
+ steps:
44
+ - uses: actions/download-artifact@v4
45
+ with:
46
+ name: dist
47
+ path: dist/
48
+ - uses: softprops/action-gh-release@v2
49
+ with:
50
+ files: dist/*
51
+ generate_release_notes: true
@@ -0,0 +1,11 @@
1
+ cff-version: 1.2.0
2
+ message: "If you use raterkit, please cite it as below."
3
+ title: "raterkit: audit a labeled dataset before you trust it"
4
+ type: software
5
+ authors:
6
+ - family-names: Mohammadi
7
+ given-names: Hadi
8
+ repository-code: "https://github.com/mohammadi-hadi/raterkit"
9
+ license: MIT
10
+ version: 0.1.0
11
+ date-released: 2026-08-05
raterkit-0.1.0/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Hadi Mohammadi
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,27 @@
1
+ PY ?= .venv/bin/python
2
+
3
+ .PHONY: demo test lint typecheck goemotions build clean
4
+
5
+ demo:
6
+ $(PY) -m raterkit.cli demo --out results
7
+ $(PY) -m raterkit.cli inject-readme README.md --results results
8
+
9
+ test:
10
+ $(PY) -m pytest -q
11
+
12
+ lint:
13
+ $(PY) -m ruff check src tests
14
+ $(PY) -m ruff format --check src tests
15
+
16
+ typecheck:
17
+ $(PY) -m mypy src
18
+
19
+ goemotions:
20
+ $(PY) examples/goemotions/fetch_data.py
21
+ $(PY) examples/goemotions/audit_goemotions.py
22
+
23
+ build:
24
+ $(PY) -m build
25
+
26
+ clean:
27
+ rm -rf dist build .pytest_cache .mypy_cache .ruff_cache
@@ -0,0 +1,269 @@
1
+ Metadata-Version: 2.4
2
+ Name: raterkit
3
+ Version: 0.1.0
4
+ Summary: Audit a labeled dataset before you trust it
5
+ Project-URL: Repository, https://github.com/mohammadi-hadi/raterkit
6
+ Project-URL: Issues, https://github.com/mohammadi-hadi/raterkit/issues
7
+ Author: Hadi Mohammadi
8
+ License-Expression: MIT
9
+ License-File: LICENSE
10
+ Keywords: annotation,data-quality,evaluation,inter-rater-reliability,krippendorff
11
+ Classifier: Development Status :: 4 - Beta
12
+ Classifier: Intended Audience :: Science/Research
13
+ Classifier: Programming Language :: Python :: 3.11
14
+ Classifier: Programming Language :: Python :: 3.12
15
+ Classifier: Programming Language :: Python :: 3.13
16
+ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
17
+ Requires-Python: >=3.11
18
+ Requires-Dist: matplotlib>=3.8
19
+ Requires-Dist: numpy>=1.26
20
+ Requires-Dist: pydantic>=2.5
21
+ Provides-Extra: dev
22
+ Requires-Dist: build>=1.2; extra == 'dev'
23
+ Requires-Dist: krippendorff>=0.6; extra == 'dev'
24
+ Requires-Dist: mypy>=1.11; extra == 'dev'
25
+ Requires-Dist: pytest>=8; extra == 'dev'
26
+ Requires-Dist: ruff>=0.6; extra == 'dev'
27
+ Requires-Dist: scipy>=1.11; extra == 'dev'
28
+ Requires-Dist: statsmodels>=0.14; extra == 'dev'
29
+ Description-Content-Type: text/markdown
30
+
31
+ # raterkit
32
+
33
+ Audit a labeled dataset before you trust it.
34
+
35
+ [![ci](https://github.com/mohammadi-hadi/raterkit/actions/workflows/ci.yml/badge.svg)](https://github.com/mohammadi-hadi/raterkit/actions/workflows/ci.yml)
36
+
37
+ Every benchmark score, every fine-tune, every "our judge agrees with humans
38
+ 92% of the time" bottoms out in a pile of human labels — produced by raters
39
+ who may have been careless, biased, tired by the end of the queue, or graded
40
+ against gold that was quietly wrong. None of that is visible in accuracy
41
+ numbers computed on top. raterkit reads the raw ratings and measures each of
42
+ these failure modes directly, with a confidence interval on every number and
43
+ a flag only when the evidence clears a stated bar.
44
+
45
+ ## The audit in one table
46
+
47
+ Eight synthetic annotation projects, seven implanted defects. Bold values are
48
+ flags; each lands on the row where its defect was implanted, and the clean
49
+ row carries none.
50
+
51
+ <!-- raterkit:demo -->
52
+ | project | implanted defect | alpha | rogue | biased | drift gap | leaks | stale gold | flags |
53
+ |---|---|---|---|---|---|---|---|---|
54
+ | clean | nothing | 0.80 | 0 | 0 | +0.00 | 0 | 0.0% | none |
55
+ | careless-crowd | every rater at 55% accuracy | **0.18** | 0 | 0 | -0.02 | 0 | 0.7% | reliability |
56
+ | one-spammer | one rater answers uniformly at random | 0.72 | **1** | 0 | +0.05 | 0 | 0.0% | rogue raters |
57
+ | leaning-rater | one rater always takes the same side of ambiguous items | 0.65 | 0 | **1** | +0.05 | 0 | - | rater bias |
58
+ | tiring-crowd | competence decays 0.97 to 0.73 over the queue | 0.64 | 0 | 0 | **+0.25** | 0 | 0.0% | drift |
59
+ | leaky-split | 12 test items duplicate a train item's text | 0.80 | 0 | 0 | +0.00 | **12** | 0.0% | split leakage |
60
+ | stale-gold | 8% of gold labels are wrong | 0.80 | 0 | 0 | +0.00 | 0 | **6.7%** | stale gold |
61
+ | easy-but-skewed | 94% of items are one class; raters 90% accurate | **0.34** | 0 | 0 | +0.12 | 0 | 0.0% | reliability |
62
+ <!-- /raterkit:demo -->
63
+
64
+ The row to sit with is `easy-but-skewed`: raters are 90% accurate and agree
65
+ with each other 81% of the time, yet alpha is 0.34 — far below the 0.667
66
+ floor. With 94% of items in one class, raters agree that often by accident;
67
+ the skew, not the raters, is doing the agreeing. Raw agreement percentages
68
+ cannot be compared across datasets with different label distributions, which
69
+ is why every agreement number here is chance-corrected.
70
+
71
+ ![rater forest](results/figures/rater_forest.png)
72
+
73
+ ![depth curve](results/figures/depth_curve.png)
74
+
75
+ `make demo` regenerates the table, the [full report](results/report.md) and
76
+ the figures from fixed seeds; CI rebuilds them from pinned dependencies and
77
+ fails if a committed number differs from what the code produces.
78
+
79
+ ## 211,225 real ratings, audited
80
+
81
+ [`examples/goemotions/`](examples/goemotions/) audits the raw crowd ratings
82
+ behind [GoEmotions](https://aclanthology.org/2020.acl-main.372/) — 211,225
83
+ ratings by 82 raters over 58,011 Reddit comments, 28 binary emotion labels
84
+ each. Committed results:
85
+ [examples/goemotions/results/goemotions.md](examples/goemotions/results/goemotions.md).
86
+
87
+ - **27 of 28 emotions sit below the 0.667 reliability floor** — the whole
88
+ 95% CI, not just the point estimate. Only *gratitude* (alpha 0.72) passes;
89
+ the median emotion manages 0.23, and *neutral* itself scores 0.26.
90
+ - **Rater quality spans 0.11 to 0.36** (rater-vs-rest alpha across the 67
91
+ raters with 300+ ratings) — the noisiest raters agree with their co-raters
92
+ at a third of the rate the best ones do, and 20 of 67 sit below 0.20.
93
+ - **The rater count was never enough for the task.** By Spearman-Brown, the
94
+ observed reliability would need ~13 raters per item to yield 0.8-reliable
95
+ aggregated labels; the dataset averages 3.6, which projects to 0.52.
96
+ - 86% of multi-rated comments are contested on at least one emotion, yet
97
+ raters marked only 1.6% of ratings "very unclear" — disagreement without
98
+ awareness of disagreement.
99
+
100
+ None of this says GoEmotions is a scandal — fine-grained emotion is
101
+ genuinely hard, and the released splits aggregate and filter these raw
102
+ ratings. It says the raw layer under a famous benchmark fails the standard
103
+ reliability bar almost everywhere, measurably and reproducibly.
104
+
105
+ ## Install
106
+
107
+ ```
108
+ pip install git+https://github.com/mohammadi-hadi/raterkit
109
+ ```
110
+
111
+ Python 3.11+. Runtime dependencies: numpy, pydantic, matplotlib.
112
+
113
+ ## Quickstart
114
+
115
+ Log one JSON object per rating — who labeled what, and optionally when, in
116
+ which split, and the item text:
117
+
118
+ ```json
119
+ {"item_id": "c41", "rater_id": "r07", "label": "toxic", "t": 1718040000, "split": "train", "text": "..."}
120
+ ```
121
+
122
+ ```python
123
+ from raterkit import load_ratings, load_gold, run_audit
124
+
125
+ audit = run_audit(load_ratings("ratings.jsonl"), gold=load_gold("gold.jsonl"))
126
+ for flag in audit.flags:
127
+ print(flag.name, "--", flag.detail)
128
+ ```
129
+
130
+ Or from the shell, with an exit code CI can gate on:
131
+
132
+ ```
133
+ raterkit report ratings.jsonl --gold gold.jsonl --out audit --fail-on-flags
134
+ ```
135
+
136
+ An optional config sets the measurement level (`nominal`, `ordinal`,
137
+ `interval`), the label order, and the target reliability. Alongside
138
+ `report.md` the audit writes `report.json` for pipelines that gate on
139
+ specific numbers. Any check that could not run is listed with the exact
140
+ fields to log to enable it — a skip describes the log file, not the raters.
141
+
142
+ An LLM judge is just another rater: give its verdicts a `rater_id` and the
143
+ same audit reports how it agrees with the humans, whether its label
144
+ distribution diverges, and whether it drifts — no special casing needed.
145
+
146
+ ## What it checks
147
+
148
+ | check | question it answers | needs |
149
+ |---|---|---|
150
+ | reliability | is agreement above the chance-corrected floor? | 2+ ratings on some items |
151
+ | rogue raters | does any rater agree with the rest credibly less than the pool? | 30+ pairs per rater |
152
+ | rater bias | does any rater use the labels differently (marginal shift)? | 30+ ratings per rater |
153
+ | drift | did agreement change between early and late work? | `t` per rating |
154
+ | split leakage | does any duplicated text span train/test? | `text` + `split` |
155
+ | stale gold | which gold labels does a unanimous crowd contradict? | a gold file |
156
+ | ambiguity | how many items are genuinely contested? | 2+ ratings on some items |
157
+ | prevalence | is raw agreement inflated by class skew? | always runs |
158
+ | rating depth | how many raters per item does your target reliability need? | always runs |
159
+
160
+ The last three are descriptive and never flag: they explain the numbers
161
+ above them.
162
+
163
+ ## How the numbers are defended
164
+
165
+ - **Cross-checked implementations.** Krippendorff's alpha (nominal, ordinal
166
+ and interval, with missing data) matches the reference `krippendorff`
167
+ package to 1e-10; Cohen's and Fleiss' kappa match statsmodels; the
168
+ chi-square survival function matches scipy to 1e-10; Benjamini-Hochberg
169
+ reproduces statsmodels decisions exactly. The library itself depends on
170
+ none of them.
171
+ - **Validation by implantation.** The synthetic projects have defect dials —
172
+ a spammer, a leaning rater, a competence drift, a gold error rate — and
173
+ the tests require each probe to recover its dial, stay monotone in it, and
174
+ stay silent on the clean project.
175
+ - **Uncertainty on every number.** Band probes trigger only when the whole
176
+ 95% bootstrap CI leaves the innocent band; the bootstrap resamples items
177
+ (never single ratings), so dependence within an item is respected.
178
+ - **Drift-checked results.** The demo table, report and README numbers are
179
+ regenerated by CI from pinned dependencies and diffed against the
180
+ committed copy.
181
+
182
+ ## Design notes
183
+
184
+ - **Chance correction is not optional.** The `easy-but-skewed` demo row and
185
+ the kappa-paradox literature both show raw agreement rewarding class skew.
186
+ All agreement numbers are Krippendorff's alpha, which handles missing
187
+ ratings, any number of raters, and nominal/ordinal/interval labels in one
188
+ framework — and reduces to the familiar coefficients where they overlap.
189
+ - **Innocent bands, not point nulls.** With enough data any rater's marginal
190
+ "significantly" differs from the pool's while meaning nothing. Band probes
191
+ state the interval a healthy project could occupy (bias within 0.10 total
192
+ variation, drift within ±0.10 alpha) and flag only when the whole CI
193
+ leaves it. Exact procedures use conventions: the 0.667 floor, zero
194
+ tolerance for cross-split duplicates. All constants live in one place
195
+ ([`probes.py`](src/raterkit/probes.py)) and are easy to disagree with.
196
+ - **Raters are compared to the pool, not to a standard.** A rater is rogue
197
+ when their CI sits below the pool's CI — a claim about this project, not
198
+ about an absolute bar their domain may not share.
199
+ - **Ambiguity is measured, not punished.** Contested items are reported
200
+ descriptively: persistent disagreement can be signal about the task, not
201
+ noise from the raters, and deserves a number rather than a flag.
202
+ - **Deterministic to the digit.** Seeded PCG64 everywhere, no BLAS in any
203
+ statistic's path: the same ratings file gives the same report on any
204
+ machine.
205
+
206
+ ## Limitations
207
+
208
+ - An audit is bounded by what was logged. No timestamps, no drift check; no
209
+ text or splits, no leakage check; no gold, no gold check.
210
+ - The innocent bands and the 0.667 floor are conventions, not derivations;
211
+ fields differ, and the constants are one file away.
212
+ - Spearman-Brown projections assume added raters behave like existing ones —
213
+ an optimistic bound, stated as such in the report.
214
+ - The rogue-rater comparison needs overlap; raters who never co-rate an item
215
+ with others cannot be tested and are counted as untested, not innocent.
216
+ - Implantation shows the probes detect the mechanisms simulated; real
217
+ annotation projects can fail in ways not simulated here.
218
+ - The GoEmotions audit measures the raw rating layer; the released benchmark
219
+ splits aggregate and filter it, so per-label alpha here is not a direct
220
+ statement about scores computed on those splits.
221
+
222
+ ## Related work
223
+
224
+ The failure modes measured here are the standard ones from the agreement and
225
+ annotation-quality literature: agreement coefficients and their proper use
226
+ ([Artstein & Poesio, 2008](https://doi.org/10.1162/coli.07-034-R2);
227
+ Krippendorff, *Content Analysis*), interpretation benchmarks
228
+ ([Landis & Koch, 1977](https://doi.org/10.2307/2529310)), the kappa paradox
229
+ under skewed prevalence
230
+ ([Feinstein & Cicchetti, 1990](https://doi.org/10.1016/0895-4356(90)90158-L)),
231
+ crowd label quality ([Snow et al., 2008](https://aclanthology.org/D08-1027/)),
232
+ learning rater competence
233
+ ([Raykar et al., 2010](https://jmlr.org/papers/v11/raykar10a.html)),
234
+ annotation error detection
235
+ ([Klie et al., 2023](https://doi.org/10.1162/coli_a_00464)), and human label
236
+ variation as signal
237
+ ([Plank, 2022](https://aclanthology.org/2022.emnlp-main.731/)). The case
238
+ study audits the GoEmotions raw ratings
239
+ ([Demszky et al., 2020](https://aclanthology.org/2020.acl-main.372/)).
240
+ raterkit's contribution is packaging the checks as one auditable tool with
241
+ uncertainty on every number and validation by implantation.
242
+
243
+ Companion projects: [judgekit](https://github.com/mohammadi-hadi/judgekit)
244
+ audits LLM judges the way this audits human raters;
245
+ [judgepanel](https://github.com/mohammadi-hadi/judgepanel) estimates how
246
+ accurate a panel of judges is when no gold labels exist at all;
247
+ [abkit](https://github.com/mohammadi-hadi/abkit) applies the same
248
+ audit-before-you-trust pattern to A/B-test readouts;
249
+ [trajectory-judge](https://github.com/mohammadi-hadi/trajectory-judge)
250
+ measures what outcome-only judges miss on agent trajectories.
251
+
252
+ ## Citation
253
+
254
+ If raterkit is useful in your work, please cite it (see
255
+ [CITATION.cff](CITATION.cff)):
256
+
257
+ ```bibtex
258
+ @software{mohammadi_raterkit,
259
+ author = {Mohammadi, Hadi},
260
+ title = {raterkit: audit a labeled dataset before you trust it},
261
+ url = {https://github.com/mohammadi-hadi/raterkit},
262
+ version = {0.1.0},
263
+ year = {2026}
264
+ }
265
+ ```
266
+
267
+ ## License
268
+
269
+ MIT
@@ -0,0 +1,239 @@
1
+ # raterkit
2
+
3
+ Audit a labeled dataset before you trust it.
4
+
5
+ [![ci](https://github.com/mohammadi-hadi/raterkit/actions/workflows/ci.yml/badge.svg)](https://github.com/mohammadi-hadi/raterkit/actions/workflows/ci.yml)
6
+
7
+ Every benchmark score, every fine-tune, every "our judge agrees with humans
8
+ 92% of the time" bottoms out in a pile of human labels — produced by raters
9
+ who may have been careless, biased, tired by the end of the queue, or graded
10
+ against gold that was quietly wrong. None of that is visible in accuracy
11
+ numbers computed on top. raterkit reads the raw ratings and measures each of
12
+ these failure modes directly, with a confidence interval on every number and
13
+ a flag only when the evidence clears a stated bar.
14
+
15
+ ## The audit in one table
16
+
17
+ Eight synthetic annotation projects, seven implanted defects. Bold values are
18
+ flags; each lands on the row where its defect was implanted, and the clean
19
+ row carries none.
20
+
21
+ <!-- raterkit:demo -->
22
+ | project | implanted defect | alpha | rogue | biased | drift gap | leaks | stale gold | flags |
23
+ |---|---|---|---|---|---|---|---|---|
24
+ | clean | nothing | 0.80 | 0 | 0 | +0.00 | 0 | 0.0% | none |
25
+ | careless-crowd | every rater at 55% accuracy | **0.18** | 0 | 0 | -0.02 | 0 | 0.7% | reliability |
26
+ | one-spammer | one rater answers uniformly at random | 0.72 | **1** | 0 | +0.05 | 0 | 0.0% | rogue raters |
27
+ | leaning-rater | one rater always takes the same side of ambiguous items | 0.65 | 0 | **1** | +0.05 | 0 | - | rater bias |
28
+ | tiring-crowd | competence decays 0.97 to 0.73 over the queue | 0.64 | 0 | 0 | **+0.25** | 0 | 0.0% | drift |
29
+ | leaky-split | 12 test items duplicate a train item's text | 0.80 | 0 | 0 | +0.00 | **12** | 0.0% | split leakage |
30
+ | stale-gold | 8% of gold labels are wrong | 0.80 | 0 | 0 | +0.00 | 0 | **6.7%** | stale gold |
31
+ | easy-but-skewed | 94% of items are one class; raters 90% accurate | **0.34** | 0 | 0 | +0.12 | 0 | 0.0% | reliability |
32
+ <!-- /raterkit:demo -->
33
+
34
+ The row to sit with is `easy-but-skewed`: raters are 90% accurate and agree
35
+ with each other 81% of the time, yet alpha is 0.34 — far below the 0.667
36
+ floor. With 94% of items in one class, raters agree that often by accident;
37
+ the skew, not the raters, is doing the agreeing. Raw agreement percentages
38
+ cannot be compared across datasets with different label distributions, which
39
+ is why every agreement number here is chance-corrected.
40
+
41
+ ![rater forest](results/figures/rater_forest.png)
42
+
43
+ ![depth curve](results/figures/depth_curve.png)
44
+
45
+ `make demo` regenerates the table, the [full report](results/report.md) and
46
+ the figures from fixed seeds; CI rebuilds them from pinned dependencies and
47
+ fails if a committed number differs from what the code produces.
48
+
49
+ ## 211,225 real ratings, audited
50
+
51
+ [`examples/goemotions/`](examples/goemotions/) audits the raw crowd ratings
52
+ behind [GoEmotions](https://aclanthology.org/2020.acl-main.372/) — 211,225
53
+ ratings by 82 raters over 58,011 Reddit comments, 28 binary emotion labels
54
+ each. Committed results:
55
+ [examples/goemotions/results/goemotions.md](examples/goemotions/results/goemotions.md).
56
+
57
+ - **27 of 28 emotions sit below the 0.667 reliability floor** — the whole
58
+ 95% CI, not just the point estimate. Only *gratitude* (alpha 0.72) passes;
59
+ the median emotion manages 0.23, and *neutral* itself scores 0.26.
60
+ - **Rater quality spans 0.11 to 0.36** (rater-vs-rest alpha across the 67
61
+ raters with 300+ ratings) — the noisiest raters agree with their co-raters
62
+ at a third of the rate the best ones do, and 20 of 67 sit below 0.20.
63
+ - **The rater count was never enough for the task.** By Spearman-Brown, the
64
+ observed reliability would need ~13 raters per item to yield 0.8-reliable
65
+ aggregated labels; the dataset averages 3.6, which projects to 0.52.
66
+ - 86% of multi-rated comments are contested on at least one emotion, yet
67
+ raters marked only 1.6% of ratings "very unclear" — disagreement without
68
+ awareness of disagreement.
69
+
70
+ None of this says GoEmotions is a scandal — fine-grained emotion is
71
+ genuinely hard, and the released splits aggregate and filter these raw
72
+ ratings. It says the raw layer under a famous benchmark fails the standard
73
+ reliability bar almost everywhere, measurably and reproducibly.
74
+
75
+ ## Install
76
+
77
+ ```
78
+ pip install git+https://github.com/mohammadi-hadi/raterkit
79
+ ```
80
+
81
+ Python 3.11+. Runtime dependencies: numpy, pydantic, matplotlib.
82
+
83
+ ## Quickstart
84
+
85
+ Log one JSON object per rating — who labeled what, and optionally when, in
86
+ which split, and the item text:
87
+
88
+ ```json
89
+ {"item_id": "c41", "rater_id": "r07", "label": "toxic", "t": 1718040000, "split": "train", "text": "..."}
90
+ ```
91
+
92
+ ```python
93
+ from raterkit import load_ratings, load_gold, run_audit
94
+
95
+ audit = run_audit(load_ratings("ratings.jsonl"), gold=load_gold("gold.jsonl"))
96
+ for flag in audit.flags:
97
+ print(flag.name, "--", flag.detail)
98
+ ```
99
+
100
+ Or from the shell, with an exit code CI can gate on:
101
+
102
+ ```
103
+ raterkit report ratings.jsonl --gold gold.jsonl --out audit --fail-on-flags
104
+ ```
105
+
106
+ An optional config sets the measurement level (`nominal`, `ordinal`,
107
+ `interval`), the label order, and the target reliability. Alongside
108
+ `report.md` the audit writes `report.json` for pipelines that gate on
109
+ specific numbers. Any check that could not run is listed with the exact
110
+ fields to log to enable it — a skip describes the log file, not the raters.
111
+
112
+ An LLM judge is just another rater: give its verdicts a `rater_id` and the
113
+ same audit reports how it agrees with the humans, whether its label
114
+ distribution diverges, and whether it drifts — no special casing needed.
115
+
116
+ ## What it checks
117
+
118
+ | check | question it answers | needs |
119
+ |---|---|---|
120
+ | reliability | is agreement above the chance-corrected floor? | 2+ ratings on some items |
121
+ | rogue raters | does any rater agree with the rest credibly less than the pool? | 30+ pairs per rater |
122
+ | rater bias | does any rater use the labels differently (marginal shift)? | 30+ ratings per rater |
123
+ | drift | did agreement change between early and late work? | `t` per rating |
124
+ | split leakage | does any duplicated text span train/test? | `text` + `split` |
125
+ | stale gold | which gold labels does a unanimous crowd contradict? | a gold file |
126
+ | ambiguity | how many items are genuinely contested? | 2+ ratings on some items |
127
+ | prevalence | is raw agreement inflated by class skew? | always runs |
128
+ | rating depth | how many raters per item does your target reliability need? | always runs |
129
+
130
+ The last three are descriptive and never flag: they explain the numbers
131
+ above them.
132
+
133
+ ## How the numbers are defended
134
+
135
+ - **Cross-checked implementations.** Krippendorff's alpha (nominal, ordinal
136
+ and interval, with missing data) matches the reference `krippendorff`
137
+ package to 1e-10; Cohen's and Fleiss' kappa match statsmodels; the
138
+ chi-square survival function matches scipy to 1e-10; Benjamini-Hochberg
139
+ reproduces statsmodels decisions exactly. The library itself depends on
140
+ none of them.
141
+ - **Validation by implantation.** The synthetic projects have defect dials —
142
+ a spammer, a leaning rater, a competence drift, a gold error rate — and
143
+ the tests require each probe to recover its dial, stay monotone in it, and
144
+ stay silent on the clean project.
145
+ - **Uncertainty on every number.** Band probes trigger only when the whole
146
+ 95% bootstrap CI leaves the innocent band; the bootstrap resamples items
147
+ (never single ratings), so dependence within an item is respected.
148
+ - **Drift-checked results.** The demo table, report and README numbers are
149
+ regenerated by CI from pinned dependencies and diffed against the
150
+ committed copy.
151
+
152
+ ## Design notes
153
+
154
+ - **Chance correction is not optional.** The `easy-but-skewed` demo row and
155
+ the kappa-paradox literature both show raw agreement rewarding class skew.
156
+ All agreement numbers are Krippendorff's alpha, which handles missing
157
+ ratings, any number of raters, and nominal/ordinal/interval labels in one
158
+ framework — and reduces to the familiar coefficients where they overlap.
159
+ - **Innocent bands, not point nulls.** With enough data any rater's marginal
160
+ "significantly" differs from the pool's while meaning nothing. Band probes
161
+ state the interval a healthy project could occupy (bias within 0.10 total
162
+ variation, drift within ±0.10 alpha) and flag only when the whole CI
163
+ leaves it. Exact procedures use conventions: the 0.667 floor, zero
164
+ tolerance for cross-split duplicates. All constants live in one place
165
+ ([`probes.py`](src/raterkit/probes.py)) and are easy to disagree with.
166
+ - **Raters are compared to the pool, not to a standard.** A rater is rogue
167
+ when their CI sits below the pool's CI — a claim about this project, not
168
+ about an absolute bar their domain may not share.
169
+ - **Ambiguity is measured, not punished.** Contested items are reported
170
+ descriptively: persistent disagreement can be signal about the task, not
171
+ noise from the raters, and deserves a number rather than a flag.
172
+ - **Deterministic to the digit.** Seeded PCG64 everywhere, no BLAS in any
173
+ statistic's path: the same ratings file gives the same report on any
174
+ machine.
175
+
176
+ ## Limitations
177
+
178
+ - An audit is bounded by what was logged. No timestamps, no drift check; no
179
+ text or splits, no leakage check; no gold, no gold check.
180
+ - The innocent bands and the 0.667 floor are conventions, not derivations;
181
+ fields differ, and the constants are one file away.
182
+ - Spearman-Brown projections assume added raters behave like existing ones —
183
+ an optimistic bound, stated as such in the report.
184
+ - The rogue-rater comparison needs overlap; raters who never co-rate an item
185
+ with others cannot be tested and are counted as untested, not innocent.
186
+ - Implantation shows the probes detect the mechanisms simulated; real
187
+ annotation projects can fail in ways not simulated here.
188
+ - The GoEmotions audit measures the raw rating layer; the released benchmark
189
+ splits aggregate and filter it, so per-label alpha here is not a direct
190
+ statement about scores computed on those splits.
191
+
192
+ ## Related work
193
+
194
+ The failure modes measured here are the standard ones from the agreement and
195
+ annotation-quality literature: agreement coefficients and their proper use
196
+ ([Artstein & Poesio, 2008](https://doi.org/10.1162/coli.07-034-R2);
197
+ Krippendorff, *Content Analysis*), interpretation benchmarks
198
+ ([Landis & Koch, 1977](https://doi.org/10.2307/2529310)), the kappa paradox
199
+ under skewed prevalence
200
+ ([Feinstein & Cicchetti, 1990](https://doi.org/10.1016/0895-4356(90)90158-L)),
201
+ crowd label quality ([Snow et al., 2008](https://aclanthology.org/D08-1027/)),
202
+ learning rater competence
203
+ ([Raykar et al., 2010](https://jmlr.org/papers/v11/raykar10a.html)),
204
+ annotation error detection
205
+ ([Klie et al., 2023](https://doi.org/10.1162/coli_a_00464)), and human label
206
+ variation as signal
207
+ ([Plank, 2022](https://aclanthology.org/2022.emnlp-main.731/)). The case
208
+ study audits the GoEmotions raw ratings
209
+ ([Demszky et al., 2020](https://aclanthology.org/2020.acl-main.372/)).
210
+ raterkit's contribution is packaging the checks as one auditable tool with
211
+ uncertainty on every number and validation by implantation.
212
+
213
+ Companion projects: [judgekit](https://github.com/mohammadi-hadi/judgekit)
214
+ audits LLM judges the way this audits human raters;
215
+ [judgepanel](https://github.com/mohammadi-hadi/judgepanel) estimates how
216
+ accurate a panel of judges is when no gold labels exist at all;
217
+ [abkit](https://github.com/mohammadi-hadi/abkit) applies the same
218
+ audit-before-you-trust pattern to A/B-test readouts;
219
+ [trajectory-judge](https://github.com/mohammadi-hadi/trajectory-judge)
220
+ measures what outcome-only judges miss on agent trajectories.
221
+
222
+ ## Citation
223
+
224
+ If raterkit is useful in your work, please cite it (see
225
+ [CITATION.cff](CITATION.cff)):
226
+
227
+ ```bibtex
228
+ @software{mohammadi_raterkit,
229
+ author = {Mohammadi, Hadi},
230
+ title = {raterkit: audit a labeled dataset before you trust it},
231
+ url = {https://github.com/mohammadi-hadi/raterkit},
232
+ version = {0.1.0},
233
+ year = {2026}
234
+ }
235
+ ```
236
+
237
+ ## License
238
+
239
+ MIT