raterkit 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- raterkit-0.1.0/.github/FUNDING.yml +1 -0
- raterkit-0.1.0/.github/workflows/ci.yml +47 -0
- raterkit-0.1.0/.github/workflows/release.yml +51 -0
- raterkit-0.1.0/CITATION.cff +11 -0
- raterkit-0.1.0/LICENSE +21 -0
- raterkit-0.1.0/Makefile +27 -0
- raterkit-0.1.0/PKG-INFO +269 -0
- raterkit-0.1.0/README.md +239 -0
- raterkit-0.1.0/examples/goemotions/audit_goemotions.py +329 -0
- raterkit-0.1.0/examples/goemotions/fetch_data.py +32 -0
- raterkit-0.1.0/examples/goemotions/results/figures/label_alpha.png +0 -0
- raterkit-0.1.0/examples/goemotions/results/goemotions.json +1014 -0
- raterkit-0.1.0/examples/goemotions/results/goemotions.md +15 -0
- raterkit-0.1.0/pyproject.toml +69 -0
- raterkit-0.1.0/requirements-results.txt +6 -0
- raterkit-0.1.0/results/demo.json +949 -0
- raterkit-0.1.0/results/figures/depth_curve.png +0 -0
- raterkit-0.1.0/results/figures/rater_forest.png +0 -0
- raterkit-0.1.0/results/report.md +242 -0
- raterkit-0.1.0/results/table.md +10 -0
- raterkit-0.1.0/src/raterkit/__init__.py +21 -0
- raterkit-0.1.0/src/raterkit/cli.py +74 -0
- raterkit-0.1.0/src/raterkit/dataset.py +117 -0
- raterkit-0.1.0/src/raterkit/demo.py +107 -0
- raterkit-0.1.0/src/raterkit/probes.py +398 -0
- raterkit-0.1.0/src/raterkit/py.typed +0 -0
- raterkit-0.1.0/src/raterkit/report.py +225 -0
- raterkit-0.1.0/src/raterkit/result.py +84 -0
- raterkit-0.1.0/src/raterkit/schema.py +104 -0
- raterkit-0.1.0/src/raterkit/stats.py +330 -0
- raterkit-0.1.0/src/raterkit/synthetic.py +118 -0
- raterkit-0.1.0/tests/test_cli.py +85 -0
- raterkit-0.1.0/tests/test_probes.py +187 -0
- raterkit-0.1.0/tests/test_schema_result.py +78 -0
- raterkit-0.1.0/tests/test_stats.py +152 -0
|
@@ -0,0 +1 @@
|
|
|
1
|
+
github: [mohammadi-hadi]
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
name: ci
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
branches: [main]
|
|
6
|
+
pull_request:
|
|
7
|
+
|
|
8
|
+
jobs:
|
|
9
|
+
test:
|
|
10
|
+
runs-on: ubuntu-latest
|
|
11
|
+
strategy:
|
|
12
|
+
matrix:
|
|
13
|
+
python-version: ["3.11", "3.12", "3.13"]
|
|
14
|
+
steps:
|
|
15
|
+
- uses: actions/checkout@v4
|
|
16
|
+
- uses: actions/setup-python@v5
|
|
17
|
+
with:
|
|
18
|
+
python-version: ${{ matrix.python-version }}
|
|
19
|
+
- name: Install
|
|
20
|
+
run: pip install -e ".[dev]"
|
|
21
|
+
- name: Lint
|
|
22
|
+
run: |
|
|
23
|
+
ruff check src tests
|
|
24
|
+
ruff format --check src tests
|
|
25
|
+
- name: Type check
|
|
26
|
+
if: matrix.python-version == '3.12'
|
|
27
|
+
run: mypy src
|
|
28
|
+
- name: Test
|
|
29
|
+
run: pytest -q
|
|
30
|
+
|
|
31
|
+
results-drift:
|
|
32
|
+
runs-on: ubuntu-latest
|
|
33
|
+
steps:
|
|
34
|
+
- uses: actions/checkout@v4
|
|
35
|
+
- uses: actions/setup-python@v5
|
|
36
|
+
with:
|
|
37
|
+
python-version: "3.12"
|
|
38
|
+
- name: Install pinned results environment
|
|
39
|
+
run: |
|
|
40
|
+
pip install -r requirements-results.txt
|
|
41
|
+
pip install -e . --no-deps
|
|
42
|
+
- name: Regenerate demo results and README table
|
|
43
|
+
run: |
|
|
44
|
+
python -m raterkit.cli demo --out results
|
|
45
|
+
python -m raterkit.cli inject-readme README.md --results results
|
|
46
|
+
- name: Fail if committed results drift
|
|
47
|
+
run: git diff --exit-code -- results/report.md results/table.md results/demo.json README.md
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
name: release
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
tags: ["v*"]
|
|
6
|
+
|
|
7
|
+
jobs:
|
|
8
|
+
build:
|
|
9
|
+
runs-on: ubuntu-latest
|
|
10
|
+
steps:
|
|
11
|
+
- uses: actions/checkout@v4
|
|
12
|
+
- uses: actions/setup-python@v5
|
|
13
|
+
with:
|
|
14
|
+
python-version: "3.12"
|
|
15
|
+
- name: Build distributions
|
|
16
|
+
run: |
|
|
17
|
+
pip install build twine
|
|
18
|
+
python -m build
|
|
19
|
+
twine check dist/*
|
|
20
|
+
- uses: actions/upload-artifact@v4
|
|
21
|
+
with:
|
|
22
|
+
name: dist
|
|
23
|
+
path: dist/
|
|
24
|
+
|
|
25
|
+
publish:
|
|
26
|
+
needs: build
|
|
27
|
+
runs-on: ubuntu-latest
|
|
28
|
+
environment: pypi
|
|
29
|
+
permissions:
|
|
30
|
+
id-token: write
|
|
31
|
+
steps:
|
|
32
|
+
- uses: actions/download-artifact@v4
|
|
33
|
+
with:
|
|
34
|
+
name: dist
|
|
35
|
+
path: dist/
|
|
36
|
+
- uses: pypa/gh-action-pypi-publish@release/v1
|
|
37
|
+
|
|
38
|
+
github-release:
|
|
39
|
+
needs: publish
|
|
40
|
+
runs-on: ubuntu-latest
|
|
41
|
+
permissions:
|
|
42
|
+
contents: write
|
|
43
|
+
steps:
|
|
44
|
+
- uses: actions/download-artifact@v4
|
|
45
|
+
with:
|
|
46
|
+
name: dist
|
|
47
|
+
path: dist/
|
|
48
|
+
- uses: softprops/action-gh-release@v2
|
|
49
|
+
with:
|
|
50
|
+
files: dist/*
|
|
51
|
+
generate_release_notes: true
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
cff-version: 1.2.0
|
|
2
|
+
message: "If you use raterkit, please cite it as below."
|
|
3
|
+
title: "raterkit: audit a labeled dataset before you trust it"
|
|
4
|
+
type: software
|
|
5
|
+
authors:
|
|
6
|
+
- family-names: Mohammadi
|
|
7
|
+
given-names: Hadi
|
|
8
|
+
repository-code: "https://github.com/mohammadi-hadi/raterkit"
|
|
9
|
+
license: MIT
|
|
10
|
+
version: 0.1.0
|
|
11
|
+
date-released: 2026-08-05
|
raterkit-0.1.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Hadi Mohammadi
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
raterkit-0.1.0/Makefile
ADDED
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
PY ?= .venv/bin/python
|
|
2
|
+
|
|
3
|
+
.PHONY: demo test lint typecheck goemotions build clean
|
|
4
|
+
|
|
5
|
+
demo:
|
|
6
|
+
$(PY) -m raterkit.cli demo --out results
|
|
7
|
+
$(PY) -m raterkit.cli inject-readme README.md --results results
|
|
8
|
+
|
|
9
|
+
test:
|
|
10
|
+
$(PY) -m pytest -q
|
|
11
|
+
|
|
12
|
+
lint:
|
|
13
|
+
$(PY) -m ruff check src tests
|
|
14
|
+
$(PY) -m ruff format --check src tests
|
|
15
|
+
|
|
16
|
+
typecheck:
|
|
17
|
+
$(PY) -m mypy src
|
|
18
|
+
|
|
19
|
+
goemotions:
|
|
20
|
+
$(PY) examples/goemotions/fetch_data.py
|
|
21
|
+
$(PY) examples/goemotions/audit_goemotions.py
|
|
22
|
+
|
|
23
|
+
build:
|
|
24
|
+
$(PY) -m build
|
|
25
|
+
|
|
26
|
+
clean:
|
|
27
|
+
rm -rf dist build .pytest_cache .mypy_cache .ruff_cache
|
raterkit-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,269 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: raterkit
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Audit a labeled dataset before you trust it
|
|
5
|
+
Project-URL: Repository, https://github.com/mohammadi-hadi/raterkit
|
|
6
|
+
Project-URL: Issues, https://github.com/mohammadi-hadi/raterkit/issues
|
|
7
|
+
Author: Hadi Mohammadi
|
|
8
|
+
License-Expression: MIT
|
|
9
|
+
License-File: LICENSE
|
|
10
|
+
Keywords: annotation,data-quality,evaluation,inter-rater-reliability,krippendorff
|
|
11
|
+
Classifier: Development Status :: 4 - Beta
|
|
12
|
+
Classifier: Intended Audience :: Science/Research
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
16
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
17
|
+
Requires-Python: >=3.11
|
|
18
|
+
Requires-Dist: matplotlib>=3.8
|
|
19
|
+
Requires-Dist: numpy>=1.26
|
|
20
|
+
Requires-Dist: pydantic>=2.5
|
|
21
|
+
Provides-Extra: dev
|
|
22
|
+
Requires-Dist: build>=1.2; extra == 'dev'
|
|
23
|
+
Requires-Dist: krippendorff>=0.6; extra == 'dev'
|
|
24
|
+
Requires-Dist: mypy>=1.11; extra == 'dev'
|
|
25
|
+
Requires-Dist: pytest>=8; extra == 'dev'
|
|
26
|
+
Requires-Dist: ruff>=0.6; extra == 'dev'
|
|
27
|
+
Requires-Dist: scipy>=1.11; extra == 'dev'
|
|
28
|
+
Requires-Dist: statsmodels>=0.14; extra == 'dev'
|
|
29
|
+
Description-Content-Type: text/markdown
|
|
30
|
+
|
|
31
|
+
# raterkit
|
|
32
|
+
|
|
33
|
+
Audit a labeled dataset before you trust it.
|
|
34
|
+
|
|
35
|
+
[](https://github.com/mohammadi-hadi/raterkit/actions/workflows/ci.yml)
|
|
36
|
+
|
|
37
|
+
Every benchmark score, every fine-tune, every "our judge agrees with humans
|
|
38
|
+
92% of the time" bottoms out in a pile of human labels — produced by raters
|
|
39
|
+
who may have been careless, biased, tired by the end of the queue, or graded
|
|
40
|
+
against gold that was quietly wrong. None of that is visible in accuracy
|
|
41
|
+
numbers computed on top. raterkit reads the raw ratings and measures each of
|
|
42
|
+
these failure modes directly, with a confidence interval on every number and
|
|
43
|
+
a flag only when the evidence clears a stated bar.
|
|
44
|
+
|
|
45
|
+
## The audit in one table
|
|
46
|
+
|
|
47
|
+
Eight synthetic annotation projects, seven implanted defects. Bold values are
|
|
48
|
+
flags; each lands on the row where its defect was implanted, and the clean
|
|
49
|
+
row carries none.
|
|
50
|
+
|
|
51
|
+
<!-- raterkit:demo -->
|
|
52
|
+
| project | implanted defect | alpha | rogue | biased | drift gap | leaks | stale gold | flags |
|
|
53
|
+
|---|---|---|---|---|---|---|---|---|
|
|
54
|
+
| clean | nothing | 0.80 | 0 | 0 | +0.00 | 0 | 0.0% | none |
|
|
55
|
+
| careless-crowd | every rater at 55% accuracy | **0.18** | 0 | 0 | -0.02 | 0 | 0.7% | reliability |
|
|
56
|
+
| one-spammer | one rater answers uniformly at random | 0.72 | **1** | 0 | +0.05 | 0 | 0.0% | rogue raters |
|
|
57
|
+
| leaning-rater | one rater always takes the same side of ambiguous items | 0.65 | 0 | **1** | +0.05 | 0 | - | rater bias |
|
|
58
|
+
| tiring-crowd | competence decays 0.97 to 0.73 over the queue | 0.64 | 0 | 0 | **+0.25** | 0 | 0.0% | drift |
|
|
59
|
+
| leaky-split | 12 test items duplicate a train item's text | 0.80 | 0 | 0 | +0.00 | **12** | 0.0% | split leakage |
|
|
60
|
+
| stale-gold | 8% of gold labels are wrong | 0.80 | 0 | 0 | +0.00 | 0 | **6.7%** | stale gold |
|
|
61
|
+
| easy-but-skewed | 94% of items are one class; raters 90% accurate | **0.34** | 0 | 0 | +0.12 | 0 | 0.0% | reliability |
|
|
62
|
+
<!-- /raterkit:demo -->
|
|
63
|
+
|
|
64
|
+
The row to sit with is `easy-but-skewed`: raters are 90% accurate and agree
|
|
65
|
+
with each other 81% of the time, yet alpha is 0.34 — far below the 0.667
|
|
66
|
+
floor. With 94% of items in one class, raters agree that often by accident;
|
|
67
|
+
the skew, not the raters, is doing the agreeing. Raw agreement percentages
|
|
68
|
+
cannot be compared across datasets with different label distributions, which
|
|
69
|
+
is why every agreement number here is chance-corrected.
|
|
70
|
+
|
|
71
|
+

|
|
72
|
+
|
|
73
|
+

|
|
74
|
+
|
|
75
|
+
`make demo` regenerates the table, the [full report](results/report.md) and
|
|
76
|
+
the figures from fixed seeds; CI rebuilds them from pinned dependencies and
|
|
77
|
+
fails if a committed number differs from what the code produces.
|
|
78
|
+
|
|
79
|
+
## 211,225 real ratings, audited
|
|
80
|
+
|
|
81
|
+
[`examples/goemotions/`](examples/goemotions/) audits the raw crowd ratings
|
|
82
|
+
behind [GoEmotions](https://aclanthology.org/2020.acl-main.372/) — 211,225
|
|
83
|
+
ratings by 82 raters over 58,011 Reddit comments, 28 binary emotion labels
|
|
84
|
+
each. Committed results:
|
|
85
|
+
[examples/goemotions/results/goemotions.md](examples/goemotions/results/goemotions.md).
|
|
86
|
+
|
|
87
|
+
- **27 of 28 emotions sit below the 0.667 reliability floor** — the whole
|
|
88
|
+
95% CI, not just the point estimate. Only *gratitude* (alpha 0.72) passes;
|
|
89
|
+
the median emotion manages 0.23, and *neutral* itself scores 0.26.
|
|
90
|
+
- **Rater quality spans 0.11 to 0.36** (rater-vs-rest alpha across the 67
|
|
91
|
+
raters with 300+ ratings) — the noisiest raters agree with their co-raters
|
|
92
|
+
at a third of the rate the best ones do, and 20 of 67 sit below 0.20.
|
|
93
|
+
- **The rater count was never enough for the task.** By Spearman-Brown, the
|
|
94
|
+
observed reliability would need ~13 raters per item to yield 0.8-reliable
|
|
95
|
+
aggregated labels; the dataset averages 3.6, which projects to 0.52.
|
|
96
|
+
- 86% of multi-rated comments are contested on at least one emotion, yet
|
|
97
|
+
raters marked only 1.6% of ratings "very unclear" — disagreement without
|
|
98
|
+
awareness of disagreement.
|
|
99
|
+
|
|
100
|
+
None of this says GoEmotions is a scandal — fine-grained emotion is
|
|
101
|
+
genuinely hard, and the released splits aggregate and filter these raw
|
|
102
|
+
ratings. It says the raw layer under a famous benchmark fails the standard
|
|
103
|
+
reliability bar almost everywhere, measurably and reproducibly.
|
|
104
|
+
|
|
105
|
+
## Install
|
|
106
|
+
|
|
107
|
+
```
|
|
108
|
+
pip install git+https://github.com/mohammadi-hadi/raterkit
|
|
109
|
+
```
|
|
110
|
+
|
|
111
|
+
Python 3.11+. Runtime dependencies: numpy, pydantic, matplotlib.
|
|
112
|
+
|
|
113
|
+
## Quickstart
|
|
114
|
+
|
|
115
|
+
Log one JSON object per rating — who labeled what, and optionally when, in
|
|
116
|
+
which split, and the item text:
|
|
117
|
+
|
|
118
|
+
```json
|
|
119
|
+
{"item_id": "c41", "rater_id": "r07", "label": "toxic", "t": 1718040000, "split": "train", "text": "..."}
|
|
120
|
+
```
|
|
121
|
+
|
|
122
|
+
```python
|
|
123
|
+
from raterkit import load_ratings, load_gold, run_audit
|
|
124
|
+
|
|
125
|
+
audit = run_audit(load_ratings("ratings.jsonl"), gold=load_gold("gold.jsonl"))
|
|
126
|
+
for flag in audit.flags:
|
|
127
|
+
print(flag.name, "--", flag.detail)
|
|
128
|
+
```
|
|
129
|
+
|
|
130
|
+
Or from the shell, with an exit code CI can gate on:
|
|
131
|
+
|
|
132
|
+
```
|
|
133
|
+
raterkit report ratings.jsonl --gold gold.jsonl --out audit --fail-on-flags
|
|
134
|
+
```
|
|
135
|
+
|
|
136
|
+
An optional config sets the measurement level (`nominal`, `ordinal`,
|
|
137
|
+
`interval`), the label order, and the target reliability. Alongside
|
|
138
|
+
`report.md` the audit writes `report.json` for pipelines that gate on
|
|
139
|
+
specific numbers. Any check that could not run is listed with the exact
|
|
140
|
+
fields to log to enable it — a skip describes the log file, not the raters.
|
|
141
|
+
|
|
142
|
+
An LLM judge is just another rater: give its verdicts a `rater_id` and the
|
|
143
|
+
same audit reports how it agrees with the humans, whether its label
|
|
144
|
+
distribution diverges, and whether it drifts — no special casing needed.
|
|
145
|
+
|
|
146
|
+
## What it checks
|
|
147
|
+
|
|
148
|
+
| check | question it answers | needs |
|
|
149
|
+
|---|---|---|
|
|
150
|
+
| reliability | is agreement above the chance-corrected floor? | 2+ ratings on some items |
|
|
151
|
+
| rogue raters | does any rater agree with the rest credibly less than the pool? | 30+ pairs per rater |
|
|
152
|
+
| rater bias | does any rater use the labels differently (marginal shift)? | 30+ ratings per rater |
|
|
153
|
+
| drift | did agreement change between early and late work? | `t` per rating |
|
|
154
|
+
| split leakage | does any duplicated text span train/test? | `text` + `split` |
|
|
155
|
+
| stale gold | which gold labels does a unanimous crowd contradict? | a gold file |
|
|
156
|
+
| ambiguity | how many items are genuinely contested? | 2+ ratings on some items |
|
|
157
|
+
| prevalence | is raw agreement inflated by class skew? | always runs |
|
|
158
|
+
| rating depth | how many raters per item does your target reliability need? | always runs |
|
|
159
|
+
|
|
160
|
+
The last three are descriptive and never flag: they explain the numbers
|
|
161
|
+
above them.
|
|
162
|
+
|
|
163
|
+
## How the numbers are defended
|
|
164
|
+
|
|
165
|
+
- **Cross-checked implementations.** Krippendorff's alpha (nominal, ordinal
|
|
166
|
+
and interval, with missing data) matches the reference `krippendorff`
|
|
167
|
+
package to 1e-10; Cohen's and Fleiss' kappa match statsmodels; the
|
|
168
|
+
chi-square survival function matches scipy to 1e-10; Benjamini-Hochberg
|
|
169
|
+
reproduces statsmodels decisions exactly. The library itself depends on
|
|
170
|
+
none of them.
|
|
171
|
+
- **Validation by implantation.** The synthetic projects have defect dials —
|
|
172
|
+
a spammer, a leaning rater, a competence drift, a gold error rate — and
|
|
173
|
+
the tests require each probe to recover its dial, stay monotone in it, and
|
|
174
|
+
stay silent on the clean project.
|
|
175
|
+
- **Uncertainty on every number.** Band probes trigger only when the whole
|
|
176
|
+
95% bootstrap CI leaves the innocent band; the bootstrap resamples items
|
|
177
|
+
(never single ratings), so dependence within an item is respected.
|
|
178
|
+
- **Drift-checked results.** The demo table, report and README numbers are
|
|
179
|
+
regenerated by CI from pinned dependencies and diffed against the
|
|
180
|
+
committed copy.
|
|
181
|
+
|
|
182
|
+
## Design notes
|
|
183
|
+
|
|
184
|
+
- **Chance correction is not optional.** The `easy-but-skewed` demo row and
|
|
185
|
+
the kappa-paradox literature both show raw agreement rewarding class skew.
|
|
186
|
+
All agreement numbers are Krippendorff's alpha, which handles missing
|
|
187
|
+
ratings, any number of raters, and nominal/ordinal/interval labels in one
|
|
188
|
+
framework — and reduces to the familiar coefficients where they overlap.
|
|
189
|
+
- **Innocent bands, not point nulls.** With enough data any rater's marginal
|
|
190
|
+
"significantly" differs from the pool's while meaning nothing. Band probes
|
|
191
|
+
state the interval a healthy project could occupy (bias within 0.10 total
|
|
192
|
+
variation, drift within ±0.10 alpha) and flag only when the whole CI
|
|
193
|
+
leaves it. Exact procedures use conventions: the 0.667 floor, zero
|
|
194
|
+
tolerance for cross-split duplicates. All constants live in one place
|
|
195
|
+
([`probes.py`](src/raterkit/probes.py)) and are easy to disagree with.
|
|
196
|
+
- **Raters are compared to the pool, not to a standard.** A rater is rogue
|
|
197
|
+
when their CI sits below the pool's CI — a claim about this project, not
|
|
198
|
+
about an absolute bar their domain may not share.
|
|
199
|
+
- **Ambiguity is measured, not punished.** Contested items are reported
|
|
200
|
+
descriptively: persistent disagreement can be signal about the task, not
|
|
201
|
+
noise from the raters, and deserves a number rather than a flag.
|
|
202
|
+
- **Deterministic to the digit.** Seeded PCG64 everywhere, no BLAS in any
|
|
203
|
+
statistic's path: the same ratings file gives the same report on any
|
|
204
|
+
machine.
|
|
205
|
+
|
|
206
|
+
## Limitations
|
|
207
|
+
|
|
208
|
+
- An audit is bounded by what was logged. No timestamps, no drift check; no
|
|
209
|
+
text or splits, no leakage check; no gold, no gold check.
|
|
210
|
+
- The innocent bands and the 0.667 floor are conventions, not derivations;
|
|
211
|
+
fields differ, and the constants are one file away.
|
|
212
|
+
- Spearman-Brown projections assume added raters behave like existing ones —
|
|
213
|
+
an optimistic bound, stated as such in the report.
|
|
214
|
+
- The rogue-rater comparison needs overlap; raters who never co-rate an item
|
|
215
|
+
with others cannot be tested and are counted as untested, not innocent.
|
|
216
|
+
- Implantation shows the probes detect the mechanisms simulated; real
|
|
217
|
+
annotation projects can fail in ways not simulated here.
|
|
218
|
+
- The GoEmotions audit measures the raw rating layer; the released benchmark
|
|
219
|
+
splits aggregate and filter it, so per-label alpha here is not a direct
|
|
220
|
+
statement about scores computed on those splits.
|
|
221
|
+
|
|
222
|
+
## Related work
|
|
223
|
+
|
|
224
|
+
The failure modes measured here are the standard ones from the agreement and
|
|
225
|
+
annotation-quality literature: agreement coefficients and their proper use
|
|
226
|
+
([Artstein & Poesio, 2008](https://doi.org/10.1162/coli.07-034-R2);
|
|
227
|
+
Krippendorff, *Content Analysis*), interpretation benchmarks
|
|
228
|
+
([Landis & Koch, 1977](https://doi.org/10.2307/2529310)), the kappa paradox
|
|
229
|
+
under skewed prevalence
|
|
230
|
+
([Feinstein & Cicchetti, 1990](https://doi.org/10.1016/0895-4356(90)90158-L)),
|
|
231
|
+
crowd label quality ([Snow et al., 2008](https://aclanthology.org/D08-1027/)),
|
|
232
|
+
learning rater competence
|
|
233
|
+
([Raykar et al., 2010](https://jmlr.org/papers/v11/raykar10a.html)),
|
|
234
|
+
annotation error detection
|
|
235
|
+
([Klie et al., 2023](https://doi.org/10.1162/coli_a_00464)), and human label
|
|
236
|
+
variation as signal
|
|
237
|
+
([Plank, 2022](https://aclanthology.org/2022.emnlp-main.731/)). The case
|
|
238
|
+
study audits the GoEmotions raw ratings
|
|
239
|
+
([Demszky et al., 2020](https://aclanthology.org/2020.acl-main.372/)).
|
|
240
|
+
raterkit's contribution is packaging the checks as one auditable tool with
|
|
241
|
+
uncertainty on every number and validation by implantation.
|
|
242
|
+
|
|
243
|
+
Companion projects: [judgekit](https://github.com/mohammadi-hadi/judgekit)
|
|
244
|
+
audits LLM judges the way this audits human raters;
|
|
245
|
+
[judgepanel](https://github.com/mohammadi-hadi/judgepanel) estimates how
|
|
246
|
+
accurate a panel of judges is when no gold labels exist at all;
|
|
247
|
+
[abkit](https://github.com/mohammadi-hadi/abkit) applies the same
|
|
248
|
+
audit-before-you-trust pattern to A/B-test readouts;
|
|
249
|
+
[trajectory-judge](https://github.com/mohammadi-hadi/trajectory-judge)
|
|
250
|
+
measures what outcome-only judges miss on agent trajectories.
|
|
251
|
+
|
|
252
|
+
## Citation
|
|
253
|
+
|
|
254
|
+
If raterkit is useful in your work, please cite it (see
|
|
255
|
+
[CITATION.cff](CITATION.cff)):
|
|
256
|
+
|
|
257
|
+
```bibtex
|
|
258
|
+
@software{mohammadi_raterkit,
|
|
259
|
+
author = {Mohammadi, Hadi},
|
|
260
|
+
title = {raterkit: audit a labeled dataset before you trust it},
|
|
261
|
+
url = {https://github.com/mohammadi-hadi/raterkit},
|
|
262
|
+
version = {0.1.0},
|
|
263
|
+
year = {2026}
|
|
264
|
+
}
|
|
265
|
+
```
|
|
266
|
+
|
|
267
|
+
## License
|
|
268
|
+
|
|
269
|
+
MIT
|
raterkit-0.1.0/README.md
ADDED
|
@@ -0,0 +1,239 @@
|
|
|
1
|
+
# raterkit
|
|
2
|
+
|
|
3
|
+
Audit a labeled dataset before you trust it.
|
|
4
|
+
|
|
5
|
+
[](https://github.com/mohammadi-hadi/raterkit/actions/workflows/ci.yml)
|
|
6
|
+
|
|
7
|
+
Every benchmark score, every fine-tune, every "our judge agrees with humans
|
|
8
|
+
92% of the time" bottoms out in a pile of human labels — produced by raters
|
|
9
|
+
who may have been careless, biased, tired by the end of the queue, or graded
|
|
10
|
+
against gold that was quietly wrong. None of that is visible in accuracy
|
|
11
|
+
numbers computed on top. raterkit reads the raw ratings and measures each of
|
|
12
|
+
these failure modes directly, with a confidence interval on every number and
|
|
13
|
+
a flag only when the evidence clears a stated bar.
|
|
14
|
+
|
|
15
|
+
## The audit in one table
|
|
16
|
+
|
|
17
|
+
Eight synthetic annotation projects, seven implanted defects. Bold values are
|
|
18
|
+
flags; each lands on the row where its defect was implanted, and the clean
|
|
19
|
+
row carries none.
|
|
20
|
+
|
|
21
|
+
<!-- raterkit:demo -->
|
|
22
|
+
| project | implanted defect | alpha | rogue | biased | drift gap | leaks | stale gold | flags |
|
|
23
|
+
|---|---|---|---|---|---|---|---|---|
|
|
24
|
+
| clean | nothing | 0.80 | 0 | 0 | +0.00 | 0 | 0.0% | none |
|
|
25
|
+
| careless-crowd | every rater at 55% accuracy | **0.18** | 0 | 0 | -0.02 | 0 | 0.7% | reliability |
|
|
26
|
+
| one-spammer | one rater answers uniformly at random | 0.72 | **1** | 0 | +0.05 | 0 | 0.0% | rogue raters |
|
|
27
|
+
| leaning-rater | one rater always takes the same side of ambiguous items | 0.65 | 0 | **1** | +0.05 | 0 | - | rater bias |
|
|
28
|
+
| tiring-crowd | competence decays 0.97 to 0.73 over the queue | 0.64 | 0 | 0 | **+0.25** | 0 | 0.0% | drift |
|
|
29
|
+
| leaky-split | 12 test items duplicate a train item's text | 0.80 | 0 | 0 | +0.00 | **12** | 0.0% | split leakage |
|
|
30
|
+
| stale-gold | 8% of gold labels are wrong | 0.80 | 0 | 0 | +0.00 | 0 | **6.7%** | stale gold |
|
|
31
|
+
| easy-but-skewed | 94% of items are one class; raters 90% accurate | **0.34** | 0 | 0 | +0.12 | 0 | 0.0% | reliability |
|
|
32
|
+
<!-- /raterkit:demo -->
|
|
33
|
+
|
|
34
|
+
The row to sit with is `easy-but-skewed`: raters are 90% accurate and agree
|
|
35
|
+
with each other 81% of the time, yet alpha is 0.34 — far below the 0.667
|
|
36
|
+
floor. With 94% of items in one class, raters agree that often by accident;
|
|
37
|
+
the skew, not the raters, is doing the agreeing. Raw agreement percentages
|
|
38
|
+
cannot be compared across datasets with different label distributions, which
|
|
39
|
+
is why every agreement number here is chance-corrected.
|
|
40
|
+
|
|
41
|
+

|
|
42
|
+
|
|
43
|
+

|
|
44
|
+
|
|
45
|
+
`make demo` regenerates the table, the [full report](results/report.md) and
|
|
46
|
+
the figures from fixed seeds; CI rebuilds them from pinned dependencies and
|
|
47
|
+
fails if a committed number differs from what the code produces.
|
|
48
|
+
|
|
49
|
+
## 211,225 real ratings, audited
|
|
50
|
+
|
|
51
|
+
[`examples/goemotions/`](examples/goemotions/) audits the raw crowd ratings
|
|
52
|
+
behind [GoEmotions](https://aclanthology.org/2020.acl-main.372/) — 211,225
|
|
53
|
+
ratings by 82 raters over 58,011 Reddit comments, 28 binary emotion labels
|
|
54
|
+
each. Committed results:
|
|
55
|
+
[examples/goemotions/results/goemotions.md](examples/goemotions/results/goemotions.md).
|
|
56
|
+
|
|
57
|
+
- **27 of 28 emotions sit below the 0.667 reliability floor** — the whole
|
|
58
|
+
95% CI, not just the point estimate. Only *gratitude* (alpha 0.72) passes;
|
|
59
|
+
the median emotion manages 0.23, and *neutral* itself scores 0.26.
|
|
60
|
+
- **Rater quality spans 0.11 to 0.36** (rater-vs-rest alpha across the 67
|
|
61
|
+
raters with 300+ ratings) — the noisiest raters agree with their co-raters
|
|
62
|
+
at a third of the rate the best ones do, and 20 of 67 sit below 0.20.
|
|
63
|
+
- **The rater count was never enough for the task.** By Spearman-Brown, the
|
|
64
|
+
observed reliability would need ~13 raters per item to yield 0.8-reliable
|
|
65
|
+
aggregated labels; the dataset averages 3.6, which projects to 0.52.
|
|
66
|
+
- 86% of multi-rated comments are contested on at least one emotion, yet
|
|
67
|
+
raters marked only 1.6% of ratings "very unclear" — disagreement without
|
|
68
|
+
awareness of disagreement.
|
|
69
|
+
|
|
70
|
+
None of this says GoEmotions is a scandal — fine-grained emotion is
|
|
71
|
+
genuinely hard, and the released splits aggregate and filter these raw
|
|
72
|
+
ratings. It says the raw layer under a famous benchmark fails the standard
|
|
73
|
+
reliability bar almost everywhere, measurably and reproducibly.
|
|
74
|
+
|
|
75
|
+
## Install
|
|
76
|
+
|
|
77
|
+
```
|
|
78
|
+
pip install git+https://github.com/mohammadi-hadi/raterkit
|
|
79
|
+
```
|
|
80
|
+
|
|
81
|
+
Python 3.11+. Runtime dependencies: numpy, pydantic, matplotlib.
|
|
82
|
+
|
|
83
|
+
## Quickstart
|
|
84
|
+
|
|
85
|
+
Log one JSON object per rating — who labeled what, and optionally when, in
|
|
86
|
+
which split, and the item text:
|
|
87
|
+
|
|
88
|
+
```json
|
|
89
|
+
{"item_id": "c41", "rater_id": "r07", "label": "toxic", "t": 1718040000, "split": "train", "text": "..."}
|
|
90
|
+
```
|
|
91
|
+
|
|
92
|
+
```python
|
|
93
|
+
from raterkit import load_ratings, load_gold, run_audit
|
|
94
|
+
|
|
95
|
+
audit = run_audit(load_ratings("ratings.jsonl"), gold=load_gold("gold.jsonl"))
|
|
96
|
+
for flag in audit.flags:
|
|
97
|
+
print(flag.name, "--", flag.detail)
|
|
98
|
+
```
|
|
99
|
+
|
|
100
|
+
Or from the shell, with an exit code CI can gate on:
|
|
101
|
+
|
|
102
|
+
```
|
|
103
|
+
raterkit report ratings.jsonl --gold gold.jsonl --out audit --fail-on-flags
|
|
104
|
+
```
|
|
105
|
+
|
|
106
|
+
An optional config sets the measurement level (`nominal`, `ordinal`,
|
|
107
|
+
`interval`), the label order, and the target reliability. Alongside
|
|
108
|
+
`report.md` the audit writes `report.json` for pipelines that gate on
|
|
109
|
+
specific numbers. Any check that could not run is listed with the exact
|
|
110
|
+
fields to log to enable it — a skip describes the log file, not the raters.
|
|
111
|
+
|
|
112
|
+
An LLM judge is just another rater: give its verdicts a `rater_id` and the
|
|
113
|
+
same audit reports how it agrees with the humans, whether its label
|
|
114
|
+
distribution diverges, and whether it drifts — no special casing needed.
|
|
115
|
+
|
|
116
|
+
## What it checks
|
|
117
|
+
|
|
118
|
+
| check | question it answers | needs |
|
|
119
|
+
|---|---|---|
|
|
120
|
+
| reliability | is agreement above the chance-corrected floor? | 2+ ratings on some items |
|
|
121
|
+
| rogue raters | does any rater agree with the rest credibly less than the pool? | 30+ pairs per rater |
|
|
122
|
+
| rater bias | does any rater use the labels differently (marginal shift)? | 30+ ratings per rater |
|
|
123
|
+
| drift | did agreement change between early and late work? | `t` per rating |
|
|
124
|
+
| split leakage | does any duplicated text span train/test? | `text` + `split` |
|
|
125
|
+
| stale gold | which gold labels does a unanimous crowd contradict? | a gold file |
|
|
126
|
+
| ambiguity | how many items are genuinely contested? | 2+ ratings on some items |
|
|
127
|
+
| prevalence | is raw agreement inflated by class skew? | always runs |
|
|
128
|
+
| rating depth | how many raters per item does your target reliability need? | always runs |
|
|
129
|
+
|
|
130
|
+
The last three are descriptive and never flag: they explain the numbers
|
|
131
|
+
above them.
|
|
132
|
+
|
|
133
|
+
## How the numbers are defended
|
|
134
|
+
|
|
135
|
+
- **Cross-checked implementations.** Krippendorff's alpha (nominal, ordinal
|
|
136
|
+
and interval, with missing data) matches the reference `krippendorff`
|
|
137
|
+
package to 1e-10; Cohen's and Fleiss' kappa match statsmodels; the
|
|
138
|
+
chi-square survival function matches scipy to 1e-10; Benjamini-Hochberg
|
|
139
|
+
reproduces statsmodels decisions exactly. The library itself depends on
|
|
140
|
+
none of them.
|
|
141
|
+
- **Validation by implantation.** The synthetic projects have defect dials —
|
|
142
|
+
a spammer, a leaning rater, a competence drift, a gold error rate — and
|
|
143
|
+
the tests require each probe to recover its dial, stay monotone in it, and
|
|
144
|
+
stay silent on the clean project.
|
|
145
|
+
- **Uncertainty on every number.** Band probes trigger only when the whole
|
|
146
|
+
95% bootstrap CI leaves the innocent band; the bootstrap resamples items
|
|
147
|
+
(never single ratings), so dependence within an item is respected.
|
|
148
|
+
- **Drift-checked results.** The demo table, report and README numbers are
|
|
149
|
+
regenerated by CI from pinned dependencies and diffed against the
|
|
150
|
+
committed copy.
|
|
151
|
+
|
|
152
|
+
## Design notes
|
|
153
|
+
|
|
154
|
+
- **Chance correction is not optional.** The `easy-but-skewed` demo row and
|
|
155
|
+
the kappa-paradox literature both show raw agreement rewarding class skew.
|
|
156
|
+
All agreement numbers are Krippendorff's alpha, which handles missing
|
|
157
|
+
ratings, any number of raters, and nominal/ordinal/interval labels in one
|
|
158
|
+
framework — and reduces to the familiar coefficients where they overlap.
|
|
159
|
+
- **Innocent bands, not point nulls.** With enough data any rater's marginal
|
|
160
|
+
"significantly" differs from the pool's while meaning nothing. Band probes
|
|
161
|
+
state the interval a healthy project could occupy (bias within 0.10 total
|
|
162
|
+
variation, drift within ±0.10 alpha) and flag only when the whole CI
|
|
163
|
+
leaves it. Exact procedures use conventions: the 0.667 floor, zero
|
|
164
|
+
tolerance for cross-split duplicates. All constants live in one place
|
|
165
|
+
([`probes.py`](src/raterkit/probes.py)) and are easy to disagree with.
|
|
166
|
+
- **Raters are compared to the pool, not to a standard.** A rater is rogue
|
|
167
|
+
when their CI sits below the pool's CI — a claim about this project, not
|
|
168
|
+
about an absolute bar their domain may not share.
|
|
169
|
+
- **Ambiguity is measured, not punished.** Contested items are reported
|
|
170
|
+
descriptively: persistent disagreement can be signal about the task, not
|
|
171
|
+
noise from the raters, and deserves a number rather than a flag.
|
|
172
|
+
- **Deterministic to the digit.** Seeded PCG64 everywhere, no BLAS in any
|
|
173
|
+
statistic's path: the same ratings file gives the same report on any
|
|
174
|
+
machine.
|
|
175
|
+
|
|
176
|
+
## Limitations
|
|
177
|
+
|
|
178
|
+
- An audit is bounded by what was logged. No timestamps, no drift check; no
|
|
179
|
+
text or splits, no leakage check; no gold, no gold check.
|
|
180
|
+
- The innocent bands and the 0.667 floor are conventions, not derivations;
|
|
181
|
+
fields differ, and the constants are one file away.
|
|
182
|
+
- Spearman-Brown projections assume added raters behave like existing ones —
|
|
183
|
+
an optimistic bound, stated as such in the report.
|
|
184
|
+
- The rogue-rater comparison needs overlap; raters who never co-rate an item
|
|
185
|
+
with others cannot be tested and are counted as untested, not innocent.
|
|
186
|
+
- Implantation shows the probes detect the mechanisms simulated; real
|
|
187
|
+
annotation projects can fail in ways not simulated here.
|
|
188
|
+
- The GoEmotions audit measures the raw rating layer; the released benchmark
|
|
189
|
+
splits aggregate and filter it, so per-label alpha here is not a direct
|
|
190
|
+
statement about scores computed on those splits.
|
|
191
|
+
|
|
192
|
+
## Related work
|
|
193
|
+
|
|
194
|
+
The failure modes measured here are the standard ones from the agreement and
|
|
195
|
+
annotation-quality literature: agreement coefficients and their proper use
|
|
196
|
+
([Artstein & Poesio, 2008](https://doi.org/10.1162/coli.07-034-R2);
|
|
197
|
+
Krippendorff, *Content Analysis*), interpretation benchmarks
|
|
198
|
+
([Landis & Koch, 1977](https://doi.org/10.2307/2529310)), the kappa paradox
|
|
199
|
+
under skewed prevalence
|
|
200
|
+
([Feinstein & Cicchetti, 1990](https://doi.org/10.1016/0895-4356(90)90158-L)),
|
|
201
|
+
crowd label quality ([Snow et al., 2008](https://aclanthology.org/D08-1027/)),
|
|
202
|
+
learning rater competence
|
|
203
|
+
([Raykar et al., 2010](https://jmlr.org/papers/v11/raykar10a.html)),
|
|
204
|
+
annotation error detection
|
|
205
|
+
([Klie et al., 2023](https://doi.org/10.1162/coli_a_00464)), and human label
|
|
206
|
+
variation as signal
|
|
207
|
+
([Plank, 2022](https://aclanthology.org/2022.emnlp-main.731/)). The case
|
|
208
|
+
study audits the GoEmotions raw ratings
|
|
209
|
+
([Demszky et al., 2020](https://aclanthology.org/2020.acl-main.372/)).
|
|
210
|
+
raterkit's contribution is packaging the checks as one auditable tool with
|
|
211
|
+
uncertainty on every number and validation by implantation.
|
|
212
|
+
|
|
213
|
+
Companion projects: [judgekit](https://github.com/mohammadi-hadi/judgekit)
|
|
214
|
+
audits LLM judges the way this audits human raters;
|
|
215
|
+
[judgepanel](https://github.com/mohammadi-hadi/judgepanel) estimates how
|
|
216
|
+
accurate a panel of judges is when no gold labels exist at all;
|
|
217
|
+
[abkit](https://github.com/mohammadi-hadi/abkit) applies the same
|
|
218
|
+
audit-before-you-trust pattern to A/B-test readouts;
|
|
219
|
+
[trajectory-judge](https://github.com/mohammadi-hadi/trajectory-judge)
|
|
220
|
+
measures what outcome-only judges miss on agent trajectories.
|
|
221
|
+
|
|
222
|
+
## Citation
|
|
223
|
+
|
|
224
|
+
If raterkit is useful in your work, please cite it (see
|
|
225
|
+
[CITATION.cff](CITATION.cff)):
|
|
226
|
+
|
|
227
|
+
```bibtex
|
|
228
|
+
@software{mohammadi_raterkit,
|
|
229
|
+
author = {Mohammadi, Hadi},
|
|
230
|
+
title = {raterkit: audit a labeled dataset before you trust it},
|
|
231
|
+
url = {https://github.com/mohammadi-hadi/raterkit},
|
|
232
|
+
version = {0.1.0},
|
|
233
|
+
year = {2026}
|
|
234
|
+
}
|
|
235
|
+
```
|
|
236
|
+
|
|
237
|
+
## License
|
|
238
|
+
|
|
239
|
+
MIT
|