arenakit 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. arenakit-0.1.0/.github/FUNDING.yml +1 -0
  2. arenakit-0.1.0/.github/workflows/ci.yml +42 -0
  3. arenakit-0.1.0/.github/workflows/release.yml +49 -0
  4. arenakit-0.1.0/CITATION.cff +11 -0
  5. arenakit-0.1.0/LICENSE +21 -0
  6. arenakit-0.1.0/Makefile +32 -0
  7. arenakit-0.1.0/PKG-INFO +321 -0
  8. arenakit-0.1.0/README.md +292 -0
  9. arenakit-0.1.0/examples/lmarena/audit_lmarena.py +138 -0
  10. arenakit-0.1.0/examples/lmarena/fetch_data.py +95 -0
  11. arenakit-0.1.0/examples/lmarena/results/figures/board.png +0 -0
  12. arenakit-0.1.0/examples/lmarena/results/figures/separation.png +0 -0
  13. arenakit-0.1.0/examples/lmarena/results/lmarena.json +2020 -0
  14. arenakit-0.1.0/examples/lmarena/results/lmarena.md +109 -0
  15. arenakit-0.1.0/examples/lmarena/results/resampling.json +29 -0
  16. arenakit-0.1.0/examples/mtbench/audit_mtbench.py +209 -0
  17. arenakit-0.1.0/examples/mtbench/fetch_data.py +47 -0
  18. arenakit-0.1.0/examples/mtbench/results/figures/gpt4_pair.png +0 -0
  19. arenakit-0.1.0/examples/mtbench/results/figures/human.png +0 -0
  20. arenakit-0.1.0/examples/mtbench/results/gpt4_pair.json +398 -0
  21. arenakit-0.1.0/examples/mtbench/results/human.json +414 -0
  22. arenakit-0.1.0/examples/mtbench/results/mtbench.md +111 -0
  23. arenakit-0.1.0/examples/mtbench/results/summary.json +57 -0
  24. arenakit-0.1.0/pyproject.toml +68 -0
  25. arenakit-0.1.0/requirements-results.txt +6 -0
  26. arenakit-0.1.0/results/demo.json +89 -0
  27. arenakit-0.1.0/results/figures/board.png +0 -0
  28. arenakit-0.1.0/results/figures/separation.png +0 -0
  29. arenakit-0.1.0/results/report.md +86 -0
  30. arenakit-0.1.0/results/table.md +11 -0
  31. arenakit-0.1.0/src/arenakit/__init__.py +30 -0
  32. arenakit-0.1.0/src/arenakit/boot.py +284 -0
  33. arenakit-0.1.0/src/arenakit/bt.py +322 -0
  34. arenakit-0.1.0/src/arenakit/cli.py +171 -0
  35. arenakit-0.1.0/src/arenakit/demo.py +163 -0
  36. arenakit-0.1.0/src/arenakit/probes.py +688 -0
  37. arenakit-0.1.0/src/arenakit/py.typed +0 -0
  38. arenakit-0.1.0/src/arenakit/report.py +227 -0
  39. arenakit-0.1.0/src/arenakit/result.py +102 -0
  40. arenakit-0.1.0/src/arenakit/schema.py +345 -0
  41. arenakit-0.1.0/src/arenakit/synthetic.py +170 -0
  42. arenakit-0.1.0/tests/test_boot.py +148 -0
  43. arenakit-0.1.0/tests/test_bt.py +162 -0
  44. arenakit-0.1.0/tests/test_cli.py +136 -0
  45. arenakit-0.1.0/tests/test_probes.py +188 -0
  46. arenakit-0.1.0/tests/test_schema.py +131 -0
@@ -0,0 +1 @@
1
+ github: [mohammadi-hadi]
@@ -0,0 +1,42 @@
1
+ name: CI
2
+
3
+ on:
4
+ push:
5
+ branches: [main]
6
+ pull_request:
7
+
8
+ jobs:
9
+ test:
10
+ runs-on: ubuntu-latest
11
+ strategy:
12
+ fail-fast: false
13
+ matrix:
14
+ python-version: ["3.11", "3.12", "3.13"]
15
+ steps:
16
+ - uses: actions/checkout@v4
17
+ - uses: actions/setup-python@v5
18
+ with:
19
+ python-version: ${{ matrix.python-version }}
20
+ - run: python -m pip install --upgrade pip
21
+ - run: pip install -e ".[dev]"
22
+ - run: ruff check src tests examples
23
+ - run: ruff format --check src tests examples
24
+ - run: mypy src
25
+ - run: pytest -q
26
+
27
+ results:
28
+ # The committed demo results are regenerated from the code and diffed. If a
29
+ # change moves a number, this fails until the results are regenerated with
30
+ # it, so the README can never quote a figure the code no longer produces.
31
+ runs-on: ubuntu-latest
32
+ steps:
33
+ - uses: actions/checkout@v4
34
+ - uses: actions/setup-python@v5
35
+ with:
36
+ python-version: "3.12"
37
+ - run: python -m pip install --upgrade pip
38
+ - run: pip install -r requirements-results.txt
39
+ - run: pip install -e . --no-deps
40
+ - run: python -m arenakit.cli demo --out results
41
+ - name: Fail if the committed results moved
42
+ run: git diff --exit-code -- results/table.md results/report.md results/demo.json
@@ -0,0 +1,49 @@
1
+ name: Release
2
+
3
+ on:
4
+ push:
5
+ tags: ["v*"]
6
+
7
+ jobs:
8
+ build:
9
+ runs-on: ubuntu-latest
10
+ steps:
11
+ - uses: actions/checkout@v4
12
+ - uses: actions/setup-python@v5
13
+ with:
14
+ python-version: "3.12"
15
+ - run: python -m pip install --upgrade pip build
16
+ - run: python -m build
17
+ - uses: actions/upload-artifact@v4
18
+ with:
19
+ name: dist
20
+ path: dist/
21
+
22
+ publish:
23
+ needs: build
24
+ runs-on: ubuntu-latest
25
+ environment: pypi
26
+ permissions:
27
+ id-token: write
28
+ steps:
29
+ - uses: actions/download-artifact@v4
30
+ with:
31
+ name: dist
32
+ path: dist/
33
+ - uses: pypa/gh-action-pypi-publish@release/v1
34
+
35
+ github-release:
36
+ needs: publish
37
+ runs-on: ubuntu-latest
38
+ permissions:
39
+ contents: write
40
+ steps:
41
+ - uses: actions/checkout@v4
42
+ - uses: actions/download-artifact@v4
43
+ with:
44
+ name: dist
45
+ path: dist/
46
+ - uses: softprops/action-gh-release@v2
47
+ with:
48
+ files: dist/*
49
+ generate_release_notes: true
@@ -0,0 +1,11 @@
1
+ cff-version: 1.2.0
2
+ message: "If you use arenakit, please cite it as below."
3
+ title: "arenakit: audit a pairwise model leaderboard before you trust its order"
4
+ type: software
5
+ authors:
6
+ - family-names: Mohammadi
7
+ given-names: Hadi
8
+ repository-code: "https://github.com/mohammadi-hadi/arenakit"
9
+ license: MIT
10
+ version: 0.1.0
11
+ date-released: 2026-08-05
arenakit-0.1.0/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Hadi Mohammadi
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,32 @@
1
+ .PHONY: install test lint results examples all clean
2
+
3
+ PY ?= .venv/bin/python
4
+
5
+ install:
6
+ python3 -m venv .venv
7
+ $(PY) -m pip install -q --upgrade pip
8
+ $(PY) -m pip install -q -e ".[dev]"
9
+
10
+ test:
11
+ $(PY) -m pytest -q
12
+
13
+ lint:
14
+ $(PY) -m ruff check src tests examples
15
+ $(PY) -m ruff format --check src tests examples
16
+ $(PY) -m mypy src
17
+
18
+ # Regenerate the committed demo results. CI runs this and fails on any diff,
19
+ # so the numbers in the README cannot drift away from the code.
20
+ results:
21
+ $(PY) -m arenakit.cli demo --out results
22
+
23
+ # The case studies download real votes, so they are not part of CI.
24
+ examples:
25
+ $(PY) examples/lmarena/audit_lmarena.py
26
+ $(PY) examples/mtbench/audit_mtbench.py
27
+
28
+ all: lint test results
29
+
30
+ clean:
31
+ rm -rf .pytest_cache .ruff_cache .mypy_cache dist
32
+ find . -name __pycache__ -type d -prune -exec rm -rf {} +
@@ -0,0 +1,321 @@
1
+ Metadata-Version: 2.4
2
+ Name: arenakit
3
+ Version: 0.1.0
4
+ Summary: Audit a pairwise model leaderboard before you trust its order
5
+ Project-URL: Repository, https://github.com/mohammadi-hadi/arenakit
6
+ Project-URL: Issues, https://github.com/mohammadi-hadi/arenakit/issues
7
+ Author: Hadi Mohammadi
8
+ License-Expression: MIT
9
+ License-File: LICENSE
10
+ Keywords: bradley-terry,leaderboard,llm-evaluation,pairwise-comparison,ranking
11
+ Classifier: Development Status :: 4 - Beta
12
+ Classifier: Intended Audience :: Science/Research
13
+ Classifier: Programming Language :: Python :: 3.11
14
+ Classifier: Programming Language :: Python :: 3.12
15
+ Classifier: Programming Language :: Python :: 3.13
16
+ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
17
+ Requires-Python: >=3.11
18
+ Requires-Dist: matplotlib>=3.8
19
+ Requires-Dist: numpy>=1.26
20
+ Requires-Dist: pydantic>=2.5
21
+ Provides-Extra: dev
22
+ Requires-Dist: build>=1.2; extra == 'dev'
23
+ Requires-Dist: mypy>=1.11; extra == 'dev'
24
+ Requires-Dist: pytest>=8; extra == 'dev'
25
+ Requires-Dist: ruff>=0.6; extra == 'dev'
26
+ Requires-Dist: scipy>=1.11; extra == 'dev'
27
+ Requires-Dist: statsmodels>=0.14; extra == 'dev'
28
+ Description-Content-Type: text/markdown
29
+
30
+ # arenakit
31
+
32
+ Audit a pairwise model leaderboard before you trust its order.
33
+
34
+ [![ci](https://github.com/mohammadi-hadi/arenakit/actions/workflows/ci.yml/badge.svg)](https://github.com/mohammadi-hadi/arenakit/actions/workflows/ci.yml)
35
+
36
+ Arena leaderboards turn pairwise votes into a ranked list, and the ranked list
37
+ is what everyone reads. But a rank is a claim, and the votes behind it support
38
+ some of those claims and not others. Two models a hair apart may be genuinely
39
+ tied. Being shown first may be worth more than the gap between neighbours. A
40
+ board can look decisive because a handful of people voted a great many times.
41
+ arenakit takes the votes, fits the same Bradley-Terry model the leaderboards
42
+ use, and reports which parts of the published order the evidence actually
43
+ carries — with an interval on every number and a flag only when the evidence
44
+ clears a stated bar.
45
+
46
+ ## The audit in one table
47
+
48
+ Nine synthetic boards, one planted defect each. Every probe in the package has
49
+ a board here that makes it fire, and the clean board fires nothing.
50
+
51
+ <!-- arenakit:demo -->
52
+ | Board | What was planted | Probes that fired |
53
+ | --- | --- | --- |
54
+ | Even board | Enough votes, spread evenly, no artefacts planted. | none |
55
+ | Too few votes | The same board on 200 votes instead of 6000. | `rank_volatility`, `separation` |
56
+ | First slot favoured | Being shown first is worth real odds, and one model takes that slot almost always. | `position_bias`, `position_reordering` |
57
+ | Draw-prone model | One weak model draws most of its battles, and half a win each is generous. | `tie_convention` |
58
+ | Rock, paper, scissors | Three models beat each other in a circle, which no single strength explains. | `intransitivity` |
59
+ | Two separate pools | Two halves of the board that never met, ranked as though they had. | `connectivity` |
60
+ | Lopsided budget | The comparison budget is spent mostly on the models at the top. | `coverage_imbalance` |
61
+ | A few power voters | Six people with strong tastes cast a third of all votes. | `separation`, `vote_clustering` |
62
+ | One loud annotator | Two equally good models, and one annotator with a firm view about them. | `annotator_influence`, `separation`, `vote_clustering` |
63
+ <!-- /arenakit:demo -->
64
+
65
+ ![board](results/figures/board.png)
66
+
67
+ ![separation](results/figures/separation.png)
68
+
69
+ The two figures are the same thin board seen two ways: on the left of each, six
70
+ models whose intervals overlap into two tiers; on the right, the gap between
71
+ each pair of neighbours measured against the evidence needed to call it real —
72
+ once on its own, and once with the whole board's worth of comparisons taken into
73
+ account.
74
+
75
+ `make results` regenerates the table, the [full report](results/report.md) and
76
+ the figures from fixed seeds. CI rebuilds them from pinned dependencies and
77
+ fails if a committed number differs from what the code produces.
78
+
79
+ ## 106,134 real arena votes, audited
80
+
81
+ [`examples/lmarena/`](examples/lmarena/) audits the public
82
+ [LMArena preference sample](https://huggingface.co/datasets/lmarena-ai/arena-human-preference-100k):
83
+ 106,134 human votes over 55 models. Committed results:
84
+ [examples/lmarena/results/lmarena.md](examples/lmarena/results/lmarena.md).
85
+
86
+ - **1 of 54 neighbouring pairs is separated** once the 54 comparisons are
87
+ controlled for together; 5 are separated judged one at a time. Only the top
88
+ model is distinguishable from its neighbour. The other 54 models form a
89
+ single block the votes cannot order — which is not the same as saying the
90
+ order is wrong, only that these votes do not establish it.
91
+ - **Closing the median unresolved gap would take roughly 8.9 million votes** at
92
+ this design, against the 106 thousand collected. For most neighbouring pairs
93
+ the honest reading is that they are tied.
94
+ - **The median model's 95% rank interval spans 7 ranks**, and
95
+ `gemma-2-9b-it-simpo` — 468 battles — spans 16.
96
+ - **Position matters, and it does not matter.** The model shown first wins
97
+ 49.53% of 65,418 decisive battles. The interval on the fitted advantage,
98
+ [-0.021, -0.001] in log-odds, excludes zero, so at this sample size the effect
99
+ is real. The whole interval also sits inside the +/-0.05 band this package
100
+ treats as immaterial, and fitting the advantage out reorders nothing.
101
+ Detectable and worth acting on are different questions, and a leaderboard
102
+ needs an answer to the second one.
103
+ - **Clustering the bootstrap on voters widens standard errors by 1.03x.** The
104
+ busiest 1% of the 49,382 voters cast 16% of all votes, so the concern is
105
+ well founded, but on this data the usual battle-level bootstrap is close
106
+ enough. That is a finding, not an assumption — and it is the kind of thing
107
+ worth checking rather than asserting either way.
108
+ - **38.4% of votes are ties**, and the ranking survives dropping them or
109
+ modelling them (Rao-Kupper theta 2.35) instead of splitting them.
110
+ - **Preference cycles land about where Bradley-Terry expects**: 3.5% of the
111
+ 14,164 decided triads are cyclic against 4.3% predicted. One strength per
112
+ model is enough to describe who beats whom on this board.
113
+
114
+ ## The same six models, judged by people and by GPT-4
115
+
116
+ [`examples/mtbench/`](examples/mtbench/) audits both halves of
117
+ [MT-Bench's judgments](https://huggingface.co/datasets/lmsys/mt_bench_human_judgments):
118
+ 3,355 votes from 65 people, and 2,400 verdicts from GPT-4 acting as a judge, on
119
+ the same models and the same questions. Committed results:
120
+ [examples/mtbench/results/mtbench.md](examples/mtbench/results/mtbench.md).
121
+
122
+ - **GPT-4 changes its mind on 15.8% of pairs when the two answers swap places.**
123
+ That figure is not estimated here; the split records those cases explicitly,
124
+ because it was collected by asking both ways round.
125
+ - **GPT-4 matches the human majority on 71.3%** of the 1,612 comparisons where
126
+ the humans reached one.
127
+ - **And it produces exactly the human leaderboard.** Kendall tau between the
128
+ two orders is +1.00. Judge noise that looks disqualifying one comparison at a
129
+ time averages out of a six-model ranking — which is a reason to be careful
130
+ about what a leaderboard proves, in both directions.
131
+ - The judge's verdicts are correlated within question: resampling questions
132
+ rather than verdicts widens its intervals by 1.48x. The human votes, spread
133
+ over 65 annotators, show no annotator carrying the board.
134
+
135
+ ## Install
136
+
137
+ ```bash
138
+ pip install arenakit
139
+ ```
140
+
141
+ Python 3.11+. Runtime dependencies are numpy, pydantic and matplotlib. The two
142
+ case studies additionally need `pyarrow` to read the published parquet files.
143
+
144
+ ## Quickstart
145
+
146
+ ```bash
147
+ arenakit audit battles.csv
148
+ ```
149
+
150
+ where `battles.csv` has `model_a`, `model_b`, `winner`, and optionally
151
+ `item_id` and `voter_id`. `model_a` is the model shown **first**.
152
+
153
+ ```python
154
+ from arenakit import Arena, load_battles, run_audit
155
+
156
+ audit = run_audit(Arena(load_battles("battles.csv")))
157
+
158
+ for row in audit.leaderboard[:5]:
159
+ print(row["rank"], row["model"], row["score"], row["rank_ci"])
160
+
161
+ for probe in audit.triggered:
162
+ print(probe.name, "-", probe.reason)
163
+ ```
164
+
165
+ Fitting on its own, without the audit:
166
+
167
+ ```python
168
+ from arenakit import fit_bt, to_elo
169
+
170
+ fit = fit_bt(arena.counts(), ties="half", order=True)
171
+ print(fit.order_log) # advantage of being shown first, in log-odds
172
+ print(to_elo(fit.scores)) # the same scores in the usual leaderboard units
173
+ ```
174
+
175
+ ## What it checks
176
+
177
+ | Probe | Question | Flags when |
178
+ | --- | --- | --- |
179
+ | `separation` | How much of the published order do the votes support? | more than a quarter of neighbouring pairs are not separated |
180
+ | `rank_volatility` | How far does each rank move under resampling? | the median rank interval spans more than 2 ranks |
181
+ | `connectivity` | Are the scores comparable at all? | the board splits into groups that never met, or a model has under 30 battles |
182
+ | `coverage_imbalance` | Was the comparison budget spent evenly? | Gini of battles per model above 0.5 |
183
+ | `position_bias` | Is being shown first worth anything? | the advantage clears 0.05 in log-odds |
184
+ | `position_reordering` | Does correcting for position change the order? | any separated pair reverses |
185
+ | `tie_convention` | Does the order survive a different tie rule? | any separated pair reverses under dropped or modelled ties |
186
+ | `intransitivity` | Can one strength per model explain who beats whom? | cycles exceed what the fit predicts by 2 points |
187
+ | `vote_clustering` | Was independent resampling ever justified? | clustered standard errors are more than 1.15x wider |
188
+ | `annotator_influence` | Is one annotator moving the board? | dropping one shifts a model by a whole rung of the ladder |
189
+
190
+ Thresholds are named constants in
191
+ [`src/arenakit/probes.py`](src/arenakit/probes.py); the table above is a
192
+ description of that file, not a second source of truth. A probe that cannot
193
+ answer its question says so and skips — for instance, the position probes stand
194
+ down when the data records which model *sorts* first rather than which was
195
+ *shown* first, which is exactly what MT-Bench's GPT-4 split does.
196
+
197
+ ## How the numbers are defended
198
+
199
+ - **Separation is not interval overlap.** Two intervals can overlap while the
200
+ difference between them is significant, so the overlap rule quietly calls
201
+ real gaps unresolved. arenakit takes the bootstrap distribution of the
202
+ *difference*, studentises it, and compares it against a critical value. There
203
+ is a test that constructs a board where the two rules disagree.
204
+ - **A leaderboard makes every comparison at once.** Ranking 55 models means 54
205
+ adjacent tests chosen by the same data that will judge them, so `separation`
206
+ reports both a marginal verdict and a simultaneous one using a max-t critical
207
+ value across the whole board. On the arena data the two answers are 5 and 1.
208
+ - **The fits are checked against a reference implementation.** Bradley-Terry is
209
+ logistic regression on signed model indicators, and the first-position term is
210
+ its intercept; both are asserted equal to `statsmodels` to 1e-5. Two-model
211
+ boards are checked against the closed form, and Rao-Kupper is checked to
212
+ collapse onto plain Bradley-Terry when there are no ties.
213
+ - **Every probe has a board built to make it fire.** `make results` audits nine
214
+ planted boards and asserts the exact set of flags each one raises, so a probe
215
+ that becomes over-eager shows up as a clean board raising a flag, and one that
216
+ goes blind shows up as a planted defect nobody noticed.
217
+ - **Reruns are identical.** One seed, fixed convergence tolerance and iteration
218
+ cap, integer matrix products where a count has to be exact, and no BLAS in any
219
+ path that produces a published number.
220
+
221
+ ## Design notes
222
+
223
+ **Sufficient statistics, not battle lists.** Every model here is fitted from a
224
+ matrix of ordered pair counts, which is all the likelihood depends on. A
225
+ bootstrap resample therefore costs three weighted `bincount` calls plus an
226
+ iteration on a 55x55 matrix, and two thousand resamples over a hundred thousand
227
+ battles take seconds rather than an afternoon.
228
+
229
+ **The default resampling is the most conservative the data allows** — voters if
230
+ they are recorded, then prompts, then battles — because treating correlated
231
+ votes as independent is the commoner mistake and the quieter one. When the
232
+ chosen column has too few distinct values to resample, the audit drops to a
233
+ coarser scheme and says so, rather than reporting the zero-width intervals a
234
+ one-annotator resample would produce.
235
+
236
+ **Disconnected boards are refused, not fudged.** Two models joined by no chain
237
+ of opponents have no estimable difference; the likelihood is flat in exactly
238
+ that direction, and any number printed for the gap comes from the prior.
239
+ `run_audit` keeps the largest connected group, reports what it dropped, and
240
+ `fit_bt` raises rather than returning a number nobody should use.
241
+
242
+ **Ties arrive as one outcome on purpose.** Real boards ship richer tie labels —
243
+ "both bad", "both good", "inconsistent" — and collapsing them changes the
244
+ ranking, so the loader refuses to guess and asks you to map them yourself. Both
245
+ case studies show the mapping in their source.
246
+
247
+ **A weak prior, so the fit always exists.** Each model plays half a virtual
248
+ battle against an average opponent, which keeps an undefeated model's score
249
+ finite. With `prior=0.0` you get the unpenalised estimate, and a clear error
250
+ when it does not exist.
251
+
252
+ ## Limitations
253
+
254
+ - **Which pairs are adjacent is decided by the point estimate.** The
255
+ simultaneous test controls the family-wise error rate across those 54
256
+ comparisons, but the comparisons themselves were chosen after seeing the data.
257
+ A fully simultaneous treatment of the ranking would be more conservative
258
+ still, so the separation counts here are, if anything, generous.
259
+ - **Bradley-Terry assumes one number per model.** `intransitivity` checks that
260
+ assumption but does not repair it; a board with genuine rock-paper-scissors
261
+ structure needs a different model, not a wider interval.
262
+ - **Half-credit ties are a convention, not a likelihood.** With `ties="half"`
263
+ the fit maximises a weighted objective rather than a probability model.
264
+ `fit_rao_kupper` gives ties a parameter instead, and `tie_convention` reports
265
+ whether the choice mattered.
266
+ - **The position term is one number for the whole board.** A judge that favours
267
+ the first slot only for long answers, or only in one language, will not show
268
+ up as a single advantage.
269
+ - **The case studies are public samples, not the live leaderboards.** The
270
+ LMArena file is a 106k-battle release covering 55 models, not the full vote
271
+ history behind the published board.
272
+
273
+ ## Related work
274
+
275
+ - Bradley and Terry (1952), *Rank Analysis of Incomplete Block Designs*
276
+ ([doi:10.2307/2334029](https://doi.org/10.2307/2334029)) — the model.
277
+ - Rao and Kupper (1967), *Ties in Paired-Comparison Experiments*
278
+ ([doi:10.1080/01621459.1967.10482901](https://doi.org/10.1080/01621459.1967.10482901))
279
+ — the tie model used here.
280
+ - Hunter (2004), *MM algorithms for generalized Bradley-Terry models*
281
+ ([doi:10.1214/aos/1079120141](https://doi.org/10.1214/aos/1079120141)) — the
282
+ fitting algorithm, including the position and tie extensions.
283
+ - Ford (1957), *Solution of a Ranking Problem from Binary Comparisons*
284
+ ([doi:10.2307/2308513](https://doi.org/10.2307/2308513)) — when a finite
285
+ estimate exists at all.
286
+ - Chiang et al. (2024), *Chatbot Arena: An Open Platform for Evaluating LLMs by
287
+ Human Preference* ([arXiv:2403.04132](https://arxiv.org/abs/2403.04132)) —
288
+ the leaderboard this package audits.
289
+ - Zheng et al. (2023), *Judging LLM-as-a-Judge with MT-Bench and Chatbot Arena*
290
+ ([arXiv:2306.05685](https://arxiv.org/abs/2306.05685)) — the source of the
291
+ human and GPT-4 judgments, and of the position-bias measurement.
292
+ - Westfall and Young (1993), *Resampling-Based Multiple Testing* — the max-t
293
+ idea behind the simultaneous critical value.
294
+
295
+ ## Companion projects
296
+
297
+ - [judgekit](https://github.com/mohammadi-hadi/judgekit) — audit an LLM judge
298
+ for bias before you trust its scores.
299
+ - [judgepanel](https://github.com/mohammadi-hadi/judgepanel) — estimate judge
300
+ accuracy without gold labels.
301
+ - [raterkit](https://github.com/mohammadi-hadi/raterkit) — audit a labeled
302
+ dataset before you trust it.
303
+ - [calikit](https://github.com/mohammadi-hadi/calikit) — audit the calibration
304
+ of a model's probabilities.
305
+ - [abkit](https://github.com/mohammadi-hadi/abkit) — audit an A/B test before
306
+ you act on it.
307
+
308
+ ## Citation
309
+
310
+ ```bibtex
311
+ @software{mohammadi_arenakit,
312
+ author = {Mohammadi, Hadi},
313
+ title = {arenakit: audit a pairwise model leaderboard before you trust its order},
314
+ year = {2026},
315
+ url = {https://github.com/mohammadi-hadi/arenakit}
316
+ }
317
+ ```
318
+
319
+ ## License
320
+
321
+ MIT