arenakit 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- arenakit-0.1.0/.github/FUNDING.yml +1 -0
- arenakit-0.1.0/.github/workflows/ci.yml +42 -0
- arenakit-0.1.0/.github/workflows/release.yml +49 -0
- arenakit-0.1.0/CITATION.cff +11 -0
- arenakit-0.1.0/LICENSE +21 -0
- arenakit-0.1.0/Makefile +32 -0
- arenakit-0.1.0/PKG-INFO +321 -0
- arenakit-0.1.0/README.md +292 -0
- arenakit-0.1.0/examples/lmarena/audit_lmarena.py +138 -0
- arenakit-0.1.0/examples/lmarena/fetch_data.py +95 -0
- arenakit-0.1.0/examples/lmarena/results/figures/board.png +0 -0
- arenakit-0.1.0/examples/lmarena/results/figures/separation.png +0 -0
- arenakit-0.1.0/examples/lmarena/results/lmarena.json +2020 -0
- arenakit-0.1.0/examples/lmarena/results/lmarena.md +109 -0
- arenakit-0.1.0/examples/lmarena/results/resampling.json +29 -0
- arenakit-0.1.0/examples/mtbench/audit_mtbench.py +209 -0
- arenakit-0.1.0/examples/mtbench/fetch_data.py +47 -0
- arenakit-0.1.0/examples/mtbench/results/figures/gpt4_pair.png +0 -0
- arenakit-0.1.0/examples/mtbench/results/figures/human.png +0 -0
- arenakit-0.1.0/examples/mtbench/results/gpt4_pair.json +398 -0
- arenakit-0.1.0/examples/mtbench/results/human.json +414 -0
- arenakit-0.1.0/examples/mtbench/results/mtbench.md +111 -0
- arenakit-0.1.0/examples/mtbench/results/summary.json +57 -0
- arenakit-0.1.0/pyproject.toml +68 -0
- arenakit-0.1.0/requirements-results.txt +6 -0
- arenakit-0.1.0/results/demo.json +89 -0
- arenakit-0.1.0/results/figures/board.png +0 -0
- arenakit-0.1.0/results/figures/separation.png +0 -0
- arenakit-0.1.0/results/report.md +86 -0
- arenakit-0.1.0/results/table.md +11 -0
- arenakit-0.1.0/src/arenakit/__init__.py +30 -0
- arenakit-0.1.0/src/arenakit/boot.py +284 -0
- arenakit-0.1.0/src/arenakit/bt.py +322 -0
- arenakit-0.1.0/src/arenakit/cli.py +171 -0
- arenakit-0.1.0/src/arenakit/demo.py +163 -0
- arenakit-0.1.0/src/arenakit/probes.py +688 -0
- arenakit-0.1.0/src/arenakit/py.typed +0 -0
- arenakit-0.1.0/src/arenakit/report.py +227 -0
- arenakit-0.1.0/src/arenakit/result.py +102 -0
- arenakit-0.1.0/src/arenakit/schema.py +345 -0
- arenakit-0.1.0/src/arenakit/synthetic.py +170 -0
- arenakit-0.1.0/tests/test_boot.py +148 -0
- arenakit-0.1.0/tests/test_bt.py +162 -0
- arenakit-0.1.0/tests/test_cli.py +136 -0
- arenakit-0.1.0/tests/test_probes.py +188 -0
- arenakit-0.1.0/tests/test_schema.py +131 -0
|
@@ -0,0 +1 @@
|
|
|
1
|
+
github: [mohammadi-hadi]
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
name: CI
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
branches: [main]
|
|
6
|
+
pull_request:
|
|
7
|
+
|
|
8
|
+
jobs:
|
|
9
|
+
test:
|
|
10
|
+
runs-on: ubuntu-latest
|
|
11
|
+
strategy:
|
|
12
|
+
fail-fast: false
|
|
13
|
+
matrix:
|
|
14
|
+
python-version: ["3.11", "3.12", "3.13"]
|
|
15
|
+
steps:
|
|
16
|
+
- uses: actions/checkout@v4
|
|
17
|
+
- uses: actions/setup-python@v5
|
|
18
|
+
with:
|
|
19
|
+
python-version: ${{ matrix.python-version }}
|
|
20
|
+
- run: python -m pip install --upgrade pip
|
|
21
|
+
- run: pip install -e ".[dev]"
|
|
22
|
+
- run: ruff check src tests examples
|
|
23
|
+
- run: ruff format --check src tests examples
|
|
24
|
+
- run: mypy src
|
|
25
|
+
- run: pytest -q
|
|
26
|
+
|
|
27
|
+
results:
|
|
28
|
+
# The committed demo results are regenerated from the code and diffed. If a
|
|
29
|
+
# change moves a number, this fails until the results are regenerated with
|
|
30
|
+
# it, so the README can never quote a figure the code no longer produces.
|
|
31
|
+
runs-on: ubuntu-latest
|
|
32
|
+
steps:
|
|
33
|
+
- uses: actions/checkout@v4
|
|
34
|
+
- uses: actions/setup-python@v5
|
|
35
|
+
with:
|
|
36
|
+
python-version: "3.12"
|
|
37
|
+
- run: python -m pip install --upgrade pip
|
|
38
|
+
- run: pip install -r requirements-results.txt
|
|
39
|
+
- run: pip install -e . --no-deps
|
|
40
|
+
- run: python -m arenakit.cli demo --out results
|
|
41
|
+
- name: Fail if the committed results moved
|
|
42
|
+
run: git diff --exit-code -- results/table.md results/report.md results/demo.json
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
name: Release
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
tags: ["v*"]
|
|
6
|
+
|
|
7
|
+
jobs:
|
|
8
|
+
build:
|
|
9
|
+
runs-on: ubuntu-latest
|
|
10
|
+
steps:
|
|
11
|
+
- uses: actions/checkout@v4
|
|
12
|
+
- uses: actions/setup-python@v5
|
|
13
|
+
with:
|
|
14
|
+
python-version: "3.12"
|
|
15
|
+
- run: python -m pip install --upgrade pip build
|
|
16
|
+
- run: python -m build
|
|
17
|
+
- uses: actions/upload-artifact@v4
|
|
18
|
+
with:
|
|
19
|
+
name: dist
|
|
20
|
+
path: dist/
|
|
21
|
+
|
|
22
|
+
publish:
|
|
23
|
+
needs: build
|
|
24
|
+
runs-on: ubuntu-latest
|
|
25
|
+
environment: pypi
|
|
26
|
+
permissions:
|
|
27
|
+
id-token: write
|
|
28
|
+
steps:
|
|
29
|
+
- uses: actions/download-artifact@v4
|
|
30
|
+
with:
|
|
31
|
+
name: dist
|
|
32
|
+
path: dist/
|
|
33
|
+
- uses: pypa/gh-action-pypi-publish@release/v1
|
|
34
|
+
|
|
35
|
+
github-release:
|
|
36
|
+
needs: publish
|
|
37
|
+
runs-on: ubuntu-latest
|
|
38
|
+
permissions:
|
|
39
|
+
contents: write
|
|
40
|
+
steps:
|
|
41
|
+
- uses: actions/checkout@v4
|
|
42
|
+
- uses: actions/download-artifact@v4
|
|
43
|
+
with:
|
|
44
|
+
name: dist
|
|
45
|
+
path: dist/
|
|
46
|
+
- uses: softprops/action-gh-release@v2
|
|
47
|
+
with:
|
|
48
|
+
files: dist/*
|
|
49
|
+
generate_release_notes: true
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
cff-version: 1.2.0
|
|
2
|
+
message: "If you use arenakit, please cite it as below."
|
|
3
|
+
title: "arenakit: audit a pairwise model leaderboard before you trust its order"
|
|
4
|
+
type: software
|
|
5
|
+
authors:
|
|
6
|
+
- family-names: Mohammadi
|
|
7
|
+
given-names: Hadi
|
|
8
|
+
repository-code: "https://github.com/mohammadi-hadi/arenakit"
|
|
9
|
+
license: MIT
|
|
10
|
+
version: 0.1.0
|
|
11
|
+
date-released: 2026-08-05
|
arenakit-0.1.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Hadi Mohammadi
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
arenakit-0.1.0/Makefile
ADDED
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
.PHONY: install test lint results examples all clean
|
|
2
|
+
|
|
3
|
+
PY ?= .venv/bin/python
|
|
4
|
+
|
|
5
|
+
install:
|
|
6
|
+
python3 -m venv .venv
|
|
7
|
+
$(PY) -m pip install -q --upgrade pip
|
|
8
|
+
$(PY) -m pip install -q -e ".[dev]"
|
|
9
|
+
|
|
10
|
+
test:
|
|
11
|
+
$(PY) -m pytest -q
|
|
12
|
+
|
|
13
|
+
lint:
|
|
14
|
+
$(PY) -m ruff check src tests examples
|
|
15
|
+
$(PY) -m ruff format --check src tests examples
|
|
16
|
+
$(PY) -m mypy src
|
|
17
|
+
|
|
18
|
+
# Regenerate the committed demo results. CI runs this and fails on any diff,
|
|
19
|
+
# so the numbers in the README cannot drift away from the code.
|
|
20
|
+
results:
|
|
21
|
+
$(PY) -m arenakit.cli demo --out results
|
|
22
|
+
|
|
23
|
+
# The case studies download real votes, so they are not part of CI.
|
|
24
|
+
examples:
|
|
25
|
+
$(PY) examples/lmarena/audit_lmarena.py
|
|
26
|
+
$(PY) examples/mtbench/audit_mtbench.py
|
|
27
|
+
|
|
28
|
+
all: lint test results
|
|
29
|
+
|
|
30
|
+
clean:
|
|
31
|
+
rm -rf .pytest_cache .ruff_cache .mypy_cache dist
|
|
32
|
+
find . -name __pycache__ -type d -prune -exec rm -rf {} +
|
arenakit-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,321 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: arenakit
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Audit a pairwise model leaderboard before you trust its order
|
|
5
|
+
Project-URL: Repository, https://github.com/mohammadi-hadi/arenakit
|
|
6
|
+
Project-URL: Issues, https://github.com/mohammadi-hadi/arenakit/issues
|
|
7
|
+
Author: Hadi Mohammadi
|
|
8
|
+
License-Expression: MIT
|
|
9
|
+
License-File: LICENSE
|
|
10
|
+
Keywords: bradley-terry,leaderboard,llm-evaluation,pairwise-comparison,ranking
|
|
11
|
+
Classifier: Development Status :: 4 - Beta
|
|
12
|
+
Classifier: Intended Audience :: Science/Research
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
16
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
17
|
+
Requires-Python: >=3.11
|
|
18
|
+
Requires-Dist: matplotlib>=3.8
|
|
19
|
+
Requires-Dist: numpy>=1.26
|
|
20
|
+
Requires-Dist: pydantic>=2.5
|
|
21
|
+
Provides-Extra: dev
|
|
22
|
+
Requires-Dist: build>=1.2; extra == 'dev'
|
|
23
|
+
Requires-Dist: mypy>=1.11; extra == 'dev'
|
|
24
|
+
Requires-Dist: pytest>=8; extra == 'dev'
|
|
25
|
+
Requires-Dist: ruff>=0.6; extra == 'dev'
|
|
26
|
+
Requires-Dist: scipy>=1.11; extra == 'dev'
|
|
27
|
+
Requires-Dist: statsmodels>=0.14; extra == 'dev'
|
|
28
|
+
Description-Content-Type: text/markdown
|
|
29
|
+
|
|
30
|
+
# arenakit
|
|
31
|
+
|
|
32
|
+
Audit a pairwise model leaderboard before you trust its order.
|
|
33
|
+
|
|
34
|
+
[](https://github.com/mohammadi-hadi/arenakit/actions/workflows/ci.yml)
|
|
35
|
+
|
|
36
|
+
Arena leaderboards turn pairwise votes into a ranked list, and the ranked list
|
|
37
|
+
is what everyone reads. But a rank is a claim, and the votes behind it support
|
|
38
|
+
some of those claims and not others. Two models a hair apart may be genuinely
|
|
39
|
+
tied. Being shown first may be worth more than the gap between neighbours. A
|
|
40
|
+
board can look decisive because a handful of people voted a great many times.
|
|
41
|
+
arenakit takes the votes, fits the same Bradley-Terry model the leaderboards
|
|
42
|
+
use, and reports which parts of the published order the evidence actually
|
|
43
|
+
carries — with an interval on every number and a flag only when the evidence
|
|
44
|
+
clears a stated bar.
|
|
45
|
+
|
|
46
|
+
## The audit in one table
|
|
47
|
+
|
|
48
|
+
Nine synthetic boards, one planted defect each. Every probe in the package has
|
|
49
|
+
a board here that makes it fire, and the clean board fires nothing.
|
|
50
|
+
|
|
51
|
+
<!-- arenakit:demo -->
|
|
52
|
+
| Board | What was planted | Probes that fired |
|
|
53
|
+
| --- | --- | --- |
|
|
54
|
+
| Even board | Enough votes, spread evenly, no artefacts planted. | none |
|
|
55
|
+
| Too few votes | The same board on 200 votes instead of 6000. | `rank_volatility`, `separation` |
|
|
56
|
+
| First slot favoured | Being shown first is worth real odds, and one model takes that slot almost always. | `position_bias`, `position_reordering` |
|
|
57
|
+
| Draw-prone model | One weak model draws most of its battles, and half a win each is generous. | `tie_convention` |
|
|
58
|
+
| Rock, paper, scissors | Three models beat each other in a circle, which no single strength explains. | `intransitivity` |
|
|
59
|
+
| Two separate pools | Two halves of the board that never met, ranked as though they had. | `connectivity` |
|
|
60
|
+
| Lopsided budget | The comparison budget is spent mostly on the models at the top. | `coverage_imbalance` |
|
|
61
|
+
| A few power voters | Six people with strong tastes cast a third of all votes. | `separation`, `vote_clustering` |
|
|
62
|
+
| One loud annotator | Two equally good models, and one annotator with a firm view about them. | `annotator_influence`, `separation`, `vote_clustering` |
|
|
63
|
+
<!-- /arenakit:demo -->
|
|
64
|
+
|
|
65
|
+

|
|
66
|
+
|
|
67
|
+

|
|
68
|
+
|
|
69
|
+
The two figures are the same thin board seen two ways: on the left of each, six
|
|
70
|
+
models whose intervals overlap into two tiers; on the right, the gap between
|
|
71
|
+
each pair of neighbours measured against the evidence needed to call it real —
|
|
72
|
+
once on its own, and once with the whole board's worth of comparisons taken into
|
|
73
|
+
account.
|
|
74
|
+
|
|
75
|
+
`make results` regenerates the table, the [full report](results/report.md) and
|
|
76
|
+
the figures from fixed seeds. CI rebuilds them from pinned dependencies and
|
|
77
|
+
fails if a committed number differs from what the code produces.
|
|
78
|
+
|
|
79
|
+
## 106,134 real arena votes, audited
|
|
80
|
+
|
|
81
|
+
[`examples/lmarena/`](examples/lmarena/) audits the public
|
|
82
|
+
[LMArena preference sample](https://huggingface.co/datasets/lmarena-ai/arena-human-preference-100k):
|
|
83
|
+
106,134 human votes over 55 models. Committed results:
|
|
84
|
+
[examples/lmarena/results/lmarena.md](examples/lmarena/results/lmarena.md).
|
|
85
|
+
|
|
86
|
+
- **1 of 54 neighbouring pairs is separated** once the 54 comparisons are
|
|
87
|
+
controlled for together; 5 are separated judged one at a time. Only the top
|
|
88
|
+
model is distinguishable from its neighbour. The other 54 models form a
|
|
89
|
+
single block the votes cannot order — which is not the same as saying the
|
|
90
|
+
order is wrong, only that these votes do not establish it.
|
|
91
|
+
- **Closing the median unresolved gap would take roughly 8.9 million votes** at
|
|
92
|
+
this design, against the 106 thousand collected. For most neighbouring pairs
|
|
93
|
+
the honest reading is that they are tied.
|
|
94
|
+
- **The median model's 95% rank interval spans 7 ranks**, and
|
|
95
|
+
`gemma-2-9b-it-simpo` — 468 battles — spans 16.
|
|
96
|
+
- **Position matters, and it does not matter.** The model shown first wins
|
|
97
|
+
49.53% of 65,418 decisive battles. The interval on the fitted advantage,
|
|
98
|
+
[-0.021, -0.001] in log-odds, excludes zero, so at this sample size the effect
|
|
99
|
+
is real. The whole interval also sits inside the +/-0.05 band this package
|
|
100
|
+
treats as immaterial, and fitting the advantage out reorders nothing.
|
|
101
|
+
Detectable and worth acting on are different questions, and a leaderboard
|
|
102
|
+
needs an answer to the second one.
|
|
103
|
+
- **Clustering the bootstrap on voters widens standard errors by 1.03x.** The
|
|
104
|
+
busiest 1% of the 49,382 voters cast 16% of all votes, so the concern is
|
|
105
|
+
well founded, but on this data the usual battle-level bootstrap is close
|
|
106
|
+
enough. That is a finding, not an assumption — and it is the kind of thing
|
|
107
|
+
worth checking rather than asserting either way.
|
|
108
|
+
- **38.4% of votes are ties**, and the ranking survives dropping them or
|
|
109
|
+
modelling them (Rao-Kupper theta 2.35) instead of splitting them.
|
|
110
|
+
- **Preference cycles land about where Bradley-Terry expects**: 3.5% of the
|
|
111
|
+
14,164 decided triads are cyclic against 4.3% predicted. One strength per
|
|
112
|
+
model is enough to describe who beats whom on this board.
|
|
113
|
+
|
|
114
|
+
## The same six models, judged by people and by GPT-4
|
|
115
|
+
|
|
116
|
+
[`examples/mtbench/`](examples/mtbench/) audits both halves of
|
|
117
|
+
[MT-Bench's judgments](https://huggingface.co/datasets/lmsys/mt_bench_human_judgments):
|
|
118
|
+
3,355 votes from 65 people, and 2,400 verdicts from GPT-4 acting as a judge, on
|
|
119
|
+
the same models and the same questions. Committed results:
|
|
120
|
+
[examples/mtbench/results/mtbench.md](examples/mtbench/results/mtbench.md).
|
|
121
|
+
|
|
122
|
+
- **GPT-4 changes its mind on 15.8% of pairs when the two answers swap places.**
|
|
123
|
+
That figure is not estimated here; the split records those cases explicitly,
|
|
124
|
+
because it was collected by asking both ways round.
|
|
125
|
+
- **GPT-4 matches the human majority on 71.3%** of the 1,612 comparisons where
|
|
126
|
+
the humans reached one.
|
|
127
|
+
- **And it produces exactly the human leaderboard.** Kendall tau between the
|
|
128
|
+
two orders is +1.00. Judge noise that looks disqualifying one comparison at a
|
|
129
|
+
time averages out of a six-model ranking — which is a reason to be careful
|
|
130
|
+
about what a leaderboard proves, in both directions.
|
|
131
|
+
- The judge's verdicts are correlated within question: resampling questions
|
|
132
|
+
rather than verdicts widens its intervals by 1.48x. The human votes, spread
|
|
133
|
+
over 65 annotators, show no annotator carrying the board.
|
|
134
|
+
|
|
135
|
+
## Install
|
|
136
|
+
|
|
137
|
+
```bash
|
|
138
|
+
pip install arenakit
|
|
139
|
+
```
|
|
140
|
+
|
|
141
|
+
Python 3.11+. Runtime dependencies are numpy, pydantic and matplotlib. The two
|
|
142
|
+
case studies additionally need `pyarrow` to read the published parquet files.
|
|
143
|
+
|
|
144
|
+
## Quickstart
|
|
145
|
+
|
|
146
|
+
```bash
|
|
147
|
+
arenakit audit battles.csv
|
|
148
|
+
```
|
|
149
|
+
|
|
150
|
+
where `battles.csv` has `model_a`, `model_b`, `winner`, and optionally
|
|
151
|
+
`item_id` and `voter_id`. `model_a` is the model shown **first**.
|
|
152
|
+
|
|
153
|
+
```python
|
|
154
|
+
from arenakit import Arena, load_battles, run_audit
|
|
155
|
+
|
|
156
|
+
audit = run_audit(Arena(load_battles("battles.csv")))
|
|
157
|
+
|
|
158
|
+
for row in audit.leaderboard[:5]:
|
|
159
|
+
print(row["rank"], row["model"], row["score"], row["rank_ci"])
|
|
160
|
+
|
|
161
|
+
for probe in audit.triggered:
|
|
162
|
+
print(probe.name, "-", probe.reason)
|
|
163
|
+
```
|
|
164
|
+
|
|
165
|
+
Fitting on its own, without the audit:
|
|
166
|
+
|
|
167
|
+
```python
|
|
168
|
+
from arenakit import fit_bt, to_elo
|
|
169
|
+
|
|
170
|
+
fit = fit_bt(arena.counts(), ties="half", order=True)
|
|
171
|
+
print(fit.order_log) # advantage of being shown first, in log-odds
|
|
172
|
+
print(to_elo(fit.scores)) # the same scores in the usual leaderboard units
|
|
173
|
+
```
|
|
174
|
+
|
|
175
|
+
## What it checks
|
|
176
|
+
|
|
177
|
+
| Probe | Question | Flags when |
|
|
178
|
+
| --- | --- | --- |
|
|
179
|
+
| `separation` | How much of the published order do the votes support? | more than a quarter of neighbouring pairs are not separated |
|
|
180
|
+
| `rank_volatility` | How far does each rank move under resampling? | the median rank interval spans more than 2 ranks |
|
|
181
|
+
| `connectivity` | Are the scores comparable at all? | the board splits into groups that never met, or a model has under 30 battles |
|
|
182
|
+
| `coverage_imbalance` | Was the comparison budget spent evenly? | Gini of battles per model above 0.5 |
|
|
183
|
+
| `position_bias` | Is being shown first worth anything? | the advantage clears 0.05 in log-odds |
|
|
184
|
+
| `position_reordering` | Does correcting for position change the order? | any separated pair reverses |
|
|
185
|
+
| `tie_convention` | Does the order survive a different tie rule? | any separated pair reverses under dropped or modelled ties |
|
|
186
|
+
| `intransitivity` | Can one strength per model explain who beats whom? | cycles exceed what the fit predicts by 2 points |
|
|
187
|
+
| `vote_clustering` | Was independent resampling ever justified? | clustered standard errors are more than 1.15x wider |
|
|
188
|
+
| `annotator_influence` | Is one annotator moving the board? | dropping one shifts a model by a whole rung of the ladder |
|
|
189
|
+
|
|
190
|
+
Thresholds are named constants in
|
|
191
|
+
[`src/arenakit/probes.py`](src/arenakit/probes.py); the table above is a
|
|
192
|
+
description of that file, not a second source of truth. A probe that cannot
|
|
193
|
+
answer its question says so and skips — for instance, the position probes stand
|
|
194
|
+
down when the data records which model *sorts* first rather than which was
|
|
195
|
+
*shown* first, which is exactly what MT-Bench's GPT-4 split does.
|
|
196
|
+
|
|
197
|
+
## How the numbers are defended
|
|
198
|
+
|
|
199
|
+
- **Separation is not interval overlap.** Two intervals can overlap while the
|
|
200
|
+
difference between them is significant, so the overlap rule quietly calls
|
|
201
|
+
real gaps unresolved. arenakit takes the bootstrap distribution of the
|
|
202
|
+
*difference*, studentises it, and compares it against a critical value. There
|
|
203
|
+
is a test that constructs a board where the two rules disagree.
|
|
204
|
+
- **A leaderboard makes every comparison at once.** Ranking 55 models means 54
|
|
205
|
+
adjacent tests chosen by the same data that will judge them, so `separation`
|
|
206
|
+
reports both a marginal verdict and a simultaneous one using a max-t critical
|
|
207
|
+
value across the whole board. On the arena data the two answers are 5 and 1.
|
|
208
|
+
- **The fits are checked against a reference implementation.** Bradley-Terry is
|
|
209
|
+
logistic regression on signed model indicators, and the first-position term is
|
|
210
|
+
its intercept; both are asserted equal to `statsmodels` to 1e-5. Two-model
|
|
211
|
+
boards are checked against the closed form, and Rao-Kupper is checked to
|
|
212
|
+
collapse onto plain Bradley-Terry when there are no ties.
|
|
213
|
+
- **Every probe has a board built to make it fire.** `make results` audits nine
|
|
214
|
+
planted boards and asserts the exact set of flags each one raises, so a probe
|
|
215
|
+
that becomes over-eager shows up as a clean board raising a flag, and one that
|
|
216
|
+
goes blind shows up as a planted defect nobody noticed.
|
|
217
|
+
- **Reruns are identical.** One seed, fixed convergence tolerance and iteration
|
|
218
|
+
cap, integer matrix products where a count has to be exact, and no BLAS in any
|
|
219
|
+
path that produces a published number.
|
|
220
|
+
|
|
221
|
+
## Design notes
|
|
222
|
+
|
|
223
|
+
**Sufficient statistics, not battle lists.** Every model here is fitted from a
|
|
224
|
+
matrix of ordered pair counts, which is all the likelihood depends on. A
|
|
225
|
+
bootstrap resample therefore costs three weighted `bincount` calls plus an
|
|
226
|
+
iteration on a 55x55 matrix, and two thousand resamples over a hundred thousand
|
|
227
|
+
battles take seconds rather than an afternoon.
|
|
228
|
+
|
|
229
|
+
**The default resampling is the most conservative the data allows** — voters if
|
|
230
|
+
they are recorded, then prompts, then battles — because treating correlated
|
|
231
|
+
votes as independent is the commoner mistake and the quieter one. When the
|
|
232
|
+
chosen column has too few distinct values to resample, the audit drops to a
|
|
233
|
+
coarser scheme and says so, rather than reporting the zero-width intervals a
|
|
234
|
+
one-annotator resample would produce.
|
|
235
|
+
|
|
236
|
+
**Disconnected boards are refused, not fudged.** Two models joined by no chain
|
|
237
|
+
of opponents have no estimable difference; the likelihood is flat in exactly
|
|
238
|
+
that direction, and any number printed for the gap comes from the prior.
|
|
239
|
+
`run_audit` keeps the largest connected group, reports what it dropped, and
|
|
240
|
+
`fit_bt` raises rather than returning a number nobody should use.
|
|
241
|
+
|
|
242
|
+
**Ties arrive as one outcome on purpose.** Real boards ship richer tie labels —
|
|
243
|
+
"both bad", "both good", "inconsistent" — and collapsing them changes the
|
|
244
|
+
ranking, so the loader refuses to guess and asks you to map them yourself. Both
|
|
245
|
+
case studies show the mapping in their source.
|
|
246
|
+
|
|
247
|
+
**A weak prior, so the fit always exists.** Each model plays half a virtual
|
|
248
|
+
battle against an average opponent, which keeps an undefeated model's score
|
|
249
|
+
finite. With `prior=0.0` you get the unpenalised estimate, and a clear error
|
|
250
|
+
when it does not exist.
|
|
251
|
+
|
|
252
|
+
## Limitations
|
|
253
|
+
|
|
254
|
+
- **Which pairs are adjacent is decided by the point estimate.** The
|
|
255
|
+
simultaneous test controls the family-wise error rate across those 54
|
|
256
|
+
comparisons, but the comparisons themselves were chosen after seeing the data.
|
|
257
|
+
A fully simultaneous treatment of the ranking would be more conservative
|
|
258
|
+
still, so the separation counts here are, if anything, generous.
|
|
259
|
+
- **Bradley-Terry assumes one number per model.** `intransitivity` checks that
|
|
260
|
+
assumption but does not repair it; a board with genuine rock-paper-scissors
|
|
261
|
+
structure needs a different model, not a wider interval.
|
|
262
|
+
- **Half-credit ties are a convention, not a likelihood.** With `ties="half"`
|
|
263
|
+
the fit maximises a weighted objective rather than a probability model.
|
|
264
|
+
`fit_rao_kupper` gives ties a parameter instead, and `tie_convention` reports
|
|
265
|
+
whether the choice mattered.
|
|
266
|
+
- **The position term is one number for the whole board.** A judge that favours
|
|
267
|
+
the first slot only for long answers, or only in one language, will not show
|
|
268
|
+
up as a single advantage.
|
|
269
|
+
- **The case studies are public samples, not the live leaderboards.** The
|
|
270
|
+
LMArena file is a 106k-battle release covering 55 models, not the full vote
|
|
271
|
+
history behind the published board.
|
|
272
|
+
|
|
273
|
+
## Related work
|
|
274
|
+
|
|
275
|
+
- Bradley and Terry (1952), *Rank Analysis of Incomplete Block Designs*
|
|
276
|
+
([doi:10.2307/2334029](https://doi.org/10.2307/2334029)) — the model.
|
|
277
|
+
- Rao and Kupper (1967), *Ties in Paired-Comparison Experiments*
|
|
278
|
+
([doi:10.1080/01621459.1967.10482901](https://doi.org/10.1080/01621459.1967.10482901))
|
|
279
|
+
— the tie model used here.
|
|
280
|
+
- Hunter (2004), *MM algorithms for generalized Bradley-Terry models*
|
|
281
|
+
([doi:10.1214/aos/1079120141](https://doi.org/10.1214/aos/1079120141)) — the
|
|
282
|
+
fitting algorithm, including the position and tie extensions.
|
|
283
|
+
- Ford (1957), *Solution of a Ranking Problem from Binary Comparisons*
|
|
284
|
+
([doi:10.2307/2308513](https://doi.org/10.2307/2308513)) — when a finite
|
|
285
|
+
estimate exists at all.
|
|
286
|
+
- Chiang et al. (2024), *Chatbot Arena: An Open Platform for Evaluating LLMs by
|
|
287
|
+
Human Preference* ([arXiv:2403.04132](https://arxiv.org/abs/2403.04132)) —
|
|
288
|
+
the leaderboard this package audits.
|
|
289
|
+
- Zheng et al. (2023), *Judging LLM-as-a-Judge with MT-Bench and Chatbot Arena*
|
|
290
|
+
([arXiv:2306.05685](https://arxiv.org/abs/2306.05685)) — the source of the
|
|
291
|
+
human and GPT-4 judgments, and of the position-bias measurement.
|
|
292
|
+
- Westfall and Young (1993), *Resampling-Based Multiple Testing* — the max-t
|
|
293
|
+
idea behind the simultaneous critical value.
|
|
294
|
+
|
|
295
|
+
## Companion projects
|
|
296
|
+
|
|
297
|
+
- [judgekit](https://github.com/mohammadi-hadi/judgekit) — audit an LLM judge
|
|
298
|
+
for bias before you trust its scores.
|
|
299
|
+
- [judgepanel](https://github.com/mohammadi-hadi/judgepanel) — estimate judge
|
|
300
|
+
accuracy without gold labels.
|
|
301
|
+
- [raterkit](https://github.com/mohammadi-hadi/raterkit) — audit a labeled
|
|
302
|
+
dataset before you trust it.
|
|
303
|
+
- [calikit](https://github.com/mohammadi-hadi/calikit) — audit the calibration
|
|
304
|
+
of a model's probabilities.
|
|
305
|
+
- [abkit](https://github.com/mohammadi-hadi/abkit) — audit an A/B test before
|
|
306
|
+
you act on it.
|
|
307
|
+
|
|
308
|
+
## Citation
|
|
309
|
+
|
|
310
|
+
```bibtex
|
|
311
|
+
@software{mohammadi_arenakit,
|
|
312
|
+
author = {Mohammadi, Hadi},
|
|
313
|
+
title = {arenakit: audit a pairwise model leaderboard before you trust its order},
|
|
314
|
+
year = {2026},
|
|
315
|
+
url = {https://github.com/mohammadi-hadi/arenakit}
|
|
316
|
+
}
|
|
317
|
+
```
|
|
318
|
+
|
|
319
|
+
## License
|
|
320
|
+
|
|
321
|
+
MIT
|