winference 0.1.0__tar.gz → 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- winference-0.2.0/LICENSE +21 -0
- {winference-0.1.0 → winference-0.2.0}/PKG-INFO +35 -35
- {winference-0.1.0 → winference-0.2.0}/README.md +32 -22
- {winference-0.1.0 → winference-0.2.0}/pyproject.toml +142 -72
- winference-0.2.0/pyproject.toml.orig +140 -0
- {winference-0.1.0 → winference-0.2.0/src}/winference/__init__.py +1 -2
- {winference-0.1.0 → winference-0.2.0/src}/winference/bradley_terry.py +20 -14
- {winference-0.1.0 → winference-0.2.0/src}/winference/calibration.py +8 -5
- {winference-0.1.0 → winference-0.2.0/src}/winference/groups.py +41 -23
- {winference-0.1.0 → winference-0.2.0/src}/winference/hodge.py +106 -29
- {winference-0.1.0 → winference-0.2.0/src}/winference/simulate.py +12 -4
- {winference-0.1.0 → winference-0.2.0/src}/winference/tournament.py +16 -7
- {winference-0.1.0 → winference-0.2.0/src}/winference/py.typed +0 -0
winference-0.2.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2025 gaurav sood
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -1,18 +1,19 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: winference
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.2.0
|
|
4
4
|
Summary: Win rate calibration under non-transitivity via Hodge decomposition and heterogeneous group testing
|
|
5
5
|
Keywords: bradley-terry,calibration,hodge-decomposition,llm-evaluation,non-transitivity,pairwise-comparison,win-rate
|
|
6
6
|
Author: gaurav sood
|
|
7
7
|
License-Expression: MIT
|
|
8
|
+
License-File: LICENSE
|
|
8
9
|
Classifier: Development Status :: 4 - Beta
|
|
9
10
|
Classifier: Intended Audience :: Developers
|
|
10
11
|
Classifier: Intended Audience :: Science/Research
|
|
11
|
-
Classifier: License :: OSI Approved :: MIT License
|
|
12
12
|
Classifier: Operating System :: OS Independent
|
|
13
13
|
Classifier: Programming Language :: Python :: 3
|
|
14
14
|
Classifier: Programming Language :: Python :: 3.12
|
|
15
15
|
Classifier: Programming Language :: Python :: 3.13
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
16
17
|
Classifier: Topic :: Scientific/Engineering
|
|
17
18
|
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
18
19
|
Classifier: Typing :: Typed
|
|
@@ -21,27 +22,16 @@ Requires-Dist: numpy>=1.22
|
|
|
21
22
|
Requires-Dist: pandas>=1.5
|
|
22
23
|
Requires-Dist: scikit-learn>=1.1
|
|
23
24
|
Requires-Dist: scipy>=1.9
|
|
24
|
-
Requires-Dist: deptry>=0.22 ; extra == 'dev'
|
|
25
|
-
Requires-Dist: pydoclint>=0.6 ; extra == 'dev'
|
|
26
|
-
Requires-Dist: pytest>=8.0 ; extra == 'dev'
|
|
27
|
-
Requires-Dist: pytest-cov>=4.0 ; extra == 'dev'
|
|
28
|
-
Requires-Dist: ruff>=0.8 ; extra == 'dev'
|
|
29
|
-
Requires-Dist: furo>=2024.0 ; extra == 'docs'
|
|
30
|
-
Requires-Dist: sphinx>=7.0 ; extra == 'docs'
|
|
31
|
-
Requires-Dist: sphinx-autodoc-typehints>=2.0 ; extra == 'docs'
|
|
32
|
-
Requires-Dist: sphinx-copybutton>=0.5 ; extra == 'docs'
|
|
33
25
|
Requires-Python: >=3.12
|
|
34
26
|
Project-URL: Documentation, https://github.com/finite-sample/winference#readme
|
|
35
27
|
Project-URL: Homepage, https://github.com/finite-sample/winference
|
|
36
28
|
Project-URL: Issues, https://github.com/finite-sample/winference/issues
|
|
37
29
|
Project-URL: Repository, https://github.com/finite-sample/winference
|
|
38
|
-
Provides-Extra: dev
|
|
39
|
-
Provides-Extra: docs
|
|
40
30
|
Description-Content-Type: text/markdown
|
|
41
31
|
|
|
42
|
-
## winference: Win rate calibration under non-transitivity
|
|
32
|
+
## winference: Win rate calibration under non-transitivity
|
|
43
33
|
|
|
44
|
-
[](https://github.com/finite-sample/winference/actions/workflows/ci.yml)
|
|
45
35
|
[](https://pypi.org/project/winference/)
|
|
46
36
|
[](https://pepy.tech/project/winference)
|
|
47
37
|
[](https://finite-sample.github.io/winference/)
|
|
@@ -71,10 +61,13 @@ Decomposes the pairwise comparison matrix into:
|
|
|
71
61
|
- **Gradient** (transitive): a potential `s_i` per model such that the
|
|
72
62
|
log-odds ≈ `s_i − s_j`. This part *can* be calibrated to a scalar ranking.
|
|
73
63
|
- **Curl** (cyclic): rock-paper-scissors structure that *cannot* be represented
|
|
74
|
-
by any linear ranking.
|
|
64
|
+
by any linear ranking. It comes from triangles of observed comparisons.
|
|
65
|
+
- **Harmonic** (global): residual flow around gaps in the comparison graph.
|
|
66
|
+
A four-player cycle with neither diagonal observed is entirely harmonic
|
|
67
|
+
when its four directed log odds and counts are equal.
|
|
75
68
|
|
|
76
|
-
Calibrate win rates from the gradient component.
|
|
77
|
-
the share of
|
|
69
|
+
Calibrate win rates from the gradient component. Add the cyclic and harmonic
|
|
70
|
+
fractions for the share of comparison-weighted variation outside the ranking.
|
|
78
71
|
|
|
79
72
|
**Use when:** Cycles persist even after conditioning on task category — the
|
|
80
73
|
non-transitivity is irreducible.
|
|
@@ -99,23 +92,25 @@ without refitting.
|
|
|
99
92
|
## Quickstart
|
|
100
93
|
|
|
101
94
|
```bash
|
|
102
|
-
pip install
|
|
103
|
-
# Then from the repo root:
|
|
104
|
-
pip install -e .
|
|
95
|
+
pip install winference
|
|
105
96
|
```
|
|
106
97
|
|
|
107
98
|
```python
|
|
108
99
|
from winference import (
|
|
109
|
-
TournamentGraph,
|
|
110
|
-
|
|
100
|
+
TournamentGraph,
|
|
101
|
+
BradleyTerry,
|
|
102
|
+
HodgeDecomposition,
|
|
103
|
+
GroupTest,
|
|
104
|
+
GroupCalibrator,
|
|
105
|
+
expected_calibration_error,
|
|
111
106
|
)
|
|
112
107
|
from winference.simulate import simulate_llm_arena
|
|
113
108
|
|
|
114
109
|
# 1. Simulate (or load) arena data
|
|
115
110
|
data = simulate_llm_arena()
|
|
116
|
-
comparisons = data["comparisons"]
|
|
117
|
-
categories
|
|
118
|
-
models
|
|
111
|
+
comparisons = data["comparisons"] # list of (model_a, model_b, a_wins)
|
|
112
|
+
categories = data["categories"] # list of category labels per comparison
|
|
113
|
+
models = data["models"]
|
|
119
114
|
|
|
120
115
|
# 2. Graph triage: is non-transitivity a problem?
|
|
121
116
|
tg = TournamentGraph(models)
|
|
@@ -123,13 +118,14 @@ for a, b, w in comparisons:
|
|
|
123
118
|
tg.add_result(a, b, w)
|
|
124
119
|
|
|
125
120
|
print(tg.summary())
|
|
126
|
-
#
|
|
121
|
+
# The default simulation has nontransitivity_index 0.5 and one cyclic triple.
|
|
127
122
|
|
|
128
123
|
# 3a. Hodge decomposition
|
|
129
124
|
hd = HodgeDecomposition(models)
|
|
130
125
|
result = hd.fit(tg.win_rate_matrix(), weights=tg.counts)
|
|
131
126
|
print(f"Transitive: {result.transitive_variance:.0%}")
|
|
132
|
-
print(f"
|
|
127
|
+
print(f"Local cycles: {result.cyclic_variance:.0%}")
|
|
128
|
+
print(f"Global cycles: {result.harmonic_variance:.0%}")
|
|
133
129
|
|
|
134
130
|
# Calibrated win probability (transitive component only)
|
|
135
131
|
p = hd.transitive_win_probability("ZetaMath", "DeltaWrite")
|
|
@@ -139,16 +135,18 @@ groups = sorted(set(categories))
|
|
|
139
135
|
gt = GroupTest(models, groups)
|
|
140
136
|
gt.fit(comparisons, categories)
|
|
141
137
|
print(gt.test_result())
|
|
142
|
-
#
|
|
138
|
+
# For the default simulation: statistic ≈ 3585.44, df = 10, reject_at_05 = True.
|
|
143
139
|
|
|
144
140
|
# Composable win rates
|
|
145
141
|
gc = GroupCalibrator(gt)
|
|
146
142
|
p_math_heavy = gc.win_probability(
|
|
147
|
-
"ZetaMath",
|
|
143
|
+
"ZetaMath",
|
|
144
|
+
"DeltaWrite",
|
|
148
145
|
target_distribution={"reasoning": 0.7, "creative_writing": 0.15, "coding": 0.15},
|
|
149
146
|
)
|
|
150
147
|
p_creative_heavy = gc.win_probability(
|
|
151
|
-
"ZetaMath",
|
|
148
|
+
"ZetaMath",
|
|
149
|
+
"DeltaWrite",
|
|
152
150
|
target_distribution={"reasoning": 0.15, "creative_writing": 0.7, "coding": 0.15},
|
|
153
151
|
)
|
|
154
152
|
```
|
|
@@ -194,13 +192,13 @@ simpler causes:
|
|
|
194
192
|
| Source | What it is | How to address |
|
|
195
193
|
|---|---|---|
|
|
196
194
|
| **Judge noise** | LLM judge gives inconsistent verdicts on the same pair | Bayesian calibration (Dawid-Skene, BWRS) |
|
|
197
|
-
| **Position bias** | Judge prefers whichever response appears first/second |
|
|
195
|
+
| **Position bias** | Judge prefers whichever response appears first/second | Randomize presentation order, average both orderings |
|
|
198
196
|
| **Style/length bias** | Judge rewards verbosity rather than quality | Regress out length/style features (cf. AlpacaEval 2.0) |
|
|
199
197
|
| **Evaluator disagreement** | Individual annotators are transitive, but they disagree with each other (Condorcet cycles) | Stratify by evaluator type, or accept the Hodge curl as measuring genuine disagreement |
|
|
200
198
|
| **Fine-grained interaction** | Strengths differ at sub-category level (algebra vs geometry within "math") | Per-prompt routing rather than aggregate ranking |
|
|
201
199
|
| **Context effects** | Evaluation of A vs B depends on what other models were seen | Session-aware experimental design |
|
|
202
200
|
|
|
203
|
-
`winference` targets the
|
|
201
|
+
`winference` targets the modeling layer (rows 4–5 in the table above). It
|
|
204
202
|
assumes you've already addressed measurement issues at the design/preprocessing
|
|
205
203
|
level.
|
|
206
204
|
|
|
@@ -228,7 +226,9 @@ Hodge decomposition of the pairwise log-odds matrix.
|
|
|
228
226
|
- `.fit(W, weights)` — decompose win-rate matrix
|
|
229
227
|
- `.transitive_win_probability(a, b)` — P(a>b) from gradient only
|
|
230
228
|
- `.worst_pairs(k)` — pairs with largest cyclic residual
|
|
231
|
-
- `.summary()` —
|
|
229
|
+
- `.summary()` — fractions of squared pairwise log odds. The optional comparison
|
|
230
|
+
counts weight both the fit and these fractions. Unobserved pairs with zero
|
|
231
|
+
count contribute zero.
|
|
232
232
|
|
|
233
233
|
### `GroupTest`
|
|
234
234
|
Likelihood-ratio test for heterogeneity across prompt groups.
|
|
@@ -257,7 +257,7 @@ Composable win rates from per-group BT.
|
|
|
257
257
|
## References
|
|
258
258
|
|
|
259
259
|
- Jiang, X., Lim, L.-H., Yao, Y., & Ye, Y. (2011). Statistical ranking and combinatorial Hodge theory. *Mathematical Programming*, 127(1), 203–244.
|
|
260
|
-
- Dittrich, R., Hatzinger, R., & Katzenbeisser, W. (1998). Modelling the effect of subject-specific covariates in paired comparison studies. *Applied Statistics*, 47(4), 511–525.
|
|
260
|
+
- Dittrich, R., Hatzinger, R., & Katzenbeisser, W. (1998). Modelling the effect of subject-specific covariates in paired comparison studies. *Applied Statistics*, 47(4), 511–525. <!-- codespell:ignore modelling -->
|
|
261
261
|
- Xu, Y., Ruis, L., Rocktäschel, T., & Kirk, R. (2025). Investigating non-transitivity in LLM-as-a-Judge. *ICML 2025*.
|
|
262
262
|
- Li, X. & Li, S. (2025). Efficient inference for covariate-adjusted Bradley-Terry model with covariate shift. *arXiv:2503.18256*.
|
|
263
263
|
- Balduzzi, D., Tuyls, K., Perolat, J., & Graepel, T. (2018). Re-evaluating evaluation. *NeurIPS*.
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
## winference: Win rate calibration under non-transitivity
|
|
1
|
+
## winference: Win rate calibration under non-transitivity
|
|
2
2
|
|
|
3
|
-
[](https://github.com/finite-sample/winference/actions/workflows/ci.yml)
|
|
4
4
|
[](https://pypi.org/project/winference/)
|
|
5
5
|
[](https://pepy.tech/project/winference)
|
|
6
6
|
[](https://finite-sample.github.io/winference/)
|
|
@@ -30,10 +30,13 @@ Decomposes the pairwise comparison matrix into:
|
|
|
30
30
|
- **Gradient** (transitive): a potential `s_i` per model such that the
|
|
31
31
|
log-odds ≈ `s_i − s_j`. This part *can* be calibrated to a scalar ranking.
|
|
32
32
|
- **Curl** (cyclic): rock-paper-scissors structure that *cannot* be represented
|
|
33
|
-
by any linear ranking.
|
|
33
|
+
by any linear ranking. It comes from triangles of observed comparisons.
|
|
34
|
+
- **Harmonic** (global): residual flow around gaps in the comparison graph.
|
|
35
|
+
A four-player cycle with neither diagonal observed is entirely harmonic
|
|
36
|
+
when its four directed log odds and counts are equal.
|
|
34
37
|
|
|
35
|
-
Calibrate win rates from the gradient component.
|
|
36
|
-
the share of
|
|
38
|
+
Calibrate win rates from the gradient component. Add the cyclic and harmonic
|
|
39
|
+
fractions for the share of comparison-weighted variation outside the ranking.
|
|
37
40
|
|
|
38
41
|
**Use when:** Cycles persist even after conditioning on task category — the
|
|
39
42
|
non-transitivity is irreducible.
|
|
@@ -58,23 +61,25 @@ without refitting.
|
|
|
58
61
|
## Quickstart
|
|
59
62
|
|
|
60
63
|
```bash
|
|
61
|
-
pip install
|
|
62
|
-
# Then from the repo root:
|
|
63
|
-
pip install -e .
|
|
64
|
+
pip install winference
|
|
64
65
|
```
|
|
65
66
|
|
|
66
67
|
```python
|
|
67
68
|
from winference import (
|
|
68
|
-
TournamentGraph,
|
|
69
|
-
|
|
69
|
+
TournamentGraph,
|
|
70
|
+
BradleyTerry,
|
|
71
|
+
HodgeDecomposition,
|
|
72
|
+
GroupTest,
|
|
73
|
+
GroupCalibrator,
|
|
74
|
+
expected_calibration_error,
|
|
70
75
|
)
|
|
71
76
|
from winference.simulate import simulate_llm_arena
|
|
72
77
|
|
|
73
78
|
# 1. Simulate (or load) arena data
|
|
74
79
|
data = simulate_llm_arena()
|
|
75
|
-
comparisons = data["comparisons"]
|
|
76
|
-
categories
|
|
77
|
-
models
|
|
80
|
+
comparisons = data["comparisons"] # list of (model_a, model_b, a_wins)
|
|
81
|
+
categories = data["categories"] # list of category labels per comparison
|
|
82
|
+
models = data["models"]
|
|
78
83
|
|
|
79
84
|
# 2. Graph triage: is non-transitivity a problem?
|
|
80
85
|
tg = TournamentGraph(models)
|
|
@@ -82,13 +87,14 @@ for a, b, w in comparisons:
|
|
|
82
87
|
tg.add_result(a, b, w)
|
|
83
88
|
|
|
84
89
|
print(tg.summary())
|
|
85
|
-
#
|
|
90
|
+
# The default simulation has nontransitivity_index 0.5 and one cyclic triple.
|
|
86
91
|
|
|
87
92
|
# 3a. Hodge decomposition
|
|
88
93
|
hd = HodgeDecomposition(models)
|
|
89
94
|
result = hd.fit(tg.win_rate_matrix(), weights=tg.counts)
|
|
90
95
|
print(f"Transitive: {result.transitive_variance:.0%}")
|
|
91
|
-
print(f"
|
|
96
|
+
print(f"Local cycles: {result.cyclic_variance:.0%}")
|
|
97
|
+
print(f"Global cycles: {result.harmonic_variance:.0%}")
|
|
92
98
|
|
|
93
99
|
# Calibrated win probability (transitive component only)
|
|
94
100
|
p = hd.transitive_win_probability("ZetaMath", "DeltaWrite")
|
|
@@ -98,16 +104,18 @@ groups = sorted(set(categories))
|
|
|
98
104
|
gt = GroupTest(models, groups)
|
|
99
105
|
gt.fit(comparisons, categories)
|
|
100
106
|
print(gt.test_result())
|
|
101
|
-
#
|
|
107
|
+
# For the default simulation: statistic ≈ 3585.44, df = 10, reject_at_05 = True.
|
|
102
108
|
|
|
103
109
|
# Composable win rates
|
|
104
110
|
gc = GroupCalibrator(gt)
|
|
105
111
|
p_math_heavy = gc.win_probability(
|
|
106
|
-
"ZetaMath",
|
|
112
|
+
"ZetaMath",
|
|
113
|
+
"DeltaWrite",
|
|
107
114
|
target_distribution={"reasoning": 0.7, "creative_writing": 0.15, "coding": 0.15},
|
|
108
115
|
)
|
|
109
116
|
p_creative_heavy = gc.win_probability(
|
|
110
|
-
"ZetaMath",
|
|
117
|
+
"ZetaMath",
|
|
118
|
+
"DeltaWrite",
|
|
111
119
|
target_distribution={"reasoning": 0.15, "creative_writing": 0.7, "coding": 0.15},
|
|
112
120
|
)
|
|
113
121
|
```
|
|
@@ -153,13 +161,13 @@ simpler causes:
|
|
|
153
161
|
| Source | What it is | How to address |
|
|
154
162
|
|---|---|---|
|
|
155
163
|
| **Judge noise** | LLM judge gives inconsistent verdicts on the same pair | Bayesian calibration (Dawid-Skene, BWRS) |
|
|
156
|
-
| **Position bias** | Judge prefers whichever response appears first/second |
|
|
164
|
+
| **Position bias** | Judge prefers whichever response appears first/second | Randomize presentation order, average both orderings |
|
|
157
165
|
| **Style/length bias** | Judge rewards verbosity rather than quality | Regress out length/style features (cf. AlpacaEval 2.0) |
|
|
158
166
|
| **Evaluator disagreement** | Individual annotators are transitive, but they disagree with each other (Condorcet cycles) | Stratify by evaluator type, or accept the Hodge curl as measuring genuine disagreement |
|
|
159
167
|
| **Fine-grained interaction** | Strengths differ at sub-category level (algebra vs geometry within "math") | Per-prompt routing rather than aggregate ranking |
|
|
160
168
|
| **Context effects** | Evaluation of A vs B depends on what other models were seen | Session-aware experimental design |
|
|
161
169
|
|
|
162
|
-
`winference` targets the
|
|
170
|
+
`winference` targets the modeling layer (rows 4–5 in the table above). It
|
|
163
171
|
assumes you've already addressed measurement issues at the design/preprocessing
|
|
164
172
|
level.
|
|
165
173
|
|
|
@@ -187,7 +195,9 @@ Hodge decomposition of the pairwise log-odds matrix.
|
|
|
187
195
|
- `.fit(W, weights)` — decompose win-rate matrix
|
|
188
196
|
- `.transitive_win_probability(a, b)` — P(a>b) from gradient only
|
|
189
197
|
- `.worst_pairs(k)` — pairs with largest cyclic residual
|
|
190
|
-
- `.summary()` —
|
|
198
|
+
- `.summary()` — fractions of squared pairwise log odds. The optional comparison
|
|
199
|
+
counts weight both the fit and these fractions. Unobserved pairs with zero
|
|
200
|
+
count contribute zero.
|
|
191
201
|
|
|
192
202
|
### `GroupTest`
|
|
193
203
|
Likelihood-ratio test for heterogeneity across prompt groups.
|
|
@@ -216,7 +226,7 @@ Composable win rates from per-group BT.
|
|
|
216
226
|
## References
|
|
217
227
|
|
|
218
228
|
- Jiang, X., Lim, L.-H., Yao, Y., & Ye, Y. (2011). Statistical ranking and combinatorial Hodge theory. *Mathematical Programming*, 127(1), 203–244.
|
|
219
|
-
- Dittrich, R., Hatzinger, R., & Katzenbeisser, W. (1998). Modelling the effect of subject-specific covariates in paired comparison studies. *Applied Statistics*, 47(4), 511–525.
|
|
229
|
+
- Dittrich, R., Hatzinger, R., & Katzenbeisser, W. (1998). Modelling the effect of subject-specific covariates in paired comparison studies. *Applied Statistics*, 47(4), 511–525. <!-- codespell:ignore modelling -->
|
|
220
230
|
- Xu, Y., Ruis, L., Rocktäschel, T., & Kirk, R. (2025). Investigating non-transitivity in LLM-as-a-Judge. *ICML 2025*.
|
|
221
231
|
- Li, X. & Li, S. (2025). Efficient inference for covariate-adjusted Bradley-Terry model with covariate shift. *arXiv:2503.18256*.
|
|
222
232
|
- Balduzzi, D., Tuyls, K., Perolat, J., & Graepel, T. (2018). Re-evaluating evaluation. *NeurIPS*.
|
|
@@ -1,20 +1,137 @@
|
|
|
1
1
|
[build-system]
|
|
2
|
-
requires = ["uv_build>=0.
|
|
2
|
+
requires = ["uv_build>=0.12.5,<0.14"]
|
|
3
3
|
build-backend = "uv_build"
|
|
4
4
|
|
|
5
5
|
[tool.uv.build-backend]
|
|
6
|
-
module-
|
|
6
|
+
module-name = "winference"
|
|
7
|
+
|
|
8
|
+
[tool.ruff]
|
|
9
|
+
line-length = 88
|
|
10
|
+
target-version = "py312"
|
|
11
|
+
|
|
12
|
+
[tool.ruff.lint]
|
|
13
|
+
external = ["DOC"]
|
|
14
|
+
select = [
|
|
15
|
+
"E",
|
|
16
|
+
"W",
|
|
17
|
+
"F",
|
|
18
|
+
"I",
|
|
19
|
+
"B",
|
|
20
|
+
"C4",
|
|
21
|
+
"UP",
|
|
22
|
+
"N",
|
|
23
|
+
"D",
|
|
24
|
+
"S",
|
|
25
|
+
"SIM",
|
|
26
|
+
"T20",
|
|
27
|
+
"PT",
|
|
28
|
+
"RUF",
|
|
29
|
+
"PTH",
|
|
30
|
+
"RET",
|
|
31
|
+
"PIE",
|
|
32
|
+
"FURB",
|
|
33
|
+
"PERF",
|
|
34
|
+
"DTZ",
|
|
35
|
+
"LOG",
|
|
36
|
+
"G",
|
|
37
|
+
"TC",
|
|
38
|
+
"FLY",
|
|
39
|
+
"RSE",
|
|
40
|
+
"SLOT",
|
|
41
|
+
"FA",
|
|
42
|
+
"A",
|
|
43
|
+
"EXE",
|
|
44
|
+
"ICN",
|
|
45
|
+
"PGH",
|
|
46
|
+
"PLE",
|
|
47
|
+
"ARG",
|
|
48
|
+
"SLF",
|
|
49
|
+
]
|
|
50
|
+
ignore = [
|
|
51
|
+
"D203",
|
|
52
|
+
"D213",
|
|
53
|
+
"W191",
|
|
54
|
+
"D206",
|
|
55
|
+
"D300",
|
|
56
|
+
]
|
|
57
|
+
|
|
58
|
+
[tool.ruff.lint.pydocstyle]
|
|
59
|
+
convention = "google"
|
|
60
|
+
|
|
61
|
+
[tool.ruff.lint.per-file-ignores]
|
|
62
|
+
"src/**" = [
|
|
63
|
+
"N803",
|
|
64
|
+
"N806",
|
|
65
|
+
]
|
|
66
|
+
"tests/**" = [
|
|
67
|
+
"S101",
|
|
68
|
+
"D",
|
|
69
|
+
"ARG",
|
|
70
|
+
"SLF",
|
|
71
|
+
"N803",
|
|
72
|
+
"N806",
|
|
73
|
+
]
|
|
74
|
+
"examples/**" = [
|
|
75
|
+
"T20",
|
|
76
|
+
"N803",
|
|
77
|
+
"N806",
|
|
78
|
+
]
|
|
79
|
+
"docs/**" = ["D"]
|
|
80
|
+
|
|
81
|
+
[tool.ruff.lint.isort]
|
|
82
|
+
known-first-party = ["winference"]
|
|
83
|
+
|
|
84
|
+
[tool.pytest.ini_options]
|
|
85
|
+
testpaths = ["tests"]
|
|
86
|
+
addopts = [
|
|
87
|
+
"-v",
|
|
88
|
+
"--tb=short",
|
|
89
|
+
"--strict-config",
|
|
90
|
+
"--strict-markers",
|
|
91
|
+
"-ra",
|
|
92
|
+
]
|
|
93
|
+
minversion = "6"
|
|
94
|
+
log_level = "INFO"
|
|
95
|
+
xfail_strict = true
|
|
96
|
+
filterwarnings = ["error"]
|
|
97
|
+
|
|
98
|
+
[tool.coverage.run]
|
|
99
|
+
source = ["src"]
|
|
100
|
+
branch = true
|
|
101
|
+
|
|
102
|
+
[tool.coverage.report]
|
|
103
|
+
exclude_lines = [
|
|
104
|
+
"pragma: no cover",
|
|
105
|
+
"if TYPE_CHECKING:",
|
|
106
|
+
"if __name__ == .__main__.:",
|
|
107
|
+
]
|
|
108
|
+
|
|
109
|
+
[tool.pydoclint]
|
|
110
|
+
style = "google"
|
|
111
|
+
arg-type-hints-in-docstring = false
|
|
112
|
+
check-return-types = false
|
|
113
|
+
check-class-attributes = false
|
|
114
|
+
allow-init-docstring = true
|
|
115
|
+
exclude = '\.venv|tests|docs'
|
|
116
|
+
check-arg-order = true
|
|
117
|
+
|
|
118
|
+
[tool.pyright]
|
|
119
|
+
include = ["src"]
|
|
120
|
+
typeCheckingMode = "standard"
|
|
121
|
+
|
|
122
|
+
[tool.preen]
|
|
123
|
+
|
|
124
|
+
[tool.codespell]
|
|
125
|
+
builtin = "clear,rare,en-GB_to_en-US"
|
|
7
126
|
|
|
8
127
|
[project]
|
|
9
128
|
name = "winference"
|
|
10
|
-
version = "0.
|
|
129
|
+
version = "0.2.0"
|
|
11
130
|
description = "Win rate calibration under non-transitivity via Hodge decomposition and heterogeneous group testing"
|
|
12
131
|
readme = "README.md"
|
|
13
132
|
license = "MIT"
|
|
133
|
+
license-files = ["LICENSE"]
|
|
14
134
|
requires-python = ">=3.12"
|
|
15
|
-
authors = [
|
|
16
|
-
{ name = "gaurav sood" },
|
|
17
|
-
]
|
|
18
135
|
keywords = [
|
|
19
136
|
"bradley-terry",
|
|
20
137
|
"calibration",
|
|
@@ -28,11 +145,11 @@ classifiers = [
|
|
|
28
145
|
"Development Status :: 4 - Beta",
|
|
29
146
|
"Intended Audience :: Developers",
|
|
30
147
|
"Intended Audience :: Science/Research",
|
|
31
|
-
"License :: OSI Approved :: MIT License",
|
|
32
148
|
"Operating System :: OS Independent",
|
|
33
149
|
"Programming Language :: Python :: 3",
|
|
34
150
|
"Programming Language :: Python :: 3.12",
|
|
35
151
|
"Programming Language :: Python :: 3.13",
|
|
152
|
+
"Programming Language :: Python :: 3.14",
|
|
36
153
|
"Topic :: Scientific/Engineering",
|
|
37
154
|
"Topic :: Scientific/Engineering :: Artificial Intelligence",
|
|
38
155
|
"Typing :: Typed",
|
|
@@ -45,20 +162,8 @@ dependencies = [
|
|
|
45
162
|
"scipy>=1.9",
|
|
46
163
|
]
|
|
47
164
|
|
|
48
|
-
[project.
|
|
49
|
-
|
|
50
|
-
"deptry>=0.22",
|
|
51
|
-
"pydoclint>=0.6",
|
|
52
|
-
"pytest>=8.0",
|
|
53
|
-
"pytest-cov>=4.0",
|
|
54
|
-
"ruff>=0.8",
|
|
55
|
-
]
|
|
56
|
-
docs = [
|
|
57
|
-
"furo>=2024.0",
|
|
58
|
-
"sphinx>=7.0",
|
|
59
|
-
"sphinx-autodoc-typehints>=2.0",
|
|
60
|
-
"sphinx-copybutton>=0.5",
|
|
61
|
-
]
|
|
165
|
+
[[project.authors]]
|
|
166
|
+
name = "gaurav sood"
|
|
62
167
|
|
|
63
168
|
[project.urls]
|
|
64
169
|
Documentation = "https://github.com/finite-sample/winference#readme"
|
|
@@ -66,57 +171,22 @@ Homepage = "https://github.com/finite-sample/winference"
|
|
|
66
171
|
Issues = "https://github.com/finite-sample/winference/issues"
|
|
67
172
|
Repository = "https://github.com/finite-sample/winference"
|
|
68
173
|
|
|
69
|
-
[
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
"
|
|
76
|
-
"B",
|
|
77
|
-
"C4",
|
|
78
|
-
"E",
|
|
79
|
-
"F",
|
|
80
|
-
"I",
|
|
81
|
-
"PTH",
|
|
82
|
-
"RUF",
|
|
83
|
-
"SIM",
|
|
84
|
-
"TCH",
|
|
85
|
-
"UP",
|
|
86
|
-
"W",
|
|
174
|
+
[dependency-groups]
|
|
175
|
+
dev = [
|
|
176
|
+
"ruff>=0.14",
|
|
177
|
+
"pyright>=1.1.390",
|
|
178
|
+
"pydoclint>=0.5",
|
|
179
|
+
"pre-commit>=4",
|
|
180
|
+
{ include-group = "test" },
|
|
87
181
|
]
|
|
88
|
-
|
|
89
|
-
"
|
|
182
|
+
test = [
|
|
183
|
+
"pytest>=8",
|
|
184
|
+
"pytest-cov>=6",
|
|
90
185
|
]
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
addopts = "-v --tb=short"
|
|
98
|
-
|
|
99
|
-
[tool.coverage.run]
|
|
100
|
-
source = ["winference"]
|
|
101
|
-
branch = true
|
|
102
|
-
|
|
103
|
-
[tool.coverage.report]
|
|
104
|
-
exclude_lines = [
|
|
105
|
-
"pragma: no cover",
|
|
106
|
-
"if TYPE_CHECKING:",
|
|
107
|
-
"if __name__ == .__main__.:",
|
|
186
|
+
docs = [
|
|
187
|
+
"sphinx>=8",
|
|
188
|
+
"py-canon @ git+https://github.com/gojiplus/py-canon@v1",
|
|
189
|
+
"furo",
|
|
190
|
+
"myst-parser",
|
|
191
|
+
"sphinx-copybutton",
|
|
108
192
|
]
|
|
109
|
-
|
|
110
|
-
[tool.pydoclint]
|
|
111
|
-
style = "google"
|
|
112
|
-
arg-type-hints-in-docstring = false
|
|
113
|
-
check-return-types = false
|
|
114
|
-
check-arg-order = true
|
|
115
|
-
allow-init-docstring = true
|
|
116
|
-
|
|
117
|
-
[tool.deptry]
|
|
118
|
-
extend_exclude = ["examples", "docs"]
|
|
119
|
-
pep621_dev_dependency_groups = ["dev", "docs"]
|
|
120
|
-
ignore = ["DEP002"]
|
|
121
|
-
[tool.deptry.per_rule_ignores]
|
|
122
|
-
DEP002 = ["matplotlib", "pandas", "scikit-learn"]
|
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["uv_build>=0.12.5,<0.14"]
|
|
3
|
+
build-backend = "uv_build"
|
|
4
|
+
|
|
5
|
+
[tool.uv.build-backend]
|
|
6
|
+
module-name = "winference"
|
|
7
|
+
|
|
8
|
+
[project]
|
|
9
|
+
name = "winference"
|
|
10
|
+
version = "0.2.0"
|
|
11
|
+
description = "Win rate calibration under non-transitivity via Hodge decomposition and heterogeneous group testing"
|
|
12
|
+
readme = "README.md"
|
|
13
|
+
license = "MIT"
|
|
14
|
+
license-files = ["LICENSE"]
|
|
15
|
+
requires-python = ">=3.12"
|
|
16
|
+
authors = [
|
|
17
|
+
{ name = "gaurav sood" },
|
|
18
|
+
]
|
|
19
|
+
keywords = [
|
|
20
|
+
"bradley-terry",
|
|
21
|
+
"calibration",
|
|
22
|
+
"hodge-decomposition",
|
|
23
|
+
"llm-evaluation",
|
|
24
|
+
"non-transitivity",
|
|
25
|
+
"pairwise-comparison",
|
|
26
|
+
"win-rate",
|
|
27
|
+
]
|
|
28
|
+
classifiers = [
|
|
29
|
+
"Development Status :: 4 - Beta",
|
|
30
|
+
"Intended Audience :: Developers",
|
|
31
|
+
"Intended Audience :: Science/Research",
|
|
32
|
+
"Operating System :: OS Independent",
|
|
33
|
+
"Programming Language :: Python :: 3",
|
|
34
|
+
"Programming Language :: Python :: 3.12",
|
|
35
|
+
"Programming Language :: Python :: 3.13",
|
|
36
|
+
"Programming Language :: Python :: 3.14",
|
|
37
|
+
"Topic :: Scientific/Engineering",
|
|
38
|
+
"Topic :: Scientific/Engineering :: Artificial Intelligence",
|
|
39
|
+
"Typing :: Typed",
|
|
40
|
+
]
|
|
41
|
+
dependencies = [
|
|
42
|
+
"matplotlib>=3.5",
|
|
43
|
+
"numpy>=1.22",
|
|
44
|
+
"pandas>=1.5",
|
|
45
|
+
"scikit-learn>=1.1",
|
|
46
|
+
"scipy>=1.9",
|
|
47
|
+
]
|
|
48
|
+
|
|
49
|
+
[project.urls]
|
|
50
|
+
Documentation = "https://github.com/finite-sample/winference#readme"
|
|
51
|
+
Homepage = "https://github.com/finite-sample/winference"
|
|
52
|
+
Issues = "https://github.com/finite-sample/winference/issues"
|
|
53
|
+
Repository = "https://github.com/finite-sample/winference"
|
|
54
|
+
|
|
55
|
+
[tool.ruff]
|
|
56
|
+
line-length = 88
|
|
57
|
+
target-version = "py312"
|
|
58
|
+
|
|
59
|
+
[tool.ruff.lint]
|
|
60
|
+
external = ["DOC"]
|
|
61
|
+
select = [
|
|
62
|
+
"E", "W", "F", "I", "B", "C4", "UP", "N", "D", "S", "SIM", "T20", "PT", "RUF",
|
|
63
|
+
"PTH", "RET", "PIE", "FURB", "PERF", "DTZ", "LOG", "G", "TC", "FLY",
|
|
64
|
+
"RSE", "SLOT", "FA", "A", "EXE", "ICN", "PGH", "PLE", "ARG", "SLF",
|
|
65
|
+
]
|
|
66
|
+
# D203/D213: the google convention the fleet standard picks. W191/D206/D300:
|
|
67
|
+
# ruff's own docs list these as always incompatible with `ruff format`, which
|
|
68
|
+
# the standard also runs.
|
|
69
|
+
ignore = ["D203", "D213", "W191", "D206", "D300"]
|
|
70
|
+
|
|
71
|
+
[tool.ruff.lint.pydocstyle]
|
|
72
|
+
convention = "google"
|
|
73
|
+
|
|
74
|
+
[tool.ruff.lint.per-file-ignores]
|
|
75
|
+
# N803/N806: matrices are capitalized throughout, as in the linear algebra the
|
|
76
|
+
# code implements -- W the win-rate matrix, Y the pairwise flow, L the graph
|
|
77
|
+
# Laplacian, C the curl. Renaming them to `w`, `y`, `l` would make the code
|
|
78
|
+
# harder to check against the papers it follows, not easier.
|
|
79
|
+
"src/**" = ["N803", "N806"]
|
|
80
|
+
"tests/**" = ["S101", "D", "ARG", "SLF", "N803", "N806"]
|
|
81
|
+
# T20: the examples are demonstration scripts whose output *is* printing.
|
|
82
|
+
"examples/**" = ["T20", "N803", "N806"]
|
|
83
|
+
"docs/**" = ["D"]
|
|
84
|
+
|
|
85
|
+
[tool.ruff.lint.isort]
|
|
86
|
+
known-first-party = ["winference"]
|
|
87
|
+
|
|
88
|
+
[tool.pytest.ini_options]
|
|
89
|
+
testpaths = ["tests"]
|
|
90
|
+
addopts = ["-v", "--tb=short", "--strict-config", "--strict-markers", "-ra"]
|
|
91
|
+
minversion = "6"
|
|
92
|
+
log_level = "INFO"
|
|
93
|
+
xfail_strict = true
|
|
94
|
+
filterwarnings = ["error"]
|
|
95
|
+
|
|
96
|
+
[tool.coverage.run]
|
|
97
|
+
source = ["src"]
|
|
98
|
+
branch = true
|
|
99
|
+
|
|
100
|
+
[tool.coverage.report]
|
|
101
|
+
exclude_lines = [
|
|
102
|
+
"pragma: no cover",
|
|
103
|
+
"if TYPE_CHECKING:",
|
|
104
|
+
"if __name__ == .__main__.:",
|
|
105
|
+
]
|
|
106
|
+
|
|
107
|
+
[tool.pydoclint]
|
|
108
|
+
style = "google"
|
|
109
|
+
arg-type-hints-in-docstring = false
|
|
110
|
+
check-return-types = false
|
|
111
|
+
check-class-attributes = false
|
|
112
|
+
allow-init-docstring = true
|
|
113
|
+
exclude = '\.venv|tests|docs'
|
|
114
|
+
check-arg-order = true
|
|
115
|
+
|
|
116
|
+
[tool.pyright]
|
|
117
|
+
include = ["src"]
|
|
118
|
+
typeCheckingMode = "standard"
|
|
119
|
+
|
|
120
|
+
[dependency-groups]
|
|
121
|
+
dev = [
|
|
122
|
+
"ruff>=0.14",
|
|
123
|
+
"pyright>=1.1.390",
|
|
124
|
+
"pydoclint>=0.5",
|
|
125
|
+
"pre-commit>=4",
|
|
126
|
+
{ include-group = "test" },
|
|
127
|
+
]
|
|
128
|
+
test = ["pytest>=8", "pytest-cov>=6"]
|
|
129
|
+
docs = [
|
|
130
|
+
"sphinx>=8",
|
|
131
|
+
"py-canon @ git+https://github.com/gojiplus/py-canon@v1",
|
|
132
|
+
"furo",
|
|
133
|
+
"myst-parser",
|
|
134
|
+
"sphinx-copybutton",
|
|
135
|
+
]
|
|
136
|
+
|
|
137
|
+
[tool.preen]
|
|
138
|
+
|
|
139
|
+
[tool.codespell]
|
|
140
|
+
builtin = "clear,rare,en-GB_to_en-US"
|
|
@@ -1,5 +1,4 @@
|
|
|
1
|
-
"""
|
|
2
|
-
winference: Win rate calibration under non-transitivity.
|
|
1
|
+
"""winference: Win rate calibration under non-transitivity.
|
|
3
2
|
|
|
4
3
|
Two approaches to calibrating win rates when preferences aren't transitive:
|
|
5
4
|
A) Hodge decomposition — separate transitive signal from cyclic residual,
|
|
@@ -1,5 +1,4 @@
|
|
|
1
|
-
"""
|
|
2
|
-
Bradley-Terry model fitting via maximum likelihood.
|
|
1
|
+
"""Bradley-Terry model fitting via maximum likelihood.
|
|
3
2
|
|
|
4
3
|
The BT model assumes P(i beats j) = sigma(theta_i - theta_j) where sigma
|
|
5
4
|
is the logistic function. Fitting recovers the strength vector theta.
|
|
@@ -16,17 +15,20 @@ from scipy.special import expit
|
|
|
16
15
|
class BradleyTerry:
|
|
17
16
|
"""Maximum-likelihood Bradley-Terry model.
|
|
18
17
|
|
|
19
|
-
Args:
|
|
20
|
-
models: Unique model identifiers.
|
|
21
|
-
|
|
22
18
|
Examples:
|
|
23
|
-
>>>
|
|
24
|
-
>>>
|
|
25
|
-
>>> bt
|
|
26
|
-
|
|
19
|
+
>>> from winference import BradleyTerry
|
|
20
|
+
>>> comparisons = [("A", "B", True)] * 7 + [("A", "B", False)] * 3
|
|
21
|
+
>>> bt = BradleyTerry(["A", "B"]).fit(comparisons)
|
|
22
|
+
>>> round(bt.win_probability("A", "B"), 2)
|
|
23
|
+
0.7
|
|
27
24
|
"""
|
|
28
25
|
|
|
29
26
|
def __init__(self, models: list[str]) -> None:
|
|
27
|
+
"""Create an unfitted model over ``models``.
|
|
28
|
+
|
|
29
|
+
Args:
|
|
30
|
+
models: Model names, in the order used for indexing.
|
|
31
|
+
"""
|
|
30
32
|
self.models = list(models)
|
|
31
33
|
self._idx: dict[str, int] = {m: i for i, m in enumerate(self.models)}
|
|
32
34
|
self.n = len(models)
|
|
@@ -43,12 +45,14 @@ class BradleyTerry:
|
|
|
43
45
|
|
|
44
46
|
Args:
|
|
45
47
|
comparisons: Each element is (model_a, model_b, a_wins).
|
|
46
|
-
reg: L2
|
|
48
|
+
reg: L2 regularization strength (prevents divergence for unbeaten models).
|
|
47
49
|
|
|
48
50
|
Returns:
|
|
49
51
|
Self for method chaining.
|
|
50
52
|
"""
|
|
51
|
-
data = np.array(
|
|
53
|
+
data = np.array(
|
|
54
|
+
[(self._idx[a], self._idx[b], float(w)) for a, b, w in comparisons]
|
|
55
|
+
)
|
|
52
56
|
idx_a = data[:, 0].astype(int)
|
|
53
57
|
idx_b = data[:, 1].astype(int)
|
|
54
58
|
outcomes = data[:, 2]
|
|
@@ -79,7 +83,9 @@ class BradleyTerry:
|
|
|
79
83
|
if not self._fitted or self.theta is None:
|
|
80
84
|
msg = "Call .fit() first"
|
|
81
85
|
raise RuntimeError(msg)
|
|
82
|
-
return float(
|
|
86
|
+
return float(
|
|
87
|
+
expit(self.theta[self._idx[model_a]] - self.theta[self._idx[model_b]])
|
|
88
|
+
)
|
|
83
89
|
|
|
84
90
|
def win_probability_matrix(self) -> NDArray[np.float64]:
|
|
85
91
|
"""NxN matrix of predicted win probabilities."""
|
|
@@ -97,8 +103,7 @@ class BradleyTerry:
|
|
|
97
103
|
if not self._fitted:
|
|
98
104
|
msg = "Call .fit() first"
|
|
99
105
|
raise RuntimeError(msg)
|
|
100
|
-
|
|
101
|
-
return preds
|
|
106
|
+
return np.array([self.win_probability(a, b) for a, b, _ in comparisons])
|
|
102
107
|
|
|
103
108
|
def strengths(self) -> dict[str, float]:
|
|
104
109
|
"""Return {model: theta} dictionary."""
|
|
@@ -113,6 +118,7 @@ class BradleyTerry:
|
|
|
113
118
|
return sorted(s, key=lambda m: s[m], reverse=True)
|
|
114
119
|
|
|
115
120
|
def __repr__(self) -> str:
|
|
121
|
+
"""Return a short description including whether the model is fitted."""
|
|
116
122
|
status = "fitted" if self._fitted else "unfitted"
|
|
117
123
|
return f"BradleyTerry(n_models={self.n}, {status})"
|
|
118
124
|
|
|
@@ -1,5 +1,4 @@
|
|
|
1
|
-
"""
|
|
2
|
-
Calibration metrics for pairwise win rate predictions.
|
|
1
|
+
"""Calibration metrics for pairwise win rate predictions.
|
|
3
2
|
|
|
4
3
|
When your model says "A beats B with probability 0.65", does A actually
|
|
5
4
|
win 65% of the time? These tools check.
|
|
@@ -25,7 +24,7 @@ def expected_calibration_error(
|
|
|
25
24
|
n_bins: Number of bins for grouping predictions.
|
|
26
25
|
|
|
27
26
|
Returns:
|
|
28
|
-
Weighted average
|
|
27
|
+
Weighted average ``|predicted - observed|`` across bins.
|
|
29
28
|
"""
|
|
30
29
|
predicted = np.asarray(predicted, dtype=float)
|
|
31
30
|
observed = np.asarray(observed, dtype=float)
|
|
@@ -62,7 +61,9 @@ def log_loss(
|
|
|
62
61
|
"""Binary cross-entropy loss. Lower is better."""
|
|
63
62
|
predicted = np.clip(np.asarray(predicted, dtype=float), 1e-10, 1 - 1e-10)
|
|
64
63
|
observed = np.asarray(observed, dtype=float)
|
|
65
|
-
return float(
|
|
64
|
+
return float(
|
|
65
|
+
-np.mean(observed * np.log(predicted) + (1 - observed) * np.log(1 - predicted))
|
|
66
|
+
)
|
|
66
67
|
|
|
67
68
|
|
|
68
69
|
def reliability_diagram(
|
|
@@ -113,7 +114,9 @@ def reliability_diagram(
|
|
|
113
114
|
return result
|
|
114
115
|
|
|
115
116
|
|
|
116
|
-
def _plot_reliability(
|
|
117
|
+
def _plot_reliability(
|
|
118
|
+
ax: Any, result: dict[str, Any], label: str, color: str | None
|
|
119
|
+
) -> None:
|
|
117
120
|
mid = result["bin_midpoints"]
|
|
118
121
|
acc = result["bin_accuracy"]
|
|
119
122
|
mask = ~np.isnan(acc)
|
|
@@ -1,5 +1,4 @@
|
|
|
1
|
-
"""
|
|
2
|
-
Heterogeneous group testing and per-group calibration.
|
|
1
|
+
"""Heterogeneous group testing and per-group calibration.
|
|
3
2
|
|
|
4
3
|
Tests whether model strengths are constant across prompt categories
|
|
5
4
|
(homogeneous BT) or differ by category (heterogeneous BT). If
|
|
@@ -26,18 +25,23 @@ from winference.bradley_terry import BradleyTerry
|
|
|
26
25
|
class GroupTest:
|
|
27
26
|
"""Likelihood-ratio test for heterogeneity across prompt groups.
|
|
28
27
|
|
|
29
|
-
Args:
|
|
30
|
-
models: Model identifiers.
|
|
31
|
-
groups: Unique group/category labels.
|
|
32
|
-
|
|
33
28
|
Examples:
|
|
34
|
-
>>>
|
|
35
|
-
>>>
|
|
36
|
-
>>>
|
|
37
|
-
|
|
29
|
+
>>> from winference import GroupTest
|
|
30
|
+
>>> comparisons = [("A", "B", True)] * 20 + [("A", "B", False)] * 20
|
|
31
|
+
>>> group_labels = ["math"] * 20 + ["creative"] * 20
|
|
32
|
+
>>> gt = GroupTest(models=["A", "B"], groups=["math", "creative"])
|
|
33
|
+
>>> _ = gt.fit(comparisons, group_labels)
|
|
34
|
+
>>> gt.test_result()["reject_at_05"] # A wins math, B wins creative
|
|
35
|
+
True
|
|
38
36
|
"""
|
|
39
37
|
|
|
40
38
|
def __init__(self, models: list[str], groups: list[str]) -> None:
|
|
39
|
+
"""Create an unfitted test over ``models`` and ``groups``.
|
|
40
|
+
|
|
41
|
+
Args:
|
|
42
|
+
models: Model names, in the order used for indexing.
|
|
43
|
+
groups: Group/category labels a comparison can belong to.
|
|
44
|
+
"""
|
|
41
45
|
self.models = list(models)
|
|
42
46
|
self.groups = list(groups)
|
|
43
47
|
self.n_models = len(models)
|
|
@@ -48,6 +52,11 @@ class GroupTest:
|
|
|
48
52
|
self._group_labels: list[str] = []
|
|
49
53
|
self._group_comparisons: dict[str, list[tuple[str, str, bool]]] = {}
|
|
50
54
|
|
|
55
|
+
@property
|
|
56
|
+
def group_labels(self) -> list[str]:
|
|
57
|
+
"""Group label of each fitted comparison, in the order supplied."""
|
|
58
|
+
return list(self._group_labels)
|
|
59
|
+
|
|
51
60
|
def fit(
|
|
52
61
|
self,
|
|
53
62
|
comparisons: list[tuple[str, str, bool]],
|
|
@@ -58,8 +67,9 @@ class GroupTest:
|
|
|
58
67
|
|
|
59
68
|
Args:
|
|
60
69
|
comparisons: List of (model_a, model_b, a_wins) tuples.
|
|
61
|
-
group_labels: Category label for each comparison, same length as
|
|
62
|
-
|
|
70
|
+
group_labels: Category label for each comparison, same length as
|
|
71
|
+
comparisons.
|
|
72
|
+
reg: Regularization for BT fitting.
|
|
63
73
|
|
|
64
74
|
Returns:
|
|
65
75
|
Self for method chaining.
|
|
@@ -77,7 +87,9 @@ class GroupTest:
|
|
|
77
87
|
self.bt_per_group = {}
|
|
78
88
|
self._group_comparisons = {}
|
|
79
89
|
for g in self.groups:
|
|
80
|
-
g_comps = [
|
|
90
|
+
g_comps = [
|
|
91
|
+
c for c, gl in zip(comparisons, group_labels, strict=True) if gl == g
|
|
92
|
+
]
|
|
81
93
|
if len(g_comps) < self.n_models:
|
|
82
94
|
continue
|
|
83
95
|
self._group_comparisons[g] = g_comps
|
|
@@ -89,7 +101,9 @@ class GroupTest:
|
|
|
89
101
|
self._group_labels = group_labels
|
|
90
102
|
return self
|
|
91
103
|
|
|
92
|
-
def _loglik(
|
|
104
|
+
def _loglik(
|
|
105
|
+
self, bt: BradleyTerry, comparisons: list[tuple[str, str, bool]]
|
|
106
|
+
) -> float:
|
|
93
107
|
"""Compute log-likelihood of data under a fitted BT model."""
|
|
94
108
|
ll = 0.0
|
|
95
109
|
for a, b, w in comparisons:
|
|
@@ -157,15 +171,17 @@ class GroupCalibrator:
|
|
|
157
171
|
|
|
158
172
|
This is the key advantage: calibration that transfers under
|
|
159
173
|
distribution shift.
|
|
160
|
-
|
|
161
|
-
Args:
|
|
162
|
-
group_test: A fitted GroupTest object.
|
|
163
|
-
|
|
164
|
-
Raises:
|
|
165
|
-
RuntimeError: If the GroupTest has not been fitted.
|
|
166
174
|
"""
|
|
167
175
|
|
|
168
176
|
def __init__(self, group_test: GroupTest) -> None:
|
|
177
|
+
"""Wrap a fitted GroupTest.
|
|
178
|
+
|
|
179
|
+
Args:
|
|
180
|
+
group_test: A fitted GroupTest object.
|
|
181
|
+
|
|
182
|
+
Raises:
|
|
183
|
+
RuntimeError: If the GroupTest has not been fitted.
|
|
184
|
+
"""
|
|
169
185
|
if group_test.bt_null is None:
|
|
170
186
|
msg = "GroupTest must be fitted"
|
|
171
187
|
raise RuntimeError(msg)
|
|
@@ -183,7 +199,7 @@ class GroupCalibrator:
|
|
|
183
199
|
model_a: First model name.
|
|
184
200
|
model_b: Second model name.
|
|
185
201
|
target_distribution: Dict mapping group to weight. Weights are
|
|
186
|
-
|
|
202
|
+
normalized internally. If None, uses the empirical distribution
|
|
187
203
|
from the training data.
|
|
188
204
|
|
|
189
205
|
Returns:
|
|
@@ -214,14 +230,16 @@ class GroupCalibrator:
|
|
|
214
230
|
for i in range(n):
|
|
215
231
|
for j in range(n):
|
|
216
232
|
if i != j:
|
|
217
|
-
W[i, j] = self.win_probability(
|
|
233
|
+
W[i, j] = self.win_probability(
|
|
234
|
+
models[i], models[j], target_distribution
|
|
235
|
+
)
|
|
218
236
|
else:
|
|
219
237
|
W[i, j] = 0.5
|
|
220
238
|
return W
|
|
221
239
|
|
|
222
240
|
def _empirical_distribution(self) -> dict[str, float]:
|
|
223
241
|
"""Group weights proportional to number of comparisons."""
|
|
224
|
-
counts = Counter(self.gt.
|
|
242
|
+
counts = Counter(self.gt.group_labels)
|
|
225
243
|
return dict(counts)
|
|
226
244
|
|
|
227
245
|
def sensitivity_analysis(
|
|
@@ -1,5 +1,4 @@
|
|
|
1
|
-
"""
|
|
2
|
-
Hodge decomposition of pairwise comparison data.
|
|
1
|
+
"""Hodge decomposition of pairwise comparison data.
|
|
3
2
|
|
|
4
3
|
Any skew-symmetric matrix of log-odds (or edge flows on the tournament
|
|
5
4
|
graph) can be orthogonally decomposed into:
|
|
@@ -24,14 +23,16 @@ Calibration strategy
|
|
|
24
23
|
1. Compute the Hodge decomposition of the empirical log-odds.
|
|
25
24
|
2. Use the gradient component s as BT strengths giving *transitive*
|
|
26
25
|
calibrated win probabilities: P_trans(i > j) = sigmoid(s_i - s_j).
|
|
27
|
-
3.
|
|
28
|
-
|
|
26
|
+
3. Add the cyclic and harmonic variance fractions to measure the share
|
|
27
|
+
of comparison-weighted variation outside a single ranking.
|
|
29
28
|
"""
|
|
30
29
|
|
|
31
30
|
from typing import NamedTuple
|
|
32
31
|
|
|
33
32
|
import numpy as np
|
|
34
33
|
from numpy.typing import NDArray
|
|
34
|
+
from scipy.sparse import coo_array
|
|
35
|
+
from scipy.sparse.linalg import lsmr
|
|
35
36
|
from scipy.special import expit, logit
|
|
36
37
|
|
|
37
38
|
|
|
@@ -47,21 +48,79 @@ class HodgeResult(NamedTuple):
|
|
|
47
48
|
harmonic_variance: float
|
|
48
49
|
|
|
49
50
|
|
|
51
|
+
def _split_residual(
|
|
52
|
+
residual: NDArray[np.float64], weights: NDArray[np.float64]
|
|
53
|
+
) -> tuple[NDArray[np.float64], NDArray[np.float64]]:
|
|
54
|
+
"""Project the observed residual onto the boundaries of observed triangles."""
|
|
55
|
+
observed = weights > 0
|
|
56
|
+
np.fill_diagonal(observed, False)
|
|
57
|
+
residual = np.where(observed, residual, 0.0)
|
|
58
|
+
# A vertex joined to every other vertex fills every cycle with triangles,
|
|
59
|
+
# including on complete tournaments. No harmonic component can remain.
|
|
60
|
+
if np.any(observed.sum(axis=1) == len(weights) - 1):
|
|
61
|
+
return residual, np.zeros_like(residual)
|
|
62
|
+
|
|
63
|
+
left, right = np.nonzero(np.triu(observed, k=1))
|
|
64
|
+
edge_index = np.zeros_like(weights, dtype=np.intp)
|
|
65
|
+
edge_index[left, right] = np.arange(len(left))
|
|
66
|
+
triangle_edges = []
|
|
67
|
+
for i, j in zip(left, right, strict=True):
|
|
68
|
+
later = np.flatnonzero(observed[i, j + 1 :] & observed[j, j + 1 :]) + j + 1
|
|
69
|
+
triangle_edges.extend(
|
|
70
|
+
(edge_index[i, j], edge_index[j, k], edge_index[i, k]) for k in later
|
|
71
|
+
)
|
|
72
|
+
if not triangle_edges:
|
|
73
|
+
return np.zeros_like(residual), residual
|
|
74
|
+
|
|
75
|
+
# In coordinates scaled by sqrt(weight), a triangle contributes signed
|
|
76
|
+
# 1/sqrt(weight) to each of its three edges. Its column space is the
|
|
77
|
+
# local component of the weighted Hodge decomposition (Jiang et al. §5.1).
|
|
78
|
+
root_weight = np.sqrt(weights[left, right])
|
|
79
|
+
rows = np.asarray(triangle_edges, dtype=np.intp).ravel()
|
|
80
|
+
cols = np.repeat(np.arange(len(triangle_edges)), 3)
|
|
81
|
+
signs = np.tile([1.0, 1.0, -1.0], len(triangle_edges))
|
|
82
|
+
boundaries = coo_array(
|
|
83
|
+
(signs / root_weight[rows], (rows, cols)),
|
|
84
|
+
shape=(len(left), len(triangle_edges)),
|
|
85
|
+
).tocsr()
|
|
86
|
+
target = root_weight * residual[left, right]
|
|
87
|
+
coefficients, stop, *_ = lsmr(
|
|
88
|
+
boundaries,
|
|
89
|
+
target,
|
|
90
|
+
atol=1e-13,
|
|
91
|
+
btol=1e-13,
|
|
92
|
+
conlim=0,
|
|
93
|
+
maxiter=5 * min(len(left), len(triangle_edges)),
|
|
94
|
+
)
|
|
95
|
+
if stop in (3, 6, 7):
|
|
96
|
+
raise RuntimeError("The weighted triangle projection did not converge")
|
|
97
|
+
local = np.zeros_like(residual)
|
|
98
|
+
local[left, right] = (boundaries @ coefficients) / root_weight
|
|
99
|
+
local[right, left] = -local[left, right]
|
|
100
|
+
return local, residual - local
|
|
101
|
+
|
|
102
|
+
|
|
50
103
|
class HodgeDecomposition:
|
|
51
104
|
"""Hodge decomposition of a pairwise comparison matrix.
|
|
52
105
|
|
|
53
|
-
Args:
|
|
54
|
-
models: Model identifiers.
|
|
55
|
-
|
|
56
106
|
Examples:
|
|
57
|
-
>>>
|
|
58
|
-
>>>
|
|
59
|
-
>>>
|
|
60
|
-
>>>
|
|
61
|
-
>>>
|
|
107
|
+
>>> import numpy as np
|
|
108
|
+
>>> from winference import HodgeDecomposition
|
|
109
|
+
>>> rps = np.array([[0.5, 0.9, 0.1], [0.1, 0.5, 0.9], [0.9, 0.1, 0.5]])
|
|
110
|
+
>>> hd = HodgeDecomposition(["A", "B", "C"])
|
|
111
|
+
>>> result = hd.fit(rps)
|
|
112
|
+
>>> result.cyclic_variance # rock-paper-scissors is pure curl
|
|
113
|
+
1.0
|
|
114
|
+
>>> hd.transitive_win_probability("A", "B") # no transitive signal left
|
|
115
|
+
0.5
|
|
62
116
|
"""
|
|
63
117
|
|
|
64
118
|
def __init__(self, models: list[str]) -> None:
|
|
119
|
+
"""Create an unfitted decomposition over ``models``.
|
|
120
|
+
|
|
121
|
+
Args:
|
|
122
|
+
models: Model names, in the order used for indexing.
|
|
123
|
+
"""
|
|
65
124
|
self.models = list(models)
|
|
66
125
|
self._idx: dict[str, int] = {m: i for i, m in enumerate(self.models)}
|
|
67
126
|
self.n = len(models)
|
|
@@ -76,24 +135,38 @@ class HodgeDecomposition:
|
|
|
76
135
|
|
|
77
136
|
Args:
|
|
78
137
|
W: Win rate matrix of shape (n, n) where W[i,j] = P(i beats j).
|
|
79
|
-
|
|
138
|
+
Observed pairs should satisfy W[i,j] + W[j,i] ~ 1. Values
|
|
139
|
+
at pairs with zero weight are ignored.
|
|
80
140
|
weights: Number of comparisons per pair of shape (n, n), used for
|
|
81
|
-
weighted least squares
|
|
141
|
+
weighted least squares and the reported variance fractions.
|
|
142
|
+
A pair with zero comparisons contributes no weight. By default
|
|
143
|
+
every pair has equal weight.
|
|
82
144
|
|
|
83
145
|
Returns:
|
|
84
|
-
HodgeResult containing decomposition components.
|
|
146
|
+
HodgeResult containing decomposition components. The gradient gives
|
|
147
|
+
fitted differences for all model pairs. Local curl and harmonic
|
|
148
|
+
flow are zero on unobserved pairs.
|
|
149
|
+
|
|
150
|
+
Raises:
|
|
151
|
+
ValueError: For invalid matrix shapes or comparison counts.
|
|
85
152
|
"""
|
|
86
153
|
n = self.n
|
|
87
154
|
|
|
88
|
-
|
|
155
|
+
if weights is None:
|
|
156
|
+
weights = np.ones((n, n))
|
|
157
|
+
weights = np.asarray(weights, dtype=float)
|
|
158
|
+
if weights.shape != (n, n) or not np.all(np.isfinite(weights) & (weights >= 0)):
|
|
159
|
+
raise ValueError(
|
|
160
|
+
"weights must be a finite, nonnegative (n, n) count matrix"
|
|
161
|
+
)
|
|
162
|
+
weights = 0.5 * weights + 0.5 * weights.T
|
|
163
|
+
if np.shape(W) != (n, n):
|
|
164
|
+
raise ValueError("W must be an (n, n) win-rate matrix")
|
|
165
|
+
W_clip = np.clip(np.where(weights > 0, W, 0.5), 0.01, 0.99)
|
|
89
166
|
Y = logit(W_clip)
|
|
90
167
|
Y = 0.5 * (Y - Y.T)
|
|
91
168
|
np.fill_diagonal(Y, 0.0)
|
|
92
169
|
|
|
93
|
-
if weights is None:
|
|
94
|
-
weights = np.ones((n, n))
|
|
95
|
-
weights = 0.5 * (weights + weights.T)
|
|
96
|
-
|
|
97
170
|
L = np.zeros((n, n))
|
|
98
171
|
div_Y = np.zeros(n)
|
|
99
172
|
for i in range(n):
|
|
@@ -112,11 +185,14 @@ class HodgeDecomposition:
|
|
|
112
185
|
s -= s.mean()
|
|
113
186
|
|
|
114
187
|
grad_flow = np.subtract.outer(s, s)
|
|
115
|
-
curl_flow = Y - grad_flow
|
|
116
|
-
|
|
188
|
+
curl_flow, harmonic_flow = _split_residual(Y - grad_flow, weights)
|
|
189
|
+
|
|
190
|
+
# The projection is orthogonal in the same weighted inner product
|
|
191
|
+
# used in the least-squares fit (Jiang et al., 2011, section 5.1).
|
|
192
|
+
edge_weights = np.triu(weights, k=1)
|
|
117
193
|
|
|
118
194
|
def sq_norm(M: NDArray[np.float64]) -> float:
|
|
119
|
-
return float(np.sum(
|
|
195
|
+
return float(np.sum(edge_weights * M**2))
|
|
120
196
|
|
|
121
197
|
total_var = sq_norm(Y)
|
|
122
198
|
grad_var = sq_norm(grad_flow)
|
|
@@ -169,23 +245,24 @@ class HodgeDecomposition:
|
|
|
169
245
|
return {m: float(self.result.potential[i]) for i, m in enumerate(self.models)}
|
|
170
246
|
|
|
171
247
|
def curl_magnitude_per_pair(self) -> NDArray[np.float64]:
|
|
172
|
-
"""
|
|
248
|
+
"""Absolute local triangle flow per pair."""
|
|
173
249
|
if self.result is None:
|
|
174
250
|
msg = "Call .fit() first"
|
|
175
251
|
raise RuntimeError(msg)
|
|
176
252
|
return np.abs(self.result.curl_flow)
|
|
177
253
|
|
|
178
254
|
def worst_pairs(self, k: int = 10) -> list[tuple[str, str, float]]:
|
|
179
|
-
"""Top-k pairs with largest
|
|
255
|
+
"""Top-k pairs with the largest absolute local triangle flow."""
|
|
180
256
|
if self.result is None:
|
|
181
257
|
msg = "Call .fit() first"
|
|
182
258
|
raise RuntimeError(msg)
|
|
183
259
|
C = np.abs(self.result.curl_flow)
|
|
184
|
-
pairs: list[tuple[str, str, float]] = []
|
|
185
260
|
n = self.n
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
261
|
+
pairs: list[tuple[str, str, float]] = [
|
|
262
|
+
(self.models[i], self.models[j], float(C[i, j]))
|
|
263
|
+
for i in range(n)
|
|
264
|
+
for j in range(i + 1, n)
|
|
265
|
+
]
|
|
189
266
|
pairs.sort(key=lambda x: x[2], reverse=True)
|
|
190
267
|
return pairs[:k]
|
|
191
268
|
|
|
@@ -1,5 +1,4 @@
|
|
|
1
|
-
"""
|
|
2
|
-
Simulate pairwise comparison data with controlled non-transitivity.
|
|
1
|
+
"""Simulate pairwise comparison data with controlled non-transitivity.
|
|
3
2
|
|
|
4
3
|
Three generators corresponding to different stories about where
|
|
5
4
|
non-transitivity comes from:
|
|
@@ -110,7 +109,9 @@ def simulate_heterogeneous(
|
|
|
110
109
|
theta[cat] = {m: float(t[i]) for i, m in enumerate(models)}
|
|
111
110
|
|
|
112
111
|
raw_weights: NDArray[np.float64] = rng.dirichlet(np.ones(n_categories) * 2)
|
|
113
|
-
category_weights = {
|
|
112
|
+
category_weights = {
|
|
113
|
+
cat: float(w) for cat, w in zip(category_names, raw_weights, strict=True)
|
|
114
|
+
}
|
|
114
115
|
|
|
115
116
|
comparisons: list[tuple[str, str, bool]] = []
|
|
116
117
|
categories: list[str] = []
|
|
@@ -211,7 +212,14 @@ def simulate_llm_arena(
|
|
|
211
212
|
Same format as simulate_heterogeneous.
|
|
212
213
|
"""
|
|
213
214
|
rng = np.random.default_rng(seed)
|
|
214
|
-
models = [
|
|
215
|
+
models = [
|
|
216
|
+
"AlphaLM",
|
|
217
|
+
"BetaChat",
|
|
218
|
+
"GammaCoder",
|
|
219
|
+
"DeltaWrite",
|
|
220
|
+
"EpsilonAll",
|
|
221
|
+
"ZetaMath",
|
|
222
|
+
]
|
|
215
223
|
categories = ["reasoning", "creative_writing", "coding"]
|
|
216
224
|
|
|
217
225
|
theta: dict[str, dict[str, float]] = {
|
|
@@ -1,7 +1,6 @@
|
|
|
1
|
-
"""
|
|
2
|
-
Tournament graph diagnostics.
|
|
1
|
+
"""Tournament graph diagnostics.
|
|
3
2
|
|
|
4
|
-
Build the directed tournament graph from pairwise outcomes and
|
|
3
|
+
Build the directed tournament graph from pairwise outcomes and analyze its
|
|
5
4
|
strongly connected component (SCC) structure. Non-trivial SCCs (size > 1)
|
|
6
5
|
indicate clusters of models that cannot be linearly ordered — i.e. pockets
|
|
7
6
|
of non-transitivity.
|
|
@@ -15,10 +14,8 @@ from numpy.typing import NDArray
|
|
|
15
14
|
class TournamentGraph:
|
|
16
15
|
"""Directed tournament graph built from pairwise comparison data.
|
|
17
16
|
|
|
18
|
-
Args:
|
|
19
|
-
models: Unique model identifiers.
|
|
20
|
-
|
|
21
17
|
Examples:
|
|
18
|
+
>>> from winference import TournamentGraph
|
|
22
19
|
>>> tg = TournamentGraph(["A", "B", "C"])
|
|
23
20
|
>>> tg.add_result("A", "B", win=True)
|
|
24
21
|
>>> tg.add_result("B", "C", win=True)
|
|
@@ -28,6 +25,11 @@ class TournamentGraph:
|
|
|
28
25
|
"""
|
|
29
26
|
|
|
30
27
|
def __init__(self, models: list[str]) -> None:
|
|
28
|
+
"""Create an empty tournament over ``models``.
|
|
29
|
+
|
|
30
|
+
Args:
|
|
31
|
+
models: Model names, in the order used for indexing.
|
|
32
|
+
"""
|
|
31
33
|
self.models = list(models)
|
|
32
34
|
self._idx: dict[str, int] = {m: i for i, m in enumerate(self.models)}
|
|
33
35
|
n = len(models)
|
|
@@ -51,7 +53,14 @@ class TournamentGraph:
|
|
|
51
53
|
col_b: str = "model_b",
|
|
52
54
|
col_win: str = "a_wins",
|
|
53
55
|
) -> None:
|
|
54
|
-
"""Bulk-load from a DataFrame with
|
|
56
|
+
"""Bulk-load from a DataFrame with model_a, model_b, a_wins (bool) columns.
|
|
57
|
+
|
|
58
|
+
Args:
|
|
59
|
+
df: Frame of comparison outcomes, one row per comparison.
|
|
60
|
+
col_a: Column holding the first model name.
|
|
61
|
+
col_b: Column holding the second model name.
|
|
62
|
+
col_win: Boolean column that is True when ``col_a`` won.
|
|
63
|
+
"""
|
|
55
64
|
for _, row in df.iterrows():
|
|
56
65
|
self.add_result(str(row[col_a]), str(row[col_b]), bool(row[col_win]))
|
|
57
66
|
|
|
File without changes
|