winference 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,269 @@
1
+ Metadata-Version: 2.4
2
+ Name: winference
3
+ Version: 0.1.0
4
+ Summary: Win rate calibration under non-transitivity via Hodge decomposition and heterogeneous group testing
5
+ Keywords: bradley-terry,calibration,hodge-decomposition,llm-evaluation,non-transitivity,pairwise-comparison,win-rate
6
+ Author: gaurav sood
7
+ License-Expression: MIT
8
+ Classifier: Development Status :: 4 - Beta
9
+ Classifier: Intended Audience :: Developers
10
+ Classifier: Intended Audience :: Science/Research
11
+ Classifier: License :: OSI Approved :: MIT License
12
+ Classifier: Operating System :: OS Independent
13
+ Classifier: Programming Language :: Python :: 3
14
+ Classifier: Programming Language :: Python :: 3.12
15
+ Classifier: Programming Language :: Python :: 3.13
16
+ Classifier: Topic :: Scientific/Engineering
17
+ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
18
+ Classifier: Typing :: Typed
19
+ Requires-Dist: matplotlib>=3.5
20
+ Requires-Dist: numpy>=1.22
21
+ Requires-Dist: pandas>=1.5
22
+ Requires-Dist: scikit-learn>=1.1
23
+ Requires-Dist: scipy>=1.9
24
+ Requires-Dist: deptry>=0.22 ; extra == 'dev'
25
+ Requires-Dist: pydoclint>=0.6 ; extra == 'dev'
26
+ Requires-Dist: pytest>=8.0 ; extra == 'dev'
27
+ Requires-Dist: pytest-cov>=4.0 ; extra == 'dev'
28
+ Requires-Dist: ruff>=0.8 ; extra == 'dev'
29
+ Requires-Dist: furo>=2024.0 ; extra == 'docs'
30
+ Requires-Dist: sphinx>=7.0 ; extra == 'docs'
31
+ Requires-Dist: sphinx-autodoc-typehints>=2.0 ; extra == 'docs'
32
+ Requires-Dist: sphinx-copybutton>=0.5 ; extra == 'docs'
33
+ Requires-Python: >=3.12
34
+ Project-URL: Documentation, https://github.com/finite-sample/winference#readme
35
+ Project-URL: Homepage, https://github.com/finite-sample/winference
36
+ Project-URL: Issues, https://github.com/finite-sample/winference/issues
37
+ Project-URL: Repository, https://github.com/finite-sample/winference
38
+ Provides-Extra: dev
39
+ Provides-Extra: docs
40
+ Description-Content-Type: text/markdown
41
+
42
+ ## winference: Win rate calibration under non-transitivity.
43
+
44
+ [![Python application](https://github.com/finite-sample/winference/actions/workflows/ci.yml/badge.svg)](https://github.com/finite-sample/winference/actions/workflows/ci.yml)
45
+ [![PyPI version](https://img.shields.io/pypi/v/winference.svg)](https://pypi.org/project/winference/)
46
+ [![Downloads](https://pepy.tech/badge/winference)](https://pepy.tech/project/winference)
47
+ [![Documentation](https://github.com/finite-sample/winference/actions/workflows/docs.yml/badge.svg)](https://finite-sample.github.io/winference/)
48
+ [![License](https://img.shields.io/github/license/finite-sample/winference)](https://github.com/finite-sample/winference/blob/main/LICENSE)
49
+
50
+
51
+ When you run an LLM arena and report "Model A beats Model B 62% of the time,"
52
+ is that number *calibrated*? And does it still hold when your users ask
53
+ different questions than your evaluation set?
54
+
55
+ If model strengths vary across task types, aggregate win rates can exhibit
56
+ **non-transitive** preferences: A beats B, B beats C, but C beats A. Standard
57
+ Bradley-Terry / Elo assumes this doesn't happen, and when it does, your
58
+ calibration breaks — especially under distribution shift.
59
+
60
+ `winference` provides two approaches to calibrating win rates in the presence of
61
+ non-transitivity, plus diagnostics to decide which one you need.
62
+
63
+ ---
64
+
65
+ ## The two approaches
66
+
67
+ ### A) Hodge decomposition → calibrate the transitive signal
68
+
69
+ Decomposes the pairwise comparison matrix into:
70
+
71
+ - **Gradient** (transitive): a potential `s_i` per model such that the
72
+ log-odds ≈ `s_i − s_j`. This part *can* be calibrated to a scalar ranking.
73
+ - **Curl** (cyclic): rock-paper-scissors structure that *cannot* be represented
74
+ by any linear ranking.
75
+
76
+ Calibrate win rates from the gradient component. Report the curl fraction as
77
+ the share of variance your calibration ignores.
78
+
79
+ **Use when:** Cycles persist even after conditioning on task category — the
80
+ non-transitivity is irreducible.
81
+
82
+ ### B) Heterogeneous groups → calibrate per category, compose
83
+
84
+ Test whether model strengths differ across prompt categories (math, creative,
85
+ coding, ...) using a **likelihood-ratio test**. If so, fit Bradley-Terry per
86
+ category. Win rates for any target distribution are then:
87
+
88
+ ```
89
+ P(A > B | π*) = Σ_k π*_k · σ(θ_{A,k} − θ_{B,k})
90
+ ```
91
+
92
+ This gives you **composable** calibration: swap in any target distribution
93
+ without refitting.
94
+
95
+ **Use when:** Non-transitivity dissolves when you condition on prompt category.
96
+
97
+ ---
98
+
99
+ ## Quickstart
100
+
101
+ ```bash
102
+ pip install numpy scipy scikit-learn matplotlib
103
+ # Then from the repo root:
104
+ pip install -e .
105
+ ```
106
+
107
+ ```python
108
+ from winference import (
109
+ TournamentGraph, BradleyTerry, HodgeDecomposition,
110
+ GroupTest, GroupCalibrator, expected_calibration_error,
111
+ )
112
+ from winference.simulate import simulate_llm_arena
113
+
114
+ # 1. Simulate (or load) arena data
115
+ data = simulate_llm_arena()
116
+ comparisons = data["comparisons"] # list of (model_a, model_b, a_wins)
117
+ categories = data["categories"] # list of category labels per comparison
118
+ models = data["models"]
119
+
120
+ # 2. Graph triage: is non-transitivity a problem?
121
+ tg = TournamentGraph(models)
122
+ for a, b, w in comparisons:
123
+ tg.add_result(a, b, w)
124
+
125
+ print(tg.summary())
126
+ # → {'nontransitivity_index': 0.83, 'cyclic_triples': 7, ...}
127
+
128
+ # 3a. Hodge decomposition
129
+ hd = HodgeDecomposition(models)
130
+ result = hd.fit(tg.win_rate_matrix(), weights=tg.counts)
131
+ print(f"Transitive: {result.transitive_variance:.0%}")
132
+ print(f"Cyclic: {result.cyclic_variance:.0%}")
133
+
134
+ # Calibrated win probability (transitive component only)
135
+ p = hd.transitive_win_probability("ZetaMath", "DeltaWrite")
136
+
137
+ # 3b. Group heterogeneity test
138
+ groups = sorted(set(categories))
139
+ gt = GroupTest(models, groups)
140
+ gt.fit(comparisons, categories)
141
+ print(gt.test_result())
142
+ # → {'statistic': 342.1, 'p_value': 1.2e-63, 'reject_at_05': True}
143
+
144
+ # Composable win rates
145
+ gc = GroupCalibrator(gt)
146
+ p_math_heavy = gc.win_probability(
147
+ "ZetaMath", "DeltaWrite",
148
+ target_distribution={"reasoning": 0.7, "creative_writing": 0.15, "coding": 0.15},
149
+ )
150
+ p_creative_heavy = gc.win_probability(
151
+ "ZetaMath", "DeltaWrite",
152
+ target_distribution={"reasoning": 0.15, "creative_writing": 0.7, "coding": 0.15},
153
+ )
154
+ ```
155
+
156
+ See `examples/quickstart.py` for the full pipeline with calibration comparison
157
+ and reliability diagrams.
158
+
159
+ ---
160
+
161
+ ## The diagnostic pipeline
162
+
163
+ ```
164
+ ┌─────────────────────────────┐
165
+ │ Build tournament graph │
166
+ │ Run Tarjan's SCC │
167
+ └──────────┬──────────────────┘
168
+ │
169
+ All SCCs size 1?
170
+ ╱ ╲
171
+ YES NO
172
+ │ │
173
+ Standard BT ┌──┴───────────────────────┐
174
+ is fine │ Condition on categories │
175
+ │ Check: do SCCs shrink? │
176
+ └──────┬───────────────┬────┘
177
+ YES NO
178
+ │ │
179
+ ┌──────┴──────┐ ┌───────┴────────┐
180
+ │ Per-group │ │ Hodge decomp │
181
+ │ BT + LRT │ │ calibrate grad │
182
+ │ + compose │ │ report curl │
183
+ └─────────────┘ └────────────────┘
184
+ ```
185
+
186
+ ---
187
+
188
+ ## Other sources of non-transitivity
189
+
190
+ Non-transitivity in pairwise comparisons doesn't always come from heterogeneous
191
+ model strengths. Before reaching for Hodge or group decomposition, rule out
192
+ simpler causes:
193
+
194
+ | Source | What it is | How to address |
195
+ |---|---|---|
196
+ | **Judge noise** | LLM judge gives inconsistent verdicts on the same pair | Bayesian calibration (Dawid-Skene, BWRS) |
197
+ | **Position bias** | Judge prefers whichever response appears first/second | Randomise presentation order, average both orderings |
198
+ | **Style/length bias** | Judge rewards verbosity rather than quality | Regress out length/style features (cf. AlpacaEval 2.0) |
199
+ | **Evaluator disagreement** | Individual annotators are transitive, but they disagree with each other (Condorcet cycles) | Stratify by evaluator type, or accept the Hodge curl as measuring genuine disagreement |
200
+ | **Fine-grained interaction** | Strengths differ at sub-category level (algebra vs geometry within "math") | Per-prompt routing rather than aggregate ranking |
201
+ | **Context effects** | Evaluation of A vs B depends on what other models were seen | Session-aware experimental design |
202
+
203
+ `winference` targets the modelling layer (rows 4–5 in the table above). It
204
+ assumes you've already addressed measurement issues at the design/preprocessing
205
+ level.
206
+
207
+ ---
208
+
209
+ ## API reference
210
+
211
+ ### `TournamentGraph`
212
+ Build a directed tournament graph and compute SCC structure.
213
+ - `.add_result(a, b, win)` — record one comparison
214
+ - `.strongly_connected_components()` — Tarjan's algorithm
215
+ - `.nontransitivity_index()` — fraction of models in non-trivial SCCs
216
+ - `.count_cyclic_triples()` — count A>B>C>A cycles
217
+ - `.summary()` — quick diagnostic dict
218
+
219
+ ### `BradleyTerry`
220
+ Standard BT model via maximum likelihood.
221
+ - `.fit(comparisons)` — fit from (a, b, a_wins) triples
222
+ - `.win_probability(a, b)` — predicted P(a beats b)
223
+ - `.strengths()` — {model: θ}
224
+ - `.rank()` — models sorted by strength
225
+
226
+ ### `HodgeDecomposition`
227
+ Hodge decomposition of the pairwise log-odds matrix.
228
+ - `.fit(W, weights)` — decompose win-rate matrix
229
+ - `.transitive_win_probability(a, b)` — P(a>b) from gradient only
230
+ - `.worst_pairs(k)` — pairs with largest cyclic residual
231
+ - `.summary()` — variance fractions (transitive / cyclic / harmonic)
232
+
233
+ ### `GroupTest`
234
+ Likelihood-ratio test for heterogeneity across prompt groups.
235
+ - `.fit(comparisons, group_labels)` — fit null + per-group BT
236
+ - `.test_result()` — {statistic, df, p_value, reject_at_05}
237
+ - `.per_group_strengths()` — {group: {model: θ}}
238
+
239
+ ### `GroupCalibrator`
240
+ Composable win rates from per-group BT.
241
+ - `.win_probability(a, b, target_distribution)` — composite P(a>b)
242
+ - `.sensitivity_analysis(a, b)` — how much does P(a>b) vary with π*?
243
+
244
+ ### Calibration utilities
245
+ - `expected_calibration_error(predicted, observed)` — ECE
246
+ - `brier_score(predicted, observed)` — Brier score
247
+ - `reliability_diagram(predicted, observed, ax=...)` — reliability plot
248
+
249
+ ### Simulators
250
+ - `simulate_transitive(...)` — pure BT (no cycles)
251
+ - `simulate_heterogeneous(...)` — per-category strengths
252
+ - `simulate_rock_paper_scissors(...)` — irreducible cyclic structure
253
+ - `simulate_llm_arena(...)` — realistic six-model arena
254
+
255
+ ---
256
+
257
+ ## References
258
+
259
+ - Jiang, X., Lim, L.-H., Yao, Y., & Ye, Y. (2011). Statistical ranking and combinatorial Hodge theory. *Mathematical Programming*, 127(1), 203–244.
260
+ - Dittrich, R., Hatzinger, R., & Katzenbeisser, W. (1998). Modelling the effect of subject-specific covariates in paired comparison studies. *Applied Statistics*, 47(4), 511–525.
261
+ - Xu, Y., Ruis, L., Rocktäschel, T., & Kirk, R. (2025). Investigating non-transitivity in LLM-as-a-Judge. *ICML 2025*.
262
+ - Li, X. & Li, S. (2025). Efficient inference for covariate-adjusted Bradley-Terry model with covariate shift. *arXiv:2503.18256*.
263
+ - Balduzzi, D., Tuyls, K., Perolat, J., & Graepel, T. (2018). Re-evaluating evaluation. *NeurIPS*.
264
+
265
+ ---
266
+
267
+ ## License
268
+
269
+ MIT
@@ -0,0 +1,228 @@
1
+ ## winference: Win rate calibration under non-transitivity.
2
+
3
+ [![Python application](https://github.com/finite-sample/winference/actions/workflows/ci.yml/badge.svg)](https://github.com/finite-sample/winference/actions/workflows/ci.yml)
4
+ [![PyPI version](https://img.shields.io/pypi/v/winference.svg)](https://pypi.org/project/winference/)
5
+ [![Downloads](https://pepy.tech/badge/winference)](https://pepy.tech/project/winference)
6
+ [![Documentation](https://github.com/finite-sample/winference/actions/workflows/docs.yml/badge.svg)](https://finite-sample.github.io/winference/)
7
+ [![License](https://img.shields.io/github/license/finite-sample/winference)](https://github.com/finite-sample/winference/blob/main/LICENSE)
8
+
9
+
10
+ When you run an LLM arena and report "Model A beats Model B 62% of the time,"
11
+ is that number *calibrated*? And does it still hold when your users ask
12
+ different questions than your evaluation set?
13
+
14
+ If model strengths vary across task types, aggregate win rates can exhibit
15
+ **non-transitive** preferences: A beats B, B beats C, but C beats A. Standard
16
+ Bradley-Terry / Elo assumes this doesn't happen, and when it does, your
17
+ calibration breaks — especially under distribution shift.
18
+
19
+ `winference` provides two approaches to calibrating win rates in the presence of
20
+ non-transitivity, plus diagnostics to decide which one you need.
21
+
22
+ ---
23
+
24
+ ## The two approaches
25
+
26
+ ### A) Hodge decomposition → calibrate the transitive signal
27
+
28
+ Decomposes the pairwise comparison matrix into:
29
+
30
+ - **Gradient** (transitive): a potential `s_i` per model such that the
31
+ log-odds ≈ `s_i − s_j`. This part *can* be calibrated to a scalar ranking.
32
+ - **Curl** (cyclic): rock-paper-scissors structure that *cannot* be represented
33
+ by any linear ranking.
34
+
35
+ Calibrate win rates from the gradient component. Report the curl fraction as
36
+ the share of variance your calibration ignores.
37
+
38
+ **Use when:** Cycles persist even after conditioning on task category — the
39
+ non-transitivity is irreducible.
40
+
41
+ ### B) Heterogeneous groups → calibrate per category, compose
42
+
43
+ Test whether model strengths differ across prompt categories (math, creative,
44
+ coding, ...) using a **likelihood-ratio test**. If so, fit Bradley-Terry per
45
+ category. Win rates for any target distribution are then:
46
+
47
+ ```
48
+ P(A > B | π*) = Σ_k π*_k · σ(θ_{A,k} − θ_{B,k})
49
+ ```
50
+
51
+ This gives you **composable** calibration: swap in any target distribution
52
+ without refitting.
53
+
54
+ **Use when:** Non-transitivity dissolves when you condition on prompt category.
55
+
56
+ ---
57
+
58
+ ## Quickstart
59
+
60
+ ```bash
61
+ pip install numpy scipy scikit-learn matplotlib
62
+ # Then from the repo root:
63
+ pip install -e .
64
+ ```
65
+
66
+ ```python
67
+ from winference import (
68
+ TournamentGraph, BradleyTerry, HodgeDecomposition,
69
+ GroupTest, GroupCalibrator, expected_calibration_error,
70
+ )
71
+ from winference.simulate import simulate_llm_arena
72
+
73
+ # 1. Simulate (or load) arena data
74
+ data = simulate_llm_arena()
75
+ comparisons = data["comparisons"] # list of (model_a, model_b, a_wins)
76
+ categories = data["categories"] # list of category labels per comparison
77
+ models = data["models"]
78
+
79
+ # 2. Graph triage: is non-transitivity a problem?
80
+ tg = TournamentGraph(models)
81
+ for a, b, w in comparisons:
82
+ tg.add_result(a, b, w)
83
+
84
+ print(tg.summary())
85
+ # → {'nontransitivity_index': 0.83, 'cyclic_triples': 7, ...}
86
+
87
+ # 3a. Hodge decomposition
88
+ hd = HodgeDecomposition(models)
89
+ result = hd.fit(tg.win_rate_matrix(), weights=tg.counts)
90
+ print(f"Transitive: {result.transitive_variance:.0%}")
91
+ print(f"Cyclic: {result.cyclic_variance:.0%}")
92
+
93
+ # Calibrated win probability (transitive component only)
94
+ p = hd.transitive_win_probability("ZetaMath", "DeltaWrite")
95
+
96
+ # 3b. Group heterogeneity test
97
+ groups = sorted(set(categories))
98
+ gt = GroupTest(models, groups)
99
+ gt.fit(comparisons, categories)
100
+ print(gt.test_result())
101
+ # → {'statistic': 342.1, 'p_value': 1.2e-63, 'reject_at_05': True}
102
+
103
+ # Composable win rates
104
+ gc = GroupCalibrator(gt)
105
+ p_math_heavy = gc.win_probability(
106
+ "ZetaMath", "DeltaWrite",
107
+ target_distribution={"reasoning": 0.7, "creative_writing": 0.15, "coding": 0.15},
108
+ )
109
+ p_creative_heavy = gc.win_probability(
110
+ "ZetaMath", "DeltaWrite",
111
+ target_distribution={"reasoning": 0.15, "creative_writing": 0.7, "coding": 0.15},
112
+ )
113
+ ```
114
+
115
+ See `examples/quickstart.py` for the full pipeline with calibration comparison
116
+ and reliability diagrams.
117
+
118
+ ---
119
+
120
+ ## The diagnostic pipeline
121
+
122
+ ```
123
+ ┌─────────────────────────────┐
124
+ │ Build tournament graph │
125
+ │ Run Tarjan's SCC │
126
+ └──────────┬──────────────────┘
127
+ │
128
+ All SCCs size 1?
129
+ ╱ ╲
130
+ YES NO
131
+ │ │
132
+ Standard BT ┌──┴───────────────────────┐
133
+ is fine │ Condition on categories │
134
+ │ Check: do SCCs shrink? │
135
+ └──────┬───────────────┬────┘
136
+ YES NO
137
+ │ │
138
+ ┌──────┴──────┐ ┌───────┴────────┐
139
+ │ Per-group │ │ Hodge decomp │
140
+ │ BT + LRT │ │ calibrate grad │
141
+ │ + compose │ │ report curl │
142
+ └─────────────┘ └────────────────┘
143
+ ```
144
+
145
+ ---
146
+
147
+ ## Other sources of non-transitivity
148
+
149
+ Non-transitivity in pairwise comparisons doesn't always come from heterogeneous
150
+ model strengths. Before reaching for Hodge or group decomposition, rule out
151
+ simpler causes:
152
+
153
+ | Source | What it is | How to address |
154
+ |---|---|---|
155
+ | **Judge noise** | LLM judge gives inconsistent verdicts on the same pair | Bayesian calibration (Dawid-Skene, BWRS) |
156
+ | **Position bias** | Judge prefers whichever response appears first/second | Randomise presentation order, average both orderings |
157
+ | **Style/length bias** | Judge rewards verbosity rather than quality | Regress out length/style features (cf. AlpacaEval 2.0) |
158
+ | **Evaluator disagreement** | Individual annotators are transitive, but they disagree with each other (Condorcet cycles) | Stratify by evaluator type, or accept the Hodge curl as measuring genuine disagreement |
159
+ | **Fine-grained interaction** | Strengths differ at sub-category level (algebra vs geometry within "math") | Per-prompt routing rather than aggregate ranking |
160
+ | **Context effects** | Evaluation of A vs B depends on what other models were seen | Session-aware experimental design |
161
+
162
+ `winference` targets the modelling layer (rows 4–5 in the table above). It
163
+ assumes you've already addressed measurement issues at the design/preprocessing
164
+ level.
165
+
166
+ ---
167
+
168
+ ## API reference
169
+
170
+ ### `TournamentGraph`
171
+ Build a directed tournament graph and compute SCC structure.
172
+ - `.add_result(a, b, win)` — record one comparison
173
+ - `.strongly_connected_components()` — Tarjan's algorithm
174
+ - `.nontransitivity_index()` — fraction of models in non-trivial SCCs
175
+ - `.count_cyclic_triples()` — count A>B>C>A cycles
176
+ - `.summary()` — quick diagnostic dict
177
+
178
+ ### `BradleyTerry`
179
+ Standard BT model via maximum likelihood.
180
+ - `.fit(comparisons)` — fit from (a, b, a_wins) triples
181
+ - `.win_probability(a, b)` — predicted P(a beats b)
182
+ - `.strengths()` — {model: θ}
183
+ - `.rank()` — models sorted by strength
184
+
185
+ ### `HodgeDecomposition`
186
+ Hodge decomposition of the pairwise log-odds matrix.
187
+ - `.fit(W, weights)` — decompose win-rate matrix
188
+ - `.transitive_win_probability(a, b)` — P(a>b) from gradient only
189
+ - `.worst_pairs(k)` — pairs with largest cyclic residual
190
+ - `.summary()` — variance fractions (transitive / cyclic / harmonic)
191
+
192
+ ### `GroupTest`
193
+ Likelihood-ratio test for heterogeneity across prompt groups.
194
+ - `.fit(comparisons, group_labels)` — fit null + per-group BT
195
+ - `.test_result()` — {statistic, df, p_value, reject_at_05}
196
+ - `.per_group_strengths()` — {group: {model: θ}}
197
+
198
+ ### `GroupCalibrator`
199
+ Composable win rates from per-group BT.
200
+ - `.win_probability(a, b, target_distribution)` — composite P(a>b)
201
+ - `.sensitivity_analysis(a, b)` — how much does P(a>b) vary with π*?
202
+
203
+ ### Calibration utilities
204
+ - `expected_calibration_error(predicted, observed)` — ECE
205
+ - `brier_score(predicted, observed)` — Brier score
206
+ - `reliability_diagram(predicted, observed, ax=...)` — reliability plot
207
+
208
+ ### Simulators
209
+ - `simulate_transitive(...)` — pure BT (no cycles)
210
+ - `simulate_heterogeneous(...)` — per-category strengths
211
+ - `simulate_rock_paper_scissors(...)` — irreducible cyclic structure
212
+ - `simulate_llm_arena(...)` — realistic six-model arena
213
+
214
+ ---
215
+
216
+ ## References
217
+
218
+ - Jiang, X., Lim, L.-H., Yao, Y., & Ye, Y. (2011). Statistical ranking and combinatorial Hodge theory. *Mathematical Programming*, 127(1), 203–244.
219
+ - Dittrich, R., Hatzinger, R., & Katzenbeisser, W. (1998). Modelling the effect of subject-specific covariates in paired comparison studies. *Applied Statistics*, 47(4), 511–525.
220
+ - Xu, Y., Ruis, L., Rocktäschel, T., & Kirk, R. (2025). Investigating non-transitivity in LLM-as-a-Judge. *ICML 2025*.
221
+ - Li, X. & Li, S. (2025). Efficient inference for covariate-adjusted Bradley-Terry model with covariate shift. *arXiv:2503.18256*.
222
+ - Balduzzi, D., Tuyls, K., Perolat, J., & Graepel, T. (2018). Re-evaluating evaluation. *NeurIPS*.
223
+
224
+ ---
225
+
226
+ ## License
227
+
228
+ MIT
@@ -0,0 +1,122 @@
1
+ [build-system]
2
+ requires = ["uv_build>=0.10.9,<0.11.0"]
3
+ build-backend = "uv_build"
4
+
5
+ [tool.uv.build-backend]
6
+ module-root = ""
7
+
8
+ [project]
9
+ name = "winference"
10
+ version = "0.1.0"
11
+ description = "Win rate calibration under non-transitivity via Hodge decomposition and heterogeneous group testing"
12
+ readme = "README.md"
13
+ license = "MIT"
14
+ requires-python = ">=3.12"
15
+ authors = [
16
+ { name = "gaurav sood" },
17
+ ]
18
+ keywords = [
19
+ "bradley-terry",
20
+ "calibration",
21
+ "hodge-decomposition",
22
+ "llm-evaluation",
23
+ "non-transitivity",
24
+ "pairwise-comparison",
25
+ "win-rate",
26
+ ]
27
+ classifiers = [
28
+ "Development Status :: 4 - Beta",
29
+ "Intended Audience :: Developers",
30
+ "Intended Audience :: Science/Research",
31
+ "License :: OSI Approved :: MIT License",
32
+ "Operating System :: OS Independent",
33
+ "Programming Language :: Python :: 3",
34
+ "Programming Language :: Python :: 3.12",
35
+ "Programming Language :: Python :: 3.13",
36
+ "Topic :: Scientific/Engineering",
37
+ "Topic :: Scientific/Engineering :: Artificial Intelligence",
38
+ "Typing :: Typed",
39
+ ]
40
+ dependencies = [
41
+ "matplotlib>=3.5",
42
+ "numpy>=1.22",
43
+ "pandas>=1.5",
44
+ "scikit-learn>=1.1",
45
+ "scipy>=1.9",
46
+ ]
47
+
48
+ [project.optional-dependencies]
49
+ dev = [
50
+ "deptry>=0.22",
51
+ "pydoclint>=0.6",
52
+ "pytest>=8.0",
53
+ "pytest-cov>=4.0",
54
+ "ruff>=0.8",
55
+ ]
56
+ docs = [
57
+ "furo>=2024.0",
58
+ "sphinx>=7.0",
59
+ "sphinx-autodoc-typehints>=2.0",
60
+ "sphinx-copybutton>=0.5",
61
+ ]
62
+
63
+ [project.urls]
64
+ Documentation = "https://github.com/finite-sample/winference#readme"
65
+ Homepage = "https://github.com/finite-sample/winference"
66
+ Issues = "https://github.com/finite-sample/winference/issues"
67
+ Repository = "https://github.com/finite-sample/winference"
68
+
69
+ [tool.ruff]
70
+ target-version = "py312"
71
+ line-length = 100
72
+
73
+ [tool.ruff.lint]
74
+ select = [
75
+ "ARG",
76
+ "B",
77
+ "C4",
78
+ "E",
79
+ "F",
80
+ "I",
81
+ "PTH",
82
+ "RUF",
83
+ "SIM",
84
+ "TCH",
85
+ "UP",
86
+ "W",
87
+ ]
88
+ ignore = [
89
+ "ARG001",
90
+ ]
91
+
92
+ [tool.ruff.lint.isort]
93
+ known-first-party = ["winference"]
94
+
95
+ [tool.pytest.ini_options]
96
+ testpaths = ["tests"]
97
+ addopts = "-v --tb=short"
98
+
99
+ [tool.coverage.run]
100
+ source = ["winference"]
101
+ branch = true
102
+
103
+ [tool.coverage.report]
104
+ exclude_lines = [
105
+ "pragma: no cover",
106
+ "if TYPE_CHECKING:",
107
+ "if __name__ == .__main__.:",
108
+ ]
109
+
110
+ [tool.pydoclint]
111
+ style = "google"
112
+ arg-type-hints-in-docstring = false
113
+ check-return-types = false
114
+ check-arg-order = true
115
+ allow-init-docstring = true
116
+
117
+ [tool.deptry]
118
+ extend_exclude = ["examples", "docs"]
119
+ pep621_dev_dependency_groups = ["dev", "docs"]
120
+ ignore = ["DEP002"]
121
+ [tool.deptry.per_rule_ignores]
122
+ DEP002 = ["matplotlib", "pandas", "scikit-learn"]
@@ -0,0 +1,44 @@
1
+ """
2
+ winference: Win rate calibration under non-transitivity.
3
+
4
+ Two approaches to calibrating win rates when preferences aren't transitive:
5
+ A) Hodge decomposition — separate transitive signal from cyclic residual,
6
+ calibrate only the transitive part.
7
+ B) Heterogeneous groups — find prompt categories where transitivity holds
8
+ within each group, calibrate per group, compose for any target distribution.
9
+ """
10
+
11
+ from importlib.metadata import version
12
+
13
+ from winference.bradley_terry import BradleyTerry
14
+ from winference.calibration import (
15
+ brier_score,
16
+ expected_calibration_error,
17
+ log_loss,
18
+ reliability_diagram,
19
+ )
20
+ from winference.groups import GroupCalibrator, GroupTest
21
+ from winference.hodge import HodgeDecomposition
22
+ from winference.simulate import (
23
+ simulate_heterogeneous,
24
+ simulate_rock_paper_scissors,
25
+ simulate_transitive,
26
+ )
27
+ from winference.tournament import TournamentGraph
28
+
29
+ __version__ = version("winference")
30
+
31
+ __all__ = [
32
+ "BradleyTerry",
33
+ "GroupCalibrator",
34
+ "GroupTest",
35
+ "HodgeDecomposition",
36
+ "TournamentGraph",
37
+ "brier_score",
38
+ "expected_calibration_error",
39
+ "log_loss",
40
+ "reliability_diagram",
41
+ "simulate_heterogeneous",
42
+ "simulate_rock_paper_scissors",
43
+ "simulate_transitive",
44
+ ]