winference 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- winference/__init__.py +44 -0
- winference/bradley_terry.py +138 -0
- winference/calibration.py +155 -0
- winference/groups.py +261 -0
- winference/hodge.py +203 -0
- winference/py.typed +0 -0
- winference/simulate.py +265 -0
- winference/tournament.py +164 -0
- winference-0.1.0.dist-info/METADATA +269 -0
- winference-0.1.0.dist-info/RECORD +11 -0
- winference-0.1.0.dist-info/WHEEL +4 -0
winference/hodge.py
ADDED
|
@@ -0,0 +1,203 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Hodge decomposition of pairwise comparison data.
|
|
3
|
+
|
|
4
|
+
Any skew-symmetric matrix of log-odds (or edge flows on the tournament
|
|
5
|
+
graph) can be orthogonally decomposed into:
|
|
6
|
+
|
|
7
|
+
Y = grad(s) + curl(C) + harmonic
|
|
8
|
+
|
|
9
|
+
where:
|
|
10
|
+
- grad(s) is the *transitive* component: there exists a potential s_i
|
|
11
|
+
for each node such that Y_ij ~ s_i - s_j. This is the part that can
|
|
12
|
+
be calibrated via a standard Bradley-Terry model.
|
|
13
|
+
- curl(C) is the *cyclic* component: it captures rock-paper-scissors
|
|
14
|
+
structure that is irreducible to any linear ranking.
|
|
15
|
+
- harmonic captures global topological structure (zero for complete
|
|
16
|
+
tournaments).
|
|
17
|
+
|
|
18
|
+
The decomposition is computed via least-squares on the combinatorial
|
|
19
|
+
Laplacian, following Jiang et al. (2011) "Statistical ranking and
|
|
20
|
+
combinatorial Hodge theory".
|
|
21
|
+
|
|
22
|
+
Calibration strategy
|
|
23
|
+
--------------------
|
|
24
|
+
1. Compute the Hodge decomposition of the empirical log-odds.
|
|
25
|
+
2. Use the gradient component s as BT strengths giving *transitive*
|
|
26
|
+
calibrated win probabilities: P_trans(i > j) = sigmoid(s_i - s_j).
|
|
27
|
+
3. Report ||curl||^2 / ||Y||^2 as the fraction of variance due to
|
|
28
|
+
non-transitive structure, the part your calibration *ignores*.
|
|
29
|
+
"""
|
|
30
|
+
|
|
31
|
+
from typing import NamedTuple
|
|
32
|
+
|
|
33
|
+
import numpy as np
|
|
34
|
+
from numpy.typing import NDArray
|
|
35
|
+
from scipy.special import expit, logit
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
class HodgeResult(NamedTuple):
|
|
39
|
+
"""Result of a Hodge decomposition."""
|
|
40
|
+
|
|
41
|
+
potential: NDArray[np.float64]
|
|
42
|
+
gradient_flow: NDArray[np.float64]
|
|
43
|
+
curl_flow: NDArray[np.float64]
|
|
44
|
+
harmonic_flow: NDArray[np.float64]
|
|
45
|
+
transitive_variance: float
|
|
46
|
+
cyclic_variance: float
|
|
47
|
+
harmonic_variance: float
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
class HodgeDecomposition:
|
|
51
|
+
"""Hodge decomposition of a pairwise comparison matrix.
|
|
52
|
+
|
|
53
|
+
Args:
|
|
54
|
+
models: Model identifiers.
|
|
55
|
+
|
|
56
|
+
Examples:
|
|
57
|
+
>>> hd = HodgeDecomposition(["A", "B", "C", "D"])
|
|
58
|
+
>>> result = hd.fit(win_rate_matrix)
|
|
59
|
+
>>> print(f"Cyclic fraction: {result.cyclic_variance:.1%}")
|
|
60
|
+
>>> # Calibrate using only the transitive part
|
|
61
|
+
>>> p_trans = hd.transitive_win_probability("A", "B")
|
|
62
|
+
"""
|
|
63
|
+
|
|
64
|
+
def __init__(self, models: list[str]) -> None:
|
|
65
|
+
self.models = list(models)
|
|
66
|
+
self._idx: dict[str, int] = {m: i for i, m in enumerate(self.models)}
|
|
67
|
+
self.n = len(models)
|
|
68
|
+
self.result: HodgeResult | None = None
|
|
69
|
+
|
|
70
|
+
def fit(
|
|
71
|
+
self,
|
|
72
|
+
W: NDArray[np.float64],
|
|
73
|
+
weights: NDArray[np.float64] | None = None,
|
|
74
|
+
) -> HodgeResult:
|
|
75
|
+
"""Decompose a win-rate matrix.
|
|
76
|
+
|
|
77
|
+
Args:
|
|
78
|
+
W: Win rate matrix of shape (n, n) where W[i,j] = P(i beats j).
|
|
79
|
+
Should satisfy W[i,j] + W[j,i] ~ 1.
|
|
80
|
+
weights: Number of comparisons per pair of shape (n, n), used for
|
|
81
|
+
weighted least squares. Default: uniform weights.
|
|
82
|
+
|
|
83
|
+
Returns:
|
|
84
|
+
HodgeResult containing decomposition components.
|
|
85
|
+
"""
|
|
86
|
+
n = self.n
|
|
87
|
+
|
|
88
|
+
W_clip = np.clip(W, 0.01, 0.99)
|
|
89
|
+
Y = logit(W_clip)
|
|
90
|
+
Y = 0.5 * (Y - Y.T)
|
|
91
|
+
np.fill_diagonal(Y, 0.0)
|
|
92
|
+
|
|
93
|
+
if weights is None:
|
|
94
|
+
weights = np.ones((n, n))
|
|
95
|
+
weights = 0.5 * (weights + weights.T)
|
|
96
|
+
|
|
97
|
+
L = np.zeros((n, n))
|
|
98
|
+
div_Y = np.zeros(n)
|
|
99
|
+
for i in range(n):
|
|
100
|
+
for j in range(n):
|
|
101
|
+
if i != j:
|
|
102
|
+
w = weights[i, j]
|
|
103
|
+
L[i, i] += w
|
|
104
|
+
L[i, j] -= w
|
|
105
|
+
div_Y[i] += w * Y[i, j]
|
|
106
|
+
|
|
107
|
+
L_red = L[1:, 1:]
|
|
108
|
+
div_red = div_Y[1:]
|
|
109
|
+
s_red = np.linalg.lstsq(L_red, div_red, rcond=None)[0]
|
|
110
|
+
s = np.zeros(n)
|
|
111
|
+
s[1:] = s_red
|
|
112
|
+
s -= s.mean()
|
|
113
|
+
|
|
114
|
+
grad_flow = np.subtract.outer(s, s)
|
|
115
|
+
curl_flow = Y - grad_flow
|
|
116
|
+
harmonic_flow = np.zeros_like(Y)
|
|
117
|
+
|
|
118
|
+
def sq_norm(M: NDArray[np.float64]) -> float:
|
|
119
|
+
return float(np.sum(np.triu(M, k=1) ** 2))
|
|
120
|
+
|
|
121
|
+
total_var = sq_norm(Y)
|
|
122
|
+
grad_var = sq_norm(grad_flow)
|
|
123
|
+
curl_var = sq_norm(curl_flow)
|
|
124
|
+
harm_var = sq_norm(harmonic_flow)
|
|
125
|
+
|
|
126
|
+
if total_var > 0:
|
|
127
|
+
tv = grad_var / total_var
|
|
128
|
+
cv = curl_var / total_var
|
|
129
|
+
hv = harm_var / total_var
|
|
130
|
+
else:
|
|
131
|
+
tv = cv = hv = 0.0
|
|
132
|
+
|
|
133
|
+
self.result = HodgeResult(
|
|
134
|
+
potential=s,
|
|
135
|
+
gradient_flow=grad_flow,
|
|
136
|
+
curl_flow=curl_flow,
|
|
137
|
+
harmonic_flow=harmonic_flow,
|
|
138
|
+
transitive_variance=tv,
|
|
139
|
+
cyclic_variance=cv,
|
|
140
|
+
harmonic_variance=hv,
|
|
141
|
+
)
|
|
142
|
+
return self.result
|
|
143
|
+
|
|
144
|
+
def transitive_win_probability(self, model_a: str, model_b: str) -> float:
|
|
145
|
+
"""P(a beats b) using only the gradient (transitive) component.
|
|
146
|
+
|
|
147
|
+
This is the win probability that *can* be calibrated to a scalar
|
|
148
|
+
ranking. The cyclic component is dropped.
|
|
149
|
+
"""
|
|
150
|
+
if self.result is None:
|
|
151
|
+
msg = "Call .fit() first"
|
|
152
|
+
raise RuntimeError(msg)
|
|
153
|
+
s = self.result.potential
|
|
154
|
+
return float(expit(s[self._idx[model_a]] - s[self._idx[model_b]]))
|
|
155
|
+
|
|
156
|
+
def transitive_win_matrix(self) -> NDArray[np.float64]:
|
|
157
|
+
"""Full NxN transitive win probability matrix."""
|
|
158
|
+
if self.result is None:
|
|
159
|
+
msg = "Call .fit() first"
|
|
160
|
+
raise RuntimeError(msg)
|
|
161
|
+
s = self.result.potential
|
|
162
|
+
return expit(np.subtract.outer(s, s))
|
|
163
|
+
|
|
164
|
+
def transitive_strengths(self) -> dict[str, float]:
|
|
165
|
+
"""Hodge potential (transitive strength) per model."""
|
|
166
|
+
if self.result is None:
|
|
167
|
+
msg = "Call .fit() first"
|
|
168
|
+
raise RuntimeError(msg)
|
|
169
|
+
return {m: float(self.result.potential[i]) for i, m in enumerate(self.models)}
|
|
170
|
+
|
|
171
|
+
def curl_magnitude_per_pair(self) -> NDArray[np.float64]:
|
|
172
|
+
"""NxN matrix of curl magnitude: how much each pair deviates from transitivity."""
|
|
173
|
+
if self.result is None:
|
|
174
|
+
msg = "Call .fit() first"
|
|
175
|
+
raise RuntimeError(msg)
|
|
176
|
+
return np.abs(self.result.curl_flow)
|
|
177
|
+
|
|
178
|
+
def worst_pairs(self, k: int = 10) -> list[tuple[str, str, float]]:
|
|
179
|
+
"""Top-k pairs with largest cyclic residual."""
|
|
180
|
+
if self.result is None:
|
|
181
|
+
msg = "Call .fit() first"
|
|
182
|
+
raise RuntimeError(msg)
|
|
183
|
+
C = np.abs(self.result.curl_flow)
|
|
184
|
+
pairs: list[tuple[str, str, float]] = []
|
|
185
|
+
n = self.n
|
|
186
|
+
for i in range(n):
|
|
187
|
+
for j in range(i + 1, n):
|
|
188
|
+
pairs.append((self.models[i], self.models[j], float(C[i, j])))
|
|
189
|
+
pairs.sort(key=lambda x: x[2], reverse=True)
|
|
190
|
+
return pairs[:k]
|
|
191
|
+
|
|
192
|
+
def summary(self) -> dict[str, float | int]:
|
|
193
|
+
"""Quick summary of the decomposition."""
|
|
194
|
+
if self.result is None:
|
|
195
|
+
msg = "Call .fit() first"
|
|
196
|
+
raise RuntimeError(msg)
|
|
197
|
+
r = self.result
|
|
198
|
+
return {
|
|
199
|
+
"transitive_variance_frac": r.transitive_variance,
|
|
200
|
+
"cyclic_variance_frac": r.cyclic_variance,
|
|
201
|
+
"harmonic_variance_frac": r.harmonic_variance,
|
|
202
|
+
"n_models": self.n,
|
|
203
|
+
}
|
winference/py.typed
ADDED
|
File without changes
|
winference/simulate.py
ADDED
|
@@ -0,0 +1,265 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Simulate pairwise comparison data with controlled non-transitivity.
|
|
3
|
+
|
|
4
|
+
Three generators corresponding to different stories about where
|
|
5
|
+
non-transitivity comes from:
|
|
6
|
+
|
|
7
|
+
1. ``simulate_transitive`` — pure BT data (no cycles).
|
|
8
|
+
2. ``simulate_heterogeneous`` — models have different strengths per
|
|
9
|
+
category. Aggregate win rates can cycle even though within-category
|
|
10
|
+
preferences are transitive.
|
|
11
|
+
3. ``simulate_rock_paper_scissors`` — irreducible cyclic structure that
|
|
12
|
+
cannot be explained by category decomposition.
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
from typing import TYPE_CHECKING, Any
|
|
16
|
+
|
|
17
|
+
import numpy as np
|
|
18
|
+
from scipy.special import expit
|
|
19
|
+
|
|
20
|
+
if TYPE_CHECKING:
|
|
21
|
+
from numpy.typing import NDArray
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def simulate_transitive(
|
|
25
|
+
n_models: int = 6,
|
|
26
|
+
n_comparisons: int = 3000,
|
|
27
|
+
strength_spread: float = 1.5,
|
|
28
|
+
seed: int = 42,
|
|
29
|
+
) -> dict[str, Any]:
|
|
30
|
+
"""Generate data from a standard BT model (fully transitive).
|
|
31
|
+
|
|
32
|
+
Args:
|
|
33
|
+
n_models: Number of models to simulate.
|
|
34
|
+
n_comparisons: Number of pairwise comparisons to generate.
|
|
35
|
+
strength_spread: Standard deviation of model strengths.
|
|
36
|
+
seed: Random seed for reproducibility.
|
|
37
|
+
|
|
38
|
+
Returns:
|
|
39
|
+
Dict with keys: comparisons (list of (model_a, model_b, a_wins)),
|
|
40
|
+
models (list of str), true_strengths (dict {model: theta}),
|
|
41
|
+
categories (list of str, all "general").
|
|
42
|
+
"""
|
|
43
|
+
rng = np.random.default_rng(seed)
|
|
44
|
+
models = [f"M{i}" for i in range(n_models)]
|
|
45
|
+
theta: NDArray[np.float64] = rng.normal(0, strength_spread, size=n_models)
|
|
46
|
+
theta = theta - theta.mean()
|
|
47
|
+
|
|
48
|
+
comparisons: list[tuple[str, str, bool]] = []
|
|
49
|
+
categories: list[str] = []
|
|
50
|
+
for _ in range(n_comparisons):
|
|
51
|
+
i, j = rng.choice(n_models, 2, replace=False)
|
|
52
|
+
p = expit(theta[i] - theta[j])
|
|
53
|
+
win = rng.random() < p
|
|
54
|
+
comparisons.append((models[i], models[j], bool(win)))
|
|
55
|
+
categories.append("general")
|
|
56
|
+
|
|
57
|
+
return {
|
|
58
|
+
"comparisons": comparisons,
|
|
59
|
+
"models": models,
|
|
60
|
+
"true_strengths": {m: float(theta[k]) for k, m in enumerate(models)},
|
|
61
|
+
"categories": categories,
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def simulate_heterogeneous(
|
|
66
|
+
n_models: int = 6,
|
|
67
|
+
n_categories: int = 3,
|
|
68
|
+
n_comparisons: int = 5000,
|
|
69
|
+
strength_spread: float = 1.5,
|
|
70
|
+
category_names: list[str] | None = None,
|
|
71
|
+
seed: int = 42,
|
|
72
|
+
) -> dict[str, Any]:
|
|
73
|
+
"""Generate data where model strengths differ by category.
|
|
74
|
+
|
|
75
|
+
Each model has a different strength in each category. Within each
|
|
76
|
+
category, preferences are transitive. But the aggregate (marginal
|
|
77
|
+
over categories) can exhibit non-transitive win rates.
|
|
78
|
+
|
|
79
|
+
Args:
|
|
80
|
+
n_models: Number of models to simulate.
|
|
81
|
+
n_categories: Number of categories.
|
|
82
|
+
n_comparisons: Number of pairwise comparisons to generate.
|
|
83
|
+
strength_spread: Standard deviation of model strengths.
|
|
84
|
+
category_names: Optional custom category names.
|
|
85
|
+
seed: Random seed for reproducibility.
|
|
86
|
+
|
|
87
|
+
Returns:
|
|
88
|
+
Dict with keys: comparisons (list of (model_a, model_b, a_wins)),
|
|
89
|
+
models (list of str), true_strengths (dict {category: {model: theta}}),
|
|
90
|
+
categories (list of str, one per comparison),
|
|
91
|
+
category_weights (dict {category: proportion}).
|
|
92
|
+
|
|
93
|
+
Raises:
|
|
94
|
+
ValueError: If category_names length doesn't match n_categories.
|
|
95
|
+
"""
|
|
96
|
+
rng = np.random.default_rng(seed)
|
|
97
|
+
models = [f"M{i}" for i in range(n_models)]
|
|
98
|
+
|
|
99
|
+
if category_names is None:
|
|
100
|
+
category_names = [f"cat_{k}" for k in range(n_categories)]
|
|
101
|
+
if len(category_names) != n_categories:
|
|
102
|
+
msg = "category_names must have length n_categories"
|
|
103
|
+
raise ValueError(msg)
|
|
104
|
+
|
|
105
|
+
theta: dict[str, dict[str, float]] = {}
|
|
106
|
+
for k, cat in enumerate(category_names):
|
|
107
|
+
t: NDArray[np.float64] = rng.normal(0, strength_spread, size=n_models)
|
|
108
|
+
t = np.roll(t, k * (n_models // n_categories))
|
|
109
|
+
t = t - t.mean()
|
|
110
|
+
theta[cat] = {m: float(t[i]) for i, m in enumerate(models)}
|
|
111
|
+
|
|
112
|
+
raw_weights: NDArray[np.float64] = rng.dirichlet(np.ones(n_categories) * 2)
|
|
113
|
+
category_weights = {cat: float(w) for cat, w in zip(category_names, raw_weights, strict=True)}
|
|
114
|
+
|
|
115
|
+
comparisons: list[tuple[str, str, bool]] = []
|
|
116
|
+
categories: list[str] = []
|
|
117
|
+
for _ in range(n_comparisons):
|
|
118
|
+
cat = rng.choice(category_names, p=raw_weights)
|
|
119
|
+
i, j = rng.choice(n_models, 2, replace=False)
|
|
120
|
+
t_cat = theta[cat]
|
|
121
|
+
p = expit(t_cat[models[i]] - t_cat[models[j]])
|
|
122
|
+
win = rng.random() < p
|
|
123
|
+
comparisons.append((models[i], models[j], bool(win)))
|
|
124
|
+
categories.append(str(cat))
|
|
125
|
+
|
|
126
|
+
return {
|
|
127
|
+
"comparisons": comparisons,
|
|
128
|
+
"models": models,
|
|
129
|
+
"true_strengths": theta,
|
|
130
|
+
"categories": categories,
|
|
131
|
+
"category_weights": category_weights,
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
def simulate_rock_paper_scissors(
|
|
136
|
+
n_models: int = 6,
|
|
137
|
+
n_comparisons: int = 5000,
|
|
138
|
+
cycle_strength: float = 0.8,
|
|
139
|
+
transitive_strength: float = 1.0,
|
|
140
|
+
seed: int = 42,
|
|
141
|
+
) -> dict[str, Any]:
|
|
142
|
+
"""Generate data with *irreducible* cyclic structure.
|
|
143
|
+
|
|
144
|
+
Models have a transitive component (some are generally better)
|
|
145
|
+
PLUS a cyclic component that cannot be explained away by categories.
|
|
146
|
+
This is the Hodge curl in action.
|
|
147
|
+
|
|
148
|
+
The cyclic component is constructed by assigning each model a position
|
|
149
|
+
on a circle and adding a rotational advantage: models beat those
|
|
150
|
+
"clockwise-adjacent" to them but lose to those "counter-clockwise".
|
|
151
|
+
|
|
152
|
+
Args:
|
|
153
|
+
n_models: Number of models to simulate.
|
|
154
|
+
n_comparisons: Number of pairwise comparisons to generate.
|
|
155
|
+
cycle_strength: Strength of the cyclic component.
|
|
156
|
+
transitive_strength: Strength of the transitive component.
|
|
157
|
+
seed: Random seed for reproducibility.
|
|
158
|
+
|
|
159
|
+
Returns:
|
|
160
|
+
Dict with keys: comparisons (list of (model_a, model_b, a_wins)),
|
|
161
|
+
models (list of str), true_transitive (dict {model: s_i} gradient component),
|
|
162
|
+
true_curl_magnitude (float).
|
|
163
|
+
"""
|
|
164
|
+
rng = np.random.default_rng(seed)
|
|
165
|
+
models = [f"M{i}" for i in range(n_models)]
|
|
166
|
+
|
|
167
|
+
s: NDArray[np.float64] = rng.normal(0, transitive_strength, size=n_models)
|
|
168
|
+
s = s - s.mean()
|
|
169
|
+
|
|
170
|
+
angles = np.linspace(0, 2 * np.pi, n_models, endpoint=False)
|
|
171
|
+
C = np.zeros((n_models, n_models))
|
|
172
|
+
for i in range(n_models):
|
|
173
|
+
for j in range(n_models):
|
|
174
|
+
C[i, j] = cycle_strength * np.sin(angles[i] - angles[j])
|
|
175
|
+
|
|
176
|
+
Y = np.subtract.outer(s, s) + C
|
|
177
|
+
|
|
178
|
+
comparisons: list[tuple[str, str, bool]] = []
|
|
179
|
+
categories: list[str] = []
|
|
180
|
+
for _ in range(n_comparisons):
|
|
181
|
+
i, j = rng.choice(n_models, 2, replace=False)
|
|
182
|
+
p = expit(Y[i, j])
|
|
183
|
+
win = rng.random() < p
|
|
184
|
+
comparisons.append((models[i], models[j], bool(win)))
|
|
185
|
+
categories.append("general")
|
|
186
|
+
|
|
187
|
+
curl_var = np.sum(np.triu(C, k=1) ** 2)
|
|
188
|
+
total_var = np.sum(np.triu(Y, k=1) ** 2)
|
|
189
|
+
|
|
190
|
+
return {
|
|
191
|
+
"comparisons": comparisons,
|
|
192
|
+
"models": models,
|
|
193
|
+
"true_transitive": {m: float(s[i]) for i, m in enumerate(models)},
|
|
194
|
+
"true_curl_magnitude": float(curl_var / total_var) if total_var > 0 else 0.0,
|
|
195
|
+
"categories": categories,
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
|
|
199
|
+
def simulate_llm_arena(
|
|
200
|
+
seed: int = 42,
|
|
201
|
+
) -> dict[str, Any]:
|
|
202
|
+
"""Simulate a realistic LLM arena scenario.
|
|
203
|
+
|
|
204
|
+
Six models with names evoking real LLMs, three task categories
|
|
205
|
+
(reasoning, creative_writing, coding) with plausible strength profiles.
|
|
206
|
+
|
|
207
|
+
Args:
|
|
208
|
+
seed: Random seed for reproducibility.
|
|
209
|
+
|
|
210
|
+
Returns:
|
|
211
|
+
Same format as simulate_heterogeneous.
|
|
212
|
+
"""
|
|
213
|
+
rng = np.random.default_rng(seed)
|
|
214
|
+
models = ["AlphaLM", "BetaChat", "GammaCoder", "DeltaWrite", "EpsilonAll", "ZetaMath"]
|
|
215
|
+
categories = ["reasoning", "creative_writing", "coding"]
|
|
216
|
+
|
|
217
|
+
theta: dict[str, dict[str, float]] = {
|
|
218
|
+
"reasoning": {
|
|
219
|
+
"AlphaLM": 0.8,
|
|
220
|
+
"BetaChat": -0.2,
|
|
221
|
+
"GammaCoder": -0.4,
|
|
222
|
+
"DeltaWrite": -1.5,
|
|
223
|
+
"EpsilonAll": 0.3,
|
|
224
|
+
"ZetaMath": 2.0,
|
|
225
|
+
},
|
|
226
|
+
"creative_writing": {
|
|
227
|
+
"AlphaLM": -0.3,
|
|
228
|
+
"BetaChat": 1.2,
|
|
229
|
+
"GammaCoder": -1.5,
|
|
230
|
+
"DeltaWrite": 2.0,
|
|
231
|
+
"EpsilonAll": 0.5,
|
|
232
|
+
"ZetaMath": -1.2,
|
|
233
|
+
},
|
|
234
|
+
"coding": {
|
|
235
|
+
"AlphaLM": 0.2,
|
|
236
|
+
"BetaChat": -1.0,
|
|
237
|
+
"GammaCoder": 2.2,
|
|
238
|
+
"DeltaWrite": -1.0,
|
|
239
|
+
"EpsilonAll": 0.3,
|
|
240
|
+
"ZetaMath": 0.4,
|
|
241
|
+
},
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
category_weights = {"reasoning": 0.35, "creative_writing": 0.35, "coding": 0.30}
|
|
245
|
+
|
|
246
|
+
n_comparisons = 8000
|
|
247
|
+
comparisons: list[tuple[str, str, bool]] = []
|
|
248
|
+
cats: list[str] = []
|
|
249
|
+
probs = [category_weights[c] for c in categories]
|
|
250
|
+
for _ in range(n_comparisons):
|
|
251
|
+
cat = rng.choice(categories, p=probs)
|
|
252
|
+
i, j = rng.choice(len(models), 2, replace=False)
|
|
253
|
+
mi, mj = models[i], models[j]
|
|
254
|
+
p = expit(theta[cat][mi] - theta[cat][mj])
|
|
255
|
+
win = rng.random() < p
|
|
256
|
+
comparisons.append((mi, mj, bool(win)))
|
|
257
|
+
cats.append(str(cat))
|
|
258
|
+
|
|
259
|
+
return {
|
|
260
|
+
"comparisons": comparisons,
|
|
261
|
+
"models": models,
|
|
262
|
+
"true_strengths": theta,
|
|
263
|
+
"categories": cats,
|
|
264
|
+
"category_weights": category_weights,
|
|
265
|
+
}
|
winference/tournament.py
ADDED
|
@@ -0,0 +1,164 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Tournament graph diagnostics.
|
|
3
|
+
|
|
4
|
+
Build the directed tournament graph from pairwise outcomes and analyse its
|
|
5
|
+
strongly connected component (SCC) structure. Non-trivial SCCs (size > 1)
|
|
6
|
+
indicate clusters of models that cannot be linearly ordered — i.e. pockets
|
|
7
|
+
of non-transitivity.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
import numpy as np
|
|
11
|
+
import pandas as pd
|
|
12
|
+
from numpy.typing import NDArray
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class TournamentGraph:
|
|
16
|
+
"""Directed tournament graph built from pairwise comparison data.
|
|
17
|
+
|
|
18
|
+
Args:
|
|
19
|
+
models: Unique model identifiers.
|
|
20
|
+
|
|
21
|
+
Examples:
|
|
22
|
+
>>> tg = TournamentGraph(["A", "B", "C"])
|
|
23
|
+
>>> tg.add_result("A", "B", win=True)
|
|
24
|
+
>>> tg.add_result("B", "C", win=True)
|
|
25
|
+
>>> tg.add_result("C", "A", win=True)
|
|
26
|
+
>>> tg.scc_sizes()
|
|
27
|
+
[3]
|
|
28
|
+
"""
|
|
29
|
+
|
|
30
|
+
def __init__(self, models: list[str]) -> None:
|
|
31
|
+
self.models = list(models)
|
|
32
|
+
self._idx: dict[str, int] = {m: i for i, m in enumerate(self.models)}
|
|
33
|
+
n = len(models)
|
|
34
|
+
self.wins: NDArray[np.float64] = np.zeros((n, n), dtype=float)
|
|
35
|
+
self.counts: NDArray[np.float64] = np.zeros((n, n), dtype=float)
|
|
36
|
+
|
|
37
|
+
def add_result(self, model_a: str, model_b: str, win: bool) -> None:
|
|
38
|
+
"""Record a single comparison outcome (model_a wins if win=True)."""
|
|
39
|
+
i, j = self._idx[model_a], self._idx[model_b]
|
|
40
|
+
self.counts[i, j] += 1
|
|
41
|
+
self.counts[j, i] += 1
|
|
42
|
+
if win:
|
|
43
|
+
self.wins[i, j] += 1
|
|
44
|
+
else:
|
|
45
|
+
self.wins[j, i] += 1
|
|
46
|
+
|
|
47
|
+
def add_results_df(
|
|
48
|
+
self,
|
|
49
|
+
df: pd.DataFrame,
|
|
50
|
+
col_a: str = "model_a",
|
|
51
|
+
col_b: str = "model_b",
|
|
52
|
+
col_win: str = "a_wins",
|
|
53
|
+
) -> None:
|
|
54
|
+
"""Bulk-load from a DataFrame with columns for model_a, model_b, a_wins (bool)."""
|
|
55
|
+
for _, row in df.iterrows():
|
|
56
|
+
self.add_result(str(row[col_a]), str(row[col_b]), bool(row[col_win]))
|
|
57
|
+
|
|
58
|
+
def win_rate_matrix(self) -> NDArray[np.float64]:
|
|
59
|
+
"""Return the NxN matrix W where W[i,j] = P(i beats j)."""
|
|
60
|
+
with np.errstate(divide="ignore", invalid="ignore"):
|
|
61
|
+
W = np.where(self.counts > 0, self.wins / self.counts, 0.5)
|
|
62
|
+
np.fill_diagonal(W, 0.5)
|
|
63
|
+
return W
|
|
64
|
+
|
|
65
|
+
def _build_adjacency(self, threshold: float = 0.5) -> dict[int, list[int]]:
|
|
66
|
+
"""Directed edge from i to j if win_rate(i,j) > threshold."""
|
|
67
|
+
W = self.win_rate_matrix()
|
|
68
|
+
adj: dict[int, list[int]] = {i: [] for i in range(len(self.models))}
|
|
69
|
+
n = len(self.models)
|
|
70
|
+
for i in range(n):
|
|
71
|
+
for j in range(n):
|
|
72
|
+
if i != j and W[i, j] > threshold:
|
|
73
|
+
adj[i].append(j)
|
|
74
|
+
return adj
|
|
75
|
+
|
|
76
|
+
def strongly_connected_components(self, threshold: float = 0.5) -> list[list[str]]:
|
|
77
|
+
"""Return SCCs via Tarjan's algorithm.
|
|
78
|
+
|
|
79
|
+
Each SCC is a list of model names. SCCs of size > 1 represent
|
|
80
|
+
non-transitive clusters.
|
|
81
|
+
"""
|
|
82
|
+
adj = self._build_adjacency(threshold)
|
|
83
|
+
n = len(self.models)
|
|
84
|
+
index_counter = [0]
|
|
85
|
+
stack: list[int] = []
|
|
86
|
+
on_stack = [False] * n
|
|
87
|
+
indices = [-1] * n
|
|
88
|
+
lowlinks = [-1] * n
|
|
89
|
+
sccs: list[list[int]] = []
|
|
90
|
+
|
|
91
|
+
def strongconnect(v: int) -> None:
|
|
92
|
+
indices[v] = lowlinks[v] = index_counter[0]
|
|
93
|
+
index_counter[0] += 1
|
|
94
|
+
stack.append(v)
|
|
95
|
+
on_stack[v] = True
|
|
96
|
+
for w in adj[v]:
|
|
97
|
+
if indices[w] == -1:
|
|
98
|
+
strongconnect(w)
|
|
99
|
+
lowlinks[v] = min(lowlinks[v], lowlinks[w])
|
|
100
|
+
elif on_stack[w]:
|
|
101
|
+
lowlinks[v] = min(lowlinks[v], indices[w])
|
|
102
|
+
if lowlinks[v] == indices[v]:
|
|
103
|
+
component: list[int] = []
|
|
104
|
+
while True:
|
|
105
|
+
w = stack.pop()
|
|
106
|
+
on_stack[w] = False
|
|
107
|
+
component.append(w)
|
|
108
|
+
if w == v:
|
|
109
|
+
break
|
|
110
|
+
sccs.append(component)
|
|
111
|
+
|
|
112
|
+
for v in range(n):
|
|
113
|
+
if indices[v] == -1:
|
|
114
|
+
strongconnect(v)
|
|
115
|
+
|
|
116
|
+
return [[self.models[i] for i in scc] for scc in sccs]
|
|
117
|
+
|
|
118
|
+
def scc_sizes(self, threshold: float = 0.5) -> list[int]:
|
|
119
|
+
"""Sorted list of SCC sizes (descending)."""
|
|
120
|
+
sccs = self.strongly_connected_components(threshold)
|
|
121
|
+
return sorted([len(s) for s in sccs], reverse=True)
|
|
122
|
+
|
|
123
|
+
def nontransitivity_index(self, threshold: float = 0.5) -> float:
|
|
124
|
+
"""Fraction of models inside non-trivial SCCs (size > 1).
|
|
125
|
+
|
|
126
|
+
0.0 = perfectly transitive, 1.0 = all models in one big cycle.
|
|
127
|
+
"""
|
|
128
|
+
sccs = self.strongly_connected_components(threshold)
|
|
129
|
+
n_in_cycles = sum(len(s) for s in sccs if len(s) > 1)
|
|
130
|
+
return n_in_cycles / len(self.models)
|
|
131
|
+
|
|
132
|
+
def count_cyclic_triples(self) -> tuple[int, int]:
|
|
133
|
+
"""Count cyclic vs total triples.
|
|
134
|
+
|
|
135
|
+
Returns:
|
|
136
|
+
Tuple of (n_cyclic, n_total). Under a transitive model n_cyclic = 0.
|
|
137
|
+
"""
|
|
138
|
+
W = self.win_rate_matrix()
|
|
139
|
+
n = len(self.models)
|
|
140
|
+
n_cyclic = 0
|
|
141
|
+
n_total = 0
|
|
142
|
+
for i in range(n):
|
|
143
|
+
for j in range(i + 1, n):
|
|
144
|
+
for k in range(j + 1, n):
|
|
145
|
+
n_total += 1
|
|
146
|
+
if (W[i, j] > 0.5 and W[j, k] > 0.5 and W[k, i] > 0.5) or (
|
|
147
|
+
W[i, k] > 0.5 and W[k, j] > 0.5 and W[j, i] > 0.5
|
|
148
|
+
):
|
|
149
|
+
n_cyclic += 1
|
|
150
|
+
return n_cyclic, n_total
|
|
151
|
+
|
|
152
|
+
def summary(self, threshold: float = 0.5) -> dict[str, int | float]:
|
|
153
|
+
"""Quick diagnostic summary."""
|
|
154
|
+
sccs = self.strongly_connected_components(threshold)
|
|
155
|
+
cyc, tot = self.count_cyclic_triples()
|
|
156
|
+
return {
|
|
157
|
+
"n_models": len(self.models),
|
|
158
|
+
"n_sccs": len(sccs),
|
|
159
|
+
"largest_scc": max(len(s) for s in sccs),
|
|
160
|
+
"nontransitivity_index": self.nontransitivity_index(threshold),
|
|
161
|
+
"cyclic_triples": cyc,
|
|
162
|
+
"total_triples": tot,
|
|
163
|
+
"cycle_fraction": cyc / tot if tot > 0 else 0.0,
|
|
164
|
+
}
|