winference 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- winference/__init__.py +44 -0
- winference/bradley_terry.py +138 -0
- winference/calibration.py +155 -0
- winference/groups.py +261 -0
- winference/hodge.py +203 -0
- winference/py.typed +0 -0
- winference/simulate.py +265 -0
- winference/tournament.py +164 -0
- winference-0.1.0.dist-info/METADATA +269 -0
- winference-0.1.0.dist-info/RECORD +11 -0
- winference-0.1.0.dist-info/WHEEL +4 -0
winference/__init__.py
ADDED
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
"""
|
|
2
|
+
winference: Win rate calibration under non-transitivity.
|
|
3
|
+
|
|
4
|
+
Two approaches to calibrating win rates when preferences aren't transitive:
|
|
5
|
+
A) Hodge decomposition — separate transitive signal from cyclic residual,
|
|
6
|
+
calibrate only the transitive part.
|
|
7
|
+
B) Heterogeneous groups — find prompt categories where transitivity holds
|
|
8
|
+
within each group, calibrate per group, compose for any target distribution.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from importlib.metadata import version
|
|
12
|
+
|
|
13
|
+
from winference.bradley_terry import BradleyTerry
|
|
14
|
+
from winference.calibration import (
|
|
15
|
+
brier_score,
|
|
16
|
+
expected_calibration_error,
|
|
17
|
+
log_loss,
|
|
18
|
+
reliability_diagram,
|
|
19
|
+
)
|
|
20
|
+
from winference.groups import GroupCalibrator, GroupTest
|
|
21
|
+
from winference.hodge import HodgeDecomposition
|
|
22
|
+
from winference.simulate import (
|
|
23
|
+
simulate_heterogeneous,
|
|
24
|
+
simulate_rock_paper_scissors,
|
|
25
|
+
simulate_transitive,
|
|
26
|
+
)
|
|
27
|
+
from winference.tournament import TournamentGraph
|
|
28
|
+
|
|
29
|
+
__version__ = version("winference")
|
|
30
|
+
|
|
31
|
+
__all__ = [
|
|
32
|
+
"BradleyTerry",
|
|
33
|
+
"GroupCalibrator",
|
|
34
|
+
"GroupTest",
|
|
35
|
+
"HodgeDecomposition",
|
|
36
|
+
"TournamentGraph",
|
|
37
|
+
"brier_score",
|
|
38
|
+
"expected_calibration_error",
|
|
39
|
+
"log_loss",
|
|
40
|
+
"reliability_diagram",
|
|
41
|
+
"simulate_heterogeneous",
|
|
42
|
+
"simulate_rock_paper_scissors",
|
|
43
|
+
"simulate_transitive",
|
|
44
|
+
]
|
|
@@ -0,0 +1,138 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Bradley-Terry model fitting via maximum likelihood.
|
|
3
|
+
|
|
4
|
+
The BT model assumes P(i beats j) = sigma(theta_i - theta_j) where sigma
|
|
5
|
+
is the logistic function. Fitting recovers the strength vector theta.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from typing import Self
|
|
9
|
+
|
|
10
|
+
import numpy as np
|
|
11
|
+
from numpy.typing import NDArray
|
|
12
|
+
from scipy.optimize import minimize
|
|
13
|
+
from scipy.special import expit
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class BradleyTerry:
|
|
17
|
+
"""Maximum-likelihood Bradley-Terry model.
|
|
18
|
+
|
|
19
|
+
Args:
|
|
20
|
+
models: Unique model identifiers.
|
|
21
|
+
|
|
22
|
+
Examples:
|
|
23
|
+
>>> bt = BradleyTerry(["A", "B", "C"])
|
|
24
|
+
>>> bt.fit(comparisons) # list of (i, j, outcome) tuples
|
|
25
|
+
>>> bt.win_probability("A", "B")
|
|
26
|
+
0.73
|
|
27
|
+
"""
|
|
28
|
+
|
|
29
|
+
def __init__(self, models: list[str]) -> None:
|
|
30
|
+
self.models = list(models)
|
|
31
|
+
self._idx: dict[str, int] = {m: i for i, m in enumerate(self.models)}
|
|
32
|
+
self.n = len(models)
|
|
33
|
+
self.theta: NDArray[np.float64] | None = None
|
|
34
|
+
self.loglik: float = 0.0
|
|
35
|
+
self._fitted = False
|
|
36
|
+
|
|
37
|
+
def fit(
|
|
38
|
+
self,
|
|
39
|
+
comparisons: list[tuple[str, str, bool]],
|
|
40
|
+
reg: float = 1e-4,
|
|
41
|
+
) -> Self:
|
|
42
|
+
"""Fit the model from a list of (model_a, model_b, a_wins) triples.
|
|
43
|
+
|
|
44
|
+
Args:
|
|
45
|
+
comparisons: Each element is (model_a, model_b, a_wins).
|
|
46
|
+
reg: L2 regularisation strength (prevents divergence for unbeaten models).
|
|
47
|
+
|
|
48
|
+
Returns:
|
|
49
|
+
Self for method chaining.
|
|
50
|
+
"""
|
|
51
|
+
data = np.array([(self._idx[a], self._idx[b], float(w)) for a, b, w in comparisons])
|
|
52
|
+
idx_a = data[:, 0].astype(int)
|
|
53
|
+
idx_b = data[:, 1].astype(int)
|
|
54
|
+
outcomes = data[:, 2]
|
|
55
|
+
|
|
56
|
+
def neg_loglik(theta: NDArray[np.float64]) -> float:
|
|
57
|
+
logits = theta[idx_a] - theta[idx_b]
|
|
58
|
+
ll = np.sum(outcomes * logits - np.logaddexp(0, logits))
|
|
59
|
+
return float(-ll + 0.5 * reg * np.sum(theta**2))
|
|
60
|
+
|
|
61
|
+
def grad(theta: NDArray[np.float64]) -> NDArray[np.float64]:
|
|
62
|
+
logits = theta[idx_a] - theta[idx_b]
|
|
63
|
+
probs = expit(logits)
|
|
64
|
+
residuals = outcomes - probs
|
|
65
|
+
g = np.zeros(self.n)
|
|
66
|
+
np.add.at(g, idx_a, residuals)
|
|
67
|
+
np.add.at(g, idx_b, -residuals)
|
|
68
|
+
return -(g - reg * theta)
|
|
69
|
+
|
|
70
|
+
x0 = np.zeros(self.n)
|
|
71
|
+
result = minimize(neg_loglik, x0, jac=grad, method="L-BFGS-B")
|
|
72
|
+
self.theta = result.x - result.x.mean()
|
|
73
|
+
self.loglik = -result.fun
|
|
74
|
+
self._fitted = True
|
|
75
|
+
return self
|
|
76
|
+
|
|
77
|
+
def win_probability(self, model_a: str, model_b: str) -> float:
|
|
78
|
+
"""Predicted P(model_a beats model_b)."""
|
|
79
|
+
if not self._fitted or self.theta is None:
|
|
80
|
+
msg = "Call .fit() first"
|
|
81
|
+
raise RuntimeError(msg)
|
|
82
|
+
return float(expit(self.theta[self._idx[model_a]] - self.theta[self._idx[model_b]]))
|
|
83
|
+
|
|
84
|
+
def win_probability_matrix(self) -> NDArray[np.float64]:
|
|
85
|
+
"""NxN matrix of predicted win probabilities."""
|
|
86
|
+
if not self._fitted or self.theta is None:
|
|
87
|
+
msg = "Call .fit() first"
|
|
88
|
+
raise RuntimeError(msg)
|
|
89
|
+
diff = self.theta[:, None] - self.theta[None, :]
|
|
90
|
+
return expit(diff)
|
|
91
|
+
|
|
92
|
+
def predicted_win_rates(
|
|
93
|
+
self,
|
|
94
|
+
comparisons: list[tuple[str, str, bool]],
|
|
95
|
+
) -> NDArray[np.float64]:
|
|
96
|
+
"""Return predicted P(a wins) for each comparison triple."""
|
|
97
|
+
if not self._fitted:
|
|
98
|
+
msg = "Call .fit() first"
|
|
99
|
+
raise RuntimeError(msg)
|
|
100
|
+
preds = np.array([self.win_probability(a, b) for a, b, _ in comparisons])
|
|
101
|
+
return preds
|
|
102
|
+
|
|
103
|
+
def strengths(self) -> dict[str, float]:
|
|
104
|
+
"""Return {model: theta} dictionary."""
|
|
105
|
+
if not self._fitted or self.theta is None:
|
|
106
|
+
msg = "Call .fit() first"
|
|
107
|
+
raise RuntimeError(msg)
|
|
108
|
+
return {m: float(self.theta[i]) for i, m in enumerate(self.models)}
|
|
109
|
+
|
|
110
|
+
def rank(self) -> list[str]:
|
|
111
|
+
"""Models sorted by decreasing strength."""
|
|
112
|
+
s = self.strengths()
|
|
113
|
+
return sorted(s, key=lambda m: s[m], reverse=True)
|
|
114
|
+
|
|
115
|
+
def __repr__(self) -> str:
|
|
116
|
+
status = "fitted" if self._fitted else "unfitted"
|
|
117
|
+
return f"BradleyTerry(n_models={self.n}, {status})"
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def fit_bt_from_matrix(
|
|
121
|
+
win_matrix: NDArray[np.float64],
|
|
122
|
+
count_matrix: NDArray[np.float64],
|
|
123
|
+
models: list[str],
|
|
124
|
+
reg: float = 1e-4,
|
|
125
|
+
) -> BradleyTerry:
|
|
126
|
+
"""Convenience: fit BT from win/count matrices rather than triples."""
|
|
127
|
+
comparisons: list[tuple[str, str, bool]] = []
|
|
128
|
+
n = len(models)
|
|
129
|
+
for i in range(n):
|
|
130
|
+
for j in range(i + 1, n):
|
|
131
|
+
if count_matrix[i, j] > 0:
|
|
132
|
+
w = round(win_matrix[i, j])
|
|
133
|
+
losses = round(count_matrix[i, j] - win_matrix[i, j])
|
|
134
|
+
comparisons.extend([(models[i], models[j], True)] * int(w))
|
|
135
|
+
comparisons.extend([(models[i], models[j], False)] * int(losses))
|
|
136
|
+
bt = BradleyTerry(models)
|
|
137
|
+
bt.fit(comparisons, reg=reg)
|
|
138
|
+
return bt
|
|
@@ -0,0 +1,155 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Calibration metrics for pairwise win rate predictions.
|
|
3
|
+
|
|
4
|
+
When your model says "A beats B with probability 0.65", does A actually
|
|
5
|
+
win 65% of the time? These tools check.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from itertools import pairwise
|
|
9
|
+
from typing import Any
|
|
10
|
+
|
|
11
|
+
import numpy as np
|
|
12
|
+
from numpy.typing import NDArray
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def expected_calibration_error(
|
|
16
|
+
predicted: NDArray[np.float64],
|
|
17
|
+
observed: NDArray[np.float64],
|
|
18
|
+
n_bins: int = 10,
|
|
19
|
+
) -> float:
|
|
20
|
+
"""Expected Calibration Error (ECE).
|
|
21
|
+
|
|
22
|
+
Args:
|
|
23
|
+
predicted: Array of predicted win probabilities, shape (n,).
|
|
24
|
+
observed: Array of binary outcomes (0 or 1), shape (n,).
|
|
25
|
+
n_bins: Number of bins for grouping predictions.
|
|
26
|
+
|
|
27
|
+
Returns:
|
|
28
|
+
Weighted average |predicted - observed| across bins.
|
|
29
|
+
"""
|
|
30
|
+
predicted = np.asarray(predicted, dtype=float)
|
|
31
|
+
observed = np.asarray(observed, dtype=float)
|
|
32
|
+
bin_edges = np.linspace(0, 1, n_bins + 1)
|
|
33
|
+
ece = 0.0
|
|
34
|
+
for lo, hi in pairwise(bin_edges):
|
|
35
|
+
mask = (predicted >= lo) & (predicted < hi)
|
|
36
|
+
if mask.sum() == 0:
|
|
37
|
+
continue
|
|
38
|
+
avg_pred = predicted[mask].mean()
|
|
39
|
+
avg_obs = observed[mask].mean()
|
|
40
|
+
weight = mask.sum() / len(predicted)
|
|
41
|
+
ece += weight * abs(avg_pred - avg_obs)
|
|
42
|
+
return float(ece)
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def brier_score(
|
|
46
|
+
predicted: NDArray[np.float64],
|
|
47
|
+
observed: NDArray[np.float64],
|
|
48
|
+
) -> float:
|
|
49
|
+
"""Brier score: mean squared error of probability predictions.
|
|
50
|
+
|
|
51
|
+
Lower is better. Perfect calibration → 0. Random guessing → 0.25.
|
|
52
|
+
"""
|
|
53
|
+
predicted = np.asarray(predicted, dtype=float)
|
|
54
|
+
observed = np.asarray(observed, dtype=float)
|
|
55
|
+
return float(np.mean((predicted - observed) ** 2))
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def log_loss(
|
|
59
|
+
predicted: NDArray[np.float64],
|
|
60
|
+
observed: NDArray[np.float64],
|
|
61
|
+
) -> float:
|
|
62
|
+
"""Binary cross-entropy loss. Lower is better."""
|
|
63
|
+
predicted = np.clip(np.asarray(predicted, dtype=float), 1e-10, 1 - 1e-10)
|
|
64
|
+
observed = np.asarray(observed, dtype=float)
|
|
65
|
+
return float(-np.mean(observed * np.log(predicted) + (1 - observed) * np.log(1 - predicted)))
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def reliability_diagram(
|
|
69
|
+
predicted: NDArray[np.float64],
|
|
70
|
+
observed: NDArray[np.float64],
|
|
71
|
+
n_bins: int = 10,
|
|
72
|
+
ax: Any = None,
|
|
73
|
+
label: str = "",
|
|
74
|
+
color: str | None = None,
|
|
75
|
+
) -> dict[str, Any]:
|
|
76
|
+
"""Reliability diagram data and optional plot.
|
|
77
|
+
|
|
78
|
+
Args:
|
|
79
|
+
predicted: Array of predicted probabilities.
|
|
80
|
+
observed: Array of binary outcomes.
|
|
81
|
+
n_bins: Number of bins.
|
|
82
|
+
ax: Matplotlib Axes. If provided, plot on it.
|
|
83
|
+
label: Label for the legend.
|
|
84
|
+
color: Color for the plot line.
|
|
85
|
+
|
|
86
|
+
Returns:
|
|
87
|
+
Dict with 'bin_midpoints', 'bin_accuracy', 'bin_counts', 'ece'.
|
|
88
|
+
"""
|
|
89
|
+
predicted = np.asarray(predicted, dtype=float)
|
|
90
|
+
observed = np.asarray(observed, dtype=float)
|
|
91
|
+
bin_edges = np.linspace(0, 1, n_bins + 1)
|
|
92
|
+
midpoints: list[float] = []
|
|
93
|
+
accuracies: list[float] = []
|
|
94
|
+
counts: list[int] = []
|
|
95
|
+
|
|
96
|
+
for lo, hi in pairwise(bin_edges):
|
|
97
|
+
mask = (predicted >= lo) & (predicted < hi)
|
|
98
|
+
c = int(mask.sum())
|
|
99
|
+
counts.append(c)
|
|
100
|
+
midpoints.append((lo + hi) / 2)
|
|
101
|
+
accuracies.append(float(observed[mask].mean()) if c > 0 else float("nan"))
|
|
102
|
+
|
|
103
|
+
result: dict[str, Any] = {
|
|
104
|
+
"bin_midpoints": np.array(midpoints),
|
|
105
|
+
"bin_accuracy": np.array(accuracies),
|
|
106
|
+
"bin_counts": np.array(counts),
|
|
107
|
+
"ece": expected_calibration_error(predicted, observed, n_bins),
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
if ax is not None:
|
|
111
|
+
_plot_reliability(ax, result, label, color)
|
|
112
|
+
|
|
113
|
+
return result
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def _plot_reliability(ax: Any, result: dict[str, Any], label: str, color: str | None) -> None:
|
|
117
|
+
mid = result["bin_midpoints"]
|
|
118
|
+
acc = result["bin_accuracy"]
|
|
119
|
+
mask = ~np.isnan(acc)
|
|
120
|
+
lbl = f"{label} (ECE={result['ece']:.3f})" if label else f"ECE={result['ece']:.3f}"
|
|
121
|
+
ax.plot(mid[mask], acc[mask], "o-", label=lbl, color=color, markersize=5)
|
|
122
|
+
existing_labels = [line.get_label() for line in ax.get_lines()]
|
|
123
|
+
if "Perfect" not in " ".join(existing_labels):
|
|
124
|
+
ax.plot([0, 1], [0, 1], "k--", alpha=0.4, label="Perfect")
|
|
125
|
+
ax.set_xlabel("Predicted win probability")
|
|
126
|
+
ax.set_ylabel("Observed win rate")
|
|
127
|
+
ax.set_xlim(0, 1)
|
|
128
|
+
ax.set_ylim(0, 1)
|
|
129
|
+
ax.legend(fontsize=8)
|
|
130
|
+
ax.set_aspect("equal")
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
def compare_calibration(
|
|
134
|
+
methods: dict[str, NDArray[np.float64]],
|
|
135
|
+
observed: NDArray[np.float64],
|
|
136
|
+
n_bins: int = 10,
|
|
137
|
+
) -> dict[str, dict[str, float]]:
|
|
138
|
+
"""Compare calibration across multiple prediction methods.
|
|
139
|
+
|
|
140
|
+
Args:
|
|
141
|
+
methods: Dict mapping method name to predicted probabilities.
|
|
142
|
+
observed: Binary outcomes.
|
|
143
|
+
n_bins: Number of bins for ECE calculation.
|
|
144
|
+
|
|
145
|
+
Returns:
|
|
146
|
+
Dict of {name: {ece, brier, logloss}}.
|
|
147
|
+
"""
|
|
148
|
+
results: dict[str, dict[str, float]] = {}
|
|
149
|
+
for name, pred in methods.items():
|
|
150
|
+
results[name] = {
|
|
151
|
+
"ece": expected_calibration_error(pred, observed, n_bins),
|
|
152
|
+
"brier": brier_score(pred, observed),
|
|
153
|
+
"logloss": log_loss(pred, observed),
|
|
154
|
+
}
|
|
155
|
+
return results
|
winference/groups.py
ADDED
|
@@ -0,0 +1,261 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Heterogeneous group testing and per-group calibration.
|
|
3
|
+
|
|
4
|
+
Tests whether model strengths are constant across prompt categories
|
|
5
|
+
(homogeneous BT) or differ by category (heterogeneous BT). If
|
|
6
|
+
heterogeneity is warranted, fits per-category BT models and provides
|
|
7
|
+
composable win rate predictions for any target distribution over categories.
|
|
8
|
+
|
|
9
|
+
The formal test is a likelihood-ratio test:
|
|
10
|
+
H0: theta_{i,k} = theta_i for all i, k (homogeneous)
|
|
11
|
+
H1: theta_{i,k} free (heterogeneous)
|
|
12
|
+
|
|
13
|
+
Lambda = -2(l0 - l1) ~ chi2 with (K-1)(N-1) degrees of freedom.
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
from collections import Counter
|
|
17
|
+
from typing import Self
|
|
18
|
+
|
|
19
|
+
import numpy as np
|
|
20
|
+
from numpy.typing import NDArray
|
|
21
|
+
from scipy import stats
|
|
22
|
+
|
|
23
|
+
from winference.bradley_terry import BradleyTerry
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
class GroupTest:
|
|
27
|
+
"""Likelihood-ratio test for heterogeneity across prompt groups.
|
|
28
|
+
|
|
29
|
+
Args:
|
|
30
|
+
models: Model identifiers.
|
|
31
|
+
groups: Unique group/category labels.
|
|
32
|
+
|
|
33
|
+
Examples:
|
|
34
|
+
>>> gt = GroupTest(models=["A","B","C"], groups=["math","creative"])
|
|
35
|
+
>>> gt.fit(comparisons, group_labels)
|
|
36
|
+
>>> print(gt.test_result())
|
|
37
|
+
{'statistic': 14.2, 'df': 2, 'p_value': 0.0008}
|
|
38
|
+
"""
|
|
39
|
+
|
|
40
|
+
def __init__(self, models: list[str], groups: list[str]) -> None:
|
|
41
|
+
self.models = list(models)
|
|
42
|
+
self.groups = list(groups)
|
|
43
|
+
self.n_models = len(models)
|
|
44
|
+
self.n_groups = len(groups)
|
|
45
|
+
self.bt_null: BradleyTerry | None = None
|
|
46
|
+
self.bt_per_group: dict[str, BradleyTerry] = {}
|
|
47
|
+
self._comparisons: list[tuple[str, str, bool]] = []
|
|
48
|
+
self._group_labels: list[str] = []
|
|
49
|
+
self._group_comparisons: dict[str, list[tuple[str, str, bool]]] = {}
|
|
50
|
+
|
|
51
|
+
def fit(
|
|
52
|
+
self,
|
|
53
|
+
comparisons: list[tuple[str, str, bool]],
|
|
54
|
+
group_labels: list[str],
|
|
55
|
+
reg: float = 1e-4,
|
|
56
|
+
) -> Self:
|
|
57
|
+
"""Fit null (pooled) and alternative (per-group) BT models.
|
|
58
|
+
|
|
59
|
+
Args:
|
|
60
|
+
comparisons: List of (model_a, model_b, a_wins) tuples.
|
|
61
|
+
group_labels: Category label for each comparison, same length as comparisons.
|
|
62
|
+
reg: Regularisation for BT fitting.
|
|
63
|
+
|
|
64
|
+
Returns:
|
|
65
|
+
Self for method chaining.
|
|
66
|
+
|
|
67
|
+
Raises:
|
|
68
|
+
ValueError: If comparisons and group_labels have different lengths.
|
|
69
|
+
"""
|
|
70
|
+
if len(comparisons) != len(group_labels):
|
|
71
|
+
msg = "comparisons and group_labels must have same length"
|
|
72
|
+
raise ValueError(msg)
|
|
73
|
+
|
|
74
|
+
self.bt_null = BradleyTerry(self.models)
|
|
75
|
+
self.bt_null.fit(comparisons, reg=reg)
|
|
76
|
+
|
|
77
|
+
self.bt_per_group = {}
|
|
78
|
+
self._group_comparisons = {}
|
|
79
|
+
for g in self.groups:
|
|
80
|
+
g_comps = [c for c, gl in zip(comparisons, group_labels, strict=True) if gl == g]
|
|
81
|
+
if len(g_comps) < self.n_models:
|
|
82
|
+
continue
|
|
83
|
+
self._group_comparisons[g] = g_comps
|
|
84
|
+
bt_g = BradleyTerry(self.models)
|
|
85
|
+
bt_g.fit(g_comps, reg=reg)
|
|
86
|
+
self.bt_per_group[g] = bt_g
|
|
87
|
+
|
|
88
|
+
self._comparisons = comparisons
|
|
89
|
+
self._group_labels = group_labels
|
|
90
|
+
return self
|
|
91
|
+
|
|
92
|
+
def _loglik(self, bt: BradleyTerry, comparisons: list[tuple[str, str, bool]]) -> float:
|
|
93
|
+
"""Compute log-likelihood of data under a fitted BT model."""
|
|
94
|
+
ll = 0.0
|
|
95
|
+
for a, b, w in comparisons:
|
|
96
|
+
p = bt.win_probability(a, b)
|
|
97
|
+
p = np.clip(p, 1e-10, 1 - 1e-10)
|
|
98
|
+
ll += np.log(p) if w else np.log(1 - p)
|
|
99
|
+
return float(ll)
|
|
100
|
+
|
|
101
|
+
def test_result(self) -> dict[str, float | int | bool]:
|
|
102
|
+
"""Likelihood-ratio test for group heterogeneity.
|
|
103
|
+
|
|
104
|
+
Returns:
|
|
105
|
+
Dict with keys: statistic (LRT statistic Lambda), df (degrees of freedom),
|
|
106
|
+
p_value (p-value under chi2 null), reject_at_05 (bool).
|
|
107
|
+
|
|
108
|
+
Raises:
|
|
109
|
+
RuntimeError: If fit() has not been called.
|
|
110
|
+
"""
|
|
111
|
+
if self.bt_null is None:
|
|
112
|
+
msg = "Call .fit() first"
|
|
113
|
+
raise RuntimeError(msg)
|
|
114
|
+
|
|
115
|
+
ll_null = self._loglik(self.bt_null, self._comparisons)
|
|
116
|
+
|
|
117
|
+
ll_alt = 0.0
|
|
118
|
+
for g, bt_g in self.bt_per_group.items():
|
|
119
|
+
ll_alt += self._loglik(bt_g, self._group_comparisons[g])
|
|
120
|
+
for g in self.groups:
|
|
121
|
+
if g not in self.bt_per_group:
|
|
122
|
+
g_comps = [
|
|
123
|
+
c
|
|
124
|
+
for c, gl in zip(self._comparisons, self._group_labels, strict=True)
|
|
125
|
+
if gl == g
|
|
126
|
+
]
|
|
127
|
+
ll_alt += self._loglik(self.bt_null, g_comps)
|
|
128
|
+
|
|
129
|
+
lrt = -2 * (ll_null - ll_alt)
|
|
130
|
+
K = len(self.bt_per_group)
|
|
131
|
+
N = self.n_models
|
|
132
|
+
df = (K - 1) * (N - 1)
|
|
133
|
+
|
|
134
|
+
if df <= 0:
|
|
135
|
+
return {"statistic": 0.0, "df": 0, "p_value": 1.0, "reject_at_05": False}
|
|
136
|
+
|
|
137
|
+
p_value = float(1 - stats.chi2.cdf(lrt, df))
|
|
138
|
+
return {
|
|
139
|
+
"statistic": float(lrt),
|
|
140
|
+
"df": df,
|
|
141
|
+
"p_value": p_value,
|
|
142
|
+
"reject_at_05": p_value < 0.05,
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
def per_group_strengths(self) -> dict[str, dict[str, float]]:
|
|
146
|
+
"""Return {group: {model: theta}} for each fitted group."""
|
|
147
|
+
return {g: bt.strengths() for g, bt in self.bt_per_group.items()}
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
class GroupCalibrator:
|
|
151
|
+
"""Composable win rate calibration using per-group BT models.
|
|
152
|
+
|
|
153
|
+
After fitting per-group BT models, compute win rates for *any*
|
|
154
|
+
target distribution over groups:
|
|
155
|
+
|
|
156
|
+
P(i > j | pi*) = sum_k pi*_k * sigmoid(theta_{i,k} - theta_{j,k})
|
|
157
|
+
|
|
158
|
+
This is the key advantage: calibration that transfers under
|
|
159
|
+
distribution shift.
|
|
160
|
+
|
|
161
|
+
Args:
|
|
162
|
+
group_test: A fitted GroupTest object.
|
|
163
|
+
|
|
164
|
+
Raises:
|
|
165
|
+
RuntimeError: If the GroupTest has not been fitted.
|
|
166
|
+
"""
|
|
167
|
+
|
|
168
|
+
def __init__(self, group_test: GroupTest) -> None:
|
|
169
|
+
if group_test.bt_null is None:
|
|
170
|
+
msg = "GroupTest must be fitted"
|
|
171
|
+
raise RuntimeError(msg)
|
|
172
|
+
self.gt = group_test
|
|
173
|
+
|
|
174
|
+
def win_probability(
|
|
175
|
+
self,
|
|
176
|
+
model_a: str,
|
|
177
|
+
model_b: str,
|
|
178
|
+
target_distribution: dict[str, float] | None = None,
|
|
179
|
+
) -> float:
|
|
180
|
+
"""Composite win probability under a target group distribution.
|
|
181
|
+
|
|
182
|
+
Args:
|
|
183
|
+
model_a: First model name.
|
|
184
|
+
model_b: Second model name.
|
|
185
|
+
target_distribution: Dict mapping group to weight. Weights are
|
|
186
|
+
normalised internally. If None, uses the empirical distribution
|
|
187
|
+
from the training data.
|
|
188
|
+
|
|
189
|
+
Returns:
|
|
190
|
+
Composite win probability P(model_a beats model_b).
|
|
191
|
+
"""
|
|
192
|
+
if target_distribution is None:
|
|
193
|
+
target_distribution = self._empirical_distribution()
|
|
194
|
+
|
|
195
|
+
total = sum(target_distribution.values())
|
|
196
|
+
pi = {g: w / total for g, w in target_distribution.items()}
|
|
197
|
+
|
|
198
|
+
p = 0.0
|
|
199
|
+
for g, weight in pi.items():
|
|
200
|
+
if g in self.gt.bt_per_group:
|
|
201
|
+
p += weight * self.gt.bt_per_group[g].win_probability(model_a, model_b)
|
|
202
|
+
elif self.gt.bt_null is not None:
|
|
203
|
+
p += weight * self.gt.bt_null.win_probability(model_a, model_b)
|
|
204
|
+
return p
|
|
205
|
+
|
|
206
|
+
def win_probability_matrix(
|
|
207
|
+
self,
|
|
208
|
+
target_distribution: dict[str, float] | None = None,
|
|
209
|
+
) -> NDArray[np.float64]:
|
|
210
|
+
"""NxN composite win probability matrix."""
|
|
211
|
+
models = self.gt.models
|
|
212
|
+
n = len(models)
|
|
213
|
+
W = np.zeros((n, n))
|
|
214
|
+
for i in range(n):
|
|
215
|
+
for j in range(n):
|
|
216
|
+
if i != j:
|
|
217
|
+
W[i, j] = self.win_probability(models[i], models[j], target_distribution)
|
|
218
|
+
else:
|
|
219
|
+
W[i, j] = 0.5
|
|
220
|
+
return W
|
|
221
|
+
|
|
222
|
+
def _empirical_distribution(self) -> dict[str, float]:
|
|
223
|
+
"""Group weights proportional to number of comparisons."""
|
|
224
|
+
counts = Counter(self.gt._group_labels)
|
|
225
|
+
return dict(counts)
|
|
226
|
+
|
|
227
|
+
def sensitivity_analysis(
|
|
228
|
+
self,
|
|
229
|
+
model_a: str,
|
|
230
|
+
model_b: str,
|
|
231
|
+
n_draws: int = 1000,
|
|
232
|
+
concentration: float = 1.0,
|
|
233
|
+
) -> dict[str, float]:
|
|
234
|
+
"""How much does P(a > b) vary as the target distribution changes?
|
|
235
|
+
|
|
236
|
+
Draws random target distributions from Dirichlet(concentration)
|
|
237
|
+
and reports the range and std of the composite win probability.
|
|
238
|
+
"""
|
|
239
|
+
groups = list(self.gt.bt_per_group.keys())
|
|
240
|
+
K = len(groups)
|
|
241
|
+
if K < 2:
|
|
242
|
+
p = self.win_probability(model_a, model_b)
|
|
243
|
+
return {"mean": p, "std": 0.0, "min": p, "max": p}
|
|
244
|
+
|
|
245
|
+
alpha = np.full(K, concentration)
|
|
246
|
+
draws = np.random.dirichlet(alpha, size=n_draws)
|
|
247
|
+
|
|
248
|
+
probs = []
|
|
249
|
+
for draw in draws:
|
|
250
|
+
pi = dict(zip(groups, draw, strict=True))
|
|
251
|
+
probs.append(self.win_probability(model_a, model_b, pi))
|
|
252
|
+
probs_arr = np.array(probs)
|
|
253
|
+
|
|
254
|
+
return {
|
|
255
|
+
"mean": float(probs_arr.mean()),
|
|
256
|
+
"std": float(probs_arr.std()),
|
|
257
|
+
"min": float(probs_arr.min()),
|
|
258
|
+
"max": float(probs_arr.max()),
|
|
259
|
+
"q05": float(np.quantile(probs_arr, 0.05)),
|
|
260
|
+
"q95": float(np.quantile(probs_arr, 0.95)),
|
|
261
|
+
}
|