pace-eval 0.2.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cme/__init__.py +11 -0
- cme/__main__.py +5 -0
- cme/analysis.py +223 -0
- cme/audit.py +145 -0
- cme/baseline.py +162 -0
- cme/cli.py +279 -0
- cme/client.py +165 -0
- cme/dynamics.py +212 -0
- cme/examples/crewai_integration.py +16 -0
- cme/examples/demo_pool.jsonl +360 -0
- cme/examples/demo_run.jsonl +480 -0
- cme/examples/langgraph_integration.py +27 -0
- cme/examples/make_demo.py +37 -0
- cme/examples/openai_quickstart.py +18 -0
- cme/experiment.py +555 -0
- cme/frontier.py +143 -0
- cme/hidden_profile.py +135 -0
- cme/proxy.py +106 -0
- cme/rebuttal.py +76 -0
- cme/report.py +197 -0
- cme/rollout_card.py +137 -0
- cme/tasks.py +266 -0
- cme/verified_debate.py +127 -0
- cme/verifier.py +92 -0
- mae_eval/__init__.py +11 -0
- mae_eval/__main__.py +5 -0
- mae_eval/auditor.py +112 -0
- mae_eval/classifier.py +62 -0
- mae_eval/cli.py +210 -0
- mae_eval/tracker.py +162 -0
- mae_eval/vector.py +91 -0
- pace/__init__.py +11 -0
- pace/__main__.py +5 -0
- pace/analysis.py +129 -0
- pace/audit.py +145 -0
- pace/baseline.py +162 -0
- pace/card.py +205 -0
- pace/cli.py +898 -0
- pace/client.py +165 -0
- pace/dynamics.py +372 -0
- pace/escape_benchmarks.py +162 -0
- pace/examples/crewai_integration.py +16 -0
- pace/examples/demo_pool.jsonl +360 -0
- pace/examples/demo_run.jsonl +480 -0
- pace/examples/langgraph_integration.py +27 -0
- pace/examples/make_demo.py +37 -0
- pace/examples/openai_quickstart.py +18 -0
- pace/experiment.py +555 -0
- pace/frontier.py +114 -0
- pace/hidden_profile.py +301 -0
- pace/importers.py +219 -0
- pace/inflate.py +213 -0
- pace/influence.py +231 -0
- pace/meta_engine.py +246 -0
- pace/middleware.py +230 -0
- pace/open_ended.py +114 -0
- pace/plotting.py +132 -0
- pace/pool_sizer.py +154 -0
- pace/protocol_v3.py +414 -0
- pace/proxy.py +106 -0
- pace/rebuttal.py +219 -0
- pace/report.py +197 -0
- pace/rollout_card.py +137 -0
- pace/schema.py +74 -0
- pace/simulator.py +199 -0
- pace/studio.html +1040 -0
- pace/tasks.py +266 -0
- pace/verified_debate.py +127 -0
- pace/verifier.py +92 -0
- pace/workbench.html +26 -0
- pace/workbench.py +119 -0
- pace_eval-0.2.2.dist-info/METADATA +205 -0
- pace_eval-0.2.2.dist-info/RECORD +83 -0
- pace_eval-0.2.2.dist-info/WHEEL +5 -0
- pace_eval-0.2.2.dist-info/entry_points.txt +4 -0
- pace_eval-0.2.2.dist-info/licenses/LICENSE +202 -0
- pace_eval-0.2.2.dist-info/top_level.txt +4 -0
- pace_lab/__init__.py +9 -0
- pace_lab/clients.py +568 -0
- pace_lab/concurrency_test.py +127 -0
- pace_lab/data.py +445 -0
- pace_lab/graders.py +626 -0
- pace_lab/strategies.py +345 -0
pace/inflate.py
ADDED
|
@@ -0,0 +1,213 @@
|
|
|
1
|
+
"""Beta-binomial sampling inflation (`pace inflate`): how much does a K-sample majority vote beat one sample, from a pool of n samples per item?
|
|
2
|
+
|
|
3
|
+
Model: item i is answered correctly with probability p_i ~ Beta(alpha, beta); the n pooled samples of an item are then Binomial(n, p_i), so the per-item success counts
|
|
4
|
+
are beta-binomial. Fit (alpha, beta) by maximum likelihood on the pool, and predict the accuracy of a vote over K samples in closed form:
|
|
5
|
+
|
|
6
|
+
acc(K) = sum_s BetaBinomial(s; K, alpha, beta) * [ 1(s > K/2) + 0.5 * 1(s = K/2) ]
|
|
7
|
+
|
|
8
|
+
This is the *binary* majority (correct versus not correct). A plurality vote over several wrong options that split their votes does at least as well, so for
|
|
9
|
+
multiple-choice tasks acc(K) is a conservative (lower) prediction of plurality-vote accuracy. Pure standard library; no numpy, no scipy.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
import math
|
|
15
|
+
from collections import defaultdict
|
|
16
|
+
from typing import Iterable, Sequence
|
|
17
|
+
|
|
18
|
+
LOG_LO, LOG_HI = math.log(1e-3), math.log(1e4)
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def _betaln(a: float, b: float) -> float:
|
|
22
|
+
return math.lgamma(a) + math.lgamma(b) - math.lgamma(a + b)
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def _log_choose(n: int, k: int) -> float:
|
|
26
|
+
return math.lgamma(n + 1) - math.lgamma(k + 1) - math.lgamma(n - k + 1)
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def beta_binomial_logpmf(s: int, n: int, a: float, b: float) -> float:
|
|
30
|
+
return _log_choose(n, s) + _betaln(s + a, n - s + b) - _betaln(a, b)
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def log_likelihood(successes: Sequence[int], n: int, a: float, b: float) -> float:
|
|
34
|
+
counts = defaultdict(int)
|
|
35
|
+
for s in successes:
|
|
36
|
+
counts[int(s)] += 1
|
|
37
|
+
return sum(c * beta_binomial_logpmf(s, n, a, b) for s, c in counts.items())
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def fit_beta_binomial(successes: Sequence[int], n: int) -> tuple[float, float]:
|
|
41
|
+
"""Maximum-likelihood (alpha, beta) from per-item success counts out of n samples (shrinking-grid search on the log scale, clamped to [1e-3, 1e4])."""
|
|
42
|
+
if not len(successes):
|
|
43
|
+
raise ValueError("no items")
|
|
44
|
+
if any(not 0 <= int(s) <= n for s in successes):
|
|
45
|
+
raise ValueError("success counts must lie in 0..n")
|
|
46
|
+
mean = min(max(sum(successes) / (n * len(successes)), 1e-3), 1 - 1e-3)
|
|
47
|
+
# start: equal-ish concentration on the log scale, then coordinate refinement
|
|
48
|
+
la, lb = math.log(mean * 2.0), math.log((1 - mean) * 2.0)
|
|
49
|
+
best = log_likelihood(successes, n, math.exp(la), math.exp(lb))
|
|
50
|
+
step = 2.0
|
|
51
|
+
for _ in range(200):
|
|
52
|
+
improved = False
|
|
53
|
+
for da in (-step, -step / 2, 0.0, step / 2, step):
|
|
54
|
+
for db in (-step, -step / 2, 0.0, step / 2, step):
|
|
55
|
+
ca, cb = min(max(la + da, LOG_LO), LOG_HI), min(max(lb + db, LOG_LO), LOG_HI)
|
|
56
|
+
ll = log_likelihood(successes, n, math.exp(ca), math.exp(cb))
|
|
57
|
+
if ll > best + 1e-12:
|
|
58
|
+
best, la, lb, improved = ll, ca, cb, True
|
|
59
|
+
if not improved:
|
|
60
|
+
step *= 0.5
|
|
61
|
+
if step < 1e-6:
|
|
62
|
+
break
|
|
63
|
+
return math.exp(la), math.exp(lb)
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def vote_accuracy(a: float, b: float, K: int, tie: float = 0.5) -> float:
|
|
67
|
+
"""Closed-form accuracy of a binary-majority vote over K independent samples under Beta(a, b) item difficulty (ties count `tie`)."""
|
|
68
|
+
if K < 1:
|
|
69
|
+
raise ValueError("K must be at least 1")
|
|
70
|
+
total = 0.0
|
|
71
|
+
for s in range(K + 1):
|
|
72
|
+
if 2 * s > K:
|
|
73
|
+
w = 1.0
|
|
74
|
+
elif 2 * s == K:
|
|
75
|
+
w = tie
|
|
76
|
+
else:
|
|
77
|
+
continue
|
|
78
|
+
total += w * math.exp(beta_binomial_logpmf(s, K, a, b))
|
|
79
|
+
return min(1.0, total)
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def empirical_vote_accuracy(successes: Sequence[int], n: int, K: int, tie: float = 0.5) -> float:
|
|
83
|
+
"""Exact expected binary-majority accuracy of a vote over K of the n pooled samples (hypergeometric over the choice of the K samples), averaged over items."""
|
|
84
|
+
if K > n:
|
|
85
|
+
raise ValueError("K cannot exceed the pool size n")
|
|
86
|
+
out = 0.0
|
|
87
|
+
denom = math.comb(n, K)
|
|
88
|
+
for s in successes:
|
|
89
|
+
acc = 0.0
|
|
90
|
+
for j in range(max(0, K - (n - s)), min(K, s) + 1):
|
|
91
|
+
w = 1.0 if 2 * j > K else (tie if 2 * j == K else 0.0)
|
|
92
|
+
if w:
|
|
93
|
+
acc += w * math.comb(s, j) * math.comb(n - s, K - j) / denom
|
|
94
|
+
out += acc
|
|
95
|
+
return out / len(successes)
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def successes_from_rows(rows: Iterable[dict]) -> tuple[list[int], int]:
|
|
99
|
+
"""Per-item success counts from rows with `item` and `correct` (one row per sample). Items with fewer samples than the most common count are dropped."""
|
|
100
|
+
by = defaultdict(list)
|
|
101
|
+
for r in rows:
|
|
102
|
+
by[str(r["item"])].append(1 if r["correct"] in (True, 1, 1.0, "1", "true", "True") else 0)
|
|
103
|
+
sizes = defaultdict(int)
|
|
104
|
+
for v in by.values():
|
|
105
|
+
sizes[len(v)] += 1
|
|
106
|
+
n = max(sizes, key=lambda k: (sizes[k], k))
|
|
107
|
+
return [sum(v) for v in by.values() if len(v) == n], n
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def inflate_report(successes: Sequence[int], n: int, ks: Sequence[int], wrong_options: float | None = None) -> dict:
|
|
111
|
+
a, b = fit_beta_binomial(successes, n)
|
|
112
|
+
p1 = a / (a + b)
|
|
113
|
+
rows = []
|
|
114
|
+
for K in ks:
|
|
115
|
+
row = {"K": K, "predicted": vote_accuracy(a, b, K)}
|
|
116
|
+
if K <= n:
|
|
117
|
+
row["pool_observed"] = empirical_vote_accuracy(successes, n, K)
|
|
118
|
+
if wrong_options is not None:
|
|
119
|
+
row["predicted_plurality"] = plurality_accuracy(a, b, K, wrong_options)
|
|
120
|
+
row["over_single_sample"] = row["predicted"] - p1
|
|
121
|
+
rows.append(row)
|
|
122
|
+
return {
|
|
123
|
+
"n_items": len(successes),
|
|
124
|
+
"pool_size": n,
|
|
125
|
+
"alpha": a,
|
|
126
|
+
"beta": b,
|
|
127
|
+
"concentration": a + b,
|
|
128
|
+
"mean_single_sample_accuracy": p1,
|
|
129
|
+
"single_sample_accuracy_observed": sum(successes) / (n * len(successes)),
|
|
130
|
+
"wrong_options": wrong_options,
|
|
131
|
+
"inflation": rows,
|
|
132
|
+
"model": "binary majority under Beta(alpha, beta) item difficulty; conservative for plurality voting over split wrong answers",
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
# ---------------------------------------------------------------------------------------------------------------------------------------------
|
|
137
|
+
# Plurality extension: wrong answers are spread over W options (uniformly), so a plurality vote can win with far fewer than K/2 correct samples.
|
|
138
|
+
# Given the number s of correct samples among K, the K-s wrong samples are Multinomial(K-s; uniform over W wrong options), independent of p_i.
|
|
139
|
+
# The vote is correct if s beats the largest wrong count; a tie with j wrong options at the maximum is credited 1/(1+j) (uniform tie-break).
|
|
140
|
+
# W = 1 recovers the binary majority above.
|
|
141
|
+
# ---------------------------------------------------------------------------------------------------------------------------------------------
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
def _poly_mul(x: list[float], y: list[float], deg: int) -> list[float]:
|
|
145
|
+
out = [0.0] * (deg + 1)
|
|
146
|
+
for i, xi in enumerate(x):
|
|
147
|
+
if xi == 0.0:
|
|
148
|
+
continue
|
|
149
|
+
for j, yj in enumerate(y):
|
|
150
|
+
if i + j > deg:
|
|
151
|
+
break
|
|
152
|
+
out[i + j] += xi * yj
|
|
153
|
+
return out
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
def _wrong_wins_prob(s: int, r: int, W: int) -> float:
|
|
157
|
+
"""P(the s correct samples win a plurality against r wrong samples spread uniformly over W options), with uniform tie-break."""
|
|
158
|
+
if r == 0:
|
|
159
|
+
return 1.0 if s > 0 else 0.0
|
|
160
|
+
if s == 0:
|
|
161
|
+
return 0.0
|
|
162
|
+
total = 0.0
|
|
163
|
+
for m in range(1, r + 1):
|
|
164
|
+
base = [1.0 / math.factorial(i) for i in range(m)] # sum_{i<m} x^i / i!
|
|
165
|
+
powers = [[1.0]]
|
|
166
|
+
for _ in range(W):
|
|
167
|
+
powers.append(_poly_mul(powers[-1], base, r))
|
|
168
|
+
for j in range(1, min(W, r // m) + 1):
|
|
169
|
+
rest = r - j * m
|
|
170
|
+
coeff = powers[W - j][rest] if rest < len(powers[W - j]) else 0.0
|
|
171
|
+
ways = math.comb(W, j) * coeff / (math.factorial(m) ** j) * math.factorial(r) / (W ** r)
|
|
172
|
+
credit = 1.0 if s > m else (1.0 / (1 + j) if s == m else 0.0)
|
|
173
|
+
total += ways * credit
|
|
174
|
+
return min(1.0, total)
|
|
175
|
+
|
|
176
|
+
|
|
177
|
+
def plurality_accuracy(a: float, b: float, K: int, W: float = 1.0) -> float:
|
|
178
|
+
"""Closed-form plurality-vote accuracy over K samples: Beta(a, b) difficulty and wrong answers spread uniformly over W options (W real; linear in W between integers)."""
|
|
179
|
+
if K < 1:
|
|
180
|
+
raise ValueError("K must be at least 1")
|
|
181
|
+
if W < 1:
|
|
182
|
+
raise ValueError("W must be at least 1")
|
|
183
|
+
|
|
184
|
+
def at(Wi: int) -> float:
|
|
185
|
+
return sum(math.exp(beta_binomial_logpmf(s, K, a, b)) * _wrong_wins_prob(s, K - s, Wi) for s in range(K + 1))
|
|
186
|
+
|
|
187
|
+
lo = int(math.floor(W))
|
|
188
|
+
if abs(W - lo) < 1e-12:
|
|
189
|
+
return min(1.0, at(lo))
|
|
190
|
+
return min(1.0, (1 - (W - lo)) * at(lo) + (W - lo) * at(lo + 1))
|
|
191
|
+
|
|
192
|
+
|
|
193
|
+
def estimate_wrong_options(rows: Iterable[dict]) -> float | None:
|
|
194
|
+
"""Effective number of wrong options W = 1 / (collision probability of two wrong samples of the same item), pooled over items.
|
|
195
|
+
Needs rows with item, correct and answer (a label). Returns None if no item has two wrong samples."""
|
|
196
|
+
by = defaultdict(list)
|
|
197
|
+
for r in rows:
|
|
198
|
+
if r["correct"] in (True, 1, 1.0, "1", "true", "True"):
|
|
199
|
+
continue
|
|
200
|
+
by[str(r["item"])].append(str(r.get("answer", "")))
|
|
201
|
+
pairs = same = 0
|
|
202
|
+
for answers in by.values():
|
|
203
|
+
n = len(answers)
|
|
204
|
+
if n < 2:
|
|
205
|
+
continue
|
|
206
|
+
counts = defaultdict(int)
|
|
207
|
+
for a in answers:
|
|
208
|
+
counts[a] += 1
|
|
209
|
+
pairs += n * (n - 1) // 2
|
|
210
|
+
same += sum(c * (c - 1) // 2 for c in counts.values())
|
|
211
|
+
if not pairs:
|
|
212
|
+
return None
|
|
213
|
+
return 1.0 / max(same / pairs, 1e-9)
|
pace/influence.py
ADDED
|
@@ -0,0 +1,231 @@
|
|
|
1
|
+
"""Peer-influence estimators for portable PACE debate logs."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import math
|
|
6
|
+
from collections import defaultdict
|
|
7
|
+
from typing import Any
|
|
8
|
+
|
|
9
|
+
import numpy as np
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def flow_identity_check(
|
|
13
|
+
wrong_r0: int, right_r0: int, wrong_to_right: int, right_to_wrong: int
|
|
14
|
+
) -> dict:
|
|
15
|
+
w, r, fic, fci = map(float, (wrong_r0, right_r0, wrong_to_right, right_to_wrong))
|
|
16
|
+
total = w + r
|
|
17
|
+
p1 = r / total if total else None
|
|
18
|
+
tau_ic = fic / w if w else 0.0
|
|
19
|
+
tau_ci = fci / r if r else 0.0
|
|
20
|
+
denom = fic * r + fci * w
|
|
21
|
+
p_star = (tau_ic / (tau_ic + tau_ci)) if tau_ic + tau_ci else None
|
|
22
|
+
lhs = (p_star - p1) if p_star is not None and p1 is not None else None
|
|
23
|
+
rhs = (r * w * (fic - fci) / (denom * total)) if denom and total else None
|
|
24
|
+
gap = abs(lhs - rhs) if lhs is not None and rhs is not None else None
|
|
25
|
+
return {
|
|
26
|
+
"p1": p1,
|
|
27
|
+
"p_star_hat": p_star,
|
|
28
|
+
"lhs": lhs,
|
|
29
|
+
"rhs": rhs,
|
|
30
|
+
"identity_gap": gap,
|
|
31
|
+
"accuracy_increased": fic > fci,
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def _records_by_agent(records: list[dict]) -> dict:
|
|
36
|
+
grouped = defaultdict(dict)
|
|
37
|
+
for row in records:
|
|
38
|
+
if not all(key in row for key in ("item", "agent", "round", "correct")):
|
|
39
|
+
continue
|
|
40
|
+
grouped[(str(row["item"]), str(row["agent"]))][int(row["round"])] = row
|
|
41
|
+
return grouped
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def dual_transition_tables(records: list[dict]) -> dict:
|
|
45
|
+
"""Return peer-count and same-answer transition cells for adjacent rounds."""
|
|
46
|
+
grouped = _records_by_agent(records)
|
|
47
|
+
by_item = defaultdict(dict)
|
|
48
|
+
for (item, agent), states in grouped.items():
|
|
49
|
+
by_item[item][agent] = states
|
|
50
|
+
correct_cells = defaultdict(lambda: [0, 0])
|
|
51
|
+
answer_cells = defaultdict(lambda: [0, 0, 0])
|
|
52
|
+
for agents in by_item.values():
|
|
53
|
+
rounds = sorted({r for state in agents.values() for r in state})
|
|
54
|
+
for current, following in zip(rounds, rounds[1:]):
|
|
55
|
+
present = [
|
|
56
|
+
state
|
|
57
|
+
for state in agents.values()
|
|
58
|
+
if current in state and following in state
|
|
59
|
+
]
|
|
60
|
+
for state in present:
|
|
61
|
+
own = int(bool(state[current]["correct"]))
|
|
62
|
+
peers_correct = (
|
|
63
|
+
sum(int(bool(other[current]["correct"])) for other in present) - own
|
|
64
|
+
)
|
|
65
|
+
key = (own, peers_correct)
|
|
66
|
+
cell = correct_cells[key]
|
|
67
|
+
cell[0] += int(bool(state[following]["correct"]))
|
|
68
|
+
cell[1] += 1
|
|
69
|
+
if (
|
|
70
|
+
"answer" in state[current]
|
|
71
|
+
and state[current].get("answer") is not None
|
|
72
|
+
):
|
|
73
|
+
agreeing = (
|
|
74
|
+
sum(
|
|
75
|
+
other[current].get("answer") == state[current].get("answer")
|
|
76
|
+
for other in present
|
|
77
|
+
)
|
|
78
|
+
- 1
|
|
79
|
+
)
|
|
80
|
+
share = round(agreeing / max(1, len(present) - 1), 2)
|
|
81
|
+
answer_cell = answer_cells[(own, share)]
|
|
82
|
+
answer_cell[0] += int(
|
|
83
|
+
state[following].get("answer") == state[current].get("answer")
|
|
84
|
+
)
|
|
85
|
+
answer_cell[1] += int(bool(state[following]["correct"]))
|
|
86
|
+
answer_cell[2] += 1
|
|
87
|
+
by_correct = {}
|
|
88
|
+
for key, (next_correct, n) in sorted(correct_cells.items()):
|
|
89
|
+
p = next_correct / n
|
|
90
|
+
by_correct[key] = {
|
|
91
|
+
"p_correct_next": p,
|
|
92
|
+
"p_switch": p if key[0] == 0 else 1.0 - p,
|
|
93
|
+
"n": n,
|
|
94
|
+
}
|
|
95
|
+
by_answer = {}
|
|
96
|
+
for key, (stay, next_correct, n) in sorted(answer_cells.items()):
|
|
97
|
+
by_answer[key] = {
|
|
98
|
+
"p_stay_own_answer": stay / n,
|
|
99
|
+
"p_correct_next": next_correct / n,
|
|
100
|
+
"n": n,
|
|
101
|
+
}
|
|
102
|
+
return {"by_correct_peers": by_correct, "by_same_answer_peers": by_answer}
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def _cell_parts(key: Any) -> tuple[int, int]:
|
|
106
|
+
if isinstance(key, tuple):
|
|
107
|
+
return int(key[0]), int(key[1])
|
|
108
|
+
parts = str(key).replace("=", " ").replace(",", " ").split()
|
|
109
|
+
return int(parts[1]), int(parts[3])
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def fit_influence_models(by_correct_peers: dict, k_agents: int) -> dict:
|
|
113
|
+
if k_agents < 2:
|
|
114
|
+
raise ValueError("k_agents must be at least 2")
|
|
115
|
+
observations = []
|
|
116
|
+
for key, cell in by_correct_peers.items():
|
|
117
|
+
own, peers_correct = _cell_parts(key)
|
|
118
|
+
opposing = peers_correct if own == 0 else (k_agents - 1 - peers_correct)
|
|
119
|
+
x = max(0.0, min(1.0, opposing / (k_agents - 1)))
|
|
120
|
+
y = float(cell.get("p_switch", cell.get("p_correct_next", 0.0)))
|
|
121
|
+
if "p_switch" not in cell and own == 1:
|
|
122
|
+
y = 1.0 - y
|
|
123
|
+
observations.append((own, x, y, float(cell.get("n", 1))))
|
|
124
|
+
if not observations:
|
|
125
|
+
raise ValueError("no transition cells")
|
|
126
|
+
models = {}
|
|
127
|
+
for name, power in (
|
|
128
|
+
("constant", 0),
|
|
129
|
+
("linear_martingale", 1),
|
|
130
|
+
("convex_amplification", 2),
|
|
131
|
+
):
|
|
132
|
+
params = {}
|
|
133
|
+
residual = 0.0
|
|
134
|
+
n_obs = 0
|
|
135
|
+
for own in (0, 1):
|
|
136
|
+
cells = [(x, y, n) for state, x, y, n in observations if state == own]
|
|
137
|
+
if not cells:
|
|
138
|
+
params["a" if own == 0 else "b"] = 0.0
|
|
139
|
+
continue
|
|
140
|
+
design = np.array([1.0 if power == 0 else x**power for x, _, _ in cells])
|
|
141
|
+
values = np.array([y for _, y, _ in cells])
|
|
142
|
+
weights = np.array([n for _, _, n in cells])
|
|
143
|
+
denom = float(np.sum(weights * design * design))
|
|
144
|
+
estimate = (
|
|
145
|
+
float(np.sum(weights * design * values) / denom) if denom else 0.0
|
|
146
|
+
)
|
|
147
|
+
estimate = max(0.0, min(1.0, estimate))
|
|
148
|
+
params["a" if own == 0 else "b"] = estimate
|
|
149
|
+
residual += float(np.sum(weights * (values - estimate * design) ** 2))
|
|
150
|
+
n_obs += len(cells)
|
|
151
|
+
a, b = params.get("a", 0.0), params.get("b", 0.0)
|
|
152
|
+
aic = n_obs * math.log(max(residual / max(1, n_obs), 1e-12)) + 4
|
|
153
|
+
models[name] = {"a": a, "b": b, "wRSS": residual, "AIC": aic, "n_cells": n_obs}
|
|
154
|
+
best = min(models, key=lambda name: models[name]["AIC"])
|
|
155
|
+
return {"models": models, "best_model": best}
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
def _replicate_id(row: dict) -> tuple[str, str]:
|
|
159
|
+
item = str(row["item"])
|
|
160
|
+
if "#" in item:
|
|
161
|
+
base, suffix = item.rsplit("#", 1)
|
|
162
|
+
return base, str(row.get("replicate", suffix))
|
|
163
|
+
if row.get("replicate") is not None:
|
|
164
|
+
return item, str(row["replicate"])
|
|
165
|
+
return item, "0"
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
def replicate_correlation_decomposition(
|
|
169
|
+
records: list[dict], round_num: int = 1
|
|
170
|
+
) -> dict:
|
|
171
|
+
grouped = defaultdict(lambda: defaultdict(dict))
|
|
172
|
+
for row in records:
|
|
173
|
+
if int(row.get("round", -1)) != round_num:
|
|
174
|
+
continue
|
|
175
|
+
base, replicate = _replicate_id(row)
|
|
176
|
+
grouped[base][replicate][str(row["agent"])] = int(bool(row["correct"]))
|
|
177
|
+
values = [
|
|
178
|
+
value
|
|
179
|
+
for reps in grouped.values()
|
|
180
|
+
for agents in reps.values()
|
|
181
|
+
for value in agents.values()
|
|
182
|
+
]
|
|
183
|
+
p_bar = float(np.mean(values)) if values else None
|
|
184
|
+
same = []
|
|
185
|
+
different = []
|
|
186
|
+
for reps in grouped.values():
|
|
187
|
+
for agents in reps.values():
|
|
188
|
+
vals = list(agents.values())
|
|
189
|
+
same.extend(a * b for i, a in enumerate(vals) for b in vals[i + 1 :])
|
|
190
|
+
rep_values = list(reps.values())
|
|
191
|
+
for i, left in enumerate(rep_values):
|
|
192
|
+
for right in rep_values[i + 1 :]:
|
|
193
|
+
different.extend(a * b for a in left.values() for b in right.values())
|
|
194
|
+
c_same = float(np.mean(same)) if same else None
|
|
195
|
+
c_diff = float(np.mean(different)) if different else None
|
|
196
|
+
denominator = p_bar * (1 - p_bar) if p_bar is not None else 0.0
|
|
197
|
+
rho_total = (
|
|
198
|
+
(c_same - p_bar**2) / denominator
|
|
199
|
+
if denominator and c_same is not None
|
|
200
|
+
else None
|
|
201
|
+
)
|
|
202
|
+
rho_item = (
|
|
203
|
+
(c_diff - p_bar**2) / denominator
|
|
204
|
+
if denominator and c_diff is not None
|
|
205
|
+
else None
|
|
206
|
+
)
|
|
207
|
+
rho_within = (
|
|
208
|
+
((c_same - c_diff) / (p_bar - c_diff))
|
|
209
|
+
if c_same is not None and c_diff is not None and p_bar != c_diff
|
|
210
|
+
else None
|
|
211
|
+
)
|
|
212
|
+
k = max(
|
|
213
|
+
(len(agents) for reps in grouped.values() for agents in reps.values()),
|
|
214
|
+
default=0,
|
|
215
|
+
)
|
|
216
|
+
clipped = min(1.0, max(0.0, rho_within)) if rho_within is not None else None
|
|
217
|
+
k_eff = k / (1 + (k - 1) * clipped) if clipped is not None and k else None
|
|
218
|
+
return {
|
|
219
|
+
"p_bar": p_bar,
|
|
220
|
+
"c_same": c_same,
|
|
221
|
+
"c_diff": c_diff,
|
|
222
|
+
"rho_total": rho_total,
|
|
223
|
+
"rho_item": rho_item,
|
|
224
|
+
"rho_within": rho_within,
|
|
225
|
+
"k_eff_within": k_eff,
|
|
226
|
+
"round1_independence_pass": abs(rho_within) < 0.05
|
|
227
|
+
if round_num == 1 and rho_within is not None
|
|
228
|
+
else None,
|
|
229
|
+
"n_items": len(grouped),
|
|
230
|
+
"n_replicated_items": sum(len(reps) >= 2 for reps in grouped.values()),
|
|
231
|
+
}
|
pace/meta_engine.py
ADDED
|
@@ -0,0 +1,246 @@
|
|
|
1
|
+
"""Native numpy meta-analysis and prospective power calculations."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import itertools
|
|
6
|
+
import math
|
|
7
|
+
|
|
8
|
+
import numpy as np
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def compute_effect_sizes(
|
|
12
|
+
p_mas: float, p_base: float, n_items: int, phi: float = 0.5
|
|
13
|
+
) -> dict:
|
|
14
|
+
if n_items < 1 or not 0 <= phi <= 1 or not (0 <= p_mas <= 1 and 0 <= p_base <= 1):
|
|
15
|
+
raise ValueError("invalid probabilities, sample size, or phi")
|
|
16
|
+
pt_m, pt_b = (
|
|
17
|
+
(p_mas * n_items + 0.5) / (n_items + 1),
|
|
18
|
+
(p_base * n_items + 0.5) / (n_items + 1),
|
|
19
|
+
)
|
|
20
|
+
log_or = math.log((pt_m / (1 - pt_m)) / (pt_b / (1 - pt_b)))
|
|
21
|
+
vm, vb = 1 / (n_items * pt_m * (1 - pt_m)), 1 / (n_items * pt_b * (1 - pt_b))
|
|
22
|
+
phi_max = math.exp(-abs(log_or) / 2)
|
|
23
|
+
v_log_or = vm + vb - 2 * phi * math.sqrt(vm * vb)
|
|
24
|
+
v_clamped = vm + vb - 2 * min(phi, phi_max) * math.sqrt(vm * vb)
|
|
25
|
+
delta = p_mas - p_base
|
|
26
|
+
se_delta = math.sqrt(
|
|
27
|
+
max(
|
|
28
|
+
0.0,
|
|
29
|
+
(
|
|
30
|
+
p_mas * (1 - p_mas)
|
|
31
|
+
+ p_base * (1 - p_base)
|
|
32
|
+
- 2 * phi * math.sqrt(p_mas * (1 - p_mas) * p_base * (1 - p_base))
|
|
33
|
+
)
|
|
34
|
+
/ n_items,
|
|
35
|
+
)
|
|
36
|
+
)
|
|
37
|
+
h = 2 * math.asin(math.sqrt(p_mas)) - 2 * math.asin(math.sqrt(p_base))
|
|
38
|
+
return {
|
|
39
|
+
"p_tilde_mas": pt_m,
|
|
40
|
+
"p_tilde_base": pt_b,
|
|
41
|
+
"log_or": log_or,
|
|
42
|
+
"v_log_or": v_log_or,
|
|
43
|
+
"phi_max": phi_max,
|
|
44
|
+
"v_log_or_clamped": v_clamped,
|
|
45
|
+
"delta": delta,
|
|
46
|
+
"se_delta": se_delta,
|
|
47
|
+
"cohen_h": h,
|
|
48
|
+
"v_cohen_h": 2 * (1 - phi) / n_items,
|
|
49
|
+
"phi_clamped": phi > phi_max,
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def shared_baseline_deff(
|
|
54
|
+
m_contrasts_sharing_baseline: int, v_mas: float, v_base: float
|
|
55
|
+
) -> dict:
|
|
56
|
+
if m_contrasts_sharing_baseline < 1 or min(v_mas, v_base) < 0:
|
|
57
|
+
raise ValueError("invalid variance or block size")
|
|
58
|
+
rho = v_base / (v_mas + v_base) if v_mas + v_base else 0.0
|
|
59
|
+
deff = 1 + (m_contrasts_sharing_baseline - 1) * rho
|
|
60
|
+
naive = (v_mas + v_base) / m_contrasts_sharing_baseline
|
|
61
|
+
return {
|
|
62
|
+
"rho_base": rho,
|
|
63
|
+
"deff": deff,
|
|
64
|
+
"naive_diagonal_variance": naive,
|
|
65
|
+
"true_shared_baseline_variance": naive * deff,
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def build_che_covariance(
|
|
70
|
+
contrasts: list[dict], rho_block: float = 0.5, block_by: str = "baseline_cell"
|
|
71
|
+
) -> np.ndarray:
|
|
72
|
+
if not 0 <= rho_block <= 1 or block_by not in {
|
|
73
|
+
"baseline_cell",
|
|
74
|
+
"benchmark",
|
|
75
|
+
"study",
|
|
76
|
+
}:
|
|
77
|
+
raise ValueError("invalid rho_block or block_by")
|
|
78
|
+
keys = {
|
|
79
|
+
"baseline_cell": lambda r: (
|
|
80
|
+
r.get("study"),
|
|
81
|
+
r.get("benchmark"),
|
|
82
|
+
r.get("model"),
|
|
83
|
+
r.get("category"),
|
|
84
|
+
),
|
|
85
|
+
"benchmark": lambda r: (r.get("study"), r.get("benchmark")),
|
|
86
|
+
"study": lambda r: (r.get("study"),),
|
|
87
|
+
}[block_by]
|
|
88
|
+
n = len(contrasts)
|
|
89
|
+
matrix = np.zeros((n, n))
|
|
90
|
+
variances = [
|
|
91
|
+
float(r.get("variance", 1 / max(1, int(float(r.get("n", 1)))))) for r in contrasts
|
|
92
|
+
]
|
|
93
|
+
for i in range(n):
|
|
94
|
+
matrix[i, i] = variances[i]
|
|
95
|
+
for j in range(i):
|
|
96
|
+
if keys(contrasts[i]) == keys(contrasts[j]):
|
|
97
|
+
matrix[i, j] = matrix[j, i] = rho_block * math.sqrt(
|
|
98
|
+
variances[i] * variances[j]
|
|
99
|
+
)
|
|
100
|
+
return matrix
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def der_simonian_laird_pool(y: np.ndarray, v: np.ndarray, clusters: list[str]) -> dict:
|
|
104
|
+
y, v = np.asarray(y, dtype=float), np.asarray(v, dtype=float)
|
|
105
|
+
if y.ndim != 1 or len(y) != len(v) or len(y) != len(clusters) or np.any(v <= 0):
|
|
106
|
+
raise ValueError("y, v, and clusters must align and v must be positive")
|
|
107
|
+
w = 1 / v
|
|
108
|
+
fixed = float(np.sum(w * y) / np.sum(w))
|
|
109
|
+
q = float(np.sum(w * (y - fixed) ** 2))
|
|
110
|
+
df = len(y) - 1
|
|
111
|
+
tau2 = max(0.0, (q - df) / (np.sum(w) - np.sum(w**2) / np.sum(w))) if df else 0.0
|
|
112
|
+
wr = 1 / (v + tau2)
|
|
113
|
+
estimate = float(np.sum(wr * y) / np.sum(wr))
|
|
114
|
+
groups = sorted(set(clusters))
|
|
115
|
+
scores = {
|
|
116
|
+
g: float(
|
|
117
|
+
np.sum(
|
|
118
|
+
wr[[i for i, c in enumerate(clusters) if c == g]]
|
|
119
|
+
* (y[[i for i, c in enumerate(clusters) if c == g]] - estimate)
|
|
120
|
+
)
|
|
121
|
+
)
|
|
122
|
+
for g in groups
|
|
123
|
+
}
|
|
124
|
+
sandwich = (
|
|
125
|
+
sum(score**2 for score in scores.values())
|
|
126
|
+
/ max(1, len(groups) - 1)
|
|
127
|
+
/ max(np.sum(wr) ** 2, 1e-12)
|
|
128
|
+
)
|
|
129
|
+
se = math.sqrt(max(0.0, sandwich))
|
|
130
|
+
df_sat = max(1.0, len(groups) - 1)
|
|
131
|
+
ci = [estimate - 1.96 * se, estimate + 1.96 * se]
|
|
132
|
+
signs = []
|
|
133
|
+
if len(groups) <= 16:
|
|
134
|
+
for signs_tuple in itertools.product((-1, 1), repeat=len(groups)):
|
|
135
|
+
signs.append(
|
|
136
|
+
abs(sum(scores[g] * sign for g, sign in zip(groups, signs_tuple)))
|
|
137
|
+
)
|
|
138
|
+
p_value = sum(value >= abs(sum(scores.values())) for value in signs) / len(
|
|
139
|
+
signs
|
|
140
|
+
)
|
|
141
|
+
else:
|
|
142
|
+
p_value = None
|
|
143
|
+
loco = []
|
|
144
|
+
for group in groups:
|
|
145
|
+
mask = np.array([c != group for c in clusters])
|
|
146
|
+
pooled = (
|
|
147
|
+
der_simonian_laird_pool(y[mask], v[mask], list(np.asarray(clusters)[mask]))
|
|
148
|
+
if mask.sum() > 1
|
|
149
|
+
else None
|
|
150
|
+
)
|
|
151
|
+
loco.append(
|
|
152
|
+
{
|
|
153
|
+
"cluster": group,
|
|
154
|
+
"estimate": None if pooled is None else pooled["estimate"],
|
|
155
|
+
}
|
|
156
|
+
)
|
|
157
|
+
return {
|
|
158
|
+
"estimate": estimate,
|
|
159
|
+
"tau2_dl": tau2,
|
|
160
|
+
"se": se,
|
|
161
|
+
"df": df_sat,
|
|
162
|
+
"ci95": ci,
|
|
163
|
+
"prediction_interval_study": [
|
|
164
|
+
estimate - 1.96 * math.sqrt(se * se + tau2),
|
|
165
|
+
estimate + 1.96 * math.sqrt(se * se + tau2),
|
|
166
|
+
],
|
|
167
|
+
"prediction_interval_single": [
|
|
168
|
+
estimate - 1.96 * math.sqrt(se * se + tau2),
|
|
169
|
+
estimate + 1.96 * math.sqrt(se * se + tau2),
|
|
170
|
+
],
|
|
171
|
+
"p_value": p_value,
|
|
172
|
+
"loco": loco,
|
|
173
|
+
"q": q,
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
|
|
177
|
+
def prospective_power_analysis(
|
|
178
|
+
p_base: float,
|
|
179
|
+
target_delta: float,
|
|
180
|
+
n_items: int | None = None,
|
|
181
|
+
target_power: float = 0.8,
|
|
182
|
+
alpha: float = 0.05,
|
|
183
|
+
phi: float = 0.5,
|
|
184
|
+
k_shared_arms: int = 1,
|
|
185
|
+
) -> dict:
|
|
186
|
+
if (
|
|
187
|
+
not 0 < p_base < 1
|
|
188
|
+
or target_delta == 0
|
|
189
|
+
or not 0 < target_power < 1
|
|
190
|
+
or not 0 < alpha < 1
|
|
191
|
+
or k_shared_arms < 1
|
|
192
|
+
):
|
|
193
|
+
raise ValueError("invalid power parameters")
|
|
194
|
+
adjusted_alpha = alpha / k_shared_arms
|
|
195
|
+
z_alpha, z_power = (
|
|
196
|
+
1.96 if adjusted_alpha <= 0.05 else 1.645,
|
|
197
|
+
0.8416 if target_power >= 0.8 else 0.5244,
|
|
198
|
+
)
|
|
199
|
+
variance = (
|
|
200
|
+
p_base * (1 - p_base)
|
|
201
|
+
+ (p_base + target_delta) * (1 - p_base - target_delta)
|
|
202
|
+
- 2
|
|
203
|
+
* phi
|
|
204
|
+
* math.sqrt(
|
|
205
|
+
p_base
|
|
206
|
+
* (1 - p_base)
|
|
207
|
+
* (p_base + target_delta)
|
|
208
|
+
* (1 - p_base - target_delta)
|
|
209
|
+
)
|
|
210
|
+
)
|
|
211
|
+
n_min = math.ceil(((z_alpha + z_power) ** 2 * variance) / target_delta**2)
|
|
212
|
+
achieved = (
|
|
213
|
+
None
|
|
214
|
+
if n_items is None
|
|
215
|
+
else min(
|
|
216
|
+
1.0,
|
|
217
|
+
max(
|
|
218
|
+
0.0,
|
|
219
|
+
0.5
|
|
220
|
+
* (
|
|
221
|
+
1
|
|
222
|
+
+ math.erf(
|
|
223
|
+
(
|
|
224
|
+
abs(target_delta)
|
|
225
|
+
* math.sqrt(n_items / max(variance, 1e-12))
|
|
226
|
+
- z_alpha
|
|
227
|
+
)
|
|
228
|
+
/ math.sqrt(2)
|
|
229
|
+
)
|
|
230
|
+
),
|
|
231
|
+
),
|
|
232
|
+
)
|
|
233
|
+
)
|
|
234
|
+
mde = (
|
|
235
|
+
None
|
|
236
|
+
if n_items is None
|
|
237
|
+
else (z_alpha + z_power) * math.sqrt(max(variance, 1e-12) / n_items)
|
|
238
|
+
)
|
|
239
|
+
return {
|
|
240
|
+
"n_min_items": n_min,
|
|
241
|
+
"achieved_power": achieved,
|
|
242
|
+
"mde": mde,
|
|
243
|
+
"alpha_adjusted": adjusted_alpha,
|
|
244
|
+
"phi": phi,
|
|
245
|
+
"n_items": n_items,
|
|
246
|
+
}
|