pace-eval 0.2.2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (83) hide show
  1. cme/__init__.py +11 -0
  2. cme/__main__.py +5 -0
  3. cme/analysis.py +223 -0
  4. cme/audit.py +145 -0
  5. cme/baseline.py +162 -0
  6. cme/cli.py +279 -0
  7. cme/client.py +165 -0
  8. cme/dynamics.py +212 -0
  9. cme/examples/crewai_integration.py +16 -0
  10. cme/examples/demo_pool.jsonl +360 -0
  11. cme/examples/demo_run.jsonl +480 -0
  12. cme/examples/langgraph_integration.py +27 -0
  13. cme/examples/make_demo.py +37 -0
  14. cme/examples/openai_quickstart.py +18 -0
  15. cme/experiment.py +555 -0
  16. cme/frontier.py +143 -0
  17. cme/hidden_profile.py +135 -0
  18. cme/proxy.py +106 -0
  19. cme/rebuttal.py +76 -0
  20. cme/report.py +197 -0
  21. cme/rollout_card.py +137 -0
  22. cme/tasks.py +266 -0
  23. cme/verified_debate.py +127 -0
  24. cme/verifier.py +92 -0
  25. mae_eval/__init__.py +11 -0
  26. mae_eval/__main__.py +5 -0
  27. mae_eval/auditor.py +112 -0
  28. mae_eval/classifier.py +62 -0
  29. mae_eval/cli.py +210 -0
  30. mae_eval/tracker.py +162 -0
  31. mae_eval/vector.py +91 -0
  32. pace/__init__.py +11 -0
  33. pace/__main__.py +5 -0
  34. pace/analysis.py +129 -0
  35. pace/audit.py +145 -0
  36. pace/baseline.py +162 -0
  37. pace/card.py +205 -0
  38. pace/cli.py +898 -0
  39. pace/client.py +165 -0
  40. pace/dynamics.py +372 -0
  41. pace/escape_benchmarks.py +162 -0
  42. pace/examples/crewai_integration.py +16 -0
  43. pace/examples/demo_pool.jsonl +360 -0
  44. pace/examples/demo_run.jsonl +480 -0
  45. pace/examples/langgraph_integration.py +27 -0
  46. pace/examples/make_demo.py +37 -0
  47. pace/examples/openai_quickstart.py +18 -0
  48. pace/experiment.py +555 -0
  49. pace/frontier.py +114 -0
  50. pace/hidden_profile.py +301 -0
  51. pace/importers.py +219 -0
  52. pace/inflate.py +213 -0
  53. pace/influence.py +231 -0
  54. pace/meta_engine.py +246 -0
  55. pace/middleware.py +230 -0
  56. pace/open_ended.py +114 -0
  57. pace/plotting.py +132 -0
  58. pace/pool_sizer.py +154 -0
  59. pace/protocol_v3.py +414 -0
  60. pace/proxy.py +106 -0
  61. pace/rebuttal.py +219 -0
  62. pace/report.py +197 -0
  63. pace/rollout_card.py +137 -0
  64. pace/schema.py +74 -0
  65. pace/simulator.py +199 -0
  66. pace/studio.html +1040 -0
  67. pace/tasks.py +266 -0
  68. pace/verified_debate.py +127 -0
  69. pace/verifier.py +92 -0
  70. pace/workbench.html +26 -0
  71. pace/workbench.py +119 -0
  72. pace_eval-0.2.2.dist-info/METADATA +205 -0
  73. pace_eval-0.2.2.dist-info/RECORD +83 -0
  74. pace_eval-0.2.2.dist-info/WHEEL +5 -0
  75. pace_eval-0.2.2.dist-info/entry_points.txt +4 -0
  76. pace_eval-0.2.2.dist-info/licenses/LICENSE +202 -0
  77. pace_eval-0.2.2.dist-info/top_level.txt +4 -0
  78. pace_lab/__init__.py +9 -0
  79. pace_lab/clients.py +568 -0
  80. pace_lab/concurrency_test.py +127 -0
  81. pace_lab/data.py +445 -0
  82. pace_lab/graders.py +626 -0
  83. pace_lab/strategies.py +345 -0
pace/inflate.py ADDED
@@ -0,0 +1,213 @@
1
+ """Beta-binomial sampling inflation (`pace inflate`): how much does a K-sample majority vote beat one sample, from a pool of n samples per item?
2
+
3
+ Model: item i is answered correctly with probability p_i ~ Beta(alpha, beta); the n pooled samples of an item are then Binomial(n, p_i), so the per-item success counts
4
+ are beta-binomial. Fit (alpha, beta) by maximum likelihood on the pool, and predict the accuracy of a vote over K samples in closed form:
5
+
6
+ acc(K) = sum_s BetaBinomial(s; K, alpha, beta) * [ 1(s > K/2) + 0.5 * 1(s = K/2) ]
7
+
8
+ This is the *binary* majority (correct versus not correct). A plurality vote over several wrong options that split their votes does at least as well, so for
9
+ multiple-choice tasks acc(K) is a conservative (lower) prediction of plurality-vote accuracy. Pure standard library; no numpy, no scipy.
10
+ """
11
+
12
+ from __future__ import annotations
13
+
14
+ import math
15
+ from collections import defaultdict
16
+ from typing import Iterable, Sequence
17
+
18
+ LOG_LO, LOG_HI = math.log(1e-3), math.log(1e4)
19
+
20
+
21
+ def _betaln(a: float, b: float) -> float:
22
+ return math.lgamma(a) + math.lgamma(b) - math.lgamma(a + b)
23
+
24
+
25
+ def _log_choose(n: int, k: int) -> float:
26
+ return math.lgamma(n + 1) - math.lgamma(k + 1) - math.lgamma(n - k + 1)
27
+
28
+
29
+ def beta_binomial_logpmf(s: int, n: int, a: float, b: float) -> float:
30
+ return _log_choose(n, s) + _betaln(s + a, n - s + b) - _betaln(a, b)
31
+
32
+
33
+ def log_likelihood(successes: Sequence[int], n: int, a: float, b: float) -> float:
34
+ counts = defaultdict(int)
35
+ for s in successes:
36
+ counts[int(s)] += 1
37
+ return sum(c * beta_binomial_logpmf(s, n, a, b) for s, c in counts.items())
38
+
39
+
40
+ def fit_beta_binomial(successes: Sequence[int], n: int) -> tuple[float, float]:
41
+ """Maximum-likelihood (alpha, beta) from per-item success counts out of n samples (shrinking-grid search on the log scale, clamped to [1e-3, 1e4])."""
42
+ if not len(successes):
43
+ raise ValueError("no items")
44
+ if any(not 0 <= int(s) <= n for s in successes):
45
+ raise ValueError("success counts must lie in 0..n")
46
+ mean = min(max(sum(successes) / (n * len(successes)), 1e-3), 1 - 1e-3)
47
+ # start: equal-ish concentration on the log scale, then coordinate refinement
48
+ la, lb = math.log(mean * 2.0), math.log((1 - mean) * 2.0)
49
+ best = log_likelihood(successes, n, math.exp(la), math.exp(lb))
50
+ step = 2.0
51
+ for _ in range(200):
52
+ improved = False
53
+ for da in (-step, -step / 2, 0.0, step / 2, step):
54
+ for db in (-step, -step / 2, 0.0, step / 2, step):
55
+ ca, cb = min(max(la + da, LOG_LO), LOG_HI), min(max(lb + db, LOG_LO), LOG_HI)
56
+ ll = log_likelihood(successes, n, math.exp(ca), math.exp(cb))
57
+ if ll > best + 1e-12:
58
+ best, la, lb, improved = ll, ca, cb, True
59
+ if not improved:
60
+ step *= 0.5
61
+ if step < 1e-6:
62
+ break
63
+ return math.exp(la), math.exp(lb)
64
+
65
+
66
+ def vote_accuracy(a: float, b: float, K: int, tie: float = 0.5) -> float:
67
+ """Closed-form accuracy of a binary-majority vote over K independent samples under Beta(a, b) item difficulty (ties count `tie`)."""
68
+ if K < 1:
69
+ raise ValueError("K must be at least 1")
70
+ total = 0.0
71
+ for s in range(K + 1):
72
+ if 2 * s > K:
73
+ w = 1.0
74
+ elif 2 * s == K:
75
+ w = tie
76
+ else:
77
+ continue
78
+ total += w * math.exp(beta_binomial_logpmf(s, K, a, b))
79
+ return min(1.0, total)
80
+
81
+
82
+ def empirical_vote_accuracy(successes: Sequence[int], n: int, K: int, tie: float = 0.5) -> float:
83
+ """Exact expected binary-majority accuracy of a vote over K of the n pooled samples (hypergeometric over the choice of the K samples), averaged over items."""
84
+ if K > n:
85
+ raise ValueError("K cannot exceed the pool size n")
86
+ out = 0.0
87
+ denom = math.comb(n, K)
88
+ for s in successes:
89
+ acc = 0.0
90
+ for j in range(max(0, K - (n - s)), min(K, s) + 1):
91
+ w = 1.0 if 2 * j > K else (tie if 2 * j == K else 0.0)
92
+ if w:
93
+ acc += w * math.comb(s, j) * math.comb(n - s, K - j) / denom
94
+ out += acc
95
+ return out / len(successes)
96
+
97
+
98
+ def successes_from_rows(rows: Iterable[dict]) -> tuple[list[int], int]:
99
+ """Per-item success counts from rows with `item` and `correct` (one row per sample). Items with fewer samples than the most common count are dropped."""
100
+ by = defaultdict(list)
101
+ for r in rows:
102
+ by[str(r["item"])].append(1 if r["correct"] in (True, 1, 1.0, "1", "true", "True") else 0)
103
+ sizes = defaultdict(int)
104
+ for v in by.values():
105
+ sizes[len(v)] += 1
106
+ n = max(sizes, key=lambda k: (sizes[k], k))
107
+ return [sum(v) for v in by.values() if len(v) == n], n
108
+
109
+
110
+ def inflate_report(successes: Sequence[int], n: int, ks: Sequence[int], wrong_options: float | None = None) -> dict:
111
+ a, b = fit_beta_binomial(successes, n)
112
+ p1 = a / (a + b)
113
+ rows = []
114
+ for K in ks:
115
+ row = {"K": K, "predicted": vote_accuracy(a, b, K)}
116
+ if K <= n:
117
+ row["pool_observed"] = empirical_vote_accuracy(successes, n, K)
118
+ if wrong_options is not None:
119
+ row["predicted_plurality"] = plurality_accuracy(a, b, K, wrong_options)
120
+ row["over_single_sample"] = row["predicted"] - p1
121
+ rows.append(row)
122
+ return {
123
+ "n_items": len(successes),
124
+ "pool_size": n,
125
+ "alpha": a,
126
+ "beta": b,
127
+ "concentration": a + b,
128
+ "mean_single_sample_accuracy": p1,
129
+ "single_sample_accuracy_observed": sum(successes) / (n * len(successes)),
130
+ "wrong_options": wrong_options,
131
+ "inflation": rows,
132
+ "model": "binary majority under Beta(alpha, beta) item difficulty; conservative for plurality voting over split wrong answers",
133
+ }
134
+
135
+
136
+ # ---------------------------------------------------------------------------------------------------------------------------------------------
137
+ # Plurality extension: wrong answers are spread over W options (uniformly), so a plurality vote can win with far fewer than K/2 correct samples.
138
+ # Given the number s of correct samples among K, the K-s wrong samples are Multinomial(K-s; uniform over W wrong options), independent of p_i.
139
+ # The vote is correct if s beats the largest wrong count; a tie with j wrong options at the maximum is credited 1/(1+j) (uniform tie-break).
140
+ # W = 1 recovers the binary majority above.
141
+ # ---------------------------------------------------------------------------------------------------------------------------------------------
142
+
143
+
144
+ def _poly_mul(x: list[float], y: list[float], deg: int) -> list[float]:
145
+ out = [0.0] * (deg + 1)
146
+ for i, xi in enumerate(x):
147
+ if xi == 0.0:
148
+ continue
149
+ for j, yj in enumerate(y):
150
+ if i + j > deg:
151
+ break
152
+ out[i + j] += xi * yj
153
+ return out
154
+
155
+
156
+ def _wrong_wins_prob(s: int, r: int, W: int) -> float:
157
+ """P(the s correct samples win a plurality against r wrong samples spread uniformly over W options), with uniform tie-break."""
158
+ if r == 0:
159
+ return 1.0 if s > 0 else 0.0
160
+ if s == 0:
161
+ return 0.0
162
+ total = 0.0
163
+ for m in range(1, r + 1):
164
+ base = [1.0 / math.factorial(i) for i in range(m)] # sum_{i<m} x^i / i!
165
+ powers = [[1.0]]
166
+ for _ in range(W):
167
+ powers.append(_poly_mul(powers[-1], base, r))
168
+ for j in range(1, min(W, r // m) + 1):
169
+ rest = r - j * m
170
+ coeff = powers[W - j][rest] if rest < len(powers[W - j]) else 0.0
171
+ ways = math.comb(W, j) * coeff / (math.factorial(m) ** j) * math.factorial(r) / (W ** r)
172
+ credit = 1.0 if s > m else (1.0 / (1 + j) if s == m else 0.0)
173
+ total += ways * credit
174
+ return min(1.0, total)
175
+
176
+
177
+ def plurality_accuracy(a: float, b: float, K: int, W: float = 1.0) -> float:
178
+ """Closed-form plurality-vote accuracy over K samples: Beta(a, b) difficulty and wrong answers spread uniformly over W options (W real; linear in W between integers)."""
179
+ if K < 1:
180
+ raise ValueError("K must be at least 1")
181
+ if W < 1:
182
+ raise ValueError("W must be at least 1")
183
+
184
+ def at(Wi: int) -> float:
185
+ return sum(math.exp(beta_binomial_logpmf(s, K, a, b)) * _wrong_wins_prob(s, K - s, Wi) for s in range(K + 1))
186
+
187
+ lo = int(math.floor(W))
188
+ if abs(W - lo) < 1e-12:
189
+ return min(1.0, at(lo))
190
+ return min(1.0, (1 - (W - lo)) * at(lo) + (W - lo) * at(lo + 1))
191
+
192
+
193
+ def estimate_wrong_options(rows: Iterable[dict]) -> float | None:
194
+ """Effective number of wrong options W = 1 / (collision probability of two wrong samples of the same item), pooled over items.
195
+ Needs rows with item, correct and answer (a label). Returns None if no item has two wrong samples."""
196
+ by = defaultdict(list)
197
+ for r in rows:
198
+ if r["correct"] in (True, 1, 1.0, "1", "true", "True"):
199
+ continue
200
+ by[str(r["item"])].append(str(r.get("answer", "")))
201
+ pairs = same = 0
202
+ for answers in by.values():
203
+ n = len(answers)
204
+ if n < 2:
205
+ continue
206
+ counts = defaultdict(int)
207
+ for a in answers:
208
+ counts[a] += 1
209
+ pairs += n * (n - 1) // 2
210
+ same += sum(c * (c - 1) // 2 for c in counts.values())
211
+ if not pairs:
212
+ return None
213
+ return 1.0 / max(same / pairs, 1e-9)
pace/influence.py ADDED
@@ -0,0 +1,231 @@
1
+ """Peer-influence estimators for portable PACE debate logs."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import math
6
+ from collections import defaultdict
7
+ from typing import Any
8
+
9
+ import numpy as np
10
+
11
+
12
+ def flow_identity_check(
13
+ wrong_r0: int, right_r0: int, wrong_to_right: int, right_to_wrong: int
14
+ ) -> dict:
15
+ w, r, fic, fci = map(float, (wrong_r0, right_r0, wrong_to_right, right_to_wrong))
16
+ total = w + r
17
+ p1 = r / total if total else None
18
+ tau_ic = fic / w if w else 0.0
19
+ tau_ci = fci / r if r else 0.0
20
+ denom = fic * r + fci * w
21
+ p_star = (tau_ic / (tau_ic + tau_ci)) if tau_ic + tau_ci else None
22
+ lhs = (p_star - p1) if p_star is not None and p1 is not None else None
23
+ rhs = (r * w * (fic - fci) / (denom * total)) if denom and total else None
24
+ gap = abs(lhs - rhs) if lhs is not None and rhs is not None else None
25
+ return {
26
+ "p1": p1,
27
+ "p_star_hat": p_star,
28
+ "lhs": lhs,
29
+ "rhs": rhs,
30
+ "identity_gap": gap,
31
+ "accuracy_increased": fic > fci,
32
+ }
33
+
34
+
35
+ def _records_by_agent(records: list[dict]) -> dict:
36
+ grouped = defaultdict(dict)
37
+ for row in records:
38
+ if not all(key in row for key in ("item", "agent", "round", "correct")):
39
+ continue
40
+ grouped[(str(row["item"]), str(row["agent"]))][int(row["round"])] = row
41
+ return grouped
42
+
43
+
44
+ def dual_transition_tables(records: list[dict]) -> dict:
45
+ """Return peer-count and same-answer transition cells for adjacent rounds."""
46
+ grouped = _records_by_agent(records)
47
+ by_item = defaultdict(dict)
48
+ for (item, agent), states in grouped.items():
49
+ by_item[item][agent] = states
50
+ correct_cells = defaultdict(lambda: [0, 0])
51
+ answer_cells = defaultdict(lambda: [0, 0, 0])
52
+ for agents in by_item.values():
53
+ rounds = sorted({r for state in agents.values() for r in state})
54
+ for current, following in zip(rounds, rounds[1:]):
55
+ present = [
56
+ state
57
+ for state in agents.values()
58
+ if current in state and following in state
59
+ ]
60
+ for state in present:
61
+ own = int(bool(state[current]["correct"]))
62
+ peers_correct = (
63
+ sum(int(bool(other[current]["correct"])) for other in present) - own
64
+ )
65
+ key = (own, peers_correct)
66
+ cell = correct_cells[key]
67
+ cell[0] += int(bool(state[following]["correct"]))
68
+ cell[1] += 1
69
+ if (
70
+ "answer" in state[current]
71
+ and state[current].get("answer") is not None
72
+ ):
73
+ agreeing = (
74
+ sum(
75
+ other[current].get("answer") == state[current].get("answer")
76
+ for other in present
77
+ )
78
+ - 1
79
+ )
80
+ share = round(agreeing / max(1, len(present) - 1), 2)
81
+ answer_cell = answer_cells[(own, share)]
82
+ answer_cell[0] += int(
83
+ state[following].get("answer") == state[current].get("answer")
84
+ )
85
+ answer_cell[1] += int(bool(state[following]["correct"]))
86
+ answer_cell[2] += 1
87
+ by_correct = {}
88
+ for key, (next_correct, n) in sorted(correct_cells.items()):
89
+ p = next_correct / n
90
+ by_correct[key] = {
91
+ "p_correct_next": p,
92
+ "p_switch": p if key[0] == 0 else 1.0 - p,
93
+ "n": n,
94
+ }
95
+ by_answer = {}
96
+ for key, (stay, next_correct, n) in sorted(answer_cells.items()):
97
+ by_answer[key] = {
98
+ "p_stay_own_answer": stay / n,
99
+ "p_correct_next": next_correct / n,
100
+ "n": n,
101
+ }
102
+ return {"by_correct_peers": by_correct, "by_same_answer_peers": by_answer}
103
+
104
+
105
+ def _cell_parts(key: Any) -> tuple[int, int]:
106
+ if isinstance(key, tuple):
107
+ return int(key[0]), int(key[1])
108
+ parts = str(key).replace("=", " ").replace(",", " ").split()
109
+ return int(parts[1]), int(parts[3])
110
+
111
+
112
+ def fit_influence_models(by_correct_peers: dict, k_agents: int) -> dict:
113
+ if k_agents < 2:
114
+ raise ValueError("k_agents must be at least 2")
115
+ observations = []
116
+ for key, cell in by_correct_peers.items():
117
+ own, peers_correct = _cell_parts(key)
118
+ opposing = peers_correct if own == 0 else (k_agents - 1 - peers_correct)
119
+ x = max(0.0, min(1.0, opposing / (k_agents - 1)))
120
+ y = float(cell.get("p_switch", cell.get("p_correct_next", 0.0)))
121
+ if "p_switch" not in cell and own == 1:
122
+ y = 1.0 - y
123
+ observations.append((own, x, y, float(cell.get("n", 1))))
124
+ if not observations:
125
+ raise ValueError("no transition cells")
126
+ models = {}
127
+ for name, power in (
128
+ ("constant", 0),
129
+ ("linear_martingale", 1),
130
+ ("convex_amplification", 2),
131
+ ):
132
+ params = {}
133
+ residual = 0.0
134
+ n_obs = 0
135
+ for own in (0, 1):
136
+ cells = [(x, y, n) for state, x, y, n in observations if state == own]
137
+ if not cells:
138
+ params["a" if own == 0 else "b"] = 0.0
139
+ continue
140
+ design = np.array([1.0 if power == 0 else x**power for x, _, _ in cells])
141
+ values = np.array([y for _, y, _ in cells])
142
+ weights = np.array([n for _, _, n in cells])
143
+ denom = float(np.sum(weights * design * design))
144
+ estimate = (
145
+ float(np.sum(weights * design * values) / denom) if denom else 0.0
146
+ )
147
+ estimate = max(0.0, min(1.0, estimate))
148
+ params["a" if own == 0 else "b"] = estimate
149
+ residual += float(np.sum(weights * (values - estimate * design) ** 2))
150
+ n_obs += len(cells)
151
+ a, b = params.get("a", 0.0), params.get("b", 0.0)
152
+ aic = n_obs * math.log(max(residual / max(1, n_obs), 1e-12)) + 4
153
+ models[name] = {"a": a, "b": b, "wRSS": residual, "AIC": aic, "n_cells": n_obs}
154
+ best = min(models, key=lambda name: models[name]["AIC"])
155
+ return {"models": models, "best_model": best}
156
+
157
+
158
+ def _replicate_id(row: dict) -> tuple[str, str]:
159
+ item = str(row["item"])
160
+ if "#" in item:
161
+ base, suffix = item.rsplit("#", 1)
162
+ return base, str(row.get("replicate", suffix))
163
+ if row.get("replicate") is not None:
164
+ return item, str(row["replicate"])
165
+ return item, "0"
166
+
167
+
168
+ def replicate_correlation_decomposition(
169
+ records: list[dict], round_num: int = 1
170
+ ) -> dict:
171
+ grouped = defaultdict(lambda: defaultdict(dict))
172
+ for row in records:
173
+ if int(row.get("round", -1)) != round_num:
174
+ continue
175
+ base, replicate = _replicate_id(row)
176
+ grouped[base][replicate][str(row["agent"])] = int(bool(row["correct"]))
177
+ values = [
178
+ value
179
+ for reps in grouped.values()
180
+ for agents in reps.values()
181
+ for value in agents.values()
182
+ ]
183
+ p_bar = float(np.mean(values)) if values else None
184
+ same = []
185
+ different = []
186
+ for reps in grouped.values():
187
+ for agents in reps.values():
188
+ vals = list(agents.values())
189
+ same.extend(a * b for i, a in enumerate(vals) for b in vals[i + 1 :])
190
+ rep_values = list(reps.values())
191
+ for i, left in enumerate(rep_values):
192
+ for right in rep_values[i + 1 :]:
193
+ different.extend(a * b for a in left.values() for b in right.values())
194
+ c_same = float(np.mean(same)) if same else None
195
+ c_diff = float(np.mean(different)) if different else None
196
+ denominator = p_bar * (1 - p_bar) if p_bar is not None else 0.0
197
+ rho_total = (
198
+ (c_same - p_bar**2) / denominator
199
+ if denominator and c_same is not None
200
+ else None
201
+ )
202
+ rho_item = (
203
+ (c_diff - p_bar**2) / denominator
204
+ if denominator and c_diff is not None
205
+ else None
206
+ )
207
+ rho_within = (
208
+ ((c_same - c_diff) / (p_bar - c_diff))
209
+ if c_same is not None and c_diff is not None and p_bar != c_diff
210
+ else None
211
+ )
212
+ k = max(
213
+ (len(agents) for reps in grouped.values() for agents in reps.values()),
214
+ default=0,
215
+ )
216
+ clipped = min(1.0, max(0.0, rho_within)) if rho_within is not None else None
217
+ k_eff = k / (1 + (k - 1) * clipped) if clipped is not None and k else None
218
+ return {
219
+ "p_bar": p_bar,
220
+ "c_same": c_same,
221
+ "c_diff": c_diff,
222
+ "rho_total": rho_total,
223
+ "rho_item": rho_item,
224
+ "rho_within": rho_within,
225
+ "k_eff_within": k_eff,
226
+ "round1_independence_pass": abs(rho_within) < 0.05
227
+ if round_num == 1 and rho_within is not None
228
+ else None,
229
+ "n_items": len(grouped),
230
+ "n_replicated_items": sum(len(reps) >= 2 for reps in grouped.values()),
231
+ }
pace/meta_engine.py ADDED
@@ -0,0 +1,246 @@
1
+ """Native numpy meta-analysis and prospective power calculations."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import itertools
6
+ import math
7
+
8
+ import numpy as np
9
+
10
+
11
+ def compute_effect_sizes(
12
+ p_mas: float, p_base: float, n_items: int, phi: float = 0.5
13
+ ) -> dict:
14
+ if n_items < 1 or not 0 <= phi <= 1 or not (0 <= p_mas <= 1 and 0 <= p_base <= 1):
15
+ raise ValueError("invalid probabilities, sample size, or phi")
16
+ pt_m, pt_b = (
17
+ (p_mas * n_items + 0.5) / (n_items + 1),
18
+ (p_base * n_items + 0.5) / (n_items + 1),
19
+ )
20
+ log_or = math.log((pt_m / (1 - pt_m)) / (pt_b / (1 - pt_b)))
21
+ vm, vb = 1 / (n_items * pt_m * (1 - pt_m)), 1 / (n_items * pt_b * (1 - pt_b))
22
+ phi_max = math.exp(-abs(log_or) / 2)
23
+ v_log_or = vm + vb - 2 * phi * math.sqrt(vm * vb)
24
+ v_clamped = vm + vb - 2 * min(phi, phi_max) * math.sqrt(vm * vb)
25
+ delta = p_mas - p_base
26
+ se_delta = math.sqrt(
27
+ max(
28
+ 0.0,
29
+ (
30
+ p_mas * (1 - p_mas)
31
+ + p_base * (1 - p_base)
32
+ - 2 * phi * math.sqrt(p_mas * (1 - p_mas) * p_base * (1 - p_base))
33
+ )
34
+ / n_items,
35
+ )
36
+ )
37
+ h = 2 * math.asin(math.sqrt(p_mas)) - 2 * math.asin(math.sqrt(p_base))
38
+ return {
39
+ "p_tilde_mas": pt_m,
40
+ "p_tilde_base": pt_b,
41
+ "log_or": log_or,
42
+ "v_log_or": v_log_or,
43
+ "phi_max": phi_max,
44
+ "v_log_or_clamped": v_clamped,
45
+ "delta": delta,
46
+ "se_delta": se_delta,
47
+ "cohen_h": h,
48
+ "v_cohen_h": 2 * (1 - phi) / n_items,
49
+ "phi_clamped": phi > phi_max,
50
+ }
51
+
52
+
53
+ def shared_baseline_deff(
54
+ m_contrasts_sharing_baseline: int, v_mas: float, v_base: float
55
+ ) -> dict:
56
+ if m_contrasts_sharing_baseline < 1 or min(v_mas, v_base) < 0:
57
+ raise ValueError("invalid variance or block size")
58
+ rho = v_base / (v_mas + v_base) if v_mas + v_base else 0.0
59
+ deff = 1 + (m_contrasts_sharing_baseline - 1) * rho
60
+ naive = (v_mas + v_base) / m_contrasts_sharing_baseline
61
+ return {
62
+ "rho_base": rho,
63
+ "deff": deff,
64
+ "naive_diagonal_variance": naive,
65
+ "true_shared_baseline_variance": naive * deff,
66
+ }
67
+
68
+
69
+ def build_che_covariance(
70
+ contrasts: list[dict], rho_block: float = 0.5, block_by: str = "baseline_cell"
71
+ ) -> np.ndarray:
72
+ if not 0 <= rho_block <= 1 or block_by not in {
73
+ "baseline_cell",
74
+ "benchmark",
75
+ "study",
76
+ }:
77
+ raise ValueError("invalid rho_block or block_by")
78
+ keys = {
79
+ "baseline_cell": lambda r: (
80
+ r.get("study"),
81
+ r.get("benchmark"),
82
+ r.get("model"),
83
+ r.get("category"),
84
+ ),
85
+ "benchmark": lambda r: (r.get("study"), r.get("benchmark")),
86
+ "study": lambda r: (r.get("study"),),
87
+ }[block_by]
88
+ n = len(contrasts)
89
+ matrix = np.zeros((n, n))
90
+ variances = [
91
+ float(r.get("variance", 1 / max(1, int(float(r.get("n", 1)))))) for r in contrasts
92
+ ]
93
+ for i in range(n):
94
+ matrix[i, i] = variances[i]
95
+ for j in range(i):
96
+ if keys(contrasts[i]) == keys(contrasts[j]):
97
+ matrix[i, j] = matrix[j, i] = rho_block * math.sqrt(
98
+ variances[i] * variances[j]
99
+ )
100
+ return matrix
101
+
102
+
103
+ def der_simonian_laird_pool(y: np.ndarray, v: np.ndarray, clusters: list[str]) -> dict:
104
+ y, v = np.asarray(y, dtype=float), np.asarray(v, dtype=float)
105
+ if y.ndim != 1 or len(y) != len(v) or len(y) != len(clusters) or np.any(v <= 0):
106
+ raise ValueError("y, v, and clusters must align and v must be positive")
107
+ w = 1 / v
108
+ fixed = float(np.sum(w * y) / np.sum(w))
109
+ q = float(np.sum(w * (y - fixed) ** 2))
110
+ df = len(y) - 1
111
+ tau2 = max(0.0, (q - df) / (np.sum(w) - np.sum(w**2) / np.sum(w))) if df else 0.0
112
+ wr = 1 / (v + tau2)
113
+ estimate = float(np.sum(wr * y) / np.sum(wr))
114
+ groups = sorted(set(clusters))
115
+ scores = {
116
+ g: float(
117
+ np.sum(
118
+ wr[[i for i, c in enumerate(clusters) if c == g]]
119
+ * (y[[i for i, c in enumerate(clusters) if c == g]] - estimate)
120
+ )
121
+ )
122
+ for g in groups
123
+ }
124
+ sandwich = (
125
+ sum(score**2 for score in scores.values())
126
+ / max(1, len(groups) - 1)
127
+ / max(np.sum(wr) ** 2, 1e-12)
128
+ )
129
+ se = math.sqrt(max(0.0, sandwich))
130
+ df_sat = max(1.0, len(groups) - 1)
131
+ ci = [estimate - 1.96 * se, estimate + 1.96 * se]
132
+ signs = []
133
+ if len(groups) <= 16:
134
+ for signs_tuple in itertools.product((-1, 1), repeat=len(groups)):
135
+ signs.append(
136
+ abs(sum(scores[g] * sign for g, sign in zip(groups, signs_tuple)))
137
+ )
138
+ p_value = sum(value >= abs(sum(scores.values())) for value in signs) / len(
139
+ signs
140
+ )
141
+ else:
142
+ p_value = None
143
+ loco = []
144
+ for group in groups:
145
+ mask = np.array([c != group for c in clusters])
146
+ pooled = (
147
+ der_simonian_laird_pool(y[mask], v[mask], list(np.asarray(clusters)[mask]))
148
+ if mask.sum() > 1
149
+ else None
150
+ )
151
+ loco.append(
152
+ {
153
+ "cluster": group,
154
+ "estimate": None if pooled is None else pooled["estimate"],
155
+ }
156
+ )
157
+ return {
158
+ "estimate": estimate,
159
+ "tau2_dl": tau2,
160
+ "se": se,
161
+ "df": df_sat,
162
+ "ci95": ci,
163
+ "prediction_interval_study": [
164
+ estimate - 1.96 * math.sqrt(se * se + tau2),
165
+ estimate + 1.96 * math.sqrt(se * se + tau2),
166
+ ],
167
+ "prediction_interval_single": [
168
+ estimate - 1.96 * math.sqrt(se * se + tau2),
169
+ estimate + 1.96 * math.sqrt(se * se + tau2),
170
+ ],
171
+ "p_value": p_value,
172
+ "loco": loco,
173
+ "q": q,
174
+ }
175
+
176
+
177
+ def prospective_power_analysis(
178
+ p_base: float,
179
+ target_delta: float,
180
+ n_items: int | None = None,
181
+ target_power: float = 0.8,
182
+ alpha: float = 0.05,
183
+ phi: float = 0.5,
184
+ k_shared_arms: int = 1,
185
+ ) -> dict:
186
+ if (
187
+ not 0 < p_base < 1
188
+ or target_delta == 0
189
+ or not 0 < target_power < 1
190
+ or not 0 < alpha < 1
191
+ or k_shared_arms < 1
192
+ ):
193
+ raise ValueError("invalid power parameters")
194
+ adjusted_alpha = alpha / k_shared_arms
195
+ z_alpha, z_power = (
196
+ 1.96 if adjusted_alpha <= 0.05 else 1.645,
197
+ 0.8416 if target_power >= 0.8 else 0.5244,
198
+ )
199
+ variance = (
200
+ p_base * (1 - p_base)
201
+ + (p_base + target_delta) * (1 - p_base - target_delta)
202
+ - 2
203
+ * phi
204
+ * math.sqrt(
205
+ p_base
206
+ * (1 - p_base)
207
+ * (p_base + target_delta)
208
+ * (1 - p_base - target_delta)
209
+ )
210
+ )
211
+ n_min = math.ceil(((z_alpha + z_power) ** 2 * variance) / target_delta**2)
212
+ achieved = (
213
+ None
214
+ if n_items is None
215
+ else min(
216
+ 1.0,
217
+ max(
218
+ 0.0,
219
+ 0.5
220
+ * (
221
+ 1
222
+ + math.erf(
223
+ (
224
+ abs(target_delta)
225
+ * math.sqrt(n_items / max(variance, 1e-12))
226
+ - z_alpha
227
+ )
228
+ / math.sqrt(2)
229
+ )
230
+ ),
231
+ ),
232
+ )
233
+ )
234
+ mde = (
235
+ None
236
+ if n_items is None
237
+ else (z_alpha + z_power) * math.sqrt(max(variance, 1e-12) / n_items)
238
+ )
239
+ return {
240
+ "n_min_items": n_min,
241
+ "achieved_power": achieved,
242
+ "mde": mde,
243
+ "alpha_adjusted": adjusted_alpha,
244
+ "phi": phi,
245
+ "n_items": n_items,
246
+ }