pace-eval 0.2.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cme/__init__.py +11 -0
- cme/__main__.py +5 -0
- cme/analysis.py +223 -0
- cme/audit.py +145 -0
- cme/baseline.py +162 -0
- cme/cli.py +279 -0
- cme/client.py +165 -0
- cme/dynamics.py +212 -0
- cme/examples/crewai_integration.py +16 -0
- cme/examples/demo_pool.jsonl +360 -0
- cme/examples/demo_run.jsonl +480 -0
- cme/examples/langgraph_integration.py +27 -0
- cme/examples/make_demo.py +37 -0
- cme/examples/openai_quickstart.py +18 -0
- cme/experiment.py +555 -0
- cme/frontier.py +143 -0
- cme/hidden_profile.py +135 -0
- cme/proxy.py +106 -0
- cme/rebuttal.py +76 -0
- cme/report.py +197 -0
- cme/rollout_card.py +137 -0
- cme/tasks.py +266 -0
- cme/verified_debate.py +127 -0
- cme/verifier.py +92 -0
- mae_eval/__init__.py +11 -0
- mae_eval/__main__.py +5 -0
- mae_eval/auditor.py +112 -0
- mae_eval/classifier.py +62 -0
- mae_eval/cli.py +210 -0
- mae_eval/tracker.py +162 -0
- mae_eval/vector.py +91 -0
- pace/__init__.py +11 -0
- pace/__main__.py +5 -0
- pace/analysis.py +129 -0
- pace/audit.py +145 -0
- pace/baseline.py +162 -0
- pace/card.py +205 -0
- pace/cli.py +898 -0
- pace/client.py +165 -0
- pace/dynamics.py +372 -0
- pace/escape_benchmarks.py +162 -0
- pace/examples/crewai_integration.py +16 -0
- pace/examples/demo_pool.jsonl +360 -0
- pace/examples/demo_run.jsonl +480 -0
- pace/examples/langgraph_integration.py +27 -0
- pace/examples/make_demo.py +37 -0
- pace/examples/openai_quickstart.py +18 -0
- pace/experiment.py +555 -0
- pace/frontier.py +114 -0
- pace/hidden_profile.py +301 -0
- pace/importers.py +219 -0
- pace/inflate.py +213 -0
- pace/influence.py +231 -0
- pace/meta_engine.py +246 -0
- pace/middleware.py +230 -0
- pace/open_ended.py +114 -0
- pace/plotting.py +132 -0
- pace/pool_sizer.py +154 -0
- pace/protocol_v3.py +414 -0
- pace/proxy.py +106 -0
- pace/rebuttal.py +219 -0
- pace/report.py +197 -0
- pace/rollout_card.py +137 -0
- pace/schema.py +74 -0
- pace/simulator.py +199 -0
- pace/studio.html +1040 -0
- pace/tasks.py +266 -0
- pace/verified_debate.py +127 -0
- pace/verifier.py +92 -0
- pace/workbench.html +26 -0
- pace/workbench.py +119 -0
- pace_eval-0.2.2.dist-info/METADATA +205 -0
- pace_eval-0.2.2.dist-info/RECORD +83 -0
- pace_eval-0.2.2.dist-info/WHEEL +5 -0
- pace_eval-0.2.2.dist-info/entry_points.txt +4 -0
- pace_eval-0.2.2.dist-info/licenses/LICENSE +202 -0
- pace_eval-0.2.2.dist-info/top_level.txt +4 -0
- pace_lab/__init__.py +9 -0
- pace_lab/clients.py +568 -0
- pace_lab/concurrency_test.py +127 -0
- pace_lab/data.py +445 -0
- pace_lab/graders.py +626 -0
- pace_lab/strategies.py +345 -0
pace/simulator.py
ADDED
|
@@ -0,0 +1,199 @@
|
|
|
1
|
+
"""Deterministic Monte Carlo generators for PACE theory diagnostics."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import numpy as np
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def _validate_probability(value: float, name: str) -> None:
|
|
9
|
+
if not 0 <= value <= 1:
|
|
10
|
+
raise ValueError("%s must be in [0, 1]" % name)
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def _majority(values: np.ndarray, n_choices: int) -> int:
|
|
14
|
+
counts = np.bincount(values.astype(int), minlength=n_choices)
|
|
15
|
+
leaders = np.flatnonzero(counts == counts.max())
|
|
16
|
+
return int(leaders[0])
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def simulate_correlated_voting(
|
|
20
|
+
n_items: int,
|
|
21
|
+
k_agents: int,
|
|
22
|
+
p_mean: float,
|
|
23
|
+
rho_peer: float,
|
|
24
|
+
n_choices: int = 2,
|
|
25
|
+
seed: int = 0,
|
|
26
|
+
) -> dict:
|
|
27
|
+
"""Simulate item difficulty as a beta latent variable and correlated answers."""
|
|
28
|
+
if n_items < 1 or k_agents < 1 or n_choices < 2:
|
|
29
|
+
raise ValueError(
|
|
30
|
+
"n_items, k_agents, and n_choices must be positive; n_choices >= 2"
|
|
31
|
+
)
|
|
32
|
+
_validate_probability(p_mean, "p_mean")
|
|
33
|
+
_validate_probability(rho_peer, "rho_peer")
|
|
34
|
+
rng = np.random.default_rng(seed)
|
|
35
|
+
concentration = max(2.0, 1.0 / max(rho_peer, 1e-9) - 1.0) if rho_peer else 1e9
|
|
36
|
+
alpha, beta = (
|
|
37
|
+
max(1e-6, p_mean * concentration),
|
|
38
|
+
max(1e-6, (1 - p_mean) * concentration),
|
|
39
|
+
)
|
|
40
|
+
truth = rng.integers(0, n_choices, n_items)
|
|
41
|
+
answers = np.empty((n_items, k_agents), dtype=int)
|
|
42
|
+
for i in range(n_items):
|
|
43
|
+
difficulty = float(rng.beta(alpha, beta))
|
|
44
|
+
for agent in range(k_agents):
|
|
45
|
+
answers[i, agent] = (
|
|
46
|
+
truth[i]
|
|
47
|
+
if rng.random() < difficulty
|
|
48
|
+
else int(
|
|
49
|
+
rng.integers(0, n_choices - 1)
|
|
50
|
+
+ (rng.integers(0, n_choices) >= truth[i])
|
|
51
|
+
)
|
|
52
|
+
% n_choices
|
|
53
|
+
)
|
|
54
|
+
correct = answers == truth[:, None]
|
|
55
|
+
p1 = float(correct.mean())
|
|
56
|
+
votes = np.array([_majority(row, n_choices) for row in answers])
|
|
57
|
+
p_vote = float(np.mean(votes == truth))
|
|
58
|
+
correlations = []
|
|
59
|
+
if k_agents > 1:
|
|
60
|
+
correlations = np.corrcoef(correct.T)[np.triu_indices(k_agents, 1)].tolist()
|
|
61
|
+
rho_total = float(np.nanmean(correlations)) if correlations else 0.0
|
|
62
|
+
k_eff = k_agents / (1 + (k_agents - 1) * rho_peer)
|
|
63
|
+
hoeffding = (
|
|
64
|
+
max(0.0, 1.0 - np.exp(-2 * k_eff * max(p_mean - 0.5, 0) ** 2))
|
|
65
|
+
if n_choices == 2
|
|
66
|
+
else max(0.0, 1.0 - np.exp(-2 * k_eff * max(p_mean - 1 / n_choices, 0) ** 2))
|
|
67
|
+
)
|
|
68
|
+
return {
|
|
69
|
+
"p1": p1,
|
|
70
|
+
"p_vote": p_vote,
|
|
71
|
+
"rho_total": rho_total,
|
|
72
|
+
"rho_peer": rho_peer,
|
|
73
|
+
"k_eff": k_eff,
|
|
74
|
+
"hoeffding_lower_bound": hoeffding,
|
|
75
|
+
"answers": answers.tolist(),
|
|
76
|
+
"truth": truth.tolist(),
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def simulate_debate_dynamics(
|
|
81
|
+
n_items: int,
|
|
82
|
+
k_agents: int,
|
|
83
|
+
t_rounds: int,
|
|
84
|
+
p1: float,
|
|
85
|
+
regime: str = "constant",
|
|
86
|
+
params: dict | None = None,
|
|
87
|
+
n_replicates: int = 1,
|
|
88
|
+
verifier_fp: float | None = None,
|
|
89
|
+
verifier_fn: float | None = None,
|
|
90
|
+
seed: int = 0,
|
|
91
|
+
) -> list[dict]:
|
|
92
|
+
"""Generate scored, round-indexed transcripts under a peer-influence regime."""
|
|
93
|
+
if min(n_items, k_agents, t_rounds, n_replicates) < 1:
|
|
94
|
+
raise ValueError("counts must be positive")
|
|
95
|
+
_validate_probability(p1, "p1")
|
|
96
|
+
if regime not in {"constant", "linear_martingale", "convex_amplification"}:
|
|
97
|
+
raise ValueError("unknown regime: %s" % regime)
|
|
98
|
+
params = params or {"a": 0.25, "b": 0.15}
|
|
99
|
+
a, b = float(params.get("a", 0.25)), float(params.get("b", 0.15))
|
|
100
|
+
if min(a, b) < 0 or max(a, b) > 1:
|
|
101
|
+
raise ValueError("regime rates must be in [0, 1]")
|
|
102
|
+
if verifier_fp is not None or verifier_fn is not None:
|
|
103
|
+
_validate_probability(float(verifier_fp or 0), "verifier_fp")
|
|
104
|
+
_validate_probability(float(verifier_fn or 0), "verifier_fn")
|
|
105
|
+
rng = np.random.default_rng(seed)
|
|
106
|
+
rows = []
|
|
107
|
+
for item_index in range(n_items):
|
|
108
|
+
for replicate in range(n_replicates):
|
|
109
|
+
states = rng.random(k_agents) < p1
|
|
110
|
+
item = (
|
|
111
|
+
"item_%03d#rep%d" % (item_index, replicate)
|
|
112
|
+
if n_replicates > 1
|
|
113
|
+
else "item_%03d" % item_index
|
|
114
|
+
)
|
|
115
|
+
for round_num in range(1, t_rounds + 1):
|
|
116
|
+
for agent in range(k_agents):
|
|
117
|
+
prompt = 100 + (round_num - 1) * k_agents * 20
|
|
118
|
+
rows.append(
|
|
119
|
+
{
|
|
120
|
+
"item": item,
|
|
121
|
+
"replicate": replicate,
|
|
122
|
+
"agent": "agent_%d" % agent,
|
|
123
|
+
"round": round_num,
|
|
124
|
+
"correct": bool(states[agent]),
|
|
125
|
+
"answer": "A" if states[agent] else "B",
|
|
126
|
+
"prompt_tokens": prompt,
|
|
127
|
+
"completion_tokens": 20,
|
|
128
|
+
}
|
|
129
|
+
)
|
|
130
|
+
if round_num == t_rounds:
|
|
131
|
+
continue
|
|
132
|
+
previous = states.copy()
|
|
133
|
+
correct_peers = int(previous.sum())
|
|
134
|
+
for agent in range(k_agents):
|
|
135
|
+
x = (
|
|
136
|
+
(correct_peers - int(previous[agent])) / max(1, k_agents - 1)
|
|
137
|
+
if previous[agent]
|
|
138
|
+
else (k_agents - 1 - correct_peers) / max(1, k_agents - 1)
|
|
139
|
+
)
|
|
140
|
+
power = (
|
|
141
|
+
0
|
|
142
|
+
if regime == "constant"
|
|
143
|
+
else 1
|
|
144
|
+
if regime == "linear_martingale"
|
|
145
|
+
else 2
|
|
146
|
+
)
|
|
147
|
+
rate = (a if not previous[agent] else b) * (
|
|
148
|
+
x**power if power else 1
|
|
149
|
+
)
|
|
150
|
+
if rng.random() < rate:
|
|
151
|
+
is_improvement = not previous[agent]
|
|
152
|
+
if (
|
|
153
|
+
verifier_fn is not None
|
|
154
|
+
and is_improvement
|
|
155
|
+
and rng.random() < verifier_fn
|
|
156
|
+
):
|
|
157
|
+
continue
|
|
158
|
+
if (
|
|
159
|
+
verifier_fp is not None
|
|
160
|
+
and not is_improvement
|
|
161
|
+
and rng.random() < verifier_fp
|
|
162
|
+
):
|
|
163
|
+
continue
|
|
164
|
+
states[agent] = not states[agent]
|
|
165
|
+
return rows
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
def simulate_scaffolding_scaling_curve(
|
|
169
|
+
budgets: list[float],
|
|
170
|
+
k_agents: int,
|
|
171
|
+
t_rounds: int,
|
|
172
|
+
c0: float,
|
|
173
|
+
l_msg: float,
|
|
174
|
+
b_half: float = 500.0,
|
|
175
|
+
p_min: float = 0.25,
|
|
176
|
+
p_max: float = 0.85,
|
|
177
|
+
) -> list[dict]:
|
|
178
|
+
"""Map total budgets to crowded MAS and SAS effective-thinking curves."""
|
|
179
|
+
if min(k_agents, t_rounds) < 1 or b_half <= 0 or p_max < p_min:
|
|
180
|
+
raise ValueError("invalid scaling parameters")
|
|
181
|
+
out = []
|
|
182
|
+
cost = (
|
|
183
|
+
k_agents * t_rounds * c0 + 0.5 * k_agents**2 * t_rounds * (t_rounds - 1) * l_msg
|
|
184
|
+
)
|
|
185
|
+
for budget in budgets:
|
|
186
|
+
b_eff = max(0.0, (float(budget) - cost) / (k_agents * t_rounds))
|
|
187
|
+
sas = max(0.0, float(budget) - c0)
|
|
188
|
+
hill = lambda b: p_min + (p_max - p_min) * b / (b + b_half)
|
|
189
|
+
out.append(
|
|
190
|
+
{
|
|
191
|
+
"budget": float(budget),
|
|
192
|
+
"c_total": cost,
|
|
193
|
+
"b_eff": b_eff,
|
|
194
|
+
"b_eff_sas": sas,
|
|
195
|
+
"p_mas": hill(b_eff),
|
|
196
|
+
"p_sas": hill(sas),
|
|
197
|
+
}
|
|
198
|
+
)
|
|
199
|
+
return out
|