pace-eval 0.2.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cme/__init__.py +11 -0
- cme/__main__.py +5 -0
- cme/analysis.py +223 -0
- cme/audit.py +145 -0
- cme/baseline.py +162 -0
- cme/cli.py +279 -0
- cme/client.py +165 -0
- cme/dynamics.py +212 -0
- cme/examples/crewai_integration.py +16 -0
- cme/examples/demo_pool.jsonl +360 -0
- cme/examples/demo_run.jsonl +480 -0
- cme/examples/langgraph_integration.py +27 -0
- cme/examples/make_demo.py +37 -0
- cme/examples/openai_quickstart.py +18 -0
- cme/experiment.py +555 -0
- cme/frontier.py +143 -0
- cme/hidden_profile.py +135 -0
- cme/proxy.py +106 -0
- cme/rebuttal.py +76 -0
- cme/report.py +197 -0
- cme/rollout_card.py +137 -0
- cme/tasks.py +266 -0
- cme/verified_debate.py +127 -0
- cme/verifier.py +92 -0
- mae_eval/__init__.py +11 -0
- mae_eval/__main__.py +5 -0
- mae_eval/auditor.py +112 -0
- mae_eval/classifier.py +62 -0
- mae_eval/cli.py +210 -0
- mae_eval/tracker.py +162 -0
- mae_eval/vector.py +91 -0
- pace/__init__.py +11 -0
- pace/__main__.py +5 -0
- pace/analysis.py +129 -0
- pace/audit.py +145 -0
- pace/baseline.py +162 -0
- pace/card.py +205 -0
- pace/cli.py +898 -0
- pace/client.py +165 -0
- pace/dynamics.py +372 -0
- pace/escape_benchmarks.py +162 -0
- pace/examples/crewai_integration.py +16 -0
- pace/examples/demo_pool.jsonl +360 -0
- pace/examples/demo_run.jsonl +480 -0
- pace/examples/langgraph_integration.py +27 -0
- pace/examples/make_demo.py +37 -0
- pace/examples/openai_quickstart.py +18 -0
- pace/experiment.py +555 -0
- pace/frontier.py +114 -0
- pace/hidden_profile.py +301 -0
- pace/importers.py +219 -0
- pace/inflate.py +213 -0
- pace/influence.py +231 -0
- pace/meta_engine.py +246 -0
- pace/middleware.py +230 -0
- pace/open_ended.py +114 -0
- pace/plotting.py +132 -0
- pace/pool_sizer.py +154 -0
- pace/protocol_v3.py +414 -0
- pace/proxy.py +106 -0
- pace/rebuttal.py +219 -0
- pace/report.py +197 -0
- pace/rollout_card.py +137 -0
- pace/schema.py +74 -0
- pace/simulator.py +199 -0
- pace/studio.html +1040 -0
- pace/tasks.py +266 -0
- pace/verified_debate.py +127 -0
- pace/verifier.py +92 -0
- pace/workbench.html +26 -0
- pace/workbench.py +119 -0
- pace_eval-0.2.2.dist-info/METADATA +205 -0
- pace_eval-0.2.2.dist-info/RECORD +83 -0
- pace_eval-0.2.2.dist-info/WHEEL +5 -0
- pace_eval-0.2.2.dist-info/entry_points.txt +4 -0
- pace_eval-0.2.2.dist-info/licenses/LICENSE +202 -0
- pace_eval-0.2.2.dist-info/top_level.txt +4 -0
- pace_lab/__init__.py +9 -0
- pace_lab/clients.py +568 -0
- pace_lab/concurrency_test.py +127 -0
- pace_lab/data.py +445 -0
- pace_lab/graders.py +626 -0
- pace_lab/strategies.py +345 -0
pace/cli.py
ADDED
|
@@ -0,0 +1,898 @@
|
|
|
1
|
+
"""pace command line: proxy | baseline | report."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import argparse
|
|
6
|
+
import json
|
|
7
|
+
import os
|
|
8
|
+
import sys
|
|
9
|
+
from collections.abc import Sequence
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def _proxy(a) -> int:
|
|
13
|
+
from .proxy import serve
|
|
14
|
+
|
|
15
|
+
srv = serve(
|
|
16
|
+
a.upstream,
|
|
17
|
+
a.log,
|
|
18
|
+
a.port,
|
|
19
|
+
a.host,
|
|
20
|
+
a.price_in,
|
|
21
|
+
a.price_out,
|
|
22
|
+
a.api_key,
|
|
23
|
+
a.item_file,
|
|
24
|
+
a.system_label,
|
|
25
|
+
)
|
|
26
|
+
print(
|
|
27
|
+
"pace proxy on http://%s:%d/v1 -> %s ; logging to %s (Ctrl+C to stop)"
|
|
28
|
+
% (a.host, a.port, a.upstream, a.log),
|
|
29
|
+
file=sys.stderr,
|
|
30
|
+
)
|
|
31
|
+
try:
|
|
32
|
+
srv.serve_forever()
|
|
33
|
+
except KeyboardInterrupt:
|
|
34
|
+
pass
|
|
35
|
+
return 0
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def _baseline(a) -> int:
|
|
39
|
+
from .baseline import OpenAICompatClient, load_bench, ollama_digest, run_baseline
|
|
40
|
+
|
|
41
|
+
items = load_bench(a.bench, a.max_items)
|
|
42
|
+
digest = ollama_digest(a.model) if "11434" in a.base_url else None
|
|
43
|
+
res = run_baseline(
|
|
44
|
+
OpenAICompatClient(a.base_url, a.api_key),
|
|
45
|
+
a.model,
|
|
46
|
+
items,
|
|
47
|
+
a.out,
|
|
48
|
+
a.k,
|
|
49
|
+
a.temperature,
|
|
50
|
+
a.seed,
|
|
51
|
+
a.max_tokens,
|
|
52
|
+
digest,
|
|
53
|
+
progress=(lambda m: print(m, file=sys.stderr)) if a.verbose else None,
|
|
54
|
+
)
|
|
55
|
+
print(json.dumps(res))
|
|
56
|
+
return 0 if res["errors"] == 0 else 1
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def _report(a) -> int:
|
|
60
|
+
from .report import build_report, plot, to_latex
|
|
61
|
+
|
|
62
|
+
try:
|
|
63
|
+
rep = build_report(
|
|
64
|
+
a.pool,
|
|
65
|
+
a.logs,
|
|
66
|
+
a.system,
|
|
67
|
+
a.price_in,
|
|
68
|
+
a.price_out,
|
|
69
|
+
a.n_boot,
|
|
70
|
+
a.perms,
|
|
71
|
+
a.seed,
|
|
72
|
+
a.k,
|
|
73
|
+
)
|
|
74
|
+
except (OSError, ValueError) as e:
|
|
75
|
+
print("error: %s" % e, file=sys.stderr)
|
|
76
|
+
return 1
|
|
77
|
+
g = rep["frontier_gap"]
|
|
78
|
+
print(
|
|
79
|
+
"items %d | pool k=%d | system accuracy %.3f at %.0f tokens/item | frontier %.3f (%s) | gap %+.3f [%+.3f, %+.3f]"
|
|
80
|
+
% (
|
|
81
|
+
rep["n_items"],
|
|
82
|
+
rep["pool_k"],
|
|
83
|
+
g["system_accuracy"],
|
|
84
|
+
g["system_mean_tokens"],
|
|
85
|
+
g["frontier_at_system_tokens"],
|
|
86
|
+
g["range_status"],
|
|
87
|
+
g["gap"],
|
|
88
|
+
g["ci_low"],
|
|
89
|
+
g["ci_high"],
|
|
90
|
+
)
|
|
91
|
+
)
|
|
92
|
+
rc = rep["resource_card"]
|
|
93
|
+
if rc:
|
|
94
|
+
print(
|
|
95
|
+
"MAS/SAS ratios: "
|
|
96
|
+
+ ", ".join(
|
|
97
|
+
"%s=%s" % (k, "n/a" if v is None else "%.3g" % v)
|
|
98
|
+
for k, v in rc["ratios_mas_over_sas"].items()
|
|
99
|
+
)
|
|
100
|
+
)
|
|
101
|
+
if a.json:
|
|
102
|
+
json.dump(rep, open(a.json, "w", encoding="utf-8"), indent=2, default=str)
|
|
103
|
+
if a.latex:
|
|
104
|
+
open(a.latex, "w", encoding="utf-8").write(to_latex(rep))
|
|
105
|
+
if a.plot:
|
|
106
|
+
plot(rep, a.plot)
|
|
107
|
+
if a.export_latex or a.export_markdown:
|
|
108
|
+
from .rebuttal import row_from_report, to_markdown
|
|
109
|
+
from .rebuttal import to_latex as rb_tex
|
|
110
|
+
|
|
111
|
+
rows = [row_from_report(rep, a.benchmark, a.model_label)]
|
|
112
|
+
if a.export_latex:
|
|
113
|
+
open(a.export_latex, "w", encoding="utf-8").write(rb_tex(rows))
|
|
114
|
+
if a.export_markdown:
|
|
115
|
+
open(a.export_markdown, "w", encoding="utf-8").write(to_markdown(rows))
|
|
116
|
+
if a.export_rollout_card:
|
|
117
|
+
from .rollout_card import write_block
|
|
118
|
+
|
|
119
|
+
for path in write_block(rep, a.export_rollout_card, a.run_id):
|
|
120
|
+
print("wrote " + path)
|
|
121
|
+
return 0
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
def _inflate(a) -> int:
|
|
125
|
+
from .inflate import estimate_wrong_options, inflate_report, successes_from_rows
|
|
126
|
+
|
|
127
|
+
try:
|
|
128
|
+
rows = [json.loads(line) for line in open(a.samples, encoding="utf-8") if line.strip()]
|
|
129
|
+
succ, n = successes_from_rows(rows)
|
|
130
|
+
ks = [int(x) for x in a.k.split(",") if x.strip()]
|
|
131
|
+
w = None
|
|
132
|
+
if a.wrong_options == "auto":
|
|
133
|
+
w = estimate_wrong_options(rows)
|
|
134
|
+
if w is None:
|
|
135
|
+
raise ValueError("--wrong-options auto needs rows with an answer label and at least one item with two wrong samples")
|
|
136
|
+
elif a.wrong_options:
|
|
137
|
+
w = float(a.wrong_options)
|
|
138
|
+
rep = inflate_report(succ, n, ks, wrong_options=w)
|
|
139
|
+
except (OSError, ValueError, KeyError) as e:
|
|
140
|
+
print("error: %s" % e, file=sys.stderr)
|
|
141
|
+
return 1
|
|
142
|
+
if a.json:
|
|
143
|
+
print(json.dumps(rep, indent=1))
|
|
144
|
+
return 0
|
|
145
|
+
print("items=%d pool=%d Beta(alpha=%.3f, beta=%.3f) single-sample accuracy %.3f (observed %.3f)" % (rep["n_items"], rep["pool_size"], rep["alpha"], rep["beta"], rep["mean_single_sample_accuracy"], rep["single_sample_accuracy_observed"]))
|
|
146
|
+
print("%5s %10s %13s %14s %18s" % ("K", "predicted", "pool_observed", "over_1_sample", "plurality(W=%s)" % (("%.2f" % rep["wrong_options"]) if rep.get("wrong_options") else "-")))
|
|
147
|
+
for r in rep["inflation"]:
|
|
148
|
+
print("%5d %10.3f %13s %14.3f %18s" % (r["K"], r["predicted"], ("%.3f" % r["pool_observed"]) if "pool_observed" in r else "n/a", r["over_single_sample"], ("%.3f" % r["predicted_plurality"]) if "predicted_plurality" in r else "n/a"))
|
|
149
|
+
print(rep["model"])
|
|
150
|
+
return 0
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
def _classify(a) -> int:
|
|
154
|
+
from mae_eval.classifier import classify_with_reason
|
|
155
|
+
|
|
156
|
+
try:
|
|
157
|
+
cfg = json.load(open(a.config, encoding="utf-8"))
|
|
158
|
+
rung, rule, reason = classify_with_reason(cfg)
|
|
159
|
+
except (OSError, ValueError, TypeError) as e:
|
|
160
|
+
print("error: %s" % e, file=sys.stderr)
|
|
161
|
+
return 1
|
|
162
|
+
print(json.dumps({"rung": rung, "rule": rule, "reason": reason}))
|
|
163
|
+
return 0
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
def _audit(a) -> int:
|
|
167
|
+
from .audit import audit, render
|
|
168
|
+
|
|
169
|
+
try:
|
|
170
|
+
card = audit(a.input, a.baseline_config, a.tolerance, a.price_in, a.price_out)
|
|
171
|
+
except (OSError, ValueError) as e:
|
|
172
|
+
print("error: %s" % e, file=sys.stderr)
|
|
173
|
+
return 1
|
|
174
|
+
if a.json:
|
|
175
|
+
print(json.dumps(card, indent=2, default=lambda x: None if x != x else str(x)))
|
|
176
|
+
else:
|
|
177
|
+
text = render(card)
|
|
178
|
+
try:
|
|
179
|
+
text.encode(sys.stdout.encoding or "utf-8")
|
|
180
|
+
except (
|
|
181
|
+
UnicodeEncodeError,
|
|
182
|
+
LookupError,
|
|
183
|
+
): # a console without box-drawing characters (for example a Windows code page): fall back to ASCII
|
|
184
|
+
text = render(card, ascii_only=True)
|
|
185
|
+
print(text)
|
|
186
|
+
return 0 if card["ok"] or not a.strict else 2
|
|
187
|
+
|
|
188
|
+
|
|
189
|
+
def _dynamics(a) -> int:
|
|
190
|
+
from .dynamics import analyse, excess_over_null, render
|
|
191
|
+
|
|
192
|
+
try:
|
|
193
|
+
res = analyse(a.log, a.n_boot, a.seed, a.first_round, a.last_round)
|
|
194
|
+
if getattr(a, "null", None):
|
|
195
|
+
null_res = analyse(a.null, a.n_boot, a.seed, a.first_round, a.last_round)
|
|
196
|
+
res["excess_over_null"] = excess_over_null(res, null_res)
|
|
197
|
+
except (OSError, ValueError) as e:
|
|
198
|
+
print("error: %s" % e, file=sys.stderr)
|
|
199
|
+
return 1
|
|
200
|
+
if a.json:
|
|
201
|
+
print(json.dumps(res, indent=2, default=str))
|
|
202
|
+
else:
|
|
203
|
+
print(render(res))
|
|
204
|
+
return 0
|
|
205
|
+
|
|
206
|
+
|
|
207
|
+
def _import_logs(a) -> int:
|
|
208
|
+
from .importers import load_and_import, write_jsonl
|
|
209
|
+
|
|
210
|
+
try:
|
|
211
|
+
count = write_jsonl(load_and_import(a.input, a.format), a.output)
|
|
212
|
+
except (OSError, ValueError, json.JSONDecodeError) as e:
|
|
213
|
+
print("error: %s" % e, file=sys.stderr)
|
|
214
|
+
return 1
|
|
215
|
+
print("wrote %d PACE records to %s" % (count, a.output))
|
|
216
|
+
return 0
|
|
217
|
+
|
|
218
|
+
|
|
219
|
+
def _card(a) -> int:
|
|
220
|
+
from .card import build_card, load_log, to_latex, to_markdown
|
|
221
|
+
|
|
222
|
+
try:
|
|
223
|
+
card = build_card(load_log(a.log))
|
|
224
|
+
text = (
|
|
225
|
+
json.dumps(card, indent=2)
|
|
226
|
+
if a.json
|
|
227
|
+
else (to_latex(card) if a.latex else to_markdown(card))
|
|
228
|
+
)
|
|
229
|
+
if a.output:
|
|
230
|
+
open(a.output, "w", encoding="utf-8").write(text)
|
|
231
|
+
else:
|
|
232
|
+
print(text, end="")
|
|
233
|
+
except (OSError, ValueError, json.JSONDecodeError) as e:
|
|
234
|
+
print("error: %s" % e, file=sys.stderr)
|
|
235
|
+
return 1
|
|
236
|
+
return 0
|
|
237
|
+
|
|
238
|
+
|
|
239
|
+
def _studio(a) -> int:
|
|
240
|
+
import webbrowser
|
|
241
|
+
from pathlib import Path
|
|
242
|
+
|
|
243
|
+
target = Path(__file__).with_name("studio.html").resolve().as_uri()
|
|
244
|
+
webbrowser.open(target)
|
|
245
|
+
print("opened PACE Studio: %s" % target)
|
|
246
|
+
return 0
|
|
247
|
+
|
|
248
|
+
|
|
249
|
+
def _workbench(a) -> int:
|
|
250
|
+
from .workbench import export_html, serve
|
|
251
|
+
|
|
252
|
+
if a.export_html:
|
|
253
|
+
export_html(a.export_html)
|
|
254
|
+
print("wrote " + a.export_html)
|
|
255
|
+
if a.serve:
|
|
256
|
+
srv = serve(a.host, a.port, a.no_browser)
|
|
257
|
+
print("PACE Workbench: http://%s:%d/" % (a.host, a.port), file=sys.stderr)
|
|
258
|
+
try:
|
|
259
|
+
srv.serve_forever()
|
|
260
|
+
except KeyboardInterrupt:
|
|
261
|
+
pass
|
|
262
|
+
return 0
|
|
263
|
+
|
|
264
|
+
|
|
265
|
+
def _card_v7(a) -> int:
|
|
266
|
+
from .workbench import card_from_file, card_v7_to_latex, card_v7_to_markdown
|
|
267
|
+
|
|
268
|
+
try:
|
|
269
|
+
card = card_from_file(a.log, a.baseline_log)
|
|
270
|
+
if a.json:
|
|
271
|
+
text = json.dumps(card, indent=2)
|
|
272
|
+
elif a.latex:
|
|
273
|
+
text = card_v7_to_latex(card)
|
|
274
|
+
else:
|
|
275
|
+
text = card_v7_to_markdown(card)
|
|
276
|
+
if a.output:
|
|
277
|
+
open(a.output, "w", encoding="utf-8").write(text)
|
|
278
|
+
else:
|
|
279
|
+
print(text, end="")
|
|
280
|
+
except (OSError, ValueError, json.JSONDecodeError) as exc:
|
|
281
|
+
print("error: %s" % exc, file=sys.stderr)
|
|
282
|
+
return 1
|
|
283
|
+
return 0
|
|
284
|
+
|
|
285
|
+
|
|
286
|
+
def _pool_size(a) -> int:
|
|
287
|
+
from .pool_sizer import compute_k_tok
|
|
288
|
+
|
|
289
|
+
result = compute_k_tok(
|
|
290
|
+
a.agents, a.rounds, a.prompt_tokens, a.msg_tokens, a.topology
|
|
291
|
+
)
|
|
292
|
+
result.pop("b_eff", None)
|
|
293
|
+
if a.json:
|
|
294
|
+
print(json.dumps(result, indent=2))
|
|
295
|
+
else:
|
|
296
|
+
print(
|
|
297
|
+
"k_tok %.3f (r=%.4f), recommended pool size %d"
|
|
298
|
+
% (result["k_tok"], result["r"], result["recommended_pool_size"])
|
|
299
|
+
)
|
|
300
|
+
return 0
|
|
301
|
+
|
|
302
|
+
|
|
303
|
+
def _simulate(a) -> int:
|
|
304
|
+
from . import simulator
|
|
305
|
+
|
|
306
|
+
if a.mode == "voting":
|
|
307
|
+
result = simulator.simulate_correlated_voting(
|
|
308
|
+
a.items, a.agents, a.p_mean, a.rho, a.choices, a.seed
|
|
309
|
+
)
|
|
310
|
+
elif a.mode == "debate":
|
|
311
|
+
result = simulator.simulate_debate_dynamics(
|
|
312
|
+
a.items,
|
|
313
|
+
a.agents,
|
|
314
|
+
a.rounds,
|
|
315
|
+
a.p_mean,
|
|
316
|
+
a.regime,
|
|
317
|
+
{"a": a.rate_a, "b": a.rate_b},
|
|
318
|
+
a.replicates,
|
|
319
|
+
a.verifier_fp,
|
|
320
|
+
a.verifier_fn,
|
|
321
|
+
a.seed,
|
|
322
|
+
)
|
|
323
|
+
else:
|
|
324
|
+
result = simulator.simulate_scaffolding_scaling_curve(
|
|
325
|
+
a.budgets, a.agents, a.rounds, a.prompt_tokens, a.msg_tokens
|
|
326
|
+
)
|
|
327
|
+
if a.out:
|
|
328
|
+
with open(a.out, "w", encoding="utf-8") as handle:
|
|
329
|
+
if isinstance(result, list):
|
|
330
|
+
handle.writelines(json.dumps(row) + "\n" for row in result)
|
|
331
|
+
else:
|
|
332
|
+
handle.write(json.dumps(result) + "\n")
|
|
333
|
+
else:
|
|
334
|
+
print(json.dumps(result, indent=2))
|
|
335
|
+
return 0
|
|
336
|
+
|
|
337
|
+
|
|
338
|
+
def _meta(a) -> int:
|
|
339
|
+
import csv
|
|
340
|
+
|
|
341
|
+
from .meta_engine import (
|
|
342
|
+
build_che_covariance,
|
|
343
|
+
compute_effect_sizes,
|
|
344
|
+
der_simonian_laird_pool,
|
|
345
|
+
)
|
|
346
|
+
|
|
347
|
+
with open(a.contrasts, encoding="utf-8", newline="") as handle:
|
|
348
|
+
rows = list(csv.DictReader(handle))
|
|
349
|
+
effects = [
|
|
350
|
+
compute_effect_sizes(
|
|
351
|
+
float(row["p_mas"]), float(row["p_base"]), int(float(row.get("n", 1))), a.phi
|
|
352
|
+
)
|
|
353
|
+
for row in rows
|
|
354
|
+
]
|
|
355
|
+
np_mod = __import__("numpy")
|
|
356
|
+
y = np_mod.array(
|
|
357
|
+
[
|
|
358
|
+
effect[a.scale] if a.scale != "delta" else effect["delta"]
|
|
359
|
+
for effect in effects
|
|
360
|
+
]
|
|
361
|
+
)
|
|
362
|
+
if a.scale == "cohen_h":
|
|
363
|
+
v = np_mod.array([effect["v_cohen_h"] for effect in effects])
|
|
364
|
+
elif a.scale == "delta":
|
|
365
|
+
v_list = []
|
|
366
|
+
for row in rows:
|
|
367
|
+
pm = float(row["p_mas"])
|
|
368
|
+
pb = float(row["p_base"])
|
|
369
|
+
n_val = max(1.0, float(row.get("n", 1)))
|
|
370
|
+
am, ab = pm * (1.0 - pm), pb * (1.0 - pb)
|
|
371
|
+
v_d = max(1e-10, (am + ab - 2.0 * a.phi * (am * ab) ** 0.5) / n_val)
|
|
372
|
+
v_list.append(v_d)
|
|
373
|
+
v = np_mod.array(v_list)
|
|
374
|
+
else:
|
|
375
|
+
v = np_mod.array([effect["v_log_or"] for effect in effects])
|
|
376
|
+
result = {
|
|
377
|
+
"estimator": "DerSimonian-Laird",
|
|
378
|
+
"scale": a.scale,
|
|
379
|
+
"effects": effects,
|
|
380
|
+
"covariance": build_che_covariance(rows, a.rho_block).tolist(),
|
|
381
|
+
}
|
|
382
|
+
if len(rows) > 1:
|
|
383
|
+
result["pooled"] = der_simonian_laird_pool(
|
|
384
|
+
y, v, [row.get("study", "study") for row in rows]
|
|
385
|
+
)
|
|
386
|
+
print(json.dumps(result, indent=2) if a.json else json.dumps(result))
|
|
387
|
+
return 0
|
|
388
|
+
|
|
389
|
+
|
|
390
|
+
def _power(a) -> int:
|
|
391
|
+
from .meta_engine import prospective_power_analysis
|
|
392
|
+
|
|
393
|
+
result = prospective_power_analysis(
|
|
394
|
+
a.p_base, a.delta, a.n_items, a.power, a.alpha, a.phi, a.shared_arms
|
|
395
|
+
)
|
|
396
|
+
print(json.dumps(result, indent=2) if a.json else json.dumps(result))
|
|
397
|
+
return 0
|
|
398
|
+
|
|
399
|
+
|
|
400
|
+
def _benchmark_gen(a) -> int:
|
|
401
|
+
from .escape_benchmarks import (
|
|
402
|
+
evaluate_with_verifier_threshold,
|
|
403
|
+
generate_clue_grid_items,
|
|
404
|
+
generate_distributed_code_items,
|
|
405
|
+
)
|
|
406
|
+
|
|
407
|
+
if a.route == "clue_grid":
|
|
408
|
+
result = generate_clue_grid_items(a.items, a.agents, a.seed)
|
|
409
|
+
elif a.route == "distributed_code":
|
|
410
|
+
result = generate_distributed_code_items(a.items, a.agents, a.seed)
|
|
411
|
+
else:
|
|
412
|
+
result = [
|
|
413
|
+
evaluate_with_verifier_threshold(0.5, 0.2, 0.1, epsilon, 0.1)
|
|
414
|
+
for epsilon in a.epsilons
|
|
415
|
+
]
|
|
416
|
+
with open(a.out, "w", encoding="utf-8") as handle:
|
|
417
|
+
handle.writelines(json.dumps(row) + "\n" for row in result)
|
|
418
|
+
return 0
|
|
419
|
+
|
|
420
|
+
|
|
421
|
+
def _aggregate_open(a) -> int:
|
|
422
|
+
from .open_ended import mbr_centroid_select
|
|
423
|
+
|
|
424
|
+
records = [
|
|
425
|
+
json.loads(line) for line in open(a.input, encoding="utf-8") if line.strip()
|
|
426
|
+
]
|
|
427
|
+
output = []
|
|
428
|
+
for row in records:
|
|
429
|
+
candidates = row.get("candidates", row.get("answers", []))
|
|
430
|
+
output.append(
|
|
431
|
+
{
|
|
432
|
+
"item": row.get("item", row.get("id", len(output))),
|
|
433
|
+
**mbr_centroid_select(candidates, a.metric, row.get("token_costs")),
|
|
434
|
+
}
|
|
435
|
+
)
|
|
436
|
+
text = "".join(json.dumps(row) + "\n" for row in output)
|
|
437
|
+
if a.out:
|
|
438
|
+
open(a.out, "w", encoding="utf-8").write(text)
|
|
439
|
+
else:
|
|
440
|
+
print(text, end="")
|
|
441
|
+
return 0
|
|
442
|
+
|
|
443
|
+
|
|
444
|
+
def _plot(a) -> int:
|
|
445
|
+
from . import plotting
|
|
446
|
+
|
|
447
|
+
if a.kind == "waterfall":
|
|
448
|
+
result = plotting.svg_waterfall_decomposition(0.4, 0.6, 0.7)
|
|
449
|
+
elif a.kind == "phase":
|
|
450
|
+
result = plotting.svg_phase_diagram()
|
|
451
|
+
elif a.kind == "frontier":
|
|
452
|
+
result = plotting.svg_frontier_curve(json.load(open(a.input, encoding="utf-8")))
|
|
453
|
+
else:
|
|
454
|
+
payload = json.load(open(a.input, encoding="utf-8"))
|
|
455
|
+
result = plotting.svg_influence_curves(
|
|
456
|
+
payload["by_correct_peers"],
|
|
457
|
+
payload.get("fit_result", payload),
|
|
458
|
+
payload.get("k_agents", 3),
|
|
459
|
+
)
|
|
460
|
+
open(a.out, "w", encoding="utf-8").write(result)
|
|
461
|
+
return 0
|
|
462
|
+
|
|
463
|
+
|
|
464
|
+
def _hidden_profile(a) -> int:
|
|
465
|
+
from .hidden_profile import generate, valid, validate
|
|
466
|
+
|
|
467
|
+
try:
|
|
468
|
+
items = generate(a.n, a.agents, a.criteria, a.split, a.margin, a.seed, a.format)
|
|
469
|
+
except ValueError as e:
|
|
470
|
+
print("error: %s" % e, file=sys.stderr)
|
|
471
|
+
return 1
|
|
472
|
+
counts = validate(items)
|
|
473
|
+
msg = (
|
|
474
|
+
"generated %d items (agents %d, criteria %d, split %d, margin %d, seed %d, format %s); decoy leads a view in violation: %d, target not leading pooled: %d, uncovered split criteria: %d"
|
|
475
|
+
% (
|
|
476
|
+
counts["items"],
|
|
477
|
+
a.agents,
|
|
478
|
+
a.criteria,
|
|
479
|
+
a.split,
|
|
480
|
+
a.margin,
|
|
481
|
+
a.seed,
|
|
482
|
+
a.format,
|
|
483
|
+
counts["decoy_not_leading_a_view"],
|
|
484
|
+
counts["target_not_leading_pooled"],
|
|
485
|
+
counts["uncovered_split_criteria"],
|
|
486
|
+
)
|
|
487
|
+
)
|
|
488
|
+
print(msg, file=sys.stderr)
|
|
489
|
+
if not valid(counts):
|
|
490
|
+
print("error: validity check failed; no output written", file=sys.stderr)
|
|
491
|
+
return 1
|
|
492
|
+
text = chr(10).join(json.dumps(it) for it in items) + chr(10)
|
|
493
|
+
if a.out:
|
|
494
|
+
with open(a.out, "w", encoding="utf-8") as fh:
|
|
495
|
+
fh.write(text)
|
|
496
|
+
else:
|
|
497
|
+
sys.stdout.write(text)
|
|
498
|
+
return 0
|
|
499
|
+
|
|
500
|
+
|
|
501
|
+
def _frontier(a) -> int:
|
|
502
|
+
from .report import frontier_report, plot_frontier, system_points
|
|
503
|
+
|
|
504
|
+
try:
|
|
505
|
+
fr = frontier_report(a.samples, a.k, a.perms, a.seed)
|
|
506
|
+
pts = system_points(a.logs) if a.logs else []
|
|
507
|
+
except (OSError, ValueError, KeyError) as e:
|
|
508
|
+
print("error: %s" % e, file=sys.stderr)
|
|
509
|
+
return 1
|
|
510
|
+
print(
|
|
511
|
+
"items %d | pool k=%d | greedy accuracy %s"
|
|
512
|
+
% (
|
|
513
|
+
fr["n_items"],
|
|
514
|
+
fr["pool_k"],
|
|
515
|
+
"n/a" if fr["greedy_accuracy"] is None else "%.3f" % fr["greedy_accuracy"],
|
|
516
|
+
)
|
|
517
|
+
)
|
|
518
|
+
print("k mean tokens SC accuracy isotonic")
|
|
519
|
+
for i, (t, x, y) in enumerate(
|
|
520
|
+
zip(fr["tokens"], fr["sc_accuracy"], fr["isotonic"]), 1
|
|
521
|
+
):
|
|
522
|
+
print("%-2d %11.0f %11.3f %8.3f" % (i, t, x, y))
|
|
523
|
+
for p in pts:
|
|
524
|
+
print(
|
|
525
|
+
"system %-12s accuracy %.3f at %.0f tokens/item (n=%d)"
|
|
526
|
+
% (p["system"], p["accuracy"], p["mean_tokens"], p["n_items"])
|
|
527
|
+
)
|
|
528
|
+
if a.json:
|
|
529
|
+
json.dump(
|
|
530
|
+
{"frontier": fr, "systems": pts},
|
|
531
|
+
open(a.json, "w", encoding="utf-8"),
|
|
532
|
+
indent=2,
|
|
533
|
+
)
|
|
534
|
+
if a.out:
|
|
535
|
+
plot_frontier(fr, pts, a.out)
|
|
536
|
+
return 0
|
|
537
|
+
|
|
538
|
+
|
|
539
|
+
def _rebuttal(a) -> int:
|
|
540
|
+
from .rebuttal import rows_from_contrasts, to_markdown
|
|
541
|
+
from .rebuttal import to_latex as rb_tex
|
|
542
|
+
|
|
543
|
+
if a.contrasts == "outputs/baseline_tier_contrasts.csv" and not os.path.exists(a.contrasts) and os.path.exists("data/baseline_tier_contrasts.csv"):
|
|
544
|
+
a.contrasts = "data/baseline_tier_contrasts.csv" # release tree layout: the corpus lives in data/
|
|
545
|
+
try:
|
|
546
|
+
rows = rows_from_contrasts(a.contrasts, a.study)
|
|
547
|
+
except (OSError, KeyError, ValueError) as e:
|
|
548
|
+
print("error: %s" % e, file=sys.stderr)
|
|
549
|
+
return 1
|
|
550
|
+
if not rows:
|
|
551
|
+
print("error: no rows", file=sys.stderr)
|
|
552
|
+
return 1
|
|
553
|
+
if a.export_latex:
|
|
554
|
+
open(a.export_latex, "w", encoding="utf-8").write(rb_tex(rows))
|
|
555
|
+
if a.export_markdown:
|
|
556
|
+
open(a.export_markdown, "w", encoding="utf-8").write(to_markdown(rows))
|
|
557
|
+
if not (a.export_latex or a.export_markdown):
|
|
558
|
+
print(to_markdown(rows))
|
|
559
|
+
return 0
|
|
560
|
+
|
|
561
|
+
|
|
562
|
+
def build_parser() -> argparse.ArgumentParser:
|
|
563
|
+
p = argparse.ArgumentParser(
|
|
564
|
+
prog="pace",
|
|
565
|
+
description="Compute-matched evaluation: measure systems against the self-consistency frontier.",
|
|
566
|
+
)
|
|
567
|
+
sub = p.add_subparsers(dest="cmd", required=True)
|
|
568
|
+
x = sub.add_parser("proxy", help="OpenAI-compatible logging proxy")
|
|
569
|
+
x.add_argument("--upstream", default="http://localhost:11434/v1")
|
|
570
|
+
x.add_argument("--port", type=int, default=8089)
|
|
571
|
+
x.add_argument("--host", default="127.0.0.1")
|
|
572
|
+
x.add_argument("--log", default="run.jsonl")
|
|
573
|
+
x.add_argument(
|
|
574
|
+
"--price-in",
|
|
575
|
+
type=float,
|
|
576
|
+
default=None,
|
|
577
|
+
help="USD per million input tokens (never bundled)",
|
|
578
|
+
)
|
|
579
|
+
x.add_argument("--price-out", type=float, default=None)
|
|
580
|
+
x.add_argument("--api-key", default=None)
|
|
581
|
+
x.add_argument(
|
|
582
|
+
"--system-label",
|
|
583
|
+
default="mas",
|
|
584
|
+
help="value of the log's system field when a request has no X-PACE-System header",
|
|
585
|
+
)
|
|
586
|
+
x.add_argument(
|
|
587
|
+
"--item-file",
|
|
588
|
+
default=None,
|
|
589
|
+
help="file whose first line is the current item id, used when a request has no X-PACE-Item header (serial runs of third-party code)",
|
|
590
|
+
)
|
|
591
|
+
x.set_defaults(func=_proxy)
|
|
592
|
+
b = sub.add_parser("baseline", help="greedy + k-sample pool, cached and resumable")
|
|
593
|
+
b.add_argument("--model", required=True)
|
|
594
|
+
b.add_argument(
|
|
595
|
+
"--bench",
|
|
596
|
+
required=True,
|
|
597
|
+
help="path to JSONL {id, question, answer}, or 'toy' (synthetic smoke test)",
|
|
598
|
+
)
|
|
599
|
+
b.add_argument("--base-url", default="http://localhost:11434/v1")
|
|
600
|
+
b.add_argument("--api-key", default="none")
|
|
601
|
+
b.add_argument("--k", type=int, default=16)
|
|
602
|
+
b.add_argument("--temperature", type=float, default=0.7)
|
|
603
|
+
b.add_argument("--seed", type=int, default=0)
|
|
604
|
+
b.add_argument("--max-tokens", type=int, default=2048)
|
|
605
|
+
b.add_argument("--max-items", type=int, default=None)
|
|
606
|
+
b.add_argument("--out", default="baseline_cache.jsonl")
|
|
607
|
+
b.add_argument("--verbose", action="store_true")
|
|
608
|
+
b.set_defaults(func=_baseline)
|
|
609
|
+
r = sub.add_parser(
|
|
610
|
+
"report", help="frontier gap, resource card, LaTeX table and figure"
|
|
611
|
+
)
|
|
612
|
+
r.add_argument("--pool", required=True, help="baseline cache from `pace baseline`")
|
|
613
|
+
r.add_argument("--logs", required=True, help="system log (calls and results)")
|
|
614
|
+
r.add_argument("--system", default="mas")
|
|
615
|
+
r.add_argument("--price-in", type=float, default=None)
|
|
616
|
+
r.add_argument("--price-out", type=float, default=None)
|
|
617
|
+
r.add_argument(
|
|
618
|
+
"--k", type=int, default=None, help="use the first k samples per item"
|
|
619
|
+
)
|
|
620
|
+
r.add_argument("--n-boot", type=int, default=2000)
|
|
621
|
+
r.add_argument("--perms", type=int, default=100)
|
|
622
|
+
r.add_argument("--seed", type=int, default=0)
|
|
623
|
+
r.add_argument("--json", default=None)
|
|
624
|
+
r.add_argument("--latex", default=None)
|
|
625
|
+
r.add_argument("--plot", default=None, help="vector figure path (.pdf)")
|
|
626
|
+
r.add_argument(
|
|
627
|
+
"--export-latex",
|
|
628
|
+
default=None,
|
|
629
|
+
help="write the nine-column rebuttal table for this run (LaTeX)",
|
|
630
|
+
)
|
|
631
|
+
r.add_argument(
|
|
632
|
+
"--export-markdown",
|
|
633
|
+
default=None,
|
|
634
|
+
help="write the nine-column rebuttal table for this run (Markdown)",
|
|
635
|
+
)
|
|
636
|
+
r.add_argument(
|
|
637
|
+
"--benchmark", default="n/a", help="benchmark label for the exported row"
|
|
638
|
+
)
|
|
639
|
+
r.add_argument(
|
|
640
|
+
"--model-label", default="n/a", help="model label for the exported row"
|
|
641
|
+
)
|
|
642
|
+
r.add_argument(
|
|
643
|
+
"--export-rollout-card",
|
|
644
|
+
default=None,
|
|
645
|
+
metavar="PREFIX",
|
|
646
|
+
help="write PREFIX.json, PREFIX.yaml and PREFIX.tex: a PACE compute block for a rollout card (arXiv:2605.12131)",
|
|
647
|
+
)
|
|
648
|
+
r.add_argument(
|
|
649
|
+
"--run-id",
|
|
650
|
+
default="run",
|
|
651
|
+
help="target id used in the rollout-card annotation rows",
|
|
652
|
+
)
|
|
653
|
+
r.set_defaults(func=_report)
|
|
654
|
+
inf = sub.add_parser("inflate", help="beta-binomial sampling inflation: predicted accuracy of a K-sample vote from a pool of samples (CPU only)")
|
|
655
|
+
inf.add_argument("--samples", required=True, help="JSONL, one row per sample with item and correct")
|
|
656
|
+
inf.add_argument("--k", default="1,3,5,9,16,24", help="comma-separated vote sizes K to predict")
|
|
657
|
+
inf.add_argument("--wrong-options", default=None, help="plurality model: number of wrong options W the wrong answers spread over (a number), or auto to estimate it from answer labels in the rows")
|
|
658
|
+
inf.add_argument("--json", action="store_true")
|
|
659
|
+
inf.set_defaults(func=_inflate)
|
|
660
|
+
c = sub.add_parser(
|
|
661
|
+
"classify",
|
|
662
|
+
help="baseline rung (A to D) of a single-agent configuration given as JSON",
|
|
663
|
+
)
|
|
664
|
+
c.add_argument(
|
|
665
|
+
"--config",
|
|
666
|
+
required=True,
|
|
667
|
+
help="JSON file; keys are listed in code/mae_eval/classifier.py",
|
|
668
|
+
)
|
|
669
|
+
c.set_defaults(func=_classify)
|
|
670
|
+
u = sub.add_parser(
|
|
671
|
+
"audit",
|
|
672
|
+
help="one-screen diagnostic card for a run log (mas and sas calls) or a baseline configuration (JSON)",
|
|
673
|
+
)
|
|
674
|
+
u.add_argument(
|
|
675
|
+
"input",
|
|
676
|
+
help="JSONL run log with 'mas' and 'sas' calls, or a JSON baseline configuration",
|
|
677
|
+
)
|
|
678
|
+
u.add_argument(
|
|
679
|
+
"--baseline-config",
|
|
680
|
+
default=None,
|
|
681
|
+
help="JSON configuration of the single-agent baseline, to classify its rung when INPUT is a log",
|
|
682
|
+
)
|
|
683
|
+
u.add_argument("--tolerance", type=float, default=0.05)
|
|
684
|
+
u.add_argument("--price-in", type=float, default=None)
|
|
685
|
+
u.add_argument("--price-out", type=float, default=None)
|
|
686
|
+
u.add_argument("--json", action="store_true")
|
|
687
|
+
u.add_argument(
|
|
688
|
+
"--strict",
|
|
689
|
+
action="store_true",
|
|
690
|
+
help="exit with status 2 when the badge is a warning",
|
|
691
|
+
)
|
|
692
|
+
u.set_defaults(func=_audit)
|
|
693
|
+
d = sub.add_parser(
|
|
694
|
+
"dynamics",
|
|
695
|
+
help="flip counts, tau rates, implied threshold, peer correlation, K_eff and tokens by round from a debate log",
|
|
696
|
+
)
|
|
697
|
+
d.add_argument(
|
|
698
|
+
"log",
|
|
699
|
+
help="JSONL with item, agent, round, correct (and optionally prompt_tokens, completion_tokens) per agent answer",
|
|
700
|
+
)
|
|
701
|
+
d.add_argument(
|
|
702
|
+
"--null",
|
|
703
|
+
default=None,
|
|
704
|
+
help="optional no-communication control JSONL log to compute excess transition rates and excess peer correlation",
|
|
705
|
+
)
|
|
706
|
+
d.add_argument("--n-boot", type=int, default=1000)
|
|
707
|
+
d.add_argument("--seed", type=int, default=0)
|
|
708
|
+
d.add_argument("--first-round", type=int, default=None)
|
|
709
|
+
d.add_argument("--last-round", type=int, default=None)
|
|
710
|
+
d.add_argument("--json", action="store_true")
|
|
711
|
+
d.set_defaults(func=_dynamics)
|
|
712
|
+
i = sub.add_parser("import-logs", help="convert an external trace to PACE JSONL")
|
|
713
|
+
i.add_argument(
|
|
714
|
+
"--format", choices=["autogen", "langgraph", "crewai", "inspect", "csv"], required=True
|
|
715
|
+
)
|
|
716
|
+
i.add_argument("--input", required=True)
|
|
717
|
+
i.add_argument("--output", required=True)
|
|
718
|
+
i.set_defaults(func=_import_logs)
|
|
719
|
+
k = sub.add_parser("card", help="render a standalone PACE Card from PACE JSONL")
|
|
720
|
+
k.add_argument("log")
|
|
721
|
+
k.add_argument("--output", default=None)
|
|
722
|
+
k.add_argument("--json", action="store_true")
|
|
723
|
+
k.add_argument("--latex", action="store_true")
|
|
724
|
+
k.set_defaults(func=_card)
|
|
725
|
+
s = sub.add_parser("studio", help="open the zero-dependency PACE Studio")
|
|
726
|
+
s.set_defaults(func=_studio)
|
|
727
|
+
wb = sub.add_parser(
|
|
728
|
+
"workbench", help="serve or export the standalone PACE Workbench"
|
|
729
|
+
)
|
|
730
|
+
wb.add_argument("--export-html", default=None)
|
|
731
|
+
wb.add_argument("--serve", action="store_true")
|
|
732
|
+
wb.add_argument("--host", default="127.0.0.1")
|
|
733
|
+
wb.add_argument("--port", type=int, default=8765)
|
|
734
|
+
wb.add_argument("--no-browser", action="store_true")
|
|
735
|
+
wb.set_defaults(func=_workbench)
|
|
736
|
+
cv7 = sub.add_parser("card-v7", help="render the seven-field PACE evaluation card")
|
|
737
|
+
cv7.add_argument("log")
|
|
738
|
+
cv7.add_argument("--baseline-log", default=None)
|
|
739
|
+
cv7.add_argument("--latex", action="store_true")
|
|
740
|
+
cv7.add_argument("--markdown", action="store_true")
|
|
741
|
+
cv7.add_argument("--json", action="store_true")
|
|
742
|
+
cv7.add_argument("--output", default=None)
|
|
743
|
+
cv7.set_defaults(func=_card_v7)
|
|
744
|
+
z = sub.add_parser(
|
|
745
|
+
"pool-size", help="constructive token budget and recommended sample pool"
|
|
746
|
+
)
|
|
747
|
+
z.add_argument("--agents", type=int, required=True)
|
|
748
|
+
z.add_argument("--rounds", type=int, required=True)
|
|
749
|
+
z.add_argument("--prompt-tokens", type=float, required=True)
|
|
750
|
+
z.add_argument("--msg-tokens", type=float, required=True)
|
|
751
|
+
z.add_argument(
|
|
752
|
+
"--topology",
|
|
753
|
+
choices=["last_round_peers", "last_round_all", "full_history_echo"],
|
|
754
|
+
default="last_round_peers",
|
|
755
|
+
)
|
|
756
|
+
z.add_argument("--json", action="store_true")
|
|
757
|
+
z.set_defaults(func=_pool_size)
|
|
758
|
+
sim = sub.add_parser("simulate", help="generate PACE theoretical simulation data")
|
|
759
|
+
sim.add_argument(
|
|
760
|
+
"--mode", choices=["voting", "debate", "scaffolding"], required=True
|
|
761
|
+
)
|
|
762
|
+
sim.add_argument("--items", type=int, default=100)
|
|
763
|
+
sim.add_argument("--agents", type=int, default=5)
|
|
764
|
+
sim.add_argument("--rounds", type=int, default=3)
|
|
765
|
+
sim.add_argument("--p-mean", type=float, default=0.7)
|
|
766
|
+
sim.add_argument("--rho", type=float, default=0.2)
|
|
767
|
+
sim.add_argument("--choices", type=int, default=2)
|
|
768
|
+
sim.add_argument("--regime", default="constant")
|
|
769
|
+
sim.add_argument("--rate-a", type=float, default=0.25)
|
|
770
|
+
sim.add_argument("--rate-b", type=float, default=0.15)
|
|
771
|
+
sim.add_argument("--replicates", type=int, default=1)
|
|
772
|
+
sim.add_argument("--verifier-fp", type=float, default=None)
|
|
773
|
+
sim.add_argument("--verifier-fn", type=float, default=None)
|
|
774
|
+
sim.add_argument("--budgets", type=float, nargs="*", default=[1000, 5000])
|
|
775
|
+
sim.add_argument("--prompt-tokens", type=float, default=100)
|
|
776
|
+
sim.add_argument("--msg-tokens", type=float, default=20)
|
|
777
|
+
sim.add_argument("--seed", type=int, default=0)
|
|
778
|
+
sim.add_argument("--out", default=None)
|
|
779
|
+
sim.set_defaults(func=_simulate)
|
|
780
|
+
meta = sub.add_parser("meta", help="native PACE meta-analysis")
|
|
781
|
+
meta.add_argument("--contrasts", required=True)
|
|
782
|
+
meta.add_argument(
|
|
783
|
+
"--scale", choices=["log_or", "delta", "cohen_h"], default="log_or"
|
|
784
|
+
)
|
|
785
|
+
meta.add_argument("--phi", type=float, default=0.5)
|
|
786
|
+
meta.add_argument("--rho-block", type=float, default=0.5)
|
|
787
|
+
meta.add_argument("--json", action="store_true")
|
|
788
|
+
meta.set_defaults(func=_meta)
|
|
789
|
+
power = sub.add_parser("power", help="prospective PACE power analysis")
|
|
790
|
+
power.add_argument("--p-base", type=float, required=True)
|
|
791
|
+
power.add_argument("--delta", type=float, required=True)
|
|
792
|
+
power.add_argument("--n-items", type=int, default=None)
|
|
793
|
+
power.add_argument("--power", type=float, default=0.8)
|
|
794
|
+
power.add_argument("--alpha", type=float, default=0.05)
|
|
795
|
+
power.add_argument("--phi", type=float, default=0.5)
|
|
796
|
+
power.add_argument("--shared-arms", type=int, default=1)
|
|
797
|
+
power.add_argument("--json", action="store_true")
|
|
798
|
+
power.set_defaults(func=_power)
|
|
799
|
+
bench = sub.add_parser(
|
|
800
|
+
"benchmark-gen", help="generate escape-route benchmark fixtures"
|
|
801
|
+
)
|
|
802
|
+
bench.add_argument(
|
|
803
|
+
"--route",
|
|
804
|
+
choices=["clue_grid", "distributed_code", "verifier_curve"],
|
|
805
|
+
required=True,
|
|
806
|
+
)
|
|
807
|
+
bench.add_argument("--items", type=int, default=30)
|
|
808
|
+
bench.add_argument("--agents", type=int, default=3)
|
|
809
|
+
bench.add_argument("--seed", type=int, default=42)
|
|
810
|
+
bench.add_argument("--epsilons", type=float, nargs="*", default=[0.01, 0.05, 0.1])
|
|
811
|
+
bench.add_argument("--out", required=True)
|
|
812
|
+
bench.set_defaults(func=_benchmark_gen)
|
|
813
|
+
open_cmd = sub.add_parser("aggregate-open", help="aggregate open-ended samples")
|
|
814
|
+
open_cmd.add_argument("--input", required=True)
|
|
815
|
+
open_cmd.add_argument(
|
|
816
|
+
"--metric",
|
|
817
|
+
choices=["ngram_cosine", "token_jaccard", "sequence_ratio", "code_structural"],
|
|
818
|
+
default="ngram_cosine",
|
|
819
|
+
)
|
|
820
|
+
open_cmd.add_argument("--out", default=None)
|
|
821
|
+
open_cmd.set_defaults(func=_aggregate_open)
|
|
822
|
+
plot = sub.add_parser("plot", help="write a pure SVG diagnostic figure")
|
|
823
|
+
plot.add_argument(
|
|
824
|
+
"--kind", choices=["waterfall", "phase", "frontier", "influence"], required=True
|
|
825
|
+
)
|
|
826
|
+
plot.add_argument("--input", default=None)
|
|
827
|
+
plot.add_argument("--out", required=True)
|
|
828
|
+
plot.set_defaults(func=_plot)
|
|
829
|
+
h = sub.add_parser(
|
|
830
|
+
"hidden-profile",
|
|
831
|
+
help="generate synthetic split-evidence (hidden-profile) items and check their validity",
|
|
832
|
+
)
|
|
833
|
+
h.add_argument("--n", type=int, default=100, help="number of items")
|
|
834
|
+
h.add_argument("--agents", type=int, default=5)
|
|
835
|
+
h.add_argument("--criteria", type=int, default=10, help="total number of criteria")
|
|
836
|
+
h.add_argument(
|
|
837
|
+
"--split",
|
|
838
|
+
type=int,
|
|
839
|
+
default=6,
|
|
840
|
+
help="number of criteria divided among the agents (the rest are shared)",
|
|
841
|
+
)
|
|
842
|
+
h.add_argument(
|
|
843
|
+
"--margin",
|
|
844
|
+
type=int,
|
|
845
|
+
default=3,
|
|
846
|
+
help="score gap per criterion between the favoured and the disfavoured candidate (at least 2)",
|
|
847
|
+
)
|
|
848
|
+
h.add_argument("--seed", type=int, default=42)
|
|
849
|
+
h.add_argument("--format", choices=["text", "numeric"], default="text")
|
|
850
|
+
h.add_argument(
|
|
851
|
+
"--out", default=None, help="output JSONL (default: standard output)"
|
|
852
|
+
)
|
|
853
|
+
h.set_defaults(func=_hidden_profile)
|
|
854
|
+
f = sub.add_parser(
|
|
855
|
+
"frontier",
|
|
856
|
+
help="self-consistency frontier of a sample pool, optionally with system points from a run log",
|
|
857
|
+
)
|
|
858
|
+
f.add_argument(
|
|
859
|
+
"--samples",
|
|
860
|
+
required=True,
|
|
861
|
+
help="baseline cache from `pace baseline` (greedy + sampled completions per item)",
|
|
862
|
+
)
|
|
863
|
+
f.add_argument(
|
|
864
|
+
"--logs",
|
|
865
|
+
default=None,
|
|
866
|
+
help="run log; every value of its 'system' field is drawn as a point",
|
|
867
|
+
)
|
|
868
|
+
f.add_argument("--k", type=int, default=None)
|
|
869
|
+
f.add_argument("--perms", type=int, default=100)
|
|
870
|
+
f.add_argument("--seed", type=int, default=0)
|
|
871
|
+
f.add_argument("--json", default=None)
|
|
872
|
+
f.add_argument("--out", default=None, help="figure path (.pdf vector, or .png)")
|
|
873
|
+
f.set_defaults(func=_frontier)
|
|
874
|
+
t = sub.add_parser(
|
|
875
|
+
"rebuttal",
|
|
876
|
+
help="nine-column rebuttal table from a contrasts CSV (default: outputs/baseline_tier_contrasts.csv, or data/baseline_tier_contrasts.csv in the release tree)",
|
|
877
|
+
)
|
|
878
|
+
t.add_argument(
|
|
879
|
+
"--contrasts",
|
|
880
|
+
"--data",
|
|
881
|
+
dest="contrasts",
|
|
882
|
+
default="outputs/baseline_tier_contrasts.csv",
|
|
883
|
+
help="contrasts CSV with columns study, category (rung), baseline, key 'Study|bench|model|arm', p_base, p_mas",
|
|
884
|
+
)
|
|
885
|
+
t.add_argument("--study", default=None)
|
|
886
|
+
t.add_argument("--export-latex", default=None)
|
|
887
|
+
t.add_argument("--export-markdown", default=None)
|
|
888
|
+
t.set_defaults(func=_rebuttal)
|
|
889
|
+
return p
|
|
890
|
+
|
|
891
|
+
|
|
892
|
+
def main(argv: Sequence[str] | None = None) -> int:
|
|
893
|
+
a = build_parser().parse_args(argv)
|
|
894
|
+
return a.func(a)
|
|
895
|
+
|
|
896
|
+
|
|
897
|
+
if __name__ == "__main__":
|
|
898
|
+
sys.exit(main())
|