pace-eval 0.2.2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (83) hide show
  1. cme/__init__.py +11 -0
  2. cme/__main__.py +5 -0
  3. cme/analysis.py +223 -0
  4. cme/audit.py +145 -0
  5. cme/baseline.py +162 -0
  6. cme/cli.py +279 -0
  7. cme/client.py +165 -0
  8. cme/dynamics.py +212 -0
  9. cme/examples/crewai_integration.py +16 -0
  10. cme/examples/demo_pool.jsonl +360 -0
  11. cme/examples/demo_run.jsonl +480 -0
  12. cme/examples/langgraph_integration.py +27 -0
  13. cme/examples/make_demo.py +37 -0
  14. cme/examples/openai_quickstart.py +18 -0
  15. cme/experiment.py +555 -0
  16. cme/frontier.py +143 -0
  17. cme/hidden_profile.py +135 -0
  18. cme/proxy.py +106 -0
  19. cme/rebuttal.py +76 -0
  20. cme/report.py +197 -0
  21. cme/rollout_card.py +137 -0
  22. cme/tasks.py +266 -0
  23. cme/verified_debate.py +127 -0
  24. cme/verifier.py +92 -0
  25. mae_eval/__init__.py +11 -0
  26. mae_eval/__main__.py +5 -0
  27. mae_eval/auditor.py +112 -0
  28. mae_eval/classifier.py +62 -0
  29. mae_eval/cli.py +210 -0
  30. mae_eval/tracker.py +162 -0
  31. mae_eval/vector.py +91 -0
  32. pace/__init__.py +11 -0
  33. pace/__main__.py +5 -0
  34. pace/analysis.py +129 -0
  35. pace/audit.py +145 -0
  36. pace/baseline.py +162 -0
  37. pace/card.py +205 -0
  38. pace/cli.py +898 -0
  39. pace/client.py +165 -0
  40. pace/dynamics.py +372 -0
  41. pace/escape_benchmarks.py +162 -0
  42. pace/examples/crewai_integration.py +16 -0
  43. pace/examples/demo_pool.jsonl +360 -0
  44. pace/examples/demo_run.jsonl +480 -0
  45. pace/examples/langgraph_integration.py +27 -0
  46. pace/examples/make_demo.py +37 -0
  47. pace/examples/openai_quickstart.py +18 -0
  48. pace/experiment.py +555 -0
  49. pace/frontier.py +114 -0
  50. pace/hidden_profile.py +301 -0
  51. pace/importers.py +219 -0
  52. pace/inflate.py +213 -0
  53. pace/influence.py +231 -0
  54. pace/meta_engine.py +246 -0
  55. pace/middleware.py +230 -0
  56. pace/open_ended.py +114 -0
  57. pace/plotting.py +132 -0
  58. pace/pool_sizer.py +154 -0
  59. pace/protocol_v3.py +414 -0
  60. pace/proxy.py +106 -0
  61. pace/rebuttal.py +219 -0
  62. pace/report.py +197 -0
  63. pace/rollout_card.py +137 -0
  64. pace/schema.py +74 -0
  65. pace/simulator.py +199 -0
  66. pace/studio.html +1040 -0
  67. pace/tasks.py +266 -0
  68. pace/verified_debate.py +127 -0
  69. pace/verifier.py +92 -0
  70. pace/workbench.html +26 -0
  71. pace/workbench.py +119 -0
  72. pace_eval-0.2.2.dist-info/METADATA +205 -0
  73. pace_eval-0.2.2.dist-info/RECORD +83 -0
  74. pace_eval-0.2.2.dist-info/WHEEL +5 -0
  75. pace_eval-0.2.2.dist-info/entry_points.txt +4 -0
  76. pace_eval-0.2.2.dist-info/licenses/LICENSE +202 -0
  77. pace_eval-0.2.2.dist-info/top_level.txt +4 -0
  78. pace_lab/__init__.py +9 -0
  79. pace_lab/clients.py +568 -0
  80. pace_lab/concurrency_test.py +127 -0
  81. pace_lab/data.py +445 -0
  82. pace_lab/graders.py +626 -0
  83. pace_lab/strategies.py +345 -0
pace/cli.py ADDED
@@ -0,0 +1,898 @@
1
+ """pace command line: proxy | baseline | report."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import argparse
6
+ import json
7
+ import os
8
+ import sys
9
+ from collections.abc import Sequence
10
+
11
+
12
+ def _proxy(a) -> int:
13
+ from .proxy import serve
14
+
15
+ srv = serve(
16
+ a.upstream,
17
+ a.log,
18
+ a.port,
19
+ a.host,
20
+ a.price_in,
21
+ a.price_out,
22
+ a.api_key,
23
+ a.item_file,
24
+ a.system_label,
25
+ )
26
+ print(
27
+ "pace proxy on http://%s:%d/v1 -> %s ; logging to %s (Ctrl+C to stop)"
28
+ % (a.host, a.port, a.upstream, a.log),
29
+ file=sys.stderr,
30
+ )
31
+ try:
32
+ srv.serve_forever()
33
+ except KeyboardInterrupt:
34
+ pass
35
+ return 0
36
+
37
+
38
+ def _baseline(a) -> int:
39
+ from .baseline import OpenAICompatClient, load_bench, ollama_digest, run_baseline
40
+
41
+ items = load_bench(a.bench, a.max_items)
42
+ digest = ollama_digest(a.model) if "11434" in a.base_url else None
43
+ res = run_baseline(
44
+ OpenAICompatClient(a.base_url, a.api_key),
45
+ a.model,
46
+ items,
47
+ a.out,
48
+ a.k,
49
+ a.temperature,
50
+ a.seed,
51
+ a.max_tokens,
52
+ digest,
53
+ progress=(lambda m: print(m, file=sys.stderr)) if a.verbose else None,
54
+ )
55
+ print(json.dumps(res))
56
+ return 0 if res["errors"] == 0 else 1
57
+
58
+
59
+ def _report(a) -> int:
60
+ from .report import build_report, plot, to_latex
61
+
62
+ try:
63
+ rep = build_report(
64
+ a.pool,
65
+ a.logs,
66
+ a.system,
67
+ a.price_in,
68
+ a.price_out,
69
+ a.n_boot,
70
+ a.perms,
71
+ a.seed,
72
+ a.k,
73
+ )
74
+ except (OSError, ValueError) as e:
75
+ print("error: %s" % e, file=sys.stderr)
76
+ return 1
77
+ g = rep["frontier_gap"]
78
+ print(
79
+ "items %d | pool k=%d | system accuracy %.3f at %.0f tokens/item | frontier %.3f (%s) | gap %+.3f [%+.3f, %+.3f]"
80
+ % (
81
+ rep["n_items"],
82
+ rep["pool_k"],
83
+ g["system_accuracy"],
84
+ g["system_mean_tokens"],
85
+ g["frontier_at_system_tokens"],
86
+ g["range_status"],
87
+ g["gap"],
88
+ g["ci_low"],
89
+ g["ci_high"],
90
+ )
91
+ )
92
+ rc = rep["resource_card"]
93
+ if rc:
94
+ print(
95
+ "MAS/SAS ratios: "
96
+ + ", ".join(
97
+ "%s=%s" % (k, "n/a" if v is None else "%.3g" % v)
98
+ for k, v in rc["ratios_mas_over_sas"].items()
99
+ )
100
+ )
101
+ if a.json:
102
+ json.dump(rep, open(a.json, "w", encoding="utf-8"), indent=2, default=str)
103
+ if a.latex:
104
+ open(a.latex, "w", encoding="utf-8").write(to_latex(rep))
105
+ if a.plot:
106
+ plot(rep, a.plot)
107
+ if a.export_latex or a.export_markdown:
108
+ from .rebuttal import row_from_report, to_markdown
109
+ from .rebuttal import to_latex as rb_tex
110
+
111
+ rows = [row_from_report(rep, a.benchmark, a.model_label)]
112
+ if a.export_latex:
113
+ open(a.export_latex, "w", encoding="utf-8").write(rb_tex(rows))
114
+ if a.export_markdown:
115
+ open(a.export_markdown, "w", encoding="utf-8").write(to_markdown(rows))
116
+ if a.export_rollout_card:
117
+ from .rollout_card import write_block
118
+
119
+ for path in write_block(rep, a.export_rollout_card, a.run_id):
120
+ print("wrote " + path)
121
+ return 0
122
+
123
+
124
+ def _inflate(a) -> int:
125
+ from .inflate import estimate_wrong_options, inflate_report, successes_from_rows
126
+
127
+ try:
128
+ rows = [json.loads(line) for line in open(a.samples, encoding="utf-8") if line.strip()]
129
+ succ, n = successes_from_rows(rows)
130
+ ks = [int(x) for x in a.k.split(",") if x.strip()]
131
+ w = None
132
+ if a.wrong_options == "auto":
133
+ w = estimate_wrong_options(rows)
134
+ if w is None:
135
+ raise ValueError("--wrong-options auto needs rows with an answer label and at least one item with two wrong samples")
136
+ elif a.wrong_options:
137
+ w = float(a.wrong_options)
138
+ rep = inflate_report(succ, n, ks, wrong_options=w)
139
+ except (OSError, ValueError, KeyError) as e:
140
+ print("error: %s" % e, file=sys.stderr)
141
+ return 1
142
+ if a.json:
143
+ print(json.dumps(rep, indent=1))
144
+ return 0
145
+ print("items=%d pool=%d Beta(alpha=%.3f, beta=%.3f) single-sample accuracy %.3f (observed %.3f)" % (rep["n_items"], rep["pool_size"], rep["alpha"], rep["beta"], rep["mean_single_sample_accuracy"], rep["single_sample_accuracy_observed"]))
146
+ print("%5s %10s %13s %14s %18s" % ("K", "predicted", "pool_observed", "over_1_sample", "plurality(W=%s)" % (("%.2f" % rep["wrong_options"]) if rep.get("wrong_options") else "-")))
147
+ for r in rep["inflation"]:
148
+ print("%5d %10.3f %13s %14.3f %18s" % (r["K"], r["predicted"], ("%.3f" % r["pool_observed"]) if "pool_observed" in r else "n/a", r["over_single_sample"], ("%.3f" % r["predicted_plurality"]) if "predicted_plurality" in r else "n/a"))
149
+ print(rep["model"])
150
+ return 0
151
+
152
+
153
+ def _classify(a) -> int:
154
+ from mae_eval.classifier import classify_with_reason
155
+
156
+ try:
157
+ cfg = json.load(open(a.config, encoding="utf-8"))
158
+ rung, rule, reason = classify_with_reason(cfg)
159
+ except (OSError, ValueError, TypeError) as e:
160
+ print("error: %s" % e, file=sys.stderr)
161
+ return 1
162
+ print(json.dumps({"rung": rung, "rule": rule, "reason": reason}))
163
+ return 0
164
+
165
+
166
+ def _audit(a) -> int:
167
+ from .audit import audit, render
168
+
169
+ try:
170
+ card = audit(a.input, a.baseline_config, a.tolerance, a.price_in, a.price_out)
171
+ except (OSError, ValueError) as e:
172
+ print("error: %s" % e, file=sys.stderr)
173
+ return 1
174
+ if a.json:
175
+ print(json.dumps(card, indent=2, default=lambda x: None if x != x else str(x)))
176
+ else:
177
+ text = render(card)
178
+ try:
179
+ text.encode(sys.stdout.encoding or "utf-8")
180
+ except (
181
+ UnicodeEncodeError,
182
+ LookupError,
183
+ ): # a console without box-drawing characters (for example a Windows code page): fall back to ASCII
184
+ text = render(card, ascii_only=True)
185
+ print(text)
186
+ return 0 if card["ok"] or not a.strict else 2
187
+
188
+
189
+ def _dynamics(a) -> int:
190
+ from .dynamics import analyse, excess_over_null, render
191
+
192
+ try:
193
+ res = analyse(a.log, a.n_boot, a.seed, a.first_round, a.last_round)
194
+ if getattr(a, "null", None):
195
+ null_res = analyse(a.null, a.n_boot, a.seed, a.first_round, a.last_round)
196
+ res["excess_over_null"] = excess_over_null(res, null_res)
197
+ except (OSError, ValueError) as e:
198
+ print("error: %s" % e, file=sys.stderr)
199
+ return 1
200
+ if a.json:
201
+ print(json.dumps(res, indent=2, default=str))
202
+ else:
203
+ print(render(res))
204
+ return 0
205
+
206
+
207
+ def _import_logs(a) -> int:
208
+ from .importers import load_and_import, write_jsonl
209
+
210
+ try:
211
+ count = write_jsonl(load_and_import(a.input, a.format), a.output)
212
+ except (OSError, ValueError, json.JSONDecodeError) as e:
213
+ print("error: %s" % e, file=sys.stderr)
214
+ return 1
215
+ print("wrote %d PACE records to %s" % (count, a.output))
216
+ return 0
217
+
218
+
219
+ def _card(a) -> int:
220
+ from .card import build_card, load_log, to_latex, to_markdown
221
+
222
+ try:
223
+ card = build_card(load_log(a.log))
224
+ text = (
225
+ json.dumps(card, indent=2)
226
+ if a.json
227
+ else (to_latex(card) if a.latex else to_markdown(card))
228
+ )
229
+ if a.output:
230
+ open(a.output, "w", encoding="utf-8").write(text)
231
+ else:
232
+ print(text, end="")
233
+ except (OSError, ValueError, json.JSONDecodeError) as e:
234
+ print("error: %s" % e, file=sys.stderr)
235
+ return 1
236
+ return 0
237
+
238
+
239
+ def _studio(a) -> int:
240
+ import webbrowser
241
+ from pathlib import Path
242
+
243
+ target = Path(__file__).with_name("studio.html").resolve().as_uri()
244
+ webbrowser.open(target)
245
+ print("opened PACE Studio: %s" % target)
246
+ return 0
247
+
248
+
249
+ def _workbench(a) -> int:
250
+ from .workbench import export_html, serve
251
+
252
+ if a.export_html:
253
+ export_html(a.export_html)
254
+ print("wrote " + a.export_html)
255
+ if a.serve:
256
+ srv = serve(a.host, a.port, a.no_browser)
257
+ print("PACE Workbench: http://%s:%d/" % (a.host, a.port), file=sys.stderr)
258
+ try:
259
+ srv.serve_forever()
260
+ except KeyboardInterrupt:
261
+ pass
262
+ return 0
263
+
264
+
265
+ def _card_v7(a) -> int:
266
+ from .workbench import card_from_file, card_v7_to_latex, card_v7_to_markdown
267
+
268
+ try:
269
+ card = card_from_file(a.log, a.baseline_log)
270
+ if a.json:
271
+ text = json.dumps(card, indent=2)
272
+ elif a.latex:
273
+ text = card_v7_to_latex(card)
274
+ else:
275
+ text = card_v7_to_markdown(card)
276
+ if a.output:
277
+ open(a.output, "w", encoding="utf-8").write(text)
278
+ else:
279
+ print(text, end="")
280
+ except (OSError, ValueError, json.JSONDecodeError) as exc:
281
+ print("error: %s" % exc, file=sys.stderr)
282
+ return 1
283
+ return 0
284
+
285
+
286
+ def _pool_size(a) -> int:
287
+ from .pool_sizer import compute_k_tok
288
+
289
+ result = compute_k_tok(
290
+ a.agents, a.rounds, a.prompt_tokens, a.msg_tokens, a.topology
291
+ )
292
+ result.pop("b_eff", None)
293
+ if a.json:
294
+ print(json.dumps(result, indent=2))
295
+ else:
296
+ print(
297
+ "k_tok %.3f (r=%.4f), recommended pool size %d"
298
+ % (result["k_tok"], result["r"], result["recommended_pool_size"])
299
+ )
300
+ return 0
301
+
302
+
303
+ def _simulate(a) -> int:
304
+ from . import simulator
305
+
306
+ if a.mode == "voting":
307
+ result = simulator.simulate_correlated_voting(
308
+ a.items, a.agents, a.p_mean, a.rho, a.choices, a.seed
309
+ )
310
+ elif a.mode == "debate":
311
+ result = simulator.simulate_debate_dynamics(
312
+ a.items,
313
+ a.agents,
314
+ a.rounds,
315
+ a.p_mean,
316
+ a.regime,
317
+ {"a": a.rate_a, "b": a.rate_b},
318
+ a.replicates,
319
+ a.verifier_fp,
320
+ a.verifier_fn,
321
+ a.seed,
322
+ )
323
+ else:
324
+ result = simulator.simulate_scaffolding_scaling_curve(
325
+ a.budgets, a.agents, a.rounds, a.prompt_tokens, a.msg_tokens
326
+ )
327
+ if a.out:
328
+ with open(a.out, "w", encoding="utf-8") as handle:
329
+ if isinstance(result, list):
330
+ handle.writelines(json.dumps(row) + "\n" for row in result)
331
+ else:
332
+ handle.write(json.dumps(result) + "\n")
333
+ else:
334
+ print(json.dumps(result, indent=2))
335
+ return 0
336
+
337
+
338
+ def _meta(a) -> int:
339
+ import csv
340
+
341
+ from .meta_engine import (
342
+ build_che_covariance,
343
+ compute_effect_sizes,
344
+ der_simonian_laird_pool,
345
+ )
346
+
347
+ with open(a.contrasts, encoding="utf-8", newline="") as handle:
348
+ rows = list(csv.DictReader(handle))
349
+ effects = [
350
+ compute_effect_sizes(
351
+ float(row["p_mas"]), float(row["p_base"]), int(float(row.get("n", 1))), a.phi
352
+ )
353
+ for row in rows
354
+ ]
355
+ np_mod = __import__("numpy")
356
+ y = np_mod.array(
357
+ [
358
+ effect[a.scale] if a.scale != "delta" else effect["delta"]
359
+ for effect in effects
360
+ ]
361
+ )
362
+ if a.scale == "cohen_h":
363
+ v = np_mod.array([effect["v_cohen_h"] for effect in effects])
364
+ elif a.scale == "delta":
365
+ v_list = []
366
+ for row in rows:
367
+ pm = float(row["p_mas"])
368
+ pb = float(row["p_base"])
369
+ n_val = max(1.0, float(row.get("n", 1)))
370
+ am, ab = pm * (1.0 - pm), pb * (1.0 - pb)
371
+ v_d = max(1e-10, (am + ab - 2.0 * a.phi * (am * ab) ** 0.5) / n_val)
372
+ v_list.append(v_d)
373
+ v = np_mod.array(v_list)
374
+ else:
375
+ v = np_mod.array([effect["v_log_or"] for effect in effects])
376
+ result = {
377
+ "estimator": "DerSimonian-Laird",
378
+ "scale": a.scale,
379
+ "effects": effects,
380
+ "covariance": build_che_covariance(rows, a.rho_block).tolist(),
381
+ }
382
+ if len(rows) > 1:
383
+ result["pooled"] = der_simonian_laird_pool(
384
+ y, v, [row.get("study", "study") for row in rows]
385
+ )
386
+ print(json.dumps(result, indent=2) if a.json else json.dumps(result))
387
+ return 0
388
+
389
+
390
+ def _power(a) -> int:
391
+ from .meta_engine import prospective_power_analysis
392
+
393
+ result = prospective_power_analysis(
394
+ a.p_base, a.delta, a.n_items, a.power, a.alpha, a.phi, a.shared_arms
395
+ )
396
+ print(json.dumps(result, indent=2) if a.json else json.dumps(result))
397
+ return 0
398
+
399
+
400
+ def _benchmark_gen(a) -> int:
401
+ from .escape_benchmarks import (
402
+ evaluate_with_verifier_threshold,
403
+ generate_clue_grid_items,
404
+ generate_distributed_code_items,
405
+ )
406
+
407
+ if a.route == "clue_grid":
408
+ result = generate_clue_grid_items(a.items, a.agents, a.seed)
409
+ elif a.route == "distributed_code":
410
+ result = generate_distributed_code_items(a.items, a.agents, a.seed)
411
+ else:
412
+ result = [
413
+ evaluate_with_verifier_threshold(0.5, 0.2, 0.1, epsilon, 0.1)
414
+ for epsilon in a.epsilons
415
+ ]
416
+ with open(a.out, "w", encoding="utf-8") as handle:
417
+ handle.writelines(json.dumps(row) + "\n" for row in result)
418
+ return 0
419
+
420
+
421
+ def _aggregate_open(a) -> int:
422
+ from .open_ended import mbr_centroid_select
423
+
424
+ records = [
425
+ json.loads(line) for line in open(a.input, encoding="utf-8") if line.strip()
426
+ ]
427
+ output = []
428
+ for row in records:
429
+ candidates = row.get("candidates", row.get("answers", []))
430
+ output.append(
431
+ {
432
+ "item": row.get("item", row.get("id", len(output))),
433
+ **mbr_centroid_select(candidates, a.metric, row.get("token_costs")),
434
+ }
435
+ )
436
+ text = "".join(json.dumps(row) + "\n" for row in output)
437
+ if a.out:
438
+ open(a.out, "w", encoding="utf-8").write(text)
439
+ else:
440
+ print(text, end="")
441
+ return 0
442
+
443
+
444
+ def _plot(a) -> int:
445
+ from . import plotting
446
+
447
+ if a.kind == "waterfall":
448
+ result = plotting.svg_waterfall_decomposition(0.4, 0.6, 0.7)
449
+ elif a.kind == "phase":
450
+ result = plotting.svg_phase_diagram()
451
+ elif a.kind == "frontier":
452
+ result = plotting.svg_frontier_curve(json.load(open(a.input, encoding="utf-8")))
453
+ else:
454
+ payload = json.load(open(a.input, encoding="utf-8"))
455
+ result = plotting.svg_influence_curves(
456
+ payload["by_correct_peers"],
457
+ payload.get("fit_result", payload),
458
+ payload.get("k_agents", 3),
459
+ )
460
+ open(a.out, "w", encoding="utf-8").write(result)
461
+ return 0
462
+
463
+
464
+ def _hidden_profile(a) -> int:
465
+ from .hidden_profile import generate, valid, validate
466
+
467
+ try:
468
+ items = generate(a.n, a.agents, a.criteria, a.split, a.margin, a.seed, a.format)
469
+ except ValueError as e:
470
+ print("error: %s" % e, file=sys.stderr)
471
+ return 1
472
+ counts = validate(items)
473
+ msg = (
474
+ "generated %d items (agents %d, criteria %d, split %d, margin %d, seed %d, format %s); decoy leads a view in violation: %d, target not leading pooled: %d, uncovered split criteria: %d"
475
+ % (
476
+ counts["items"],
477
+ a.agents,
478
+ a.criteria,
479
+ a.split,
480
+ a.margin,
481
+ a.seed,
482
+ a.format,
483
+ counts["decoy_not_leading_a_view"],
484
+ counts["target_not_leading_pooled"],
485
+ counts["uncovered_split_criteria"],
486
+ )
487
+ )
488
+ print(msg, file=sys.stderr)
489
+ if not valid(counts):
490
+ print("error: validity check failed; no output written", file=sys.stderr)
491
+ return 1
492
+ text = chr(10).join(json.dumps(it) for it in items) + chr(10)
493
+ if a.out:
494
+ with open(a.out, "w", encoding="utf-8") as fh:
495
+ fh.write(text)
496
+ else:
497
+ sys.stdout.write(text)
498
+ return 0
499
+
500
+
501
+ def _frontier(a) -> int:
502
+ from .report import frontier_report, plot_frontier, system_points
503
+
504
+ try:
505
+ fr = frontier_report(a.samples, a.k, a.perms, a.seed)
506
+ pts = system_points(a.logs) if a.logs else []
507
+ except (OSError, ValueError, KeyError) as e:
508
+ print("error: %s" % e, file=sys.stderr)
509
+ return 1
510
+ print(
511
+ "items %d | pool k=%d | greedy accuracy %s"
512
+ % (
513
+ fr["n_items"],
514
+ fr["pool_k"],
515
+ "n/a" if fr["greedy_accuracy"] is None else "%.3f" % fr["greedy_accuracy"],
516
+ )
517
+ )
518
+ print("k mean tokens SC accuracy isotonic")
519
+ for i, (t, x, y) in enumerate(
520
+ zip(fr["tokens"], fr["sc_accuracy"], fr["isotonic"]), 1
521
+ ):
522
+ print("%-2d %11.0f %11.3f %8.3f" % (i, t, x, y))
523
+ for p in pts:
524
+ print(
525
+ "system %-12s accuracy %.3f at %.0f tokens/item (n=%d)"
526
+ % (p["system"], p["accuracy"], p["mean_tokens"], p["n_items"])
527
+ )
528
+ if a.json:
529
+ json.dump(
530
+ {"frontier": fr, "systems": pts},
531
+ open(a.json, "w", encoding="utf-8"),
532
+ indent=2,
533
+ )
534
+ if a.out:
535
+ plot_frontier(fr, pts, a.out)
536
+ return 0
537
+
538
+
539
+ def _rebuttal(a) -> int:
540
+ from .rebuttal import rows_from_contrasts, to_markdown
541
+ from .rebuttal import to_latex as rb_tex
542
+
543
+ if a.contrasts == "outputs/baseline_tier_contrasts.csv" and not os.path.exists(a.contrasts) and os.path.exists("data/baseline_tier_contrasts.csv"):
544
+ a.contrasts = "data/baseline_tier_contrasts.csv" # release tree layout: the corpus lives in data/
545
+ try:
546
+ rows = rows_from_contrasts(a.contrasts, a.study)
547
+ except (OSError, KeyError, ValueError) as e:
548
+ print("error: %s" % e, file=sys.stderr)
549
+ return 1
550
+ if not rows:
551
+ print("error: no rows", file=sys.stderr)
552
+ return 1
553
+ if a.export_latex:
554
+ open(a.export_latex, "w", encoding="utf-8").write(rb_tex(rows))
555
+ if a.export_markdown:
556
+ open(a.export_markdown, "w", encoding="utf-8").write(to_markdown(rows))
557
+ if not (a.export_latex or a.export_markdown):
558
+ print(to_markdown(rows))
559
+ return 0
560
+
561
+
562
+ def build_parser() -> argparse.ArgumentParser:
563
+ p = argparse.ArgumentParser(
564
+ prog="pace",
565
+ description="Compute-matched evaluation: measure systems against the self-consistency frontier.",
566
+ )
567
+ sub = p.add_subparsers(dest="cmd", required=True)
568
+ x = sub.add_parser("proxy", help="OpenAI-compatible logging proxy")
569
+ x.add_argument("--upstream", default="http://localhost:11434/v1")
570
+ x.add_argument("--port", type=int, default=8089)
571
+ x.add_argument("--host", default="127.0.0.1")
572
+ x.add_argument("--log", default="run.jsonl")
573
+ x.add_argument(
574
+ "--price-in",
575
+ type=float,
576
+ default=None,
577
+ help="USD per million input tokens (never bundled)",
578
+ )
579
+ x.add_argument("--price-out", type=float, default=None)
580
+ x.add_argument("--api-key", default=None)
581
+ x.add_argument(
582
+ "--system-label",
583
+ default="mas",
584
+ help="value of the log's system field when a request has no X-PACE-System header",
585
+ )
586
+ x.add_argument(
587
+ "--item-file",
588
+ default=None,
589
+ help="file whose first line is the current item id, used when a request has no X-PACE-Item header (serial runs of third-party code)",
590
+ )
591
+ x.set_defaults(func=_proxy)
592
+ b = sub.add_parser("baseline", help="greedy + k-sample pool, cached and resumable")
593
+ b.add_argument("--model", required=True)
594
+ b.add_argument(
595
+ "--bench",
596
+ required=True,
597
+ help="path to JSONL {id, question, answer}, or 'toy' (synthetic smoke test)",
598
+ )
599
+ b.add_argument("--base-url", default="http://localhost:11434/v1")
600
+ b.add_argument("--api-key", default="none")
601
+ b.add_argument("--k", type=int, default=16)
602
+ b.add_argument("--temperature", type=float, default=0.7)
603
+ b.add_argument("--seed", type=int, default=0)
604
+ b.add_argument("--max-tokens", type=int, default=2048)
605
+ b.add_argument("--max-items", type=int, default=None)
606
+ b.add_argument("--out", default="baseline_cache.jsonl")
607
+ b.add_argument("--verbose", action="store_true")
608
+ b.set_defaults(func=_baseline)
609
+ r = sub.add_parser(
610
+ "report", help="frontier gap, resource card, LaTeX table and figure"
611
+ )
612
+ r.add_argument("--pool", required=True, help="baseline cache from `pace baseline`")
613
+ r.add_argument("--logs", required=True, help="system log (calls and results)")
614
+ r.add_argument("--system", default="mas")
615
+ r.add_argument("--price-in", type=float, default=None)
616
+ r.add_argument("--price-out", type=float, default=None)
617
+ r.add_argument(
618
+ "--k", type=int, default=None, help="use the first k samples per item"
619
+ )
620
+ r.add_argument("--n-boot", type=int, default=2000)
621
+ r.add_argument("--perms", type=int, default=100)
622
+ r.add_argument("--seed", type=int, default=0)
623
+ r.add_argument("--json", default=None)
624
+ r.add_argument("--latex", default=None)
625
+ r.add_argument("--plot", default=None, help="vector figure path (.pdf)")
626
+ r.add_argument(
627
+ "--export-latex",
628
+ default=None,
629
+ help="write the nine-column rebuttal table for this run (LaTeX)",
630
+ )
631
+ r.add_argument(
632
+ "--export-markdown",
633
+ default=None,
634
+ help="write the nine-column rebuttal table for this run (Markdown)",
635
+ )
636
+ r.add_argument(
637
+ "--benchmark", default="n/a", help="benchmark label for the exported row"
638
+ )
639
+ r.add_argument(
640
+ "--model-label", default="n/a", help="model label for the exported row"
641
+ )
642
+ r.add_argument(
643
+ "--export-rollout-card",
644
+ default=None,
645
+ metavar="PREFIX",
646
+ help="write PREFIX.json, PREFIX.yaml and PREFIX.tex: a PACE compute block for a rollout card (arXiv:2605.12131)",
647
+ )
648
+ r.add_argument(
649
+ "--run-id",
650
+ default="run",
651
+ help="target id used in the rollout-card annotation rows",
652
+ )
653
+ r.set_defaults(func=_report)
654
+ inf = sub.add_parser("inflate", help="beta-binomial sampling inflation: predicted accuracy of a K-sample vote from a pool of samples (CPU only)")
655
+ inf.add_argument("--samples", required=True, help="JSONL, one row per sample with item and correct")
656
+ inf.add_argument("--k", default="1,3,5,9,16,24", help="comma-separated vote sizes K to predict")
657
+ inf.add_argument("--wrong-options", default=None, help="plurality model: number of wrong options W the wrong answers spread over (a number), or auto to estimate it from answer labels in the rows")
658
+ inf.add_argument("--json", action="store_true")
659
+ inf.set_defaults(func=_inflate)
660
+ c = sub.add_parser(
661
+ "classify",
662
+ help="baseline rung (A to D) of a single-agent configuration given as JSON",
663
+ )
664
+ c.add_argument(
665
+ "--config",
666
+ required=True,
667
+ help="JSON file; keys are listed in code/mae_eval/classifier.py",
668
+ )
669
+ c.set_defaults(func=_classify)
670
+ u = sub.add_parser(
671
+ "audit",
672
+ help="one-screen diagnostic card for a run log (mas and sas calls) or a baseline configuration (JSON)",
673
+ )
674
+ u.add_argument(
675
+ "input",
676
+ help="JSONL run log with 'mas' and 'sas' calls, or a JSON baseline configuration",
677
+ )
678
+ u.add_argument(
679
+ "--baseline-config",
680
+ default=None,
681
+ help="JSON configuration of the single-agent baseline, to classify its rung when INPUT is a log",
682
+ )
683
+ u.add_argument("--tolerance", type=float, default=0.05)
684
+ u.add_argument("--price-in", type=float, default=None)
685
+ u.add_argument("--price-out", type=float, default=None)
686
+ u.add_argument("--json", action="store_true")
687
+ u.add_argument(
688
+ "--strict",
689
+ action="store_true",
690
+ help="exit with status 2 when the badge is a warning",
691
+ )
692
+ u.set_defaults(func=_audit)
693
+ d = sub.add_parser(
694
+ "dynamics",
695
+ help="flip counts, tau rates, implied threshold, peer correlation, K_eff and tokens by round from a debate log",
696
+ )
697
+ d.add_argument(
698
+ "log",
699
+ help="JSONL with item, agent, round, correct (and optionally prompt_tokens, completion_tokens) per agent answer",
700
+ )
701
+ d.add_argument(
702
+ "--null",
703
+ default=None,
704
+ help="optional no-communication control JSONL log to compute excess transition rates and excess peer correlation",
705
+ )
706
+ d.add_argument("--n-boot", type=int, default=1000)
707
+ d.add_argument("--seed", type=int, default=0)
708
+ d.add_argument("--first-round", type=int, default=None)
709
+ d.add_argument("--last-round", type=int, default=None)
710
+ d.add_argument("--json", action="store_true")
711
+ d.set_defaults(func=_dynamics)
712
+ i = sub.add_parser("import-logs", help="convert an external trace to PACE JSONL")
713
+ i.add_argument(
714
+ "--format", choices=["autogen", "langgraph", "crewai", "inspect", "csv"], required=True
715
+ )
716
+ i.add_argument("--input", required=True)
717
+ i.add_argument("--output", required=True)
718
+ i.set_defaults(func=_import_logs)
719
+ k = sub.add_parser("card", help="render a standalone PACE Card from PACE JSONL")
720
+ k.add_argument("log")
721
+ k.add_argument("--output", default=None)
722
+ k.add_argument("--json", action="store_true")
723
+ k.add_argument("--latex", action="store_true")
724
+ k.set_defaults(func=_card)
725
+ s = sub.add_parser("studio", help="open the zero-dependency PACE Studio")
726
+ s.set_defaults(func=_studio)
727
+ wb = sub.add_parser(
728
+ "workbench", help="serve or export the standalone PACE Workbench"
729
+ )
730
+ wb.add_argument("--export-html", default=None)
731
+ wb.add_argument("--serve", action="store_true")
732
+ wb.add_argument("--host", default="127.0.0.1")
733
+ wb.add_argument("--port", type=int, default=8765)
734
+ wb.add_argument("--no-browser", action="store_true")
735
+ wb.set_defaults(func=_workbench)
736
+ cv7 = sub.add_parser("card-v7", help="render the seven-field PACE evaluation card")
737
+ cv7.add_argument("log")
738
+ cv7.add_argument("--baseline-log", default=None)
739
+ cv7.add_argument("--latex", action="store_true")
740
+ cv7.add_argument("--markdown", action="store_true")
741
+ cv7.add_argument("--json", action="store_true")
742
+ cv7.add_argument("--output", default=None)
743
+ cv7.set_defaults(func=_card_v7)
744
+ z = sub.add_parser(
745
+ "pool-size", help="constructive token budget and recommended sample pool"
746
+ )
747
+ z.add_argument("--agents", type=int, required=True)
748
+ z.add_argument("--rounds", type=int, required=True)
749
+ z.add_argument("--prompt-tokens", type=float, required=True)
750
+ z.add_argument("--msg-tokens", type=float, required=True)
751
+ z.add_argument(
752
+ "--topology",
753
+ choices=["last_round_peers", "last_round_all", "full_history_echo"],
754
+ default="last_round_peers",
755
+ )
756
+ z.add_argument("--json", action="store_true")
757
+ z.set_defaults(func=_pool_size)
758
+ sim = sub.add_parser("simulate", help="generate PACE theoretical simulation data")
759
+ sim.add_argument(
760
+ "--mode", choices=["voting", "debate", "scaffolding"], required=True
761
+ )
762
+ sim.add_argument("--items", type=int, default=100)
763
+ sim.add_argument("--agents", type=int, default=5)
764
+ sim.add_argument("--rounds", type=int, default=3)
765
+ sim.add_argument("--p-mean", type=float, default=0.7)
766
+ sim.add_argument("--rho", type=float, default=0.2)
767
+ sim.add_argument("--choices", type=int, default=2)
768
+ sim.add_argument("--regime", default="constant")
769
+ sim.add_argument("--rate-a", type=float, default=0.25)
770
+ sim.add_argument("--rate-b", type=float, default=0.15)
771
+ sim.add_argument("--replicates", type=int, default=1)
772
+ sim.add_argument("--verifier-fp", type=float, default=None)
773
+ sim.add_argument("--verifier-fn", type=float, default=None)
774
+ sim.add_argument("--budgets", type=float, nargs="*", default=[1000, 5000])
775
+ sim.add_argument("--prompt-tokens", type=float, default=100)
776
+ sim.add_argument("--msg-tokens", type=float, default=20)
777
+ sim.add_argument("--seed", type=int, default=0)
778
+ sim.add_argument("--out", default=None)
779
+ sim.set_defaults(func=_simulate)
780
+ meta = sub.add_parser("meta", help="native PACE meta-analysis")
781
+ meta.add_argument("--contrasts", required=True)
782
+ meta.add_argument(
783
+ "--scale", choices=["log_or", "delta", "cohen_h"], default="log_or"
784
+ )
785
+ meta.add_argument("--phi", type=float, default=0.5)
786
+ meta.add_argument("--rho-block", type=float, default=0.5)
787
+ meta.add_argument("--json", action="store_true")
788
+ meta.set_defaults(func=_meta)
789
+ power = sub.add_parser("power", help="prospective PACE power analysis")
790
+ power.add_argument("--p-base", type=float, required=True)
791
+ power.add_argument("--delta", type=float, required=True)
792
+ power.add_argument("--n-items", type=int, default=None)
793
+ power.add_argument("--power", type=float, default=0.8)
794
+ power.add_argument("--alpha", type=float, default=0.05)
795
+ power.add_argument("--phi", type=float, default=0.5)
796
+ power.add_argument("--shared-arms", type=int, default=1)
797
+ power.add_argument("--json", action="store_true")
798
+ power.set_defaults(func=_power)
799
+ bench = sub.add_parser(
800
+ "benchmark-gen", help="generate escape-route benchmark fixtures"
801
+ )
802
+ bench.add_argument(
803
+ "--route",
804
+ choices=["clue_grid", "distributed_code", "verifier_curve"],
805
+ required=True,
806
+ )
807
+ bench.add_argument("--items", type=int, default=30)
808
+ bench.add_argument("--agents", type=int, default=3)
809
+ bench.add_argument("--seed", type=int, default=42)
810
+ bench.add_argument("--epsilons", type=float, nargs="*", default=[0.01, 0.05, 0.1])
811
+ bench.add_argument("--out", required=True)
812
+ bench.set_defaults(func=_benchmark_gen)
813
+ open_cmd = sub.add_parser("aggregate-open", help="aggregate open-ended samples")
814
+ open_cmd.add_argument("--input", required=True)
815
+ open_cmd.add_argument(
816
+ "--metric",
817
+ choices=["ngram_cosine", "token_jaccard", "sequence_ratio", "code_structural"],
818
+ default="ngram_cosine",
819
+ )
820
+ open_cmd.add_argument("--out", default=None)
821
+ open_cmd.set_defaults(func=_aggregate_open)
822
+ plot = sub.add_parser("plot", help="write a pure SVG diagnostic figure")
823
+ plot.add_argument(
824
+ "--kind", choices=["waterfall", "phase", "frontier", "influence"], required=True
825
+ )
826
+ plot.add_argument("--input", default=None)
827
+ plot.add_argument("--out", required=True)
828
+ plot.set_defaults(func=_plot)
829
+ h = sub.add_parser(
830
+ "hidden-profile",
831
+ help="generate synthetic split-evidence (hidden-profile) items and check their validity",
832
+ )
833
+ h.add_argument("--n", type=int, default=100, help="number of items")
834
+ h.add_argument("--agents", type=int, default=5)
835
+ h.add_argument("--criteria", type=int, default=10, help="total number of criteria")
836
+ h.add_argument(
837
+ "--split",
838
+ type=int,
839
+ default=6,
840
+ help="number of criteria divided among the agents (the rest are shared)",
841
+ )
842
+ h.add_argument(
843
+ "--margin",
844
+ type=int,
845
+ default=3,
846
+ help="score gap per criterion between the favoured and the disfavoured candidate (at least 2)",
847
+ )
848
+ h.add_argument("--seed", type=int, default=42)
849
+ h.add_argument("--format", choices=["text", "numeric"], default="text")
850
+ h.add_argument(
851
+ "--out", default=None, help="output JSONL (default: standard output)"
852
+ )
853
+ h.set_defaults(func=_hidden_profile)
854
+ f = sub.add_parser(
855
+ "frontier",
856
+ help="self-consistency frontier of a sample pool, optionally with system points from a run log",
857
+ )
858
+ f.add_argument(
859
+ "--samples",
860
+ required=True,
861
+ help="baseline cache from `pace baseline` (greedy + sampled completions per item)",
862
+ )
863
+ f.add_argument(
864
+ "--logs",
865
+ default=None,
866
+ help="run log; every value of its 'system' field is drawn as a point",
867
+ )
868
+ f.add_argument("--k", type=int, default=None)
869
+ f.add_argument("--perms", type=int, default=100)
870
+ f.add_argument("--seed", type=int, default=0)
871
+ f.add_argument("--json", default=None)
872
+ f.add_argument("--out", default=None, help="figure path (.pdf vector, or .png)")
873
+ f.set_defaults(func=_frontier)
874
+ t = sub.add_parser(
875
+ "rebuttal",
876
+ help="nine-column rebuttal table from a contrasts CSV (default: outputs/baseline_tier_contrasts.csv, or data/baseline_tier_contrasts.csv in the release tree)",
877
+ )
878
+ t.add_argument(
879
+ "--contrasts",
880
+ "--data",
881
+ dest="contrasts",
882
+ default="outputs/baseline_tier_contrasts.csv",
883
+ help="contrasts CSV with columns study, category (rung), baseline, key 'Study|bench|model|arm', p_base, p_mas",
884
+ )
885
+ t.add_argument("--study", default=None)
886
+ t.add_argument("--export-latex", default=None)
887
+ t.add_argument("--export-markdown", default=None)
888
+ t.set_defaults(func=_rebuttal)
889
+ return p
890
+
891
+
892
+ def main(argv: Sequence[str] | None = None) -> int:
893
+ a = build_parser().parse_args(argv)
894
+ return a.func(a)
895
+
896
+
897
+ if __name__ == "__main__":
898
+ sys.exit(main())