pace-eval 0.2.2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (83) hide show
  1. cme/__init__.py +11 -0
  2. cme/__main__.py +5 -0
  3. cme/analysis.py +223 -0
  4. cme/audit.py +145 -0
  5. cme/baseline.py +162 -0
  6. cme/cli.py +279 -0
  7. cme/client.py +165 -0
  8. cme/dynamics.py +212 -0
  9. cme/examples/crewai_integration.py +16 -0
  10. cme/examples/demo_pool.jsonl +360 -0
  11. cme/examples/demo_run.jsonl +480 -0
  12. cme/examples/langgraph_integration.py +27 -0
  13. cme/examples/make_demo.py +37 -0
  14. cme/examples/openai_quickstart.py +18 -0
  15. cme/experiment.py +555 -0
  16. cme/frontier.py +143 -0
  17. cme/hidden_profile.py +135 -0
  18. cme/proxy.py +106 -0
  19. cme/rebuttal.py +76 -0
  20. cme/report.py +197 -0
  21. cme/rollout_card.py +137 -0
  22. cme/tasks.py +266 -0
  23. cme/verified_debate.py +127 -0
  24. cme/verifier.py +92 -0
  25. mae_eval/__init__.py +11 -0
  26. mae_eval/__main__.py +5 -0
  27. mae_eval/auditor.py +112 -0
  28. mae_eval/classifier.py +62 -0
  29. mae_eval/cli.py +210 -0
  30. mae_eval/tracker.py +162 -0
  31. mae_eval/vector.py +91 -0
  32. pace/__init__.py +11 -0
  33. pace/__main__.py +5 -0
  34. pace/analysis.py +129 -0
  35. pace/audit.py +145 -0
  36. pace/baseline.py +162 -0
  37. pace/card.py +205 -0
  38. pace/cli.py +898 -0
  39. pace/client.py +165 -0
  40. pace/dynamics.py +372 -0
  41. pace/escape_benchmarks.py +162 -0
  42. pace/examples/crewai_integration.py +16 -0
  43. pace/examples/demo_pool.jsonl +360 -0
  44. pace/examples/demo_run.jsonl +480 -0
  45. pace/examples/langgraph_integration.py +27 -0
  46. pace/examples/make_demo.py +37 -0
  47. pace/examples/openai_quickstart.py +18 -0
  48. pace/experiment.py +555 -0
  49. pace/frontier.py +114 -0
  50. pace/hidden_profile.py +301 -0
  51. pace/importers.py +219 -0
  52. pace/inflate.py +213 -0
  53. pace/influence.py +231 -0
  54. pace/meta_engine.py +246 -0
  55. pace/middleware.py +230 -0
  56. pace/open_ended.py +114 -0
  57. pace/plotting.py +132 -0
  58. pace/pool_sizer.py +154 -0
  59. pace/protocol_v3.py +414 -0
  60. pace/proxy.py +106 -0
  61. pace/rebuttal.py +219 -0
  62. pace/report.py +197 -0
  63. pace/rollout_card.py +137 -0
  64. pace/schema.py +74 -0
  65. pace/simulator.py +199 -0
  66. pace/studio.html +1040 -0
  67. pace/tasks.py +266 -0
  68. pace/verified_debate.py +127 -0
  69. pace/verifier.py +92 -0
  70. pace/workbench.html +26 -0
  71. pace/workbench.py +119 -0
  72. pace_eval-0.2.2.dist-info/METADATA +205 -0
  73. pace_eval-0.2.2.dist-info/RECORD +83 -0
  74. pace_eval-0.2.2.dist-info/WHEEL +5 -0
  75. pace_eval-0.2.2.dist-info/entry_points.txt +4 -0
  76. pace_eval-0.2.2.dist-info/licenses/LICENSE +202 -0
  77. pace_eval-0.2.2.dist-info/top_level.txt +4 -0
  78. pace_lab/__init__.py +9 -0
  79. pace_lab/clients.py +568 -0
  80. pace_lab/concurrency_test.py +127 -0
  81. pace_lab/data.py +445 -0
  82. pace_lab/graders.py +626 -0
  83. pace_lab/strategies.py +345 -0
cme/__init__.py ADDED
@@ -0,0 +1,11 @@
1
+ """cme: measure a multi-agent system against the self-consistency frontier at its measured token cost.
2
+
3
+ Commands: `cme proxy` (logging proxy), `cme baseline` (greedy + sample pool), `cme report` (frontier gap, resource card, table, figure).
4
+ Built on mae_eval (resource vector and parity auditor). Python API: cme.frontier_gap, cme.build_report, cme.resource_card, cme.wrap (log a client's usage without a proxy), cme.track (label items and rounds).
5
+ """
6
+ from .frontier import Pool, frontier_at, frontier_curve, frontier_gap, pav_increasing
7
+ from .report import build_report, to_latex
8
+ from .client import wrap, track, log_result
9
+
10
+ __version__ = "0.1.0"
11
+ __all__ = ["Pool", "frontier_at", "frontier_curve", "frontier_gap", "pav_increasing", "build_report", "to_latex", "wrap", "track", "log_result"]
cme/__main__.py ADDED
@@ -0,0 +1,5 @@
1
+ import sys
2
+
3
+ from .cli import main
4
+
5
+ sys.exit(main())
cme/analysis.py ADDED
@@ -0,0 +1,223 @@
1
+ """Load a confirmatory cell (one model x one task) and estimate frontier gaps per strategy.
2
+
3
+ A cell directory holds calls.jsonl (every call) and results.jsonl (one row per strategy x item). The pool frontier is rebuilt from the
4
+ cached pool calls with the same scorer used at run time, so re-scoring never needs a model. Correctness of a pool sample is mapped to
5
+ the label 'G' (wrong answers keep their normalised text) so plurality voting uses the scorer's notion of correct.
6
+
7
+ Estimand (docs/PREREGISTRATION.md section 5): gap(s) = accuracy(s) - F(mean tokens of s), F the isotonic self-consistency frontier,
8
+ with a paired bootstrap over items. `pool="views"` builds the frontier from the independent split-evidence agents (S2v) instead.
9
+ """
10
+
11
+ from __future__ import annotations
12
+
13
+ import json
14
+ import os
15
+
16
+ import numpy as np
17
+
18
+ from . import experiment as E
19
+ from . import tasks as T
20
+ from .frontier import Pool, frontier_curve, frontier_gap, pav_increasing
21
+
22
+
23
+ def read_results(cell_dir: str) -> dict[str, dict[str, dict]]:
24
+ out: dict[str, dict[str, dict]] = {}
25
+ p = os.path.join(cell_dir, "results.jsonl")
26
+ if os.path.exists(p):
27
+ for l in open(p, encoding="utf-8"):
28
+ if l.strip():
29
+ r = json.loads(l)
30
+ out.setdefault(r["strategy"], {})[r["item"]] = r
31
+ return out
32
+
33
+
34
+ def build_pool(
35
+ store: E.Store,
36
+ model: str,
37
+ items: list[dict],
38
+ pool: str = "pool",
39
+ k: int = 16,
40
+ extension_path: str | None = None,
41
+ ) -> Pool:
42
+ """Pool of per-item sampled answers and token counts. pool='pool' uses the 16 full-evidence samples; 'views' the 5 split-evidence agents."""
43
+ extra = {}
44
+ if extension_path and os.path.exists(extension_path):
45
+ for line in open(extension_path, encoding="utf-8"):
46
+ if line.strip():
47
+ row = json.loads(line)
48
+ extra[row["key"]] = row
49
+ pitems = {}
50
+ for it in items:
51
+ if it["kind"] == "code":
52
+ continue
53
+ keys = (
54
+ ["pool|%s|%s|%d" % (model, it["id"], i) for i in range(k)]
55
+ if pool == "pool"
56
+ else ["r1v|%s|%s|%d" % (model, it["id"], i) for i in range(k)]
57
+ )
58
+ recs = [store.d.get(x) or extra.get(x) for x in keys]
59
+ if any(
60
+ r is None
61
+ or r.get("prompt_tokens") is None
62
+ or r.get("completion_tokens") is None
63
+ for r in recs
64
+ ):
65
+ continue
66
+ pitems[it["id"]] = {
67
+ "gold": "G",
68
+ "answers": [
69
+ "G"
70
+ if T.score(it, r["text"])
71
+ else ((E.ans_of(it, r) + "_wrong") if E.ans_of(it, r) else "")
72
+ for r in recs
73
+ ],
74
+ "tokens": [r["prompt_tokens"] + r["completion_tokens"] for r in recs],
75
+ }
76
+ return Pool(pitems, k) if pitems else None
77
+
78
+
79
+ def cell_gaps(
80
+ cell_dir: str,
81
+ model: str,
82
+ items: list[dict],
83
+ strategies: list[str] | None = None,
84
+ pool: str = "pool",
85
+ k: int | None = None,
86
+ n_boot: int = 2000,
87
+ n_perm: int = 100,
88
+ seed: int = 0,
89
+ tokens: str = "total",
90
+ extension_path: str | None = None,
91
+ ) -> dict:
92
+ """Frontier, per-strategy accuracy, mean tokens and gap with a bootstrap interval. tokens='completion' is the sensitivity analysis."""
93
+ store = E.Store(os.path.join(cell_dir, "calls.jsonl"))
94
+ k = k or (16 if pool == "pool" else 5)
95
+ P = build_pool(store, model, items, pool, k, extension_path=extension_path)
96
+ if P is None:
97
+ return {"error": "no complete pool in %s" % cell_dir}
98
+ if tokens == "completion": # recompute pool tokens with completion tokens only
99
+ for it in P.items:
100
+ keys = [
101
+ "pool|%s|%s|%d" % (model, it, i)
102
+ if pool == "pool"
103
+ else "r1v|%s|%s|%d" % (model, it, i)
104
+ for i in range(k)
105
+ ]
106
+ P.items[it]["tokens"] = [store.d[x]["completion_tokens"] for x in keys]
107
+ ids, acc, tok = P.curves(n_perm, seed)
108
+ res = read_results(cell_dir)
109
+ out = {
110
+ "model": model,
111
+ "pool": pool,
112
+ "k": k,
113
+ "n_pool_items": len(ids),
114
+ "tokens": tokens,
115
+ "strategies": {},
116
+ }
117
+ tk, ak = frontier_curve(acc, tok)
118
+ out["frontier"] = {
119
+ "tokens": tk.tolist(),
120
+ "sc_accuracy": ak.tolist(),
121
+ "isotonic": pav_increasing(ak).tolist(),
122
+ }
123
+ for s, rows in sorted(res.items()):
124
+ if strategies and s not in strategies:
125
+ continue
126
+ common = [i for i in ids if i in rows]
127
+ if len(common) < 5:
128
+ continue
129
+ pos = {i: n for n, i in enumerate(ids)}
130
+ sel = [pos[i] for i in common]
131
+ use = (
132
+ (lambda r: r["completion_tokens"])
133
+ if tokens == "completion"
134
+ else (lambda r: r["prompt_tokens"] + r["completion_tokens"])
135
+ )
136
+ sc = np.array([float(rows[i]["correct"]) for i in common])
137
+ st = np.array([float(use(rows[i])) for i in common])
138
+ g = frontier_gap(acc[sel], tok[sel], sc, st, n_boot=n_boot, seed=seed)
139
+ g["n_items"] = len(common)
140
+ out["strategies"][s] = g
141
+ return out
142
+
143
+
144
+ def difference_of_gaps(
145
+ cell_dir: str,
146
+ model: str,
147
+ items: list[dict],
148
+ a: str,
149
+ b: str,
150
+ n_boot: int = 2000,
151
+ seed: int = 0,
152
+ n_perm: int = 100,
153
+ ) -> dict | None:
154
+ """Paired bootstrap of gap(a) - gap(b) on the items both strategies finished (used for H3: S4 versus S3)."""
155
+ store = E.Store(os.path.join(cell_dir, "calls.jsonl"))
156
+ P = build_pool(store, model, items, "pool", 16)
157
+ res = read_results(cell_dir)
158
+ if P is None or a not in res or b not in res:
159
+ return None
160
+ ids, acc, tok = P.curves(n_perm, seed)
161
+ common = [i for i in ids if i in res[a] and i in res[b]]
162
+ if len(common) < 5:
163
+ return None
164
+ pos = {i: n for n, i in enumerate(ids)}
165
+ sel = np.array([pos[i] for i in common])
166
+ sa = np.array([float(res[a][i]["correct"]) for i in common])
167
+ ta = np.array(
168
+ [
169
+ float(res[a][i]["prompt_tokens"] + res[a][i]["completion_tokens"])
170
+ for i in common
171
+ ]
172
+ )
173
+ sb = np.array([float(res[b][i]["correct"]) for i in common])
174
+ tb = np.array(
175
+ [
176
+ float(res[b][i]["prompt_tokens"] + res[b][i]["completion_tokens"])
177
+ for i in common
178
+ ]
179
+ )
180
+ from .frontier import frontier_at
181
+
182
+ rng = np.random.default_rng(seed)
183
+ n = len(common)
184
+ A, T_ = acc[sel], tok[sel]
185
+
186
+ def one(ix):
187
+ tk, ak = frontier_curve(A[ix], T_[ix])
188
+ ga = sa[ix].mean() - frontier_at(tk, ak, float(ta[ix].mean()))[0]
189
+ gb = sb[ix].mean() - frontier_at(tk, ak, float(tb[ix].mean()))[0]
190
+ return ga - gb
191
+
192
+ est = one(np.arange(n))
193
+ boots = np.array([one(rng.integers(0, n, n)) for _ in range(n_boot)])
194
+ lo, hi = np.quantile(boots, [0.025, 0.975])
195
+ p_two = float(
196
+ min(1.0, 2 * (min(np.sum(boots <= 0), np.sum(boots >= 0)) + 1) / (n_boot + 1))
197
+ ) # bootstrap two-sided p for H0: difference = 0
198
+ return {
199
+ "difference": float(est),
200
+ "ci_low": float(lo),
201
+ "ci_high": float(hi),
202
+ "p_two_sided": p_two,
203
+ "n_items": n,
204
+ "a": a,
205
+ "b": b,
206
+ }
207
+
208
+
209
+ def holm(pvals: dict[str, float], alpha: float = 0.05) -> dict[str, dict]:
210
+ """Holm step-down: returns adjusted p-values and rejection flags."""
211
+ order = sorted(pvals, key=pvals.get)
212
+ m = len(order)
213
+ out, running = {}, 0.0
214
+ for i, name in enumerate(order):
215
+ adj = min(1.0, (m - i) * pvals[name])
216
+ running = max(running, adj)
217
+ out[name] = {"p": pvals[name], "p_holm": running, "reject": running < alpha}
218
+ return out
219
+
220
+
221
+ def one_sided_p(boots_gap: np.ndarray) -> float:
222
+ """Bootstrap p-value for H0: gap <= 0 (share of resamples with gap <= 0)."""
223
+ return float((np.sum(boots_gap <= 0) + 1) / (len(boots_gap) + 1))
cme/audit.py ADDED
@@ -0,0 +1,145 @@
1
+ """cme audit: a one-screen diagnostic card for a multi-agent evaluation (standard library plus mae_eval).
2
+
3
+ cme audit run_trace.jsonl # a run log with "mas" and "sas" calls: measured MAS/SAS resource ratios and parity
4
+ cme audit run_trace.jsonl --baseline-config sas.json # plus the rung of the single-agent baseline (keys: code/mae_eval/classifier.py)
5
+ cme audit sas.json # a baseline configuration only: its rung and whether a matched resource is declared
6
+
7
+ Badge rules (stated on the card):
8
+ PASS: COMPUTE-MATCHED a log was given, token parity holds within the tolerance (r_token ratio within 1 +/- tolerance), and, if a baseline configuration
9
+ was given, its rung is C or D; or only a configuration was given and its rung is C (rule R2) or D.
10
+ WARNING: BASELINE NOT COMPUTE-MATCHED anything else: parity violated or not measurable, or the baseline is rung A or B, or rung C with the matching unverified (R3).
11
+ Rung A or B does not make a baseline wrong; it means the comparison does not separate coordination from extra computation. Nothing here estimates tokens.
12
+ """
13
+ from __future__ import annotations
14
+
15
+ import json
16
+ import math
17
+ from typing import Dict, Optional
18
+
19
+ BIBTEX = """@unpublished{cme2026,
20
+ title = {More Agents, More Samples, or More Thinking? Untangling Coordination, Ensembling, and Test-Time Compute in Multi-Agent {LLMs}},
21
+ author = {Sajjadi Mohammadabadi, Seyed Mahmoud},
22
+ year = {2026},
23
+ note = {Manuscript in preparation}
24
+ }"""
25
+
26
+
27
+ def _fmt(r: Optional[float]) -> str:
28
+ if r is None:
29
+ return "n/a"
30
+ return "inf" if math.isinf(r) else "%.2f\u00d7" % r
31
+
32
+
33
+ def load_input(path: str):
34
+ """Return ('config', dict) for a single JSON object without a 'system' key, else ('log', {system: [records]})."""
35
+ text = open(path, encoding="utf-8").read()
36
+ try:
37
+ obj = json.loads(text)
38
+ if isinstance(obj, dict) and "system" not in obj:
39
+ return "config", obj
40
+ except json.JSONDecodeError:
41
+ pass
42
+ from mae_eval.cli import TraceError, load_trace
43
+ recs = []
44
+ for no, line in enumerate(text.splitlines(), 1):
45
+ if line.strip():
46
+ r = json.loads(line)
47
+ if "correct" in r and "prompt_tokens" not in r: # a per-item result line written by cme.log_result or by hand
48
+ continue
49
+ recs.append({k: v for k, v in r.items() if v is not None}) # null = unreported (proxy and wrap logs): treat as absent, never as 0 by guess
50
+ import tempfile, os
51
+ fd, tmp = tempfile.mkstemp(suffix=".jsonl")
52
+ try:
53
+ with os.fdopen(fd, "w", encoding="utf-8") as fh:
54
+ fh.write("\n".join(json.dumps(r) for r in recs) + "\n")
55
+ try:
56
+ return "log", load_trace(tmp)
57
+ except TraceError as e:
58
+ raise ValueError(str(e))
59
+ finally:
60
+ os.unlink(tmp)
61
+
62
+
63
+ def audit(path: str, baseline_config: Optional[str] = None, tolerance: float = 0.05, price_in: Optional[float] = None, price_out: Optional[float] = None) -> Dict:
64
+ from mae_eval.auditor import EvaluationAuditor
65
+ from mae_eval.classifier import classify_with_reason
66
+ from mae_eval.cli import session_from_records
67
+ from mae_eval.tracker import ResourceTracker
68
+ kind, data = load_input(path)
69
+ card: Dict = {"input": kind, "tolerance": tolerance, "rung": None, "rule": None, "rung_reason": None, "ratios": None, "parity": None, "notes": []}
70
+ cfg = None
71
+ if kind == "config":
72
+ cfg = data
73
+ elif baseline_config:
74
+ cfg = json.load(open(baseline_config, encoding="utf-8"))
75
+ if cfg is not None:
76
+ card["rung"], card["rule"], card["rung_reason"] = classify_with_reason(cfg)
77
+ if kind == "log":
78
+ if "mas" not in data or "sas" not in data:
79
+ card["notes"].append("the log needs both 'mas' and 'sas' calls to compute ratios; systems found: %s" % ", ".join(sorted(data)))
80
+ else:
81
+ priced = price_in is not None and price_out is not None
82
+ tracker = ResourceTracker("model", "cme-audit")
83
+ if priced:
84
+ tracker.register_pricing(price_in, price_out)
85
+ vec = {k: session_from_records(v, tracker).get_resource_vector(allow_missing_pricing=not priced) for k, v in data.items() if k in ("mas", "sas")}
86
+ aud = EvaluationAuditor(vec["mas"], vec["sas"])
87
+ r = aud.ratios()
88
+ card["ratios"] = {"r_token": r["r_token"], "r_lat": r["r_lat"], "r_cost": r["r_cost"]}
89
+ tok = r["r_token"]
90
+ card["parity"] = bool(tok is not None and not math.isinf(tok) and abs(tok - 1.0) <= tolerance)
91
+ if not priced:
92
+ card["notes"].append("no prices given: cost ratio not computed")
93
+ elif cfg is not None:
94
+ card["notes"].append("a configuration alone has no measured tokens: no resource ratios")
95
+ if card["rung"] is None:
96
+ card["notes"].append("baseline rung not classified: pass --baseline-config or a configuration file")
97
+ rung, rule = card["rung"], card["rule"]
98
+ if kind == "log":
99
+ ok = bool(card["parity"]) and (rung in (None, "C", "D"))
100
+ if rung is None:
101
+ ok = False # parity alone with an unknown baseline is not enough to call the comparison matched
102
+ else:
103
+ ok = rung == "D" or (rung == "C" and rule == "R2")
104
+ card["badge"] = "PASS: COMPUTE-MATCHED" if ok else "WARNING: BASELINE NOT COMPUTE-MATCHED"
105
+ card["ok"] = ok
106
+ return card
107
+
108
+
109
+ def render(card: Dict, ascii_only: bool = False) -> str:
110
+ w = 66
111
+ line = "\u2500" * w
112
+ out = ["\u250c" + line + "\u2510"]
113
+
114
+ def row(s=""):
115
+ import textwrap
116
+ for part in textwrap.wrap(s, w - 3, subsequent_indent=" ", drop_whitespace=False)[:] or [""]:
117
+ out.append("\u2502 " + part.ljust(w - 1) + "\u2502")
118
+ row("CME audit")
119
+ out.append("\u251c" + line + "\u2524")
120
+ if card["rung"]:
121
+ cat = {"A": "Category A (single greedy pass)", "B": "Category B (sampled, unmatched tokens)", "C": "Category C (resource-matched or test-time scaling)",
122
+ "D": "Category D (budget sweep)"}[card["rung"]]
123
+ row("Baseline rung : %s [%s]" % (cat, card["rule"]))
124
+ row(" %s" % card["rung_reason"])
125
+ else:
126
+ row("Baseline rung : not classified")
127
+ r = card["ratios"]
128
+ if r:
129
+ row("MAS / SAS : R_token %s R_lat %s R_cost %s" % (_fmt(r["r_token"]), _fmt(r["r_lat"]), _fmt(r["r_cost"])))
130
+ row("Token parity : %s (tolerance %.0f%%)" % ("holds" if card["parity"] else "violated", 100 * card["tolerance"]))
131
+ else:
132
+ row("MAS / SAS : not available")
133
+ row()
134
+ row("[%s]" % card["badge"])
135
+ for n in card["notes"]:
136
+ row("note: " + n)
137
+ out.append("\u2514" + line + "\u2518")
138
+ out.append("")
139
+ out.append("To cite (BibTeX; the manuscript is in preparation):")
140
+ out.append(BIBTEX)
141
+ text = "\n".join(out)
142
+ if ascii_only:
143
+ for u, c in (("\u250c", "+"), ("\u2510", "+"), ("\u2514", "+"), ("\u2518", "+"), ("\u251c", "+"), ("\u2524", "+"), ("\u2500", "-"), ("\u2502", "|"), ("\u00d7", "x")):
144
+ text = text.replace(u, c)
145
+ return text
cme/baseline.py ADDED
@@ -0,0 +1,162 @@
1
+ """cme baseline: a cached, resumable greedy run plus a pool of independent sampled completions.
2
+
3
+ Every call is appended to a JSONL cache as soon as it finishes, so an interrupted run resumes where it stopped (a record is keyed by
4
+ item id, arm and index). Sampled arms use temperature > 0 and a distinct seed per (item, index); the greedy arm uses temperature 0.
5
+ Token counts come from the endpoint's usage field. Reasoning tokens are recorded only when the endpoint reports them; otherwise the
6
+ field is null and nothing is inferred.
7
+
8
+ Benchmarks: a JSONL file with one object per line {"id", "question", "answer"}; or the built-in `toy` set (synthetic arithmetic, for
9
+ smoke tests only; it says nothing about any model).
10
+ """
11
+ from __future__ import annotations
12
+
13
+ import hashlib
14
+ import json
15
+ import os
16
+ import re
17
+ import time
18
+ import urllib.request
19
+ from typing import Callable, Dict, Iterable, List, Optional
20
+
21
+ PROMPT = ("Solve the problem. Think step by step, then finish with a final line of the form 'Answer: <answer>'.\n\nProblem: {q}")
22
+
23
+
24
+ def toy_bench(n: int = 20) -> List[dict]:
25
+ """Synthetic arithmetic items. Not a benchmark: used only to exercise the pipeline."""
26
+ return [{"id": "toy%03d" % i, "question": "What is %d + %d * %d?" % (i + 3, i + 2, 4), "answer": str(i + 3 + (i + 2) * 4)} for i in range(n)]
27
+
28
+
29
+ def load_bench(spec: str, limit: Optional[int] = None) -> List[dict]:
30
+ if spec == "toy":
31
+ items = toy_bench(limit or 20)
32
+ else:
33
+ items = [json.loads(l) for l in open(spec, encoding="utf-8") if l.strip()]
34
+ for it in items:
35
+ if not {"id", "question", "answer"} <= set(it):
36
+ raise ValueError("bench items need id, question and answer")
37
+ return items[:limit] if limit else items
38
+
39
+
40
+ _ANS = re.compile(r"answer\s*[:=]\s*(.+)", re.I)
41
+
42
+
43
+ def extract_answer(text: str) -> str:
44
+ """Last 'Answer: x' line, else the last number or standalone letter A-J, normalised (trimmed, lower-cased, no $ , or trailing period)."""
45
+ m = _ANS.findall(text or "")
46
+ cand = m[-1] if m else ((re.findall(r"-?\d[\d,]*\.?\d*", text or "") or [""])[-1])
47
+ cand = cand.strip().strip("*$ ").rstrip(".").replace(",", "").lower()
48
+ cand = re.sub(r"^\\boxed\{(.*)\}$", r"\1", cand)
49
+ return cand
50
+
51
+
52
+ def norm_gold(a: str) -> str:
53
+ return str(a).strip().rstrip(".").replace(",", "").lower()
54
+
55
+
56
+ class OpenAICompatClient:
57
+ """Minimal client for /v1/chat/completions (Ollama, vLLM, llama.cpp server, OpenAI-compatible hosts). Standard library only."""
58
+
59
+ def __init__(self, base_url: str = "http://localhost:11434/v1", api_key: str = "none", timeout: float = 600.0):
60
+ self.base, self.key, self.timeout = base_url.rstrip("/"), api_key, timeout
61
+
62
+ def chat(self, model: str, prompt: str, temperature: float, seed: int, max_tokens: int = 2048) -> dict:
63
+ body = json.dumps({"model": model, "messages": [{"role": "user", "content": prompt}], "temperature": temperature, "seed": seed,
64
+ "max_tokens": max_tokens, "stream": False}).encode()
65
+ req = urllib.request.Request(self.base + "/chat/completions", data=body,
66
+ headers={"Content-Type": "application/json", "Authorization": "Bearer " + self.key})
67
+ t0 = time.time()
68
+ with urllib.request.urlopen(req, timeout=self.timeout) as r:
69
+ data = json.loads(r.read())
70
+ lat = time.time() - t0
71
+ msg = data["choices"][0]["message"]
72
+ usage = data.get("usage", {})
73
+ det = usage.get("completion_tokens_details") or {}
74
+ return {"text": msg.get("content") or "", "prompt_tokens": usage.get("prompt_tokens"), "completion_tokens": usage.get("completion_tokens"),
75
+ "reasoning_tokens": det.get("reasoning_tokens"), "latency": lat}
76
+
77
+
78
+ def ollama_digest(model: str, host: str = "http://localhost:11434") -> Optional[str]:
79
+ """Best-effort model digest from Ollama /api/tags; None when unavailable."""
80
+ try:
81
+ with urllib.request.urlopen(host + "/api/tags", timeout=5) as r:
82
+ for m in json.loads(r.read()).get("models", []):
83
+ if m.get("name") == model or m.get("model") == model:
84
+ return m.get("digest")
85
+ except Exception:
86
+ return None
87
+ return None
88
+
89
+
90
+ def read_cache(path: str) -> Dict[tuple, dict]:
91
+ done: Dict[tuple, dict] = {}
92
+ if os.path.exists(path):
93
+ for l in open(path, encoding="utf-8"):
94
+ if l.strip():
95
+ r = json.loads(l)
96
+ done[(r["item"], r["arm"], r["idx"])] = r
97
+ return done
98
+
99
+
100
+ def sample_seed(base_seed: int, item: str, idx: int) -> int:
101
+ """Distinct, reproducible seed per (item, sample index)."""
102
+ return int(hashlib.sha256(("%d|%s|%d" % (base_seed, item, idx)).encode()).hexdigest()[:8], 16)
103
+
104
+
105
+ def run_baseline(client, model: str, items: Iterable[dict], out_path: str, k: int = 16, temperature: float = 0.7, seed: int = 0,
106
+ max_tokens: int = 2048, digest: Optional[str] = None, progress: Optional[Callable[[str], None]] = None) -> dict:
107
+ """Run greedy + k sampled completions per item, appending to out_path. Returns counts: new, cached, errors."""
108
+ if temperature <= 0:
109
+ raise ValueError("sampled arms need temperature > 0")
110
+ done = read_cache(out_path)
111
+ new = cached = errors = 0
112
+ with open(out_path, "a", encoding="utf-8") as fh:
113
+ for it in items:
114
+ gold = norm_gold(it["answer"])
115
+ jobs = [("greedy", 0, 0.0)] + [("sample", j, temperature) for j in range(k)]
116
+ for arm, j, temp in jobs:
117
+ if (it["id"], arm, j) in done:
118
+ cached += 1
119
+ continue
120
+ s = sample_seed(seed, it["id"], j) if arm == "sample" else seed
121
+ try:
122
+ r = client.chat(model, PROMPT.format(q=it["question"]), temp, s, max_tokens)
123
+ except Exception as e: # network or server error: skip and report, do not cache
124
+ errors += 1
125
+ if progress:
126
+ progress("error %s %s %d: %s" % (it["id"], arm, j, e))
127
+ continue
128
+ ans = extract_answer(r["text"])
129
+ rec = {"item": it["id"], "arm": arm, "idx": j, "seed": s, "temperature": temp, "model": model, "model_digest": digest,
130
+ "answer": ans, "gold": gold, "correct": ans == gold, "prompt_tokens": r["prompt_tokens"],
131
+ "completion_tokens": r["completion_tokens"], "reasoning_tokens": r["reasoning_tokens"], "latency": r["latency"]}
132
+ fh.write(json.dumps(rec) + "\n")
133
+ fh.flush()
134
+ new += 1
135
+ if progress:
136
+ progress("item %s done (new %d, cached %d, errors %d)" % (it["id"], new, cached, errors))
137
+ return {"new": new, "cached": cached, "errors": errors}
138
+
139
+
140
+ def load_pool(path: str, k: Optional[int] = None):
141
+ """Build a frontier.Pool and the greedy records from a baseline cache. Items lacking k samples are dropped (reported)."""
142
+ from .frontier import Pool
143
+ rows = list(read_cache(path).values())
144
+ by: Dict[str, dict] = {}
145
+ greedy: Dict[str, dict] = {}
146
+ for r in rows:
147
+ if r["arm"] == "greedy":
148
+ greedy[r["item"]] = r
149
+ else:
150
+ d = by.setdefault(r["item"], {"gold": r["gold"], "samples": {}})
151
+ d["samples"][r["idx"]] = r
152
+ need = k or max((len(d["samples"]) for d in by.values()), default=0)
153
+ items = {}
154
+ dropped = 0
155
+ for it, d in by.items():
156
+ ss = [d["samples"][i] for i in sorted(d["samples"])]
157
+ if len(ss) < need or any(s["prompt_tokens"] is None or s["completion_tokens"] is None for s in ss):
158
+ dropped += 1
159
+ continue
160
+ items[it] = {"gold": d["gold"], "answers": [s["answer"] for s in ss[:need]],
161
+ "tokens": [s["prompt_tokens"] + s["completion_tokens"] for s in ss[:need]]}
162
+ return Pool(items, need), greedy, dropped