pace-eval 0.2.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cme/__init__.py +11 -0
- cme/__main__.py +5 -0
- cme/analysis.py +223 -0
- cme/audit.py +145 -0
- cme/baseline.py +162 -0
- cme/cli.py +279 -0
- cme/client.py +165 -0
- cme/dynamics.py +212 -0
- cme/examples/crewai_integration.py +16 -0
- cme/examples/demo_pool.jsonl +360 -0
- cme/examples/demo_run.jsonl +480 -0
- cme/examples/langgraph_integration.py +27 -0
- cme/examples/make_demo.py +37 -0
- cme/examples/openai_quickstart.py +18 -0
- cme/experiment.py +555 -0
- cme/frontier.py +143 -0
- cme/hidden_profile.py +135 -0
- cme/proxy.py +106 -0
- cme/rebuttal.py +76 -0
- cme/report.py +197 -0
- cme/rollout_card.py +137 -0
- cme/tasks.py +266 -0
- cme/verified_debate.py +127 -0
- cme/verifier.py +92 -0
- mae_eval/__init__.py +11 -0
- mae_eval/__main__.py +5 -0
- mae_eval/auditor.py +112 -0
- mae_eval/classifier.py +62 -0
- mae_eval/cli.py +210 -0
- mae_eval/tracker.py +162 -0
- mae_eval/vector.py +91 -0
- pace/__init__.py +11 -0
- pace/__main__.py +5 -0
- pace/analysis.py +129 -0
- pace/audit.py +145 -0
- pace/baseline.py +162 -0
- pace/card.py +205 -0
- pace/cli.py +898 -0
- pace/client.py +165 -0
- pace/dynamics.py +372 -0
- pace/escape_benchmarks.py +162 -0
- pace/examples/crewai_integration.py +16 -0
- pace/examples/demo_pool.jsonl +360 -0
- pace/examples/demo_run.jsonl +480 -0
- pace/examples/langgraph_integration.py +27 -0
- pace/examples/make_demo.py +37 -0
- pace/examples/openai_quickstart.py +18 -0
- pace/experiment.py +555 -0
- pace/frontier.py +114 -0
- pace/hidden_profile.py +301 -0
- pace/importers.py +219 -0
- pace/inflate.py +213 -0
- pace/influence.py +231 -0
- pace/meta_engine.py +246 -0
- pace/middleware.py +230 -0
- pace/open_ended.py +114 -0
- pace/plotting.py +132 -0
- pace/pool_sizer.py +154 -0
- pace/protocol_v3.py +414 -0
- pace/proxy.py +106 -0
- pace/rebuttal.py +219 -0
- pace/report.py +197 -0
- pace/rollout_card.py +137 -0
- pace/schema.py +74 -0
- pace/simulator.py +199 -0
- pace/studio.html +1040 -0
- pace/tasks.py +266 -0
- pace/verified_debate.py +127 -0
- pace/verifier.py +92 -0
- pace/workbench.html +26 -0
- pace/workbench.py +119 -0
- pace_eval-0.2.2.dist-info/METADATA +205 -0
- pace_eval-0.2.2.dist-info/RECORD +83 -0
- pace_eval-0.2.2.dist-info/WHEEL +5 -0
- pace_eval-0.2.2.dist-info/entry_points.txt +4 -0
- pace_eval-0.2.2.dist-info/licenses/LICENSE +202 -0
- pace_eval-0.2.2.dist-info/top_level.txt +4 -0
- pace_lab/__init__.py +9 -0
- pace_lab/clients.py +568 -0
- pace_lab/concurrency_test.py +127 -0
- pace_lab/data.py +445 -0
- pace_lab/graders.py +626 -0
- pace_lab/strategies.py +345 -0
cme/__init__.py
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
"""cme: measure a multi-agent system against the self-consistency frontier at its measured token cost.
|
|
2
|
+
|
|
3
|
+
Commands: `cme proxy` (logging proxy), `cme baseline` (greedy + sample pool), `cme report` (frontier gap, resource card, table, figure).
|
|
4
|
+
Built on mae_eval (resource vector and parity auditor). Python API: cme.frontier_gap, cme.build_report, cme.resource_card, cme.wrap (log a client's usage without a proxy), cme.track (label items and rounds).
|
|
5
|
+
"""
|
|
6
|
+
from .frontier import Pool, frontier_at, frontier_curve, frontier_gap, pav_increasing
|
|
7
|
+
from .report import build_report, to_latex
|
|
8
|
+
from .client import wrap, track, log_result
|
|
9
|
+
|
|
10
|
+
__version__ = "0.1.0"
|
|
11
|
+
__all__ = ["Pool", "frontier_at", "frontier_curve", "frontier_gap", "pav_increasing", "build_report", "to_latex", "wrap", "track", "log_result"]
|
cme/__main__.py
ADDED
cme/analysis.py
ADDED
|
@@ -0,0 +1,223 @@
|
|
|
1
|
+
"""Load a confirmatory cell (one model x one task) and estimate frontier gaps per strategy.
|
|
2
|
+
|
|
3
|
+
A cell directory holds calls.jsonl (every call) and results.jsonl (one row per strategy x item). The pool frontier is rebuilt from the
|
|
4
|
+
cached pool calls with the same scorer used at run time, so re-scoring never needs a model. Correctness of a pool sample is mapped to
|
|
5
|
+
the label 'G' (wrong answers keep their normalised text) so plurality voting uses the scorer's notion of correct.
|
|
6
|
+
|
|
7
|
+
Estimand (docs/PREREGISTRATION.md section 5): gap(s) = accuracy(s) - F(mean tokens of s), F the isotonic self-consistency frontier,
|
|
8
|
+
with a paired bootstrap over items. `pool="views"` builds the frontier from the independent split-evidence agents (S2v) instead.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
import json
|
|
14
|
+
import os
|
|
15
|
+
|
|
16
|
+
import numpy as np
|
|
17
|
+
|
|
18
|
+
from . import experiment as E
|
|
19
|
+
from . import tasks as T
|
|
20
|
+
from .frontier import Pool, frontier_curve, frontier_gap, pav_increasing
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def read_results(cell_dir: str) -> dict[str, dict[str, dict]]:
|
|
24
|
+
out: dict[str, dict[str, dict]] = {}
|
|
25
|
+
p = os.path.join(cell_dir, "results.jsonl")
|
|
26
|
+
if os.path.exists(p):
|
|
27
|
+
for l in open(p, encoding="utf-8"):
|
|
28
|
+
if l.strip():
|
|
29
|
+
r = json.loads(l)
|
|
30
|
+
out.setdefault(r["strategy"], {})[r["item"]] = r
|
|
31
|
+
return out
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def build_pool(
|
|
35
|
+
store: E.Store,
|
|
36
|
+
model: str,
|
|
37
|
+
items: list[dict],
|
|
38
|
+
pool: str = "pool",
|
|
39
|
+
k: int = 16,
|
|
40
|
+
extension_path: str | None = None,
|
|
41
|
+
) -> Pool:
|
|
42
|
+
"""Pool of per-item sampled answers and token counts. pool='pool' uses the 16 full-evidence samples; 'views' the 5 split-evidence agents."""
|
|
43
|
+
extra = {}
|
|
44
|
+
if extension_path and os.path.exists(extension_path):
|
|
45
|
+
for line in open(extension_path, encoding="utf-8"):
|
|
46
|
+
if line.strip():
|
|
47
|
+
row = json.loads(line)
|
|
48
|
+
extra[row["key"]] = row
|
|
49
|
+
pitems = {}
|
|
50
|
+
for it in items:
|
|
51
|
+
if it["kind"] == "code":
|
|
52
|
+
continue
|
|
53
|
+
keys = (
|
|
54
|
+
["pool|%s|%s|%d" % (model, it["id"], i) for i in range(k)]
|
|
55
|
+
if pool == "pool"
|
|
56
|
+
else ["r1v|%s|%s|%d" % (model, it["id"], i) for i in range(k)]
|
|
57
|
+
)
|
|
58
|
+
recs = [store.d.get(x) or extra.get(x) for x in keys]
|
|
59
|
+
if any(
|
|
60
|
+
r is None
|
|
61
|
+
or r.get("prompt_tokens") is None
|
|
62
|
+
or r.get("completion_tokens") is None
|
|
63
|
+
for r in recs
|
|
64
|
+
):
|
|
65
|
+
continue
|
|
66
|
+
pitems[it["id"]] = {
|
|
67
|
+
"gold": "G",
|
|
68
|
+
"answers": [
|
|
69
|
+
"G"
|
|
70
|
+
if T.score(it, r["text"])
|
|
71
|
+
else ((E.ans_of(it, r) + "_wrong") if E.ans_of(it, r) else "")
|
|
72
|
+
for r in recs
|
|
73
|
+
],
|
|
74
|
+
"tokens": [r["prompt_tokens"] + r["completion_tokens"] for r in recs],
|
|
75
|
+
}
|
|
76
|
+
return Pool(pitems, k) if pitems else None
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def cell_gaps(
|
|
80
|
+
cell_dir: str,
|
|
81
|
+
model: str,
|
|
82
|
+
items: list[dict],
|
|
83
|
+
strategies: list[str] | None = None,
|
|
84
|
+
pool: str = "pool",
|
|
85
|
+
k: int | None = None,
|
|
86
|
+
n_boot: int = 2000,
|
|
87
|
+
n_perm: int = 100,
|
|
88
|
+
seed: int = 0,
|
|
89
|
+
tokens: str = "total",
|
|
90
|
+
extension_path: str | None = None,
|
|
91
|
+
) -> dict:
|
|
92
|
+
"""Frontier, per-strategy accuracy, mean tokens and gap with a bootstrap interval. tokens='completion' is the sensitivity analysis."""
|
|
93
|
+
store = E.Store(os.path.join(cell_dir, "calls.jsonl"))
|
|
94
|
+
k = k or (16 if pool == "pool" else 5)
|
|
95
|
+
P = build_pool(store, model, items, pool, k, extension_path=extension_path)
|
|
96
|
+
if P is None:
|
|
97
|
+
return {"error": "no complete pool in %s" % cell_dir}
|
|
98
|
+
if tokens == "completion": # recompute pool tokens with completion tokens only
|
|
99
|
+
for it in P.items:
|
|
100
|
+
keys = [
|
|
101
|
+
"pool|%s|%s|%d" % (model, it, i)
|
|
102
|
+
if pool == "pool"
|
|
103
|
+
else "r1v|%s|%s|%d" % (model, it, i)
|
|
104
|
+
for i in range(k)
|
|
105
|
+
]
|
|
106
|
+
P.items[it]["tokens"] = [store.d[x]["completion_tokens"] for x in keys]
|
|
107
|
+
ids, acc, tok = P.curves(n_perm, seed)
|
|
108
|
+
res = read_results(cell_dir)
|
|
109
|
+
out = {
|
|
110
|
+
"model": model,
|
|
111
|
+
"pool": pool,
|
|
112
|
+
"k": k,
|
|
113
|
+
"n_pool_items": len(ids),
|
|
114
|
+
"tokens": tokens,
|
|
115
|
+
"strategies": {},
|
|
116
|
+
}
|
|
117
|
+
tk, ak = frontier_curve(acc, tok)
|
|
118
|
+
out["frontier"] = {
|
|
119
|
+
"tokens": tk.tolist(),
|
|
120
|
+
"sc_accuracy": ak.tolist(),
|
|
121
|
+
"isotonic": pav_increasing(ak).tolist(),
|
|
122
|
+
}
|
|
123
|
+
for s, rows in sorted(res.items()):
|
|
124
|
+
if strategies and s not in strategies:
|
|
125
|
+
continue
|
|
126
|
+
common = [i for i in ids if i in rows]
|
|
127
|
+
if len(common) < 5:
|
|
128
|
+
continue
|
|
129
|
+
pos = {i: n for n, i in enumerate(ids)}
|
|
130
|
+
sel = [pos[i] for i in common]
|
|
131
|
+
use = (
|
|
132
|
+
(lambda r: r["completion_tokens"])
|
|
133
|
+
if tokens == "completion"
|
|
134
|
+
else (lambda r: r["prompt_tokens"] + r["completion_tokens"])
|
|
135
|
+
)
|
|
136
|
+
sc = np.array([float(rows[i]["correct"]) for i in common])
|
|
137
|
+
st = np.array([float(use(rows[i])) for i in common])
|
|
138
|
+
g = frontier_gap(acc[sel], tok[sel], sc, st, n_boot=n_boot, seed=seed)
|
|
139
|
+
g["n_items"] = len(common)
|
|
140
|
+
out["strategies"][s] = g
|
|
141
|
+
return out
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
def difference_of_gaps(
|
|
145
|
+
cell_dir: str,
|
|
146
|
+
model: str,
|
|
147
|
+
items: list[dict],
|
|
148
|
+
a: str,
|
|
149
|
+
b: str,
|
|
150
|
+
n_boot: int = 2000,
|
|
151
|
+
seed: int = 0,
|
|
152
|
+
n_perm: int = 100,
|
|
153
|
+
) -> dict | None:
|
|
154
|
+
"""Paired bootstrap of gap(a) - gap(b) on the items both strategies finished (used for H3: S4 versus S3)."""
|
|
155
|
+
store = E.Store(os.path.join(cell_dir, "calls.jsonl"))
|
|
156
|
+
P = build_pool(store, model, items, "pool", 16)
|
|
157
|
+
res = read_results(cell_dir)
|
|
158
|
+
if P is None or a not in res or b not in res:
|
|
159
|
+
return None
|
|
160
|
+
ids, acc, tok = P.curves(n_perm, seed)
|
|
161
|
+
common = [i for i in ids if i in res[a] and i in res[b]]
|
|
162
|
+
if len(common) < 5:
|
|
163
|
+
return None
|
|
164
|
+
pos = {i: n for n, i in enumerate(ids)}
|
|
165
|
+
sel = np.array([pos[i] for i in common])
|
|
166
|
+
sa = np.array([float(res[a][i]["correct"]) for i in common])
|
|
167
|
+
ta = np.array(
|
|
168
|
+
[
|
|
169
|
+
float(res[a][i]["prompt_tokens"] + res[a][i]["completion_tokens"])
|
|
170
|
+
for i in common
|
|
171
|
+
]
|
|
172
|
+
)
|
|
173
|
+
sb = np.array([float(res[b][i]["correct"]) for i in common])
|
|
174
|
+
tb = np.array(
|
|
175
|
+
[
|
|
176
|
+
float(res[b][i]["prompt_tokens"] + res[b][i]["completion_tokens"])
|
|
177
|
+
for i in common
|
|
178
|
+
]
|
|
179
|
+
)
|
|
180
|
+
from .frontier import frontier_at
|
|
181
|
+
|
|
182
|
+
rng = np.random.default_rng(seed)
|
|
183
|
+
n = len(common)
|
|
184
|
+
A, T_ = acc[sel], tok[sel]
|
|
185
|
+
|
|
186
|
+
def one(ix):
|
|
187
|
+
tk, ak = frontier_curve(A[ix], T_[ix])
|
|
188
|
+
ga = sa[ix].mean() - frontier_at(tk, ak, float(ta[ix].mean()))[0]
|
|
189
|
+
gb = sb[ix].mean() - frontier_at(tk, ak, float(tb[ix].mean()))[0]
|
|
190
|
+
return ga - gb
|
|
191
|
+
|
|
192
|
+
est = one(np.arange(n))
|
|
193
|
+
boots = np.array([one(rng.integers(0, n, n)) for _ in range(n_boot)])
|
|
194
|
+
lo, hi = np.quantile(boots, [0.025, 0.975])
|
|
195
|
+
p_two = float(
|
|
196
|
+
min(1.0, 2 * (min(np.sum(boots <= 0), np.sum(boots >= 0)) + 1) / (n_boot + 1))
|
|
197
|
+
) # bootstrap two-sided p for H0: difference = 0
|
|
198
|
+
return {
|
|
199
|
+
"difference": float(est),
|
|
200
|
+
"ci_low": float(lo),
|
|
201
|
+
"ci_high": float(hi),
|
|
202
|
+
"p_two_sided": p_two,
|
|
203
|
+
"n_items": n,
|
|
204
|
+
"a": a,
|
|
205
|
+
"b": b,
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
|
|
209
|
+
def holm(pvals: dict[str, float], alpha: float = 0.05) -> dict[str, dict]:
|
|
210
|
+
"""Holm step-down: returns adjusted p-values and rejection flags."""
|
|
211
|
+
order = sorted(pvals, key=pvals.get)
|
|
212
|
+
m = len(order)
|
|
213
|
+
out, running = {}, 0.0
|
|
214
|
+
for i, name in enumerate(order):
|
|
215
|
+
adj = min(1.0, (m - i) * pvals[name])
|
|
216
|
+
running = max(running, adj)
|
|
217
|
+
out[name] = {"p": pvals[name], "p_holm": running, "reject": running < alpha}
|
|
218
|
+
return out
|
|
219
|
+
|
|
220
|
+
|
|
221
|
+
def one_sided_p(boots_gap: np.ndarray) -> float:
|
|
222
|
+
"""Bootstrap p-value for H0: gap <= 0 (share of resamples with gap <= 0)."""
|
|
223
|
+
return float((np.sum(boots_gap <= 0) + 1) / (len(boots_gap) + 1))
|
cme/audit.py
ADDED
|
@@ -0,0 +1,145 @@
|
|
|
1
|
+
"""cme audit: a one-screen diagnostic card for a multi-agent evaluation (standard library plus mae_eval).
|
|
2
|
+
|
|
3
|
+
cme audit run_trace.jsonl # a run log with "mas" and "sas" calls: measured MAS/SAS resource ratios and parity
|
|
4
|
+
cme audit run_trace.jsonl --baseline-config sas.json # plus the rung of the single-agent baseline (keys: code/mae_eval/classifier.py)
|
|
5
|
+
cme audit sas.json # a baseline configuration only: its rung and whether a matched resource is declared
|
|
6
|
+
|
|
7
|
+
Badge rules (stated on the card):
|
|
8
|
+
PASS: COMPUTE-MATCHED a log was given, token parity holds within the tolerance (r_token ratio within 1 +/- tolerance), and, if a baseline configuration
|
|
9
|
+
was given, its rung is C or D; or only a configuration was given and its rung is C (rule R2) or D.
|
|
10
|
+
WARNING: BASELINE NOT COMPUTE-MATCHED anything else: parity violated or not measurable, or the baseline is rung A or B, or rung C with the matching unverified (R3).
|
|
11
|
+
Rung A or B does not make a baseline wrong; it means the comparison does not separate coordination from extra computation. Nothing here estimates tokens.
|
|
12
|
+
"""
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
import json
|
|
16
|
+
import math
|
|
17
|
+
from typing import Dict, Optional
|
|
18
|
+
|
|
19
|
+
BIBTEX = """@unpublished{cme2026,
|
|
20
|
+
title = {More Agents, More Samples, or More Thinking? Untangling Coordination, Ensembling, and Test-Time Compute in Multi-Agent {LLMs}},
|
|
21
|
+
author = {Sajjadi Mohammadabadi, Seyed Mahmoud},
|
|
22
|
+
year = {2026},
|
|
23
|
+
note = {Manuscript in preparation}
|
|
24
|
+
}"""
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def _fmt(r: Optional[float]) -> str:
|
|
28
|
+
if r is None:
|
|
29
|
+
return "n/a"
|
|
30
|
+
return "inf" if math.isinf(r) else "%.2f\u00d7" % r
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def load_input(path: str):
|
|
34
|
+
"""Return ('config', dict) for a single JSON object without a 'system' key, else ('log', {system: [records]})."""
|
|
35
|
+
text = open(path, encoding="utf-8").read()
|
|
36
|
+
try:
|
|
37
|
+
obj = json.loads(text)
|
|
38
|
+
if isinstance(obj, dict) and "system" not in obj:
|
|
39
|
+
return "config", obj
|
|
40
|
+
except json.JSONDecodeError:
|
|
41
|
+
pass
|
|
42
|
+
from mae_eval.cli import TraceError, load_trace
|
|
43
|
+
recs = []
|
|
44
|
+
for no, line in enumerate(text.splitlines(), 1):
|
|
45
|
+
if line.strip():
|
|
46
|
+
r = json.loads(line)
|
|
47
|
+
if "correct" in r and "prompt_tokens" not in r: # a per-item result line written by cme.log_result or by hand
|
|
48
|
+
continue
|
|
49
|
+
recs.append({k: v for k, v in r.items() if v is not None}) # null = unreported (proxy and wrap logs): treat as absent, never as 0 by guess
|
|
50
|
+
import tempfile, os
|
|
51
|
+
fd, tmp = tempfile.mkstemp(suffix=".jsonl")
|
|
52
|
+
try:
|
|
53
|
+
with os.fdopen(fd, "w", encoding="utf-8") as fh:
|
|
54
|
+
fh.write("\n".join(json.dumps(r) for r in recs) + "\n")
|
|
55
|
+
try:
|
|
56
|
+
return "log", load_trace(tmp)
|
|
57
|
+
except TraceError as e:
|
|
58
|
+
raise ValueError(str(e))
|
|
59
|
+
finally:
|
|
60
|
+
os.unlink(tmp)
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def audit(path: str, baseline_config: Optional[str] = None, tolerance: float = 0.05, price_in: Optional[float] = None, price_out: Optional[float] = None) -> Dict:
|
|
64
|
+
from mae_eval.auditor import EvaluationAuditor
|
|
65
|
+
from mae_eval.classifier import classify_with_reason
|
|
66
|
+
from mae_eval.cli import session_from_records
|
|
67
|
+
from mae_eval.tracker import ResourceTracker
|
|
68
|
+
kind, data = load_input(path)
|
|
69
|
+
card: Dict = {"input": kind, "tolerance": tolerance, "rung": None, "rule": None, "rung_reason": None, "ratios": None, "parity": None, "notes": []}
|
|
70
|
+
cfg = None
|
|
71
|
+
if kind == "config":
|
|
72
|
+
cfg = data
|
|
73
|
+
elif baseline_config:
|
|
74
|
+
cfg = json.load(open(baseline_config, encoding="utf-8"))
|
|
75
|
+
if cfg is not None:
|
|
76
|
+
card["rung"], card["rule"], card["rung_reason"] = classify_with_reason(cfg)
|
|
77
|
+
if kind == "log":
|
|
78
|
+
if "mas" not in data or "sas" not in data:
|
|
79
|
+
card["notes"].append("the log needs both 'mas' and 'sas' calls to compute ratios; systems found: %s" % ", ".join(sorted(data)))
|
|
80
|
+
else:
|
|
81
|
+
priced = price_in is not None and price_out is not None
|
|
82
|
+
tracker = ResourceTracker("model", "cme-audit")
|
|
83
|
+
if priced:
|
|
84
|
+
tracker.register_pricing(price_in, price_out)
|
|
85
|
+
vec = {k: session_from_records(v, tracker).get_resource_vector(allow_missing_pricing=not priced) for k, v in data.items() if k in ("mas", "sas")}
|
|
86
|
+
aud = EvaluationAuditor(vec["mas"], vec["sas"])
|
|
87
|
+
r = aud.ratios()
|
|
88
|
+
card["ratios"] = {"r_token": r["r_token"], "r_lat": r["r_lat"], "r_cost": r["r_cost"]}
|
|
89
|
+
tok = r["r_token"]
|
|
90
|
+
card["parity"] = bool(tok is not None and not math.isinf(tok) and abs(tok - 1.0) <= tolerance)
|
|
91
|
+
if not priced:
|
|
92
|
+
card["notes"].append("no prices given: cost ratio not computed")
|
|
93
|
+
elif cfg is not None:
|
|
94
|
+
card["notes"].append("a configuration alone has no measured tokens: no resource ratios")
|
|
95
|
+
if card["rung"] is None:
|
|
96
|
+
card["notes"].append("baseline rung not classified: pass --baseline-config or a configuration file")
|
|
97
|
+
rung, rule = card["rung"], card["rule"]
|
|
98
|
+
if kind == "log":
|
|
99
|
+
ok = bool(card["parity"]) and (rung in (None, "C", "D"))
|
|
100
|
+
if rung is None:
|
|
101
|
+
ok = False # parity alone with an unknown baseline is not enough to call the comparison matched
|
|
102
|
+
else:
|
|
103
|
+
ok = rung == "D" or (rung == "C" and rule == "R2")
|
|
104
|
+
card["badge"] = "PASS: COMPUTE-MATCHED" if ok else "WARNING: BASELINE NOT COMPUTE-MATCHED"
|
|
105
|
+
card["ok"] = ok
|
|
106
|
+
return card
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def render(card: Dict, ascii_only: bool = False) -> str:
|
|
110
|
+
w = 66
|
|
111
|
+
line = "\u2500" * w
|
|
112
|
+
out = ["\u250c" + line + "\u2510"]
|
|
113
|
+
|
|
114
|
+
def row(s=""):
|
|
115
|
+
import textwrap
|
|
116
|
+
for part in textwrap.wrap(s, w - 3, subsequent_indent=" ", drop_whitespace=False)[:] or [""]:
|
|
117
|
+
out.append("\u2502 " + part.ljust(w - 1) + "\u2502")
|
|
118
|
+
row("CME audit")
|
|
119
|
+
out.append("\u251c" + line + "\u2524")
|
|
120
|
+
if card["rung"]:
|
|
121
|
+
cat = {"A": "Category A (single greedy pass)", "B": "Category B (sampled, unmatched tokens)", "C": "Category C (resource-matched or test-time scaling)",
|
|
122
|
+
"D": "Category D (budget sweep)"}[card["rung"]]
|
|
123
|
+
row("Baseline rung : %s [%s]" % (cat, card["rule"]))
|
|
124
|
+
row(" %s" % card["rung_reason"])
|
|
125
|
+
else:
|
|
126
|
+
row("Baseline rung : not classified")
|
|
127
|
+
r = card["ratios"]
|
|
128
|
+
if r:
|
|
129
|
+
row("MAS / SAS : R_token %s R_lat %s R_cost %s" % (_fmt(r["r_token"]), _fmt(r["r_lat"]), _fmt(r["r_cost"])))
|
|
130
|
+
row("Token parity : %s (tolerance %.0f%%)" % ("holds" if card["parity"] else "violated", 100 * card["tolerance"]))
|
|
131
|
+
else:
|
|
132
|
+
row("MAS / SAS : not available")
|
|
133
|
+
row()
|
|
134
|
+
row("[%s]" % card["badge"])
|
|
135
|
+
for n in card["notes"]:
|
|
136
|
+
row("note: " + n)
|
|
137
|
+
out.append("\u2514" + line + "\u2518")
|
|
138
|
+
out.append("")
|
|
139
|
+
out.append("To cite (BibTeX; the manuscript is in preparation):")
|
|
140
|
+
out.append(BIBTEX)
|
|
141
|
+
text = "\n".join(out)
|
|
142
|
+
if ascii_only:
|
|
143
|
+
for u, c in (("\u250c", "+"), ("\u2510", "+"), ("\u2514", "+"), ("\u2518", "+"), ("\u251c", "+"), ("\u2524", "+"), ("\u2500", "-"), ("\u2502", "|"), ("\u00d7", "x")):
|
|
144
|
+
text = text.replace(u, c)
|
|
145
|
+
return text
|
cme/baseline.py
ADDED
|
@@ -0,0 +1,162 @@
|
|
|
1
|
+
"""cme baseline: a cached, resumable greedy run plus a pool of independent sampled completions.
|
|
2
|
+
|
|
3
|
+
Every call is appended to a JSONL cache as soon as it finishes, so an interrupted run resumes where it stopped (a record is keyed by
|
|
4
|
+
item id, arm and index). Sampled arms use temperature > 0 and a distinct seed per (item, index); the greedy arm uses temperature 0.
|
|
5
|
+
Token counts come from the endpoint's usage field. Reasoning tokens are recorded only when the endpoint reports them; otherwise the
|
|
6
|
+
field is null and nothing is inferred.
|
|
7
|
+
|
|
8
|
+
Benchmarks: a JSONL file with one object per line {"id", "question", "answer"}; or the built-in `toy` set (synthetic arithmetic, for
|
|
9
|
+
smoke tests only; it says nothing about any model).
|
|
10
|
+
"""
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
import hashlib
|
|
14
|
+
import json
|
|
15
|
+
import os
|
|
16
|
+
import re
|
|
17
|
+
import time
|
|
18
|
+
import urllib.request
|
|
19
|
+
from typing import Callable, Dict, Iterable, List, Optional
|
|
20
|
+
|
|
21
|
+
PROMPT = ("Solve the problem. Think step by step, then finish with a final line of the form 'Answer: <answer>'.\n\nProblem: {q}")
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def toy_bench(n: int = 20) -> List[dict]:
|
|
25
|
+
"""Synthetic arithmetic items. Not a benchmark: used only to exercise the pipeline."""
|
|
26
|
+
return [{"id": "toy%03d" % i, "question": "What is %d + %d * %d?" % (i + 3, i + 2, 4), "answer": str(i + 3 + (i + 2) * 4)} for i in range(n)]
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def load_bench(spec: str, limit: Optional[int] = None) -> List[dict]:
|
|
30
|
+
if spec == "toy":
|
|
31
|
+
items = toy_bench(limit or 20)
|
|
32
|
+
else:
|
|
33
|
+
items = [json.loads(l) for l in open(spec, encoding="utf-8") if l.strip()]
|
|
34
|
+
for it in items:
|
|
35
|
+
if not {"id", "question", "answer"} <= set(it):
|
|
36
|
+
raise ValueError("bench items need id, question and answer")
|
|
37
|
+
return items[:limit] if limit else items
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
_ANS = re.compile(r"answer\s*[:=]\s*(.+)", re.I)
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def extract_answer(text: str) -> str:
|
|
44
|
+
"""Last 'Answer: x' line, else the last number or standalone letter A-J, normalised (trimmed, lower-cased, no $ , or trailing period)."""
|
|
45
|
+
m = _ANS.findall(text or "")
|
|
46
|
+
cand = m[-1] if m else ((re.findall(r"-?\d[\d,]*\.?\d*", text or "") or [""])[-1])
|
|
47
|
+
cand = cand.strip().strip("*$ ").rstrip(".").replace(",", "").lower()
|
|
48
|
+
cand = re.sub(r"^\\boxed\{(.*)\}$", r"\1", cand)
|
|
49
|
+
return cand
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def norm_gold(a: str) -> str:
|
|
53
|
+
return str(a).strip().rstrip(".").replace(",", "").lower()
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
class OpenAICompatClient:
|
|
57
|
+
"""Minimal client for /v1/chat/completions (Ollama, vLLM, llama.cpp server, OpenAI-compatible hosts). Standard library only."""
|
|
58
|
+
|
|
59
|
+
def __init__(self, base_url: str = "http://localhost:11434/v1", api_key: str = "none", timeout: float = 600.0):
|
|
60
|
+
self.base, self.key, self.timeout = base_url.rstrip("/"), api_key, timeout
|
|
61
|
+
|
|
62
|
+
def chat(self, model: str, prompt: str, temperature: float, seed: int, max_tokens: int = 2048) -> dict:
|
|
63
|
+
body = json.dumps({"model": model, "messages": [{"role": "user", "content": prompt}], "temperature": temperature, "seed": seed,
|
|
64
|
+
"max_tokens": max_tokens, "stream": False}).encode()
|
|
65
|
+
req = urllib.request.Request(self.base + "/chat/completions", data=body,
|
|
66
|
+
headers={"Content-Type": "application/json", "Authorization": "Bearer " + self.key})
|
|
67
|
+
t0 = time.time()
|
|
68
|
+
with urllib.request.urlopen(req, timeout=self.timeout) as r:
|
|
69
|
+
data = json.loads(r.read())
|
|
70
|
+
lat = time.time() - t0
|
|
71
|
+
msg = data["choices"][0]["message"]
|
|
72
|
+
usage = data.get("usage", {})
|
|
73
|
+
det = usage.get("completion_tokens_details") or {}
|
|
74
|
+
return {"text": msg.get("content") or "", "prompt_tokens": usage.get("prompt_tokens"), "completion_tokens": usage.get("completion_tokens"),
|
|
75
|
+
"reasoning_tokens": det.get("reasoning_tokens"), "latency": lat}
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def ollama_digest(model: str, host: str = "http://localhost:11434") -> Optional[str]:
|
|
79
|
+
"""Best-effort model digest from Ollama /api/tags; None when unavailable."""
|
|
80
|
+
try:
|
|
81
|
+
with urllib.request.urlopen(host + "/api/tags", timeout=5) as r:
|
|
82
|
+
for m in json.loads(r.read()).get("models", []):
|
|
83
|
+
if m.get("name") == model or m.get("model") == model:
|
|
84
|
+
return m.get("digest")
|
|
85
|
+
except Exception:
|
|
86
|
+
return None
|
|
87
|
+
return None
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def read_cache(path: str) -> Dict[tuple, dict]:
|
|
91
|
+
done: Dict[tuple, dict] = {}
|
|
92
|
+
if os.path.exists(path):
|
|
93
|
+
for l in open(path, encoding="utf-8"):
|
|
94
|
+
if l.strip():
|
|
95
|
+
r = json.loads(l)
|
|
96
|
+
done[(r["item"], r["arm"], r["idx"])] = r
|
|
97
|
+
return done
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def sample_seed(base_seed: int, item: str, idx: int) -> int:
|
|
101
|
+
"""Distinct, reproducible seed per (item, sample index)."""
|
|
102
|
+
return int(hashlib.sha256(("%d|%s|%d" % (base_seed, item, idx)).encode()).hexdigest()[:8], 16)
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def run_baseline(client, model: str, items: Iterable[dict], out_path: str, k: int = 16, temperature: float = 0.7, seed: int = 0,
|
|
106
|
+
max_tokens: int = 2048, digest: Optional[str] = None, progress: Optional[Callable[[str], None]] = None) -> dict:
|
|
107
|
+
"""Run greedy + k sampled completions per item, appending to out_path. Returns counts: new, cached, errors."""
|
|
108
|
+
if temperature <= 0:
|
|
109
|
+
raise ValueError("sampled arms need temperature > 0")
|
|
110
|
+
done = read_cache(out_path)
|
|
111
|
+
new = cached = errors = 0
|
|
112
|
+
with open(out_path, "a", encoding="utf-8") as fh:
|
|
113
|
+
for it in items:
|
|
114
|
+
gold = norm_gold(it["answer"])
|
|
115
|
+
jobs = [("greedy", 0, 0.0)] + [("sample", j, temperature) for j in range(k)]
|
|
116
|
+
for arm, j, temp in jobs:
|
|
117
|
+
if (it["id"], arm, j) in done:
|
|
118
|
+
cached += 1
|
|
119
|
+
continue
|
|
120
|
+
s = sample_seed(seed, it["id"], j) if arm == "sample" else seed
|
|
121
|
+
try:
|
|
122
|
+
r = client.chat(model, PROMPT.format(q=it["question"]), temp, s, max_tokens)
|
|
123
|
+
except Exception as e: # network or server error: skip and report, do not cache
|
|
124
|
+
errors += 1
|
|
125
|
+
if progress:
|
|
126
|
+
progress("error %s %s %d: %s" % (it["id"], arm, j, e))
|
|
127
|
+
continue
|
|
128
|
+
ans = extract_answer(r["text"])
|
|
129
|
+
rec = {"item": it["id"], "arm": arm, "idx": j, "seed": s, "temperature": temp, "model": model, "model_digest": digest,
|
|
130
|
+
"answer": ans, "gold": gold, "correct": ans == gold, "prompt_tokens": r["prompt_tokens"],
|
|
131
|
+
"completion_tokens": r["completion_tokens"], "reasoning_tokens": r["reasoning_tokens"], "latency": r["latency"]}
|
|
132
|
+
fh.write(json.dumps(rec) + "\n")
|
|
133
|
+
fh.flush()
|
|
134
|
+
new += 1
|
|
135
|
+
if progress:
|
|
136
|
+
progress("item %s done (new %d, cached %d, errors %d)" % (it["id"], new, cached, errors))
|
|
137
|
+
return {"new": new, "cached": cached, "errors": errors}
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
def load_pool(path: str, k: Optional[int] = None):
|
|
141
|
+
"""Build a frontier.Pool and the greedy records from a baseline cache. Items lacking k samples are dropped (reported)."""
|
|
142
|
+
from .frontier import Pool
|
|
143
|
+
rows = list(read_cache(path).values())
|
|
144
|
+
by: Dict[str, dict] = {}
|
|
145
|
+
greedy: Dict[str, dict] = {}
|
|
146
|
+
for r in rows:
|
|
147
|
+
if r["arm"] == "greedy":
|
|
148
|
+
greedy[r["item"]] = r
|
|
149
|
+
else:
|
|
150
|
+
d = by.setdefault(r["item"], {"gold": r["gold"], "samples": {}})
|
|
151
|
+
d["samples"][r["idx"]] = r
|
|
152
|
+
need = k or max((len(d["samples"]) for d in by.values()), default=0)
|
|
153
|
+
items = {}
|
|
154
|
+
dropped = 0
|
|
155
|
+
for it, d in by.items():
|
|
156
|
+
ss = [d["samples"][i] for i in sorted(d["samples"])]
|
|
157
|
+
if len(ss) < need or any(s["prompt_tokens"] is None or s["completion_tokens"] is None for s in ss):
|
|
158
|
+
dropped += 1
|
|
159
|
+
continue
|
|
160
|
+
items[it] = {"gold": d["gold"], "answers": [s["answer"] for s in ss[:need]],
|
|
161
|
+
"tokens": [s["prompt_tokens"] + s["completion_tokens"] for s in ss[:need]]}
|
|
162
|
+
return Pool(items, need), greedy, dropped
|