pace-eval 0.2.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cme/__init__.py +11 -0
- cme/__main__.py +5 -0
- cme/analysis.py +223 -0
- cme/audit.py +145 -0
- cme/baseline.py +162 -0
- cme/cli.py +279 -0
- cme/client.py +165 -0
- cme/dynamics.py +212 -0
- cme/examples/crewai_integration.py +16 -0
- cme/examples/demo_pool.jsonl +360 -0
- cme/examples/demo_run.jsonl +480 -0
- cme/examples/langgraph_integration.py +27 -0
- cme/examples/make_demo.py +37 -0
- cme/examples/openai_quickstart.py +18 -0
- cme/experiment.py +555 -0
- cme/frontier.py +143 -0
- cme/hidden_profile.py +135 -0
- cme/proxy.py +106 -0
- cme/rebuttal.py +76 -0
- cme/report.py +197 -0
- cme/rollout_card.py +137 -0
- cme/tasks.py +266 -0
- cme/verified_debate.py +127 -0
- cme/verifier.py +92 -0
- mae_eval/__init__.py +11 -0
- mae_eval/__main__.py +5 -0
- mae_eval/auditor.py +112 -0
- mae_eval/classifier.py +62 -0
- mae_eval/cli.py +210 -0
- mae_eval/tracker.py +162 -0
- mae_eval/vector.py +91 -0
- pace/__init__.py +11 -0
- pace/__main__.py +5 -0
- pace/analysis.py +129 -0
- pace/audit.py +145 -0
- pace/baseline.py +162 -0
- pace/card.py +205 -0
- pace/cli.py +898 -0
- pace/client.py +165 -0
- pace/dynamics.py +372 -0
- pace/escape_benchmarks.py +162 -0
- pace/examples/crewai_integration.py +16 -0
- pace/examples/demo_pool.jsonl +360 -0
- pace/examples/demo_run.jsonl +480 -0
- pace/examples/langgraph_integration.py +27 -0
- pace/examples/make_demo.py +37 -0
- pace/examples/openai_quickstart.py +18 -0
- pace/experiment.py +555 -0
- pace/frontier.py +114 -0
- pace/hidden_profile.py +301 -0
- pace/importers.py +219 -0
- pace/inflate.py +213 -0
- pace/influence.py +231 -0
- pace/meta_engine.py +246 -0
- pace/middleware.py +230 -0
- pace/open_ended.py +114 -0
- pace/plotting.py +132 -0
- pace/pool_sizer.py +154 -0
- pace/protocol_v3.py +414 -0
- pace/proxy.py +106 -0
- pace/rebuttal.py +219 -0
- pace/report.py +197 -0
- pace/rollout_card.py +137 -0
- pace/schema.py +74 -0
- pace/simulator.py +199 -0
- pace/studio.html +1040 -0
- pace/tasks.py +266 -0
- pace/verified_debate.py +127 -0
- pace/verifier.py +92 -0
- pace/workbench.html +26 -0
- pace/workbench.py +119 -0
- pace_eval-0.2.2.dist-info/METADATA +205 -0
- pace_eval-0.2.2.dist-info/RECORD +83 -0
- pace_eval-0.2.2.dist-info/WHEEL +5 -0
- pace_eval-0.2.2.dist-info/entry_points.txt +4 -0
- pace_eval-0.2.2.dist-info/licenses/LICENSE +202 -0
- pace_eval-0.2.2.dist-info/top_level.txt +4 -0
- pace_lab/__init__.py +9 -0
- pace_lab/clients.py +568 -0
- pace_lab/concurrency_test.py +127 -0
- pace_lab/data.py +445 -0
- pace_lab/graders.py +626 -0
- pace_lab/strategies.py +345 -0
pace/analysis.py
ADDED
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
"""Load a confirmatory cell (one model x one task) and estimate frontier gaps per strategy.
|
|
2
|
+
|
|
3
|
+
A cell directory holds calls.jsonl (every call) and results.jsonl (one row per strategy x item). The pool frontier is rebuilt from the
|
|
4
|
+
cached pool calls with the same scorer used at run time, so re-scoring never needs a model. Correctness of a pool sample is mapped to
|
|
5
|
+
the label 'G' (wrong answers keep their normalised text) so plurality voting uses the scorer's notion of correct.
|
|
6
|
+
|
|
7
|
+
Estimand (docs/PREREGISTRATION.md section 5): gap(s) = accuracy(s) - F(mean tokens of s), F the isotonic self-consistency frontier,
|
|
8
|
+
with a paired bootstrap over items. `pool="views"` builds the frontier from the independent split-evidence agents (S2v) instead.
|
|
9
|
+
"""
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import json
|
|
13
|
+
import os
|
|
14
|
+
from typing import Dict, List, Optional
|
|
15
|
+
|
|
16
|
+
import numpy as np
|
|
17
|
+
|
|
18
|
+
from . import experiment as E
|
|
19
|
+
from . import tasks as T
|
|
20
|
+
from .frontier import Pool, frontier_curve, frontier_gap, pav_increasing
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def read_results(cell_dir: str) -> Dict[str, Dict[str, dict]]:
|
|
24
|
+
out: Dict[str, Dict[str, dict]] = {}
|
|
25
|
+
p = os.path.join(cell_dir, "results.jsonl")
|
|
26
|
+
if os.path.exists(p):
|
|
27
|
+
for l in open(p, encoding="utf-8"):
|
|
28
|
+
if l.strip():
|
|
29
|
+
r = json.loads(l)
|
|
30
|
+
out.setdefault(r["strategy"], {})[r["item"]] = r
|
|
31
|
+
return out
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def build_pool(store: E.Store, model: str, items: List[dict], pool: str = "pool", k: int = 16) -> Pool:
|
|
35
|
+
"""Pool of per-item sampled answers and token counts. pool='pool' uses the 16 full-evidence samples; 'views' the 5 split-evidence agents."""
|
|
36
|
+
pitems = {}
|
|
37
|
+
for it in items:
|
|
38
|
+
if it["kind"] == "code":
|
|
39
|
+
continue
|
|
40
|
+
keys = ["pool|%s|%s|%d" % (model, it["id"], i) for i in range(k)] if pool == "pool" else ["r1v|%s|%s|%d" % (model, it["id"], i) for i in range(k)]
|
|
41
|
+
recs = [store.d.get(x) for x in keys]
|
|
42
|
+
if any(r is None or r.get("prompt_tokens") is None or r.get("completion_tokens") is None for r in recs):
|
|
43
|
+
continue
|
|
44
|
+
pitems[it["id"]] = {"gold": "G", "answers": ["G" if T.score(it, r["text"]) else (E.ans_of(it, r) or "?") + "_wrong" for r in recs],
|
|
45
|
+
"tokens": [r["prompt_tokens"] + r["completion_tokens"] for r in recs]}
|
|
46
|
+
return Pool(pitems, k) if pitems else None
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def cell_gaps(cell_dir: str, model: str, items: List[dict], strategies: Optional[List[str]] = None, pool: str = "pool", k: Optional[int] = None,
|
|
50
|
+
n_boot: int = 2000, n_perm: int = 100, seed: int = 0, tokens: str = "total") -> dict:
|
|
51
|
+
"""Frontier, per-strategy accuracy, mean tokens and gap with a bootstrap interval. tokens='completion' is the sensitivity analysis."""
|
|
52
|
+
store = E.Store(os.path.join(cell_dir, "calls.jsonl"))
|
|
53
|
+
k = k or (16 if pool == "pool" else 5)
|
|
54
|
+
P = build_pool(store, model, items, pool, k)
|
|
55
|
+
if P is None:
|
|
56
|
+
return {"error": "no complete pool in %s" % cell_dir}
|
|
57
|
+
if tokens == "completion": # recompute pool tokens with completion tokens only
|
|
58
|
+
for it in P.items:
|
|
59
|
+
keys = ["pool|%s|%s|%d" % (model, it, i) if pool == "pool" else "r1v|%s|%s|%d" % (model, it, i) for i in range(k)]
|
|
60
|
+
P.items[it]["tokens"] = [store.d[x]["completion_tokens"] for x in keys]
|
|
61
|
+
ids, acc, tok = P.curves(n_perm, seed)
|
|
62
|
+
res = read_results(cell_dir)
|
|
63
|
+
out = {"model": model, "pool": pool, "k": k, "n_pool_items": len(ids), "tokens": tokens, "strategies": {}}
|
|
64
|
+
tk, ak = frontier_curve(acc, tok)
|
|
65
|
+
out["frontier"] = {"tokens": tk.tolist(), "sc_accuracy": ak.tolist(), "isotonic": pav_increasing(ak).tolist()}
|
|
66
|
+
for s, rows in sorted(res.items()):
|
|
67
|
+
if strategies and s not in strategies:
|
|
68
|
+
continue
|
|
69
|
+
common = [i for i in ids if i in rows]
|
|
70
|
+
if len(common) < 5:
|
|
71
|
+
continue
|
|
72
|
+
pos = {i: n for n, i in enumerate(ids)}
|
|
73
|
+
sel = [pos[i] for i in common]
|
|
74
|
+
use = (lambda r: r["completion_tokens"]) if tokens == "completion" else (lambda r: r["prompt_tokens"] + r["completion_tokens"])
|
|
75
|
+
sc = np.array([float(rows[i]["correct"]) for i in common])
|
|
76
|
+
st = np.array([float(use(rows[i])) for i in common])
|
|
77
|
+
g = frontier_gap(acc[sel], tok[sel], sc, st, n_boot=n_boot, seed=seed)
|
|
78
|
+
g["n_items"] = len(common)
|
|
79
|
+
out["strategies"][s] = g
|
|
80
|
+
return out
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def difference_of_gaps(cell_dir: str, model: str, items: List[dict], a: str, b: str, n_boot: int = 2000, seed: int = 0, n_perm: int = 100) -> Optional[dict]:
|
|
84
|
+
"""Paired bootstrap of gap(a) - gap(b) on the items both strategies finished (used for H3: S4 versus S3)."""
|
|
85
|
+
store = E.Store(os.path.join(cell_dir, "calls.jsonl"))
|
|
86
|
+
P = build_pool(store, model, items, "pool", 16)
|
|
87
|
+
res = read_results(cell_dir)
|
|
88
|
+
if P is None or a not in res or b not in res:
|
|
89
|
+
return None
|
|
90
|
+
ids, acc, tok = P.curves(n_perm, seed)
|
|
91
|
+
common = [i for i in ids if i in res[a] and i in res[b]]
|
|
92
|
+
if len(common) < 5:
|
|
93
|
+
return None
|
|
94
|
+
pos = {i: n for n, i in enumerate(ids)}
|
|
95
|
+
sel = np.array([pos[i] for i in common])
|
|
96
|
+
sa = np.array([float(res[a][i]["correct"]) for i in common]); ta = np.array([float(res[a][i]["prompt_tokens"] + res[a][i]["completion_tokens"]) for i in common])
|
|
97
|
+
sb = np.array([float(res[b][i]["correct"]) for i in common]); tb = np.array([float(res[b][i]["prompt_tokens"] + res[b][i]["completion_tokens"]) for i in common])
|
|
98
|
+
from .frontier import frontier_at
|
|
99
|
+
rng = np.random.default_rng(seed)
|
|
100
|
+
n = len(common)
|
|
101
|
+
A, T_ = acc[sel], tok[sel]
|
|
102
|
+
|
|
103
|
+
def one(ix):
|
|
104
|
+
tk, ak = frontier_curve(A[ix], T_[ix])
|
|
105
|
+
ga = sa[ix].mean() - frontier_at(tk, ak, float(ta[ix].mean()))[0]
|
|
106
|
+
gb = sb[ix].mean() - frontier_at(tk, ak, float(tb[ix].mean()))[0]
|
|
107
|
+
return ga - gb
|
|
108
|
+
est = one(np.arange(n))
|
|
109
|
+
boots = np.array([one(rng.integers(0, n, n)) for _ in range(n_boot)])
|
|
110
|
+
lo, hi = np.quantile(boots, [0.025, 0.975])
|
|
111
|
+
p_two = float(min(1.0, 2 * (min(np.sum(boots <= 0), np.sum(boots >= 0)) + 1) / (n_boot + 1))) # bootstrap two-sided p for H0: difference = 0
|
|
112
|
+
return {"difference": float(est), "ci_low": float(lo), "ci_high": float(hi), "p_two_sided": p_two, "n_items": n, "a": a, "b": b}
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def holm(pvals: Dict[str, float], alpha: float = 0.05) -> Dict[str, dict]:
|
|
116
|
+
"""Holm step-down: returns adjusted p-values and rejection flags."""
|
|
117
|
+
order = sorted(pvals, key=pvals.get)
|
|
118
|
+
m = len(order)
|
|
119
|
+
out, running = {}, 0.0
|
|
120
|
+
for i, name in enumerate(order):
|
|
121
|
+
adj = min(1.0, (m - i) * pvals[name])
|
|
122
|
+
running = max(running, adj)
|
|
123
|
+
out[name] = {"p": pvals[name], "p_holm": running, "reject": running < alpha}
|
|
124
|
+
return out
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def one_sided_p(boots_gap: np.ndarray) -> float:
|
|
128
|
+
"""Bootstrap p-value for H0: gap <= 0 (share of resamples with gap <= 0)."""
|
|
129
|
+
return float((np.sum(boots_gap <= 0) + 1) / (len(boots_gap) + 1))
|
pace/audit.py
ADDED
|
@@ -0,0 +1,145 @@
|
|
|
1
|
+
"""pace audit: a one-screen diagnostic card for a multi-agent evaluation (standard library plus mae_eval).
|
|
2
|
+
|
|
3
|
+
pace audit run_trace.jsonl # a run log with "mas" and "sas" calls: measured MAS/SAS resource ratios and parity
|
|
4
|
+
pace audit run_trace.jsonl --baseline-config sas.json # plus the rung of the single-agent baseline (keys: code/mae_eval/classifier.py)
|
|
5
|
+
pace audit sas.json # a baseline configuration only: its rung and whether a matched resource is declared
|
|
6
|
+
|
|
7
|
+
Badge rules (stated on the card):
|
|
8
|
+
PASS: COMPUTE-MATCHED a log was given, token parity holds within the tolerance (r_token ratio within 1 +/- tolerance), and, if a baseline configuration
|
|
9
|
+
was given, its rung is C or D; or only a configuration was given and its rung is C (rule R2) or D.
|
|
10
|
+
WARNING: BASELINE NOT COMPUTE-MATCHED anything else: parity violated or not measurable, or the baseline is rung A or B, or rung C with the matching unverified (R3).
|
|
11
|
+
Rung A or B does not make a baseline wrong; it means the comparison does not separate coordination from extra computation. Nothing here estimates tokens.
|
|
12
|
+
"""
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
import json
|
|
16
|
+
import math
|
|
17
|
+
from typing import Dict, Optional
|
|
18
|
+
|
|
19
|
+
BIBTEX = """@unpublished{cme2026,
|
|
20
|
+
title = {More Agents, More Samples, or More Thinking? Untangling Coordination, Ensembling, and Test-Time Compute in Multi-Agent {LLMs}},
|
|
21
|
+
author = {Sajjadi Mohammadabadi, Seyed Mahmoud},
|
|
22
|
+
year = {2026},
|
|
23
|
+
note = {Manuscript in preparation}
|
|
24
|
+
}"""
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def _fmt(r: Optional[float]) -> str:
|
|
28
|
+
if r is None:
|
|
29
|
+
return "n/a"
|
|
30
|
+
return "inf" if math.isinf(r) else "%.2f\u00d7" % r
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def load_input(path: str):
|
|
34
|
+
"""Return ('config', dict) for a single JSON object without a 'system' key, else ('log', {system: [records]})."""
|
|
35
|
+
text = open(path, encoding="utf-8").read()
|
|
36
|
+
try:
|
|
37
|
+
obj = json.loads(text)
|
|
38
|
+
if isinstance(obj, dict) and "system" not in obj:
|
|
39
|
+
return "config", obj
|
|
40
|
+
except json.JSONDecodeError:
|
|
41
|
+
pass
|
|
42
|
+
from mae_eval.cli import TraceError, load_trace
|
|
43
|
+
recs = []
|
|
44
|
+
for no, line in enumerate(text.splitlines(), 1):
|
|
45
|
+
if line.strip():
|
|
46
|
+
r = json.loads(line)
|
|
47
|
+
if "correct" in r and "prompt_tokens" not in r: # a per-item result line written by pace.log_result or by hand
|
|
48
|
+
continue
|
|
49
|
+
recs.append({k: v for k, v in r.items() if v is not None}) # null = unreported (proxy and wrap logs): treat as absent, never as 0 by guess
|
|
50
|
+
import tempfile, os
|
|
51
|
+
fd, tmp = tempfile.mkstemp(suffix=".jsonl")
|
|
52
|
+
try:
|
|
53
|
+
with os.fdopen(fd, "w", encoding="utf-8") as fh:
|
|
54
|
+
fh.write("\n".join(json.dumps(r) for r in recs) + "\n")
|
|
55
|
+
try:
|
|
56
|
+
return "log", load_trace(tmp)
|
|
57
|
+
except TraceError as e:
|
|
58
|
+
raise ValueError(str(e))
|
|
59
|
+
finally:
|
|
60
|
+
os.unlink(tmp)
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def audit(path: str, baseline_config: Optional[str] = None, tolerance: float = 0.05, price_in: Optional[float] = None, price_out: Optional[float] = None) -> Dict:
|
|
64
|
+
from mae_eval.auditor import EvaluationAuditor
|
|
65
|
+
from mae_eval.classifier import classify_with_reason
|
|
66
|
+
from mae_eval.cli import session_from_records
|
|
67
|
+
from mae_eval.tracker import ResourceTracker
|
|
68
|
+
kind, data = load_input(path)
|
|
69
|
+
card: Dict = {"input": kind, "tolerance": tolerance, "rung": None, "rule": None, "rung_reason": None, "ratios": None, "parity": None, "notes": []}
|
|
70
|
+
cfg = None
|
|
71
|
+
if kind == "config":
|
|
72
|
+
cfg = data
|
|
73
|
+
elif baseline_config:
|
|
74
|
+
cfg = json.load(open(baseline_config, encoding="utf-8"))
|
|
75
|
+
if cfg is not None:
|
|
76
|
+
card["rung"], card["rule"], card["rung_reason"] = classify_with_reason(cfg)
|
|
77
|
+
if kind == "log":
|
|
78
|
+
if "mas" not in data or "sas" not in data:
|
|
79
|
+
card["notes"].append("the log needs both 'mas' and 'sas' calls to compute ratios; systems found: %s" % ", ".join(sorted(data)))
|
|
80
|
+
else:
|
|
81
|
+
priced = price_in is not None and price_out is not None
|
|
82
|
+
tracker = ResourceTracker("model", "cme-audit")
|
|
83
|
+
if priced:
|
|
84
|
+
tracker.register_pricing(price_in, price_out)
|
|
85
|
+
vec = {k: session_from_records(v, tracker).get_resource_vector(allow_missing_pricing=not priced) for k, v in data.items() if k in ("mas", "sas")}
|
|
86
|
+
aud = EvaluationAuditor(vec["mas"], vec["sas"])
|
|
87
|
+
r = aud.ratios()
|
|
88
|
+
card["ratios"] = {"r_token": r["r_token"], "r_lat": r["r_lat"], "r_cost": r["r_cost"]}
|
|
89
|
+
tok = r["r_token"]
|
|
90
|
+
card["parity"] = bool(tok is not None and not math.isinf(tok) and abs(tok - 1.0) <= tolerance)
|
|
91
|
+
if not priced:
|
|
92
|
+
card["notes"].append("no prices given: cost ratio not computed")
|
|
93
|
+
elif cfg is not None:
|
|
94
|
+
card["notes"].append("a configuration alone has no measured tokens: no resource ratios")
|
|
95
|
+
if card["rung"] is None:
|
|
96
|
+
card["notes"].append("baseline rung not classified: pass --baseline-config or a configuration file")
|
|
97
|
+
rung, rule = card["rung"], card["rule"]
|
|
98
|
+
if kind == "log":
|
|
99
|
+
ok = bool(card["parity"]) and (rung in (None, "C", "D"))
|
|
100
|
+
if rung is None:
|
|
101
|
+
ok = False # parity alone with an unknown baseline is not enough to call the comparison matched
|
|
102
|
+
else:
|
|
103
|
+
ok = rung == "D" or (rung == "C" and rule == "R2")
|
|
104
|
+
card["badge"] = "PASS: COMPUTE-MATCHED" if ok else "WARNING: BASELINE NOT COMPUTE-MATCHED"
|
|
105
|
+
card["ok"] = ok
|
|
106
|
+
return card
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def render(card: Dict, ascii_only: bool = False) -> str:
|
|
110
|
+
w = 66
|
|
111
|
+
line = "\u2500" * w
|
|
112
|
+
out = ["\u250c" + line + "\u2510"]
|
|
113
|
+
|
|
114
|
+
def row(s=""):
|
|
115
|
+
import textwrap
|
|
116
|
+
for part in textwrap.wrap(s, w - 3, subsequent_indent=" ", drop_whitespace=False)[:] or [""]:
|
|
117
|
+
out.append("\u2502 " + part.ljust(w - 1) + "\u2502")
|
|
118
|
+
row("PACE audit")
|
|
119
|
+
out.append("\u251c" + line + "\u2524")
|
|
120
|
+
if card["rung"]:
|
|
121
|
+
cat = {"A": "Category A (single greedy pass)", "B": "Category B (sampled, unmatched tokens)", "C": "Category C (resource-matched or test-time scaling)",
|
|
122
|
+
"D": "Category D (budget sweep)"}[card["rung"]]
|
|
123
|
+
row("Baseline rung : %s [%s]" % (cat, card["rule"]))
|
|
124
|
+
row(" %s" % card["rung_reason"])
|
|
125
|
+
else:
|
|
126
|
+
row("Baseline rung : not classified")
|
|
127
|
+
r = card["ratios"]
|
|
128
|
+
if r:
|
|
129
|
+
row("MAS / SAS : R_token %s R_lat %s R_cost %s" % (_fmt(r["r_token"]), _fmt(r["r_lat"]), _fmt(r["r_cost"])))
|
|
130
|
+
row("Token parity : %s (tolerance %.0f%%)" % ("holds" if card["parity"] else "violated", 100 * card["tolerance"]))
|
|
131
|
+
else:
|
|
132
|
+
row("MAS / SAS : not available")
|
|
133
|
+
row()
|
|
134
|
+
row("[%s]" % card["badge"])
|
|
135
|
+
for n in card["notes"]:
|
|
136
|
+
row("note: " + n)
|
|
137
|
+
out.append("\u2514" + line + "\u2518")
|
|
138
|
+
out.append("")
|
|
139
|
+
out.append("To cite (BibTeX; the manuscript is in preparation):")
|
|
140
|
+
out.append(BIBTEX)
|
|
141
|
+
text = "\n".join(out)
|
|
142
|
+
if ascii_only:
|
|
143
|
+
for u, c in (("\u250c", "+"), ("\u2510", "+"), ("\u2514", "+"), ("\u2518", "+"), ("\u251c", "+"), ("\u2524", "+"), ("\u2500", "-"), ("\u2502", "|"), ("\u00d7", "x")):
|
|
144
|
+
text = text.replace(u, c)
|
|
145
|
+
return text
|
pace/baseline.py
ADDED
|
@@ -0,0 +1,162 @@
|
|
|
1
|
+
"""pace baseline: a cached, resumable greedy run plus a pool of independent sampled completions.
|
|
2
|
+
|
|
3
|
+
Every call is appended to a JSONL cache as soon as it finishes, so an interrupted run resumes where it stopped (a record is keyed by
|
|
4
|
+
item id, arm and index). Sampled arms use temperature > 0 and a distinct seed per (item, index); the greedy arm uses temperature 0.
|
|
5
|
+
Token counts come from the endpoint's usage field. Reasoning tokens are recorded only when the endpoint reports them; otherwise the
|
|
6
|
+
field is null and nothing is inferred.
|
|
7
|
+
|
|
8
|
+
Benchmarks: a JSONL file with one object per line {"id", "question", "answer"}; or the built-in `toy` set (synthetic arithmetic, for
|
|
9
|
+
smoke tests only; it says nothing about any model).
|
|
10
|
+
"""
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
import hashlib
|
|
14
|
+
import json
|
|
15
|
+
import os
|
|
16
|
+
import re
|
|
17
|
+
import time
|
|
18
|
+
import urllib.request
|
|
19
|
+
from typing import Callable, Dict, Iterable, List, Optional
|
|
20
|
+
|
|
21
|
+
PROMPT = ("Solve the problem. Think step by step, then finish with a final line of the form 'Answer: <answer>'.\n\nProblem: {q}")
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def toy_bench(n: int = 20) -> List[dict]:
|
|
25
|
+
"""Synthetic arithmetic items. Not a benchmark: used only to exercise the pipeline."""
|
|
26
|
+
return [{"id": "toy%03d" % i, "question": "What is %d + %d * %d?" % (i + 3, i + 2, 4), "answer": str(i + 3 + (i + 2) * 4)} for i in range(n)]
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def load_bench(spec: str, limit: Optional[int] = None) -> List[dict]:
|
|
30
|
+
if spec == "toy":
|
|
31
|
+
items = toy_bench(limit or 20)
|
|
32
|
+
else:
|
|
33
|
+
items = [json.loads(l) for l in open(spec, encoding="utf-8") if l.strip()]
|
|
34
|
+
for it in items:
|
|
35
|
+
if not {"id", "question", "answer"} <= set(it):
|
|
36
|
+
raise ValueError("bench items need id, question and answer")
|
|
37
|
+
return items[:limit] if limit else items
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
_ANS = re.compile(r"answer\s*[:=]\s*(.+)", re.I)
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def extract_answer(text: str) -> str:
|
|
44
|
+
"""Last 'Answer: x' line, else the last number or standalone letter A-J, normalised (trimmed, lower-cased, no $ , or trailing period)."""
|
|
45
|
+
m = _ANS.findall(text or "")
|
|
46
|
+
cand = m[-1] if m else ((re.findall(r"-?\d[\d,]*\.?\d*", text or "") or [""])[-1])
|
|
47
|
+
cand = cand.strip().strip("*$ ").rstrip(".").replace(",", "").lower()
|
|
48
|
+
cand = re.sub(r"^\\boxed\{(.*)\}$", r"\1", cand)
|
|
49
|
+
return cand
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def norm_gold(a: str) -> str:
|
|
53
|
+
return str(a).strip().rstrip(".").replace(",", "").lower()
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
class OpenAICompatClient:
|
|
57
|
+
"""Minimal client for /v1/chat/completions (Ollama, vLLM, llama.cpp server, OpenAI-compatible hosts). Standard library only."""
|
|
58
|
+
|
|
59
|
+
def __init__(self, base_url: str = "http://localhost:11434/v1", api_key: str = "none", timeout: float = 600.0):
|
|
60
|
+
self.base, self.key, self.timeout = base_url.rstrip("/"), api_key, timeout
|
|
61
|
+
|
|
62
|
+
def chat(self, model: str, prompt: str, temperature: float, seed: int, max_tokens: int = 2048) -> dict:
|
|
63
|
+
body = json.dumps({"model": model, "messages": [{"role": "user", "content": prompt}], "temperature": temperature, "seed": seed,
|
|
64
|
+
"max_tokens": max_tokens, "stream": False}).encode()
|
|
65
|
+
req = urllib.request.Request(self.base + "/chat/completions", data=body,
|
|
66
|
+
headers={"Content-Type": "application/json", "Authorization": "Bearer " + self.key})
|
|
67
|
+
t0 = time.time()
|
|
68
|
+
with urllib.request.urlopen(req, timeout=self.timeout) as r:
|
|
69
|
+
data = json.loads(r.read())
|
|
70
|
+
lat = time.time() - t0
|
|
71
|
+
msg = data["choices"][0]["message"]
|
|
72
|
+
usage = data.get("usage", {})
|
|
73
|
+
det = usage.get("completion_tokens_details") or {}
|
|
74
|
+
return {"text": msg.get("content") or "", "prompt_tokens": usage.get("prompt_tokens"), "completion_tokens": usage.get("completion_tokens"),
|
|
75
|
+
"reasoning_tokens": det.get("reasoning_tokens"), "latency": lat}
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def ollama_digest(model: str, host: str = "http://localhost:11434") -> Optional[str]:
|
|
79
|
+
"""Best-effort model digest from Ollama /api/tags; None when unavailable."""
|
|
80
|
+
try:
|
|
81
|
+
with urllib.request.urlopen(host + "/api/tags", timeout=5) as r:
|
|
82
|
+
for m in json.loads(r.read()).get("models", []):
|
|
83
|
+
if m.get("name") == model or m.get("model") == model:
|
|
84
|
+
return m.get("digest")
|
|
85
|
+
except Exception:
|
|
86
|
+
return None
|
|
87
|
+
return None
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def read_cache(path: str) -> Dict[tuple, dict]:
|
|
91
|
+
done: Dict[tuple, dict] = {}
|
|
92
|
+
if os.path.exists(path):
|
|
93
|
+
for l in open(path, encoding="utf-8"):
|
|
94
|
+
if l.strip():
|
|
95
|
+
r = json.loads(l)
|
|
96
|
+
done[(r["item"], r["arm"], r["idx"])] = r
|
|
97
|
+
return done
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def sample_seed(base_seed: int, item: str, idx: int) -> int:
|
|
101
|
+
"""Distinct, reproducible seed per (item, sample index)."""
|
|
102
|
+
return int(hashlib.sha256(("%d|%s|%d" % (base_seed, item, idx)).encode()).hexdigest()[:8], 16)
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def run_baseline(client, model: str, items: Iterable[dict], out_path: str, k: int = 16, temperature: float = 0.7, seed: int = 0,
|
|
106
|
+
max_tokens: int = 2048, digest: Optional[str] = None, progress: Optional[Callable[[str], None]] = None) -> dict:
|
|
107
|
+
"""Run greedy + k sampled completions per item, appending to out_path. Returns counts: new, cached, errors."""
|
|
108
|
+
if temperature <= 0:
|
|
109
|
+
raise ValueError("sampled arms need temperature > 0")
|
|
110
|
+
done = read_cache(out_path)
|
|
111
|
+
new = cached = errors = 0
|
|
112
|
+
with open(out_path, "a", encoding="utf-8") as fh:
|
|
113
|
+
for it in items:
|
|
114
|
+
gold = norm_gold(it["answer"])
|
|
115
|
+
jobs = [("greedy", 0, 0.0)] + [("sample", j, temperature) for j in range(k)]
|
|
116
|
+
for arm, j, temp in jobs:
|
|
117
|
+
if (it["id"], arm, j) in done:
|
|
118
|
+
cached += 1
|
|
119
|
+
continue
|
|
120
|
+
s = sample_seed(seed, it["id"], j) if arm == "sample" else seed
|
|
121
|
+
try:
|
|
122
|
+
r = client.chat(model, PROMPT.format(q=it["question"]), temp, s, max_tokens)
|
|
123
|
+
except Exception as e: # network or server error: skip and report, do not cache
|
|
124
|
+
errors += 1
|
|
125
|
+
if progress:
|
|
126
|
+
progress("error %s %s %d: %s" % (it["id"], arm, j, e))
|
|
127
|
+
continue
|
|
128
|
+
ans = extract_answer(r["text"])
|
|
129
|
+
rec = {"item": it["id"], "arm": arm, "idx": j, "seed": s, "temperature": temp, "model": model, "model_digest": digest,
|
|
130
|
+
"answer": ans, "gold": gold, "correct": ans == gold, "prompt_tokens": r["prompt_tokens"],
|
|
131
|
+
"completion_tokens": r["completion_tokens"], "reasoning_tokens": r["reasoning_tokens"], "latency": r["latency"]}
|
|
132
|
+
fh.write(json.dumps(rec) + "\n")
|
|
133
|
+
fh.flush()
|
|
134
|
+
new += 1
|
|
135
|
+
if progress:
|
|
136
|
+
progress("item %s done (new %d, cached %d, errors %d)" % (it["id"], new, cached, errors))
|
|
137
|
+
return {"new": new, "cached": cached, "errors": errors}
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
def load_pool(path: str, k: Optional[int] = None):
|
|
141
|
+
"""Build a frontier.Pool and the greedy records from a baseline cache. Items lacking k samples are dropped (reported)."""
|
|
142
|
+
from .frontier import Pool
|
|
143
|
+
rows = list(read_cache(path).values())
|
|
144
|
+
by: Dict[str, dict] = {}
|
|
145
|
+
greedy: Dict[str, dict] = {}
|
|
146
|
+
for r in rows:
|
|
147
|
+
if r["arm"] == "greedy":
|
|
148
|
+
greedy[r["item"]] = r
|
|
149
|
+
else:
|
|
150
|
+
d = by.setdefault(r["item"], {"gold": r["gold"], "samples": {}})
|
|
151
|
+
d["samples"][r["idx"]] = r
|
|
152
|
+
need = k or max((len(d["samples"]) for d in by.values()), default=0)
|
|
153
|
+
items = {}
|
|
154
|
+
dropped = 0
|
|
155
|
+
for it, d in by.items():
|
|
156
|
+
ss = [d["samples"][i] for i in sorted(d["samples"])]
|
|
157
|
+
if len(ss) < need or any(s["prompt_tokens"] is None or s["completion_tokens"] is None for s in ss):
|
|
158
|
+
dropped += 1
|
|
159
|
+
continue
|
|
160
|
+
items[it] = {"gold": d["gold"], "answers": [s["answer"] for s in ss[:need]],
|
|
161
|
+
"tokens": [s["prompt_tokens"] + s["completion_tokens"] for s in ss[:need]]}
|
|
162
|
+
return Pool(items, need), greedy, dropped
|
pace/card.py
ADDED
|
@@ -0,0 +1,205 @@
|
|
|
1
|
+
"""Standalone PACE resource vectors and card exports."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
from collections import defaultdict
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
from typing import Any
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def load_log(path: str) -> list[dict[str, Any]]:
|
|
12
|
+
rows = [
|
|
13
|
+
json.loads(line)
|
|
14
|
+
for line in Path(path).read_text(encoding="utf-8").splitlines()
|
|
15
|
+
if line.strip()
|
|
16
|
+
]
|
|
17
|
+
if not rows:
|
|
18
|
+
raise ValueError("empty PACE log")
|
|
19
|
+
required = {"item", "agent", "round", "correct"}
|
|
20
|
+
if not all(required <= set(row) for row in rows):
|
|
21
|
+
raise ValueError("PACE logs require item, agent, round, and correct")
|
|
22
|
+
return rows
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def resource_vector(rows: list[dict[str, Any]]) -> dict[str, float | None]:
|
|
26
|
+
items = {str(row["item"]) for row in rows}
|
|
27
|
+
n_items = max(1, len(items))
|
|
28
|
+
calls = len(rows)
|
|
29
|
+
rounds = {int(row["round"]) for row in rows}
|
|
30
|
+
# Independent trajectories per item = calls made in the item's first round, before any communication (an agent that answers again in a later round is the same trajectory, not a new sample)
|
|
31
|
+
first_round, later_calls = {}, 0
|
|
32
|
+
for row in rows:
|
|
33
|
+
first_round.setdefault(str(row["item"]), int(row["round"]))
|
|
34
|
+
first_round[str(row["item"])] = min(first_round[str(row["item"])], int(row["round"]))
|
|
35
|
+
trajectories = {}
|
|
36
|
+
for row in rows:
|
|
37
|
+
key = str(row["item"])
|
|
38
|
+
if int(row["round"]) == first_round[key]:
|
|
39
|
+
trajectories[key] = trajectories.get(key, 0) + 1
|
|
40
|
+
else:
|
|
41
|
+
later_calls += 1
|
|
42
|
+
token_values = [
|
|
43
|
+
float(row["prompt_tokens"]) + float(row["completion_tokens"])
|
|
44
|
+
for row in rows
|
|
45
|
+
if row.get("prompt_tokens") is not None
|
|
46
|
+
and row.get("completion_tokens") is not None
|
|
47
|
+
]
|
|
48
|
+
reason_values = [
|
|
49
|
+
float(row["reasoning_tokens"])
|
|
50
|
+
for row in rows
|
|
51
|
+
if row.get("reasoning_tokens") is not None
|
|
52
|
+
]
|
|
53
|
+
cost_values = [
|
|
54
|
+
float(row["cost_usd"])
|
|
55
|
+
for row in rows
|
|
56
|
+
if row.get("cost_usd") is not None
|
|
57
|
+
]
|
|
58
|
+
lat_values = [
|
|
59
|
+
float(row["latency_s"])
|
|
60
|
+
for row in rows
|
|
61
|
+
if row.get("latency_s") is not None
|
|
62
|
+
]
|
|
63
|
+
return {
|
|
64
|
+
"r_reason": (sum(reason_values) / n_items) if reason_values else None,
|
|
65
|
+
"r_token": (sum(token_values) / n_items) if token_values else None,
|
|
66
|
+
"r_cost": (sum(cost_values) / n_items) if cost_values else None,
|
|
67
|
+
"r_lat": (sum(lat_values) / n_items) if lat_values else None,
|
|
68
|
+
"r_sample": sum(trajectories.values()) / n_items,
|
|
69
|
+
"r_sys": float(
|
|
70
|
+
sum(1 for row in rows if row.get("tool_calls")) / n_items
|
|
71
|
+
),
|
|
72
|
+
"r_call": calls / n_items,
|
|
73
|
+
"r_round": float(max(rounds) - min(rounds) + 1),
|
|
74
|
+
"r_context": later_calls / n_items,
|
|
75
|
+
"r_scaff": float(
|
|
76
|
+
sum(1 for row in rows if row.get("scaffold") or row.get("tool_calls"))
|
|
77
|
+
),
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def classify_baseline(vector: dict[str, float | None]) -> str:
|
|
82
|
+
"""Rung (A, B or C) that this log would occupy as a single-agent baseline, following Table 7 of the paper; "n/a" if the log is a multi-agent system.
|
|
83
|
+
|
|
84
|
+
A: one greedy-style pass (one trajectory, one round, no later-round context). B: several independent trajectories in one round (sampled; no named
|
|
85
|
+
resource is known to be matched). C: one trajectory refined over several rounds (a named resource, the iteration count, may be matched; confirm with
|
|
86
|
+
`pace classify`). Rung D (the Pareto frontier) is a property of a family of runs and cannot be read from one log.
|
|
87
|
+
"""
|
|
88
|
+
return classify_baseline_detail(vector)[0]
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def classify_baseline_detail(vector: dict[str, float | None]) -> tuple[str, str]:
|
|
92
|
+
r_round = float(vector.get("r_round") or 0.0)
|
|
93
|
+
r_sample = float(vector.get("r_sample") or 0.0)
|
|
94
|
+
r_context = float(vector.get("r_context") or 0.0)
|
|
95
|
+
if r_round <= 1 and r_sample <= 1 and r_context <= 0:
|
|
96
|
+
return "A", "one trajectory, one round"
|
|
97
|
+
if r_round <= 1 and r_context <= 0:
|
|
98
|
+
return "B", "%.3g independent trajectories in one round, no resource known to be matched" % r_sample
|
|
99
|
+
if r_sample <= 1:
|
|
100
|
+
return "C", "one trajectory over %.3g rounds; C only if the iteration ceiling is matched" % r_round
|
|
101
|
+
return "n/a", "several trajectories with later-round communication: a multi-agent system, not a single-agent baseline"
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def feasibility(vector: dict[str, float | None]) -> dict[str, Any]:
|
|
105
|
+
if vector.get("r_token") is None:
|
|
106
|
+
return {
|
|
107
|
+
"status": "insufficient_data",
|
|
108
|
+
"active_axes": [k for k, v in vector.items() if v is not None and v > 0],
|
|
109
|
+
"direction": "missing token logs",
|
|
110
|
+
}
|
|
111
|
+
positive = [key for key, value in vector.items() if value is not None and value > 0]
|
|
112
|
+
return {
|
|
113
|
+
"status": "measured_only" if positive else "insufficient_data",
|
|
114
|
+
"active_axes": positive,
|
|
115
|
+
"direction": "requires baseline comparison for directional feasibility",
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
def transition_matrix(rows: list[dict[str, Any]]) -> dict[str, dict[str, float]]:
|
|
120
|
+
grouped = defaultdict(dict)
|
|
121
|
+
for row in rows:
|
|
122
|
+
grouped[(str(row["item"]), str(row["agent"]))][int(row["round"])] = bool(
|
|
123
|
+
row["correct"]
|
|
124
|
+
)
|
|
125
|
+
counts = defaultdict(lambda: [0, 0])
|
|
126
|
+
by_item = defaultdict(dict)
|
|
127
|
+
for (item, agent), states in grouped.items():
|
|
128
|
+
by_item[item][agent] = states
|
|
129
|
+
for agents in by_item.values():
|
|
130
|
+
rounds = sorted({round_no for states in agents.values() for round_no in states})
|
|
131
|
+
for current, following in zip(rounds, rounds[1:]):
|
|
132
|
+
present = [
|
|
133
|
+
states
|
|
134
|
+
for states in agents.values()
|
|
135
|
+
if current in states and following in states
|
|
136
|
+
]
|
|
137
|
+
for states in present:
|
|
138
|
+
peers_right = sum(int(other[current]) for other in present) - int(
|
|
139
|
+
states[current]
|
|
140
|
+
)
|
|
141
|
+
key = "own=%d,peers_right=%d" % (int(states[current]), peers_right)
|
|
142
|
+
counts[key][0] += int(states[following])
|
|
143
|
+
counts[key][1] += 1
|
|
144
|
+
return {
|
|
145
|
+
key: {"p_next": good / total, "n": total}
|
|
146
|
+
for key, (good, total) in counts.items()
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
def build_card(rows: list[dict[str, Any]]) -> dict[str, Any]:
|
|
151
|
+
vector = resource_vector(rows)
|
|
152
|
+
return {
|
|
153
|
+
"schema": "pace-card/v1",
|
|
154
|
+
"n_records": len(rows),
|
|
155
|
+
"n_items": len({str(row["item"]) for row in rows}),
|
|
156
|
+
"resource_vector": vector,
|
|
157
|
+
"baseline": classify_baseline(vector),
|
|
158
|
+
"baseline_reason": classify_baseline_detail(vector)[1],
|
|
159
|
+
"feasibility": feasibility(vector),
|
|
160
|
+
"transition_matrix": transition_matrix(rows),
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
def to_markdown(card: dict[str, Any]) -> str:
|
|
165
|
+
lines = [
|
|
166
|
+
"# PACE Card",
|
|
167
|
+
"",
|
|
168
|
+
"| Field | Value |",
|
|
169
|
+
"|---|---:|",
|
|
170
|
+
"| Baseline | %s |" % card["baseline"],
|
|
171
|
+
"| Items | %s |" % card["n_items"],
|
|
172
|
+
]
|
|
173
|
+
lines += [
|
|
174
|
+
"| %s | %s |" % (key, ("%.3f" % value if value is not None else "n/a"))
|
|
175
|
+
for key, value in card["resource_vector"].items()
|
|
176
|
+
]
|
|
177
|
+
lines += [
|
|
178
|
+
"",
|
|
179
|
+
"## Transition matrix",
|
|
180
|
+
"",
|
|
181
|
+
"| State | P(correct at t+1) | n |",
|
|
182
|
+
"|---|---:|---:|",
|
|
183
|
+
]
|
|
184
|
+
lines += [
|
|
185
|
+
"| %s | %.3f | %d |" % (key, value["p_next"], value["n"])
|
|
186
|
+
for key, value in card["transition_matrix"].items()
|
|
187
|
+
]
|
|
188
|
+
return "\n".join(lines) + "\n"
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
def to_latex(card: dict[str, Any]) -> str:
|
|
192
|
+
rows = [
|
|
193
|
+
"\\begin{tcolorbox}[title={PACE Card}]",
|
|
194
|
+
"\\textbf{Baseline:} %s\\par" % card["baseline"],
|
|
195
|
+
"\\begin{tabular}{lr}",
|
|
196
|
+
"\\toprule",
|
|
197
|
+
"Axis & Value\\\\",
|
|
198
|
+
"\\midrule",
|
|
199
|
+
]
|
|
200
|
+
rows += [
|
|
201
|
+
"%s & %s\\\\" % (key.replace("_", "\\_"), ("%.3f" % value if value is not None else "n/a"))
|
|
202
|
+
for key, value in card["resource_vector"].items()
|
|
203
|
+
]
|
|
204
|
+
rows += ["\\bottomrule", "\\end{tabular}", "\\end{tcolorbox}"]
|
|
205
|
+
return "\n".join(rows) + "\n"
|