pace-eval 0.2.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cme/__init__.py +11 -0
- cme/__main__.py +5 -0
- cme/analysis.py +223 -0
- cme/audit.py +145 -0
- cme/baseline.py +162 -0
- cme/cli.py +279 -0
- cme/client.py +165 -0
- cme/dynamics.py +212 -0
- cme/examples/crewai_integration.py +16 -0
- cme/examples/demo_pool.jsonl +360 -0
- cme/examples/demo_run.jsonl +480 -0
- cme/examples/langgraph_integration.py +27 -0
- cme/examples/make_demo.py +37 -0
- cme/examples/openai_quickstart.py +18 -0
- cme/experiment.py +555 -0
- cme/frontier.py +143 -0
- cme/hidden_profile.py +135 -0
- cme/proxy.py +106 -0
- cme/rebuttal.py +76 -0
- cme/report.py +197 -0
- cme/rollout_card.py +137 -0
- cme/tasks.py +266 -0
- cme/verified_debate.py +127 -0
- cme/verifier.py +92 -0
- mae_eval/__init__.py +11 -0
- mae_eval/__main__.py +5 -0
- mae_eval/auditor.py +112 -0
- mae_eval/classifier.py +62 -0
- mae_eval/cli.py +210 -0
- mae_eval/tracker.py +162 -0
- mae_eval/vector.py +91 -0
- pace/__init__.py +11 -0
- pace/__main__.py +5 -0
- pace/analysis.py +129 -0
- pace/audit.py +145 -0
- pace/baseline.py +162 -0
- pace/card.py +205 -0
- pace/cli.py +898 -0
- pace/client.py +165 -0
- pace/dynamics.py +372 -0
- pace/escape_benchmarks.py +162 -0
- pace/examples/crewai_integration.py +16 -0
- pace/examples/demo_pool.jsonl +360 -0
- pace/examples/demo_run.jsonl +480 -0
- pace/examples/langgraph_integration.py +27 -0
- pace/examples/make_demo.py +37 -0
- pace/examples/openai_quickstart.py +18 -0
- pace/experiment.py +555 -0
- pace/frontier.py +114 -0
- pace/hidden_profile.py +301 -0
- pace/importers.py +219 -0
- pace/inflate.py +213 -0
- pace/influence.py +231 -0
- pace/meta_engine.py +246 -0
- pace/middleware.py +230 -0
- pace/open_ended.py +114 -0
- pace/plotting.py +132 -0
- pace/pool_sizer.py +154 -0
- pace/protocol_v3.py +414 -0
- pace/proxy.py +106 -0
- pace/rebuttal.py +219 -0
- pace/report.py +197 -0
- pace/rollout_card.py +137 -0
- pace/schema.py +74 -0
- pace/simulator.py +199 -0
- pace/studio.html +1040 -0
- pace/tasks.py +266 -0
- pace/verified_debate.py +127 -0
- pace/verifier.py +92 -0
- pace/workbench.html +26 -0
- pace/workbench.py +119 -0
- pace_eval-0.2.2.dist-info/METADATA +205 -0
- pace_eval-0.2.2.dist-info/RECORD +83 -0
- pace_eval-0.2.2.dist-info/WHEEL +5 -0
- pace_eval-0.2.2.dist-info/entry_points.txt +4 -0
- pace_eval-0.2.2.dist-info/licenses/LICENSE +202 -0
- pace_eval-0.2.2.dist-info/top_level.txt +4 -0
- pace_lab/__init__.py +9 -0
- pace_lab/clients.py +568 -0
- pace_lab/concurrency_test.py +127 -0
- pace_lab/data.py +445 -0
- pace_lab/graders.py +626 -0
- pace_lab/strategies.py +345 -0
pace/rebuttal.py
ADDED
|
@@ -0,0 +1,219 @@
|
|
|
1
|
+
"""Nine-column rebuttal table: Study | Benchmark | Model | MA configuration | Baseline rung | MA acc | Baseline acc | Raw gap | Compute-normalized gap.
|
|
2
|
+
|
|
3
|
+
Two sources:
|
|
4
|
+
* the literature contrasts (outputs/baseline_tier_contrasts.csv: study, category = rung A/B/C, baseline, key 'Study|bench|model|arm',
|
|
5
|
+
p_base, p_mas): one row per (configuration, baseline). Raw gap = MA acc - baseline acc. Compute-normalized gap = MA acc minus the
|
|
6
|
+
strongest single-agent baseline reported for the same configuration (the highest baseline accuracy over its rungs), so it equals the raw
|
|
7
|
+
gap when the row already uses the strongest baseline and is lower when a stronger one exists. It is a within-study comparison, not a
|
|
8
|
+
frontier estimate.
|
|
9
|
+
* a `pace report` run: one row; rung from the baseline classifier (the greedy arm is rung A), raw gap against greedy, and the
|
|
10
|
+
compute-normalized gap is the frontier gap at the system's measured tokens.
|
|
11
|
+
Unreported cells print as n/a.
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
import csv
|
|
17
|
+
import re
|
|
18
|
+
|
|
19
|
+
COLUMNS = [
|
|
20
|
+
"Study",
|
|
21
|
+
"Benchmark",
|
|
22
|
+
"Model",
|
|
23
|
+
"MA configuration",
|
|
24
|
+
"Baseline rung",
|
|
25
|
+
"MA acc",
|
|
26
|
+
"Baseline acc",
|
|
27
|
+
"Raw gap",
|
|
28
|
+
"Compute-normalized gap",
|
|
29
|
+
]
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def _f(x: float | None, signed: bool = False) -> str:
|
|
33
|
+
if x is None:
|
|
34
|
+
return "n/a"
|
|
35
|
+
return ("%+.3f" if signed else "%.3f") % x
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def rows_from_contrasts(path: str, study: str | None = None) -> list[dict[str, str]]:
|
|
39
|
+
raw = list(csv.DictReader(open(path, encoding="utf-8")))
|
|
40
|
+
best: dict[str, float] = {}
|
|
41
|
+
for r in raw:
|
|
42
|
+
best[r["key"]] = max(best.get(r["key"], -1.0), float(r["p_base"]))
|
|
43
|
+
out = []
|
|
44
|
+
for r in raw:
|
|
45
|
+
if study and r["study"] != study:
|
|
46
|
+
continue
|
|
47
|
+
_, bench, model, arm = r["key"].split("|", 3)
|
|
48
|
+
pb, pm = float(r["p_base"]), float(r["p_mas"])
|
|
49
|
+
out.append(
|
|
50
|
+
{
|
|
51
|
+
"Study": r["study"],
|
|
52
|
+
"Benchmark": bench,
|
|
53
|
+
"Model": model,
|
|
54
|
+
"MA configuration": arm,
|
|
55
|
+
"Baseline rung": r["category"],
|
|
56
|
+
"MA acc": _f(pm),
|
|
57
|
+
"Baseline acc": _f(pb),
|
|
58
|
+
"Raw gap": _f(pm - pb, True),
|
|
59
|
+
"Compute-normalized gap": _f(pm - best[r["key"]], True),
|
|
60
|
+
}
|
|
61
|
+
)
|
|
62
|
+
return out
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def row_from_report(
|
|
66
|
+
rep: dict, benchmark: str = "n/a", model: str = "n/a", config: dict | None = None
|
|
67
|
+
) -> dict[str, str]:
|
|
68
|
+
"""One row for a `pace report` result. `config` describes the baseline (default: the greedy arm, rung A)."""
|
|
69
|
+
from mae_eval.classifier import classify_baseline_rung
|
|
70
|
+
|
|
71
|
+
g = rep["frontier_gap"]
|
|
72
|
+
rung = classify_baseline_rung(config or {"n_samples": 1})
|
|
73
|
+
ga = rep.get("greedy_accuracy")
|
|
74
|
+
return {
|
|
75
|
+
"Study": "this run",
|
|
76
|
+
"Benchmark": benchmark,
|
|
77
|
+
"Model": model,
|
|
78
|
+
"MA configuration": rep.get("system", "n/a"),
|
|
79
|
+
"Baseline rung": rung,
|
|
80
|
+
"MA acc": _f(g["system_accuracy"]),
|
|
81
|
+
"Baseline acc": _f(ga),
|
|
82
|
+
"Raw gap": _f(None if ga is None else g["system_accuracy"] - ga, True),
|
|
83
|
+
"Compute-normalized gap": _f(g["gap"], True)
|
|
84
|
+
+ " [%s, %s]" % (_f(g["ci_low"], True), _f(g["ci_high"], True)),
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def to_markdown(rows: list[dict[str, str]]) -> str:
|
|
89
|
+
lines = ["| " + " | ".join(COLUMNS) + " |", "|" + "---|" * len(COLUMNS)]
|
|
90
|
+
lines += ["| " + " | ".join(r[c] for c in COLUMNS) + " |" for r in rows]
|
|
91
|
+
return "\n".join(lines) + "\n"
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def _tex(s: str) -> str:
|
|
95
|
+
for a, b in (("&", "\\&"), ("%", "\\%"), ("_", "\\_"), ("#", "\\#"), ("$", "\\$")):
|
|
96
|
+
s = s.replace(a, b)
|
|
97
|
+
return s
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def _signed(s: str) -> str:
|
|
101
|
+
return re.sub(r"([+-])(\d)", r"$\1$\2", s)
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def to_latex(
|
|
105
|
+
rows: list[dict[str, str]],
|
|
106
|
+
caption: str = "Contrasts by baseline rung.",
|
|
107
|
+
label: str = "tab:rebuttal",
|
|
108
|
+
) -> str:
|
|
109
|
+
head = " & ".join(c for c in COLUMNS) + r" \\"
|
|
110
|
+
body = [
|
|
111
|
+
" & ".join(
|
|
112
|
+
_signed(_tex(r[c]))
|
|
113
|
+
if c in ("Raw gap", "Compute-normalized gap")
|
|
114
|
+
else _tex(r[c])
|
|
115
|
+
for c in COLUMNS
|
|
116
|
+
)
|
|
117
|
+
+ r" \\"
|
|
118
|
+
for r in rows
|
|
119
|
+
]
|
|
120
|
+
return (
|
|
121
|
+
"\n".join(
|
|
122
|
+
[
|
|
123
|
+
r"\begin{table}[t]",
|
|
124
|
+
r"\centering",
|
|
125
|
+
r"\scriptsize",
|
|
126
|
+
r"\caption{%s}" % caption,
|
|
127
|
+
r"\label{%s}" % label,
|
|
128
|
+
r"\resizebox{\textwidth}{!}{%",
|
|
129
|
+
r"\begin{tabular}{lllllrrrr}",
|
|
130
|
+
r"\toprule",
|
|
131
|
+
head,
|
|
132
|
+
r"\midrule",
|
|
133
|
+
]
|
|
134
|
+
+ body
|
|
135
|
+
+ [r"\bottomrule", r"\end{tabular}}", r"\end{table}"]
|
|
136
|
+
)
|
|
137
|
+
+ "\n"
|
|
138
|
+
)
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
def audit_contrasts_noise_band(rows: list[dict], phi: float = 0.50) -> dict:
|
|
142
|
+
"""Summarize paired-binomial noise bands, effect sizes, and baseline cells."""
|
|
143
|
+
import math
|
|
144
|
+
|
|
145
|
+
if not 0 <= phi <= 1:
|
|
146
|
+
raise ValueError("phi must be between 0 and 1")
|
|
147
|
+
details = []
|
|
148
|
+
counts = {
|
|
149
|
+
"below_-1.96_se": 0,
|
|
150
|
+
"within_+-1.96_se": 0,
|
|
151
|
+
"above_+1.96_se": 0,
|
|
152
|
+
"below_-1.0_se": 0,
|
|
153
|
+
"within_+-1.0_se": 0,
|
|
154
|
+
"above_+1.0_se": 0,
|
|
155
|
+
}
|
|
156
|
+
for row in rows:
|
|
157
|
+
pm, pb, n = float(row["p_mas"]), float(row["p_base"]), float(row.get("n", 1))
|
|
158
|
+
vm, vb = pm * (1 - pm), pb * (1 - pb)
|
|
159
|
+
se = math.sqrt(max(0.0, (vm + vb - 2 * phi * math.sqrt(max(0.0, vm * vb))) / n))
|
|
160
|
+
delta = pm - pb
|
|
161
|
+
for scale, low, middle, high in (
|
|
162
|
+
(1.96, "below_-1.96_se", "within_+-1.96_se", "above_+1.96_se"),
|
|
163
|
+
(1.0, "below_-1.0_se", "within_+-1.0_se", "above_+1.0_se"),
|
|
164
|
+
):
|
|
165
|
+
if delta < -scale * se:
|
|
166
|
+
counts[low] += 1
|
|
167
|
+
elif delta > scale * se:
|
|
168
|
+
counts[high] += 1
|
|
169
|
+
else:
|
|
170
|
+
counts[middle] += 1
|
|
171
|
+
h = 2 * math.asin(math.sqrt(max(0.0, min(1.0, pm)))) - 2 * math.asin(
|
|
172
|
+
math.sqrt(max(0.0, min(1.0, pb)))
|
|
173
|
+
)
|
|
174
|
+
p11 = min(pm, pb)
|
|
175
|
+
p10, p01, p00 = pm - p11, pb - p11, 1 - pm - pb + p11
|
|
176
|
+
if p10 * p01 == 0:
|
|
177
|
+
log_or = float("inf") if p11 * p00 > 0 else 0.0
|
|
178
|
+
else:
|
|
179
|
+
log_or = math.log(max(1e-300, (p11 * p00) / (p10 * p01)))
|
|
180
|
+
phi_max = 0.0 if math.isinf(log_or) else math.exp(-abs(log_or) / 2)
|
|
181
|
+
details.append(
|
|
182
|
+
{
|
|
183
|
+
"delta": delta,
|
|
184
|
+
"se_delta": se,
|
|
185
|
+
"cohens_h": h,
|
|
186
|
+
"log_or": log_or,
|
|
187
|
+
"phi_max": phi_max,
|
|
188
|
+
"phi_binds": phi > phi_max,
|
|
189
|
+
}
|
|
190
|
+
)
|
|
191
|
+
cells = {}
|
|
192
|
+
for row in rows:
|
|
193
|
+
key = tuple(
|
|
194
|
+
row.get(name, "") for name in ("study", "benchmark", "model", "category")
|
|
195
|
+
)
|
|
196
|
+
cell = cells.setdefault(key, {"n": 0, "p_mas": [], "p_base": []})
|
|
197
|
+
cell["n"] += int(row.get("n", 1))
|
|
198
|
+
cell["p_mas"].append(float(row["p_mas"]))
|
|
199
|
+
cell["p_base"].append(float(row["p_base"]))
|
|
200
|
+
summaries = [
|
|
201
|
+
{
|
|
202
|
+
"study": key[0],
|
|
203
|
+
"benchmark": key[1],
|
|
204
|
+
"model": key[2],
|
|
205
|
+
"category": key[3],
|
|
206
|
+
"n": value["n"],
|
|
207
|
+
"p_mas": sum(value["p_mas"]) / len(value["p_mas"]),
|
|
208
|
+
"p_base": sum(value["p_base"]) / len(value["p_base"]),
|
|
209
|
+
}
|
|
210
|
+
for key, value in cells.items()
|
|
211
|
+
]
|
|
212
|
+
return {
|
|
213
|
+
"phi": phi,
|
|
214
|
+
"n_contrasts": len(rows),
|
|
215
|
+
"counts": counts,
|
|
216
|
+
"contrasts": details,
|
|
217
|
+
"phi_clamp_count": sum(item["phi_binds"] for item in details),
|
|
218
|
+
"baseline_cells": summaries,
|
|
219
|
+
}
|
pace/report.py
ADDED
|
@@ -0,0 +1,197 @@
|
|
|
1
|
+
"""pace report: frontier gap, per-item resource card, LaTeX table and vector figure from a baseline pool and a system log.
|
|
2
|
+
|
|
3
|
+
Inputs
|
|
4
|
+
--pool baseline cache written by `pace baseline` (greedy + sampled completions per item)
|
|
5
|
+
--logs JSONL of the system under test: call records (proxy format: system, item, prompt_tokens, completion_tokens, optional
|
|
6
|
+
reasoning_tokens, start, end, cost, tool_calls) and result records {"item", "system", "correct": true|false}
|
|
7
|
+
(a record that has "correct" and no token fields is a result).
|
|
8
|
+
The resource card reports per-item means for the system (MAS) and the greedy baseline (SAS) and passes them to mae_eval's auditor.
|
|
9
|
+
A ratio is n/a when both sides are 0 or when either side is unreported.
|
|
10
|
+
"""
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
import json
|
|
14
|
+
from typing import Dict, List, Optional
|
|
15
|
+
|
|
16
|
+
import numpy as np
|
|
17
|
+
|
|
18
|
+
from mae_eval.auditor import EvaluationAuditor
|
|
19
|
+
from mae_eval.vector import DIMENSIONS, ResourceVector
|
|
20
|
+
|
|
21
|
+
from .baseline import load_pool
|
|
22
|
+
from .frontier import frontier_curve, frontier_gap, frontier_at, pav_increasing
|
|
23
|
+
|
|
24
|
+
OKABE_ITO = {"blue": "#0072B2", "orange": "#E69F00", "green": "#009E73", "verm": "#D55E00", "grey": "#666666"}
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def read_run(path: str, system: str):
|
|
28
|
+
calls: Dict[str, List[dict]] = {}
|
|
29
|
+
results: Dict[str, bool] = {}
|
|
30
|
+
skipped = 0
|
|
31
|
+
for l in open(path, encoding="utf-8"):
|
|
32
|
+
if not l.strip():
|
|
33
|
+
continue
|
|
34
|
+
r = json.loads(l)
|
|
35
|
+
if r.get("system", system) != system:
|
|
36
|
+
continue
|
|
37
|
+
if "correct" in r and "prompt_tokens" not in r:
|
|
38
|
+
results[str(r["item"])] = bool(r["correct"])
|
|
39
|
+
elif r.get("prompt_tokens") is None or r.get("completion_tokens") is None:
|
|
40
|
+
skipped += 1 # usage missing: counted, never guessed
|
|
41
|
+
else:
|
|
42
|
+
calls.setdefault(str(r.get("item", "")), []).append(r)
|
|
43
|
+
return calls, results, skipped
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def _mean(xs: List[Optional[float]]) -> Optional[float]:
|
|
47
|
+
return None if (not xs or any(x is None for x in xs)) else float(sum(xs) / len(xs))
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def _vector(per_item: List[List[dict]], price_in: Optional[float], price_out: Optional[float]) -> ResourceVector:
|
|
51
|
+
"""Per-item mean resource vector from a list (one entry per item) of call-record lists."""
|
|
52
|
+
tok, reas, cost, lat, samp, sysc = [], [], [], [], [], []
|
|
53
|
+
for calls in per_item:
|
|
54
|
+
tok.append(sum(c["prompt_tokens"] + c["completion_tokens"] for c in calls))
|
|
55
|
+
reas.append(None if any(c.get("reasoning_tokens") is None for c in calls) else sum(c["reasoning_tokens"] for c in calls))
|
|
56
|
+
if price_in is not None and price_out is not None:
|
|
57
|
+
cost.append(sum((c["prompt_tokens"] * price_in + c["completion_tokens"] * price_out) / 1e6 for c in calls))
|
|
58
|
+
elif all(c.get("cost") is not None for c in calls):
|
|
59
|
+
cost.append(sum(c["cost"] for c in calls))
|
|
60
|
+
else:
|
|
61
|
+
cost.append(None)
|
|
62
|
+
if all(c.get("start") is not None and c.get("end") is not None for c in calls):
|
|
63
|
+
lat.append(max(c["end"] for c in calls) - min(c["start"] for c in calls)) # makespan, concurrency-aware
|
|
64
|
+
else:
|
|
65
|
+
lat.append(None if any(c.get("latency") is None for c in calls) else sum(c["latency"] for c in calls))
|
|
66
|
+
samp.append(sum(int(c.get("n_completions", 1)) for c in calls))
|
|
67
|
+
sysc.append(sum(int(c.get("tool_calls", 0)) for c in calls))
|
|
68
|
+
return ResourceVector(r_reason=_mean(reas), r_token=_mean(tok), r_cost=_mean(cost), r_lat=_mean(lat), r_sample=_mean(samp), r_sys=_mean(sysc))
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def build_report(pool_path: str, log_path: str, system: str = "mas", price_in: Optional[float] = None, price_out: Optional[float] = None,
|
|
72
|
+
n_boot: int = 2000, n_perm: int = 100, seed: int = 0, k: Optional[int] = None) -> dict:
|
|
73
|
+
pool, greedy, dropped = load_pool(pool_path, k)
|
|
74
|
+
ids, acc, tok = pool.curves(n_perm, seed)
|
|
75
|
+
calls, results, skipped = read_run(log_path, system)
|
|
76
|
+
common = [i for i in ids if i in results and i in calls]
|
|
77
|
+
if not common:
|
|
78
|
+
raise ValueError("no item has both a pool and a system result; check item ids")
|
|
79
|
+
pos = {i: n for n, i in enumerate(ids)}
|
|
80
|
+
sel = [pos[i] for i in common]
|
|
81
|
+
sys_correct = np.array([float(results[i]) for i in common])
|
|
82
|
+
sys_tokens = np.array([float(sum(c["prompt_tokens"] + c["completion_tokens"] for c in calls[i])) for i in common])
|
|
83
|
+
gap = frontier_gap(acc[sel], tok[sel], sys_correct, sys_tokens, n_boot=n_boot, seed=seed)
|
|
84
|
+
tk, ak = frontier_curve(acc[sel], tok[sel])
|
|
85
|
+
fit = pav_increasing(ak)
|
|
86
|
+
sas_items = [[{"prompt_tokens": greedy[i]["prompt_tokens"], "completion_tokens": greedy[i]["completion_tokens"],
|
|
87
|
+
"reasoning_tokens": greedy[i].get("reasoning_tokens"), "latency": greedy[i]["latency"]}] for i in common if i in greedy]
|
|
88
|
+
card: dict = {}
|
|
89
|
+
if len(sas_items) == len(common) and all(c[0]["prompt_tokens"] is not None for c in sas_items):
|
|
90
|
+
sas_v = _vector(sas_items, price_in, price_out)
|
|
91
|
+
mas_v = _vector([calls[i] for i in common], price_in, price_out)
|
|
92
|
+
aud = EvaluationAuditor(mas_v, sas_v)
|
|
93
|
+
card = {"mas_per_item": mas_v.to_dict(), "sas_per_item": sas_v.to_dict(), "ratios_mas_over_sas": aud.ratios(),
|
|
94
|
+
"control_states": {d: s.value for d, s in aud.control_states().items()}}
|
|
95
|
+
greedy_acc = float(np.mean([greedy[i]["correct"] for i in common if i in greedy])) if any(i in greedy for i in common) else None
|
|
96
|
+
return {"n_items": len(common), "pool_k": pool.K, "pool_items_dropped_incomplete": dropped, "system_calls_missing_usage": skipped,
|
|
97
|
+
"frontier": {"tokens": tk.tolist(), "sc_accuracy": ak.tolist(), "isotonic": fit.tolist()}, "greedy_accuracy": greedy_acc,
|
|
98
|
+
"frontier_gap": gap, "resource_card": card, "system": system}
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def to_latex(rep: dict, caption: str = "Frontier gap and per-item resource card.", label: str = "tab:cme_report") -> str:
|
|
102
|
+
g = rep["frontier_gap"]
|
|
103
|
+
rows = [r"\begin{table}[h]", r"\centering", r"\small", r"\caption{%s}" % caption, r"\label{%s}" % label, r"\begin{tabular}{lrr}", r"\toprule",
|
|
104
|
+
r"Quantity & System & Greedy baseline \\", r"\midrule"]
|
|
105
|
+
rc = rep.get("resource_card") or {}
|
|
106
|
+
for d in DIMENSIONS:
|
|
107
|
+
if rc:
|
|
108
|
+
m, s = rc["mas_per_item"][d], rc["sas_per_item"][d]
|
|
109
|
+
fmt = lambda x: "n/a" if x is None else ("%.4g" % x)
|
|
110
|
+
rows.append(r"%s & %s & %s \\" % (d.replace("_", r"\_"), fmt(m), fmt(s)))
|
|
111
|
+
rows += [r"\midrule", r"Accuracy & %.3f & %s \\" % (g["system_accuracy"], "n/a" if rep["greedy_accuracy"] is None else "%.3f" % rep["greedy_accuracy"]),
|
|
112
|
+
r"Self-consistency frontier at system tokens & \multicolumn{2}{r}{%.3f (%s)} \\" % (g["frontier_at_system_tokens"], g["range_status"]),
|
|
113
|
+
r"Frontier gap (95\%% bootstrap CI) & \multicolumn{2}{r}{$%+.3f$ [$%+.3f$, $%+.3f$]} \\" % (g["gap"], g["ci_low"], g["ci_high"]),
|
|
114
|
+
r"\bottomrule", r"\end{tabular}", r"\end{table}"]
|
|
115
|
+
return "\n".join(rows) + "\n"
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def plot(rep: dict, out_path: str) -> None:
|
|
119
|
+
import matplotlib
|
|
120
|
+
matplotlib.use("Agg")
|
|
121
|
+
import matplotlib.pyplot as plt
|
|
122
|
+
import matplotlib.ticker
|
|
123
|
+
matplotlib.rcParams["pdf.fonttype"] = 42
|
|
124
|
+
f, g = rep["frontier"], rep["frontier_gap"]
|
|
125
|
+
fig, ax = plt.subplots(figsize=(4.6, 3.2))
|
|
126
|
+
ax.plot(f["tokens"], f["sc_accuracy"], "o", color=OKABE_ITO["grey"], ms=3.5, label="Self-consistency, k = 1 to %d" % rep["pool_k"])
|
|
127
|
+
ax.plot(f["tokens"], f["isotonic"], "-", color=OKABE_ITO["blue"], lw=1.6, label="Isotonic frontier")
|
|
128
|
+
if rep["greedy_accuracy"] is not None:
|
|
129
|
+
ax.plot([f["tokens"][0]], [rep["greedy_accuracy"]], "s", color=OKABE_ITO["green"], label="Greedy")
|
|
130
|
+
ax.plot([g["system_mean_tokens"]], [g["system_accuracy"]], "D", color=OKABE_ITO["verm"], label="System (%s)" % rep["system"])
|
|
131
|
+
ax.vlines(g["system_mean_tokens"], g["frontier_at_system_tokens"], g["system_accuracy"], colors=OKABE_ITO["verm"], lw=1.2, linestyles=":")
|
|
132
|
+
ax.set_xscale("log")
|
|
133
|
+
ax.xaxis.set_minor_formatter(matplotlib.ticker.NullFormatter())
|
|
134
|
+
ax.set_xlabel("Mean tokens per item (prompt + completion)")
|
|
135
|
+
ax.set_ylabel("Accuracy")
|
|
136
|
+
ax.set_title("gap %+.3f [%+.3f, %+.3f]" % (g["gap"], g["ci_low"], g["ci_high"]), fontsize=9)
|
|
137
|
+
ax.legend(fontsize=7, frameon=False)
|
|
138
|
+
ax.spines[["top", "right"]].set_visible(False)
|
|
139
|
+
fig.tight_layout()
|
|
140
|
+
fig.savefig(out_path, bbox_inches="tight")
|
|
141
|
+
plt.close(fig)
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
OI_CYCLE = ["#D55E00", "#009E73", "#E69F00", "#CC79A7", "#56B4E9", "#F0E442", "#000000"] # Okabe-Ito, skipping the frontier blue
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
def frontier_report(pool_path: str, k: Optional[int] = None, n_perm: int = 100, seed: int = 0) -> dict:
|
|
148
|
+
"""Self-consistency frontier of a sample pool: points for k = 1..K, isotonic fit, greedy accuracy. No system needed."""
|
|
149
|
+
pool, greedy, dropped = load_pool(pool_path, k)
|
|
150
|
+
ids, acc, tok = pool.curves(n_perm, seed)
|
|
151
|
+
tk, ak = frontier_curve(acc, tok)
|
|
152
|
+
gacc = float(np.mean([greedy[i]["correct"] for i in ids if i in greedy])) if any(i in greedy for i in ids) else None
|
|
153
|
+
return {"n_items": len(ids), "pool_k": pool.K, "pool_items_dropped_incomplete": dropped, "tokens": tk.tolist(), "sc_accuracy": ak.tolist(),
|
|
154
|
+
"isotonic": pav_increasing(ak).tolist(), "greedy_accuracy": gacc}
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
def system_points(log_path: str) -> List[dict]:
|
|
158
|
+
"""Accuracy and mean tokens per item of every system in a run log (systems are the values of the 'system' field)."""
|
|
159
|
+
names = []
|
|
160
|
+
for l in open(log_path, encoding="utf-8"):
|
|
161
|
+
if l.strip():
|
|
162
|
+
s = json.loads(l).get("system")
|
|
163
|
+
if s and s not in names:
|
|
164
|
+
names.append(s)
|
|
165
|
+
out = []
|
|
166
|
+
for s in names:
|
|
167
|
+
calls, results, _ = read_run(log_path, s)
|
|
168
|
+
common = [i for i in results if i in calls]
|
|
169
|
+
if common:
|
|
170
|
+
out.append({"system": s, "accuracy": float(np.mean([results[i] for i in common])), "n_items": len(common),
|
|
171
|
+
"mean_tokens": float(np.mean([sum(c["prompt_tokens"] + c["completion_tokens"] for c in calls[i]) for i in common]))})
|
|
172
|
+
return out
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
def plot_frontier(fr: dict, points: List[dict], out_path: str) -> None:
|
|
176
|
+
"""Frontier curve with system points (Okabe-Ito palette); .pdf is vector, .png is raster."""
|
|
177
|
+
import matplotlib
|
|
178
|
+
matplotlib.use("Agg")
|
|
179
|
+
import matplotlib.pyplot as plt
|
|
180
|
+
import matplotlib.ticker
|
|
181
|
+
matplotlib.rcParams["pdf.fonttype"] = 42
|
|
182
|
+
fig, ax = plt.subplots(figsize=(5.4, 3.5))
|
|
183
|
+
ax.plot(fr["tokens"], fr["sc_accuracy"], "o", color="#666666", ms=3.5, label="self-consistency, k = 1 to %d" % fr["pool_k"])
|
|
184
|
+
ax.plot(fr["tokens"], fr["isotonic"], "-", color="#0072B2", lw=1.6, label="isotonic frontier")
|
|
185
|
+
if fr.get("greedy_accuracy") is not None:
|
|
186
|
+
ax.plot([fr["tokens"][0]], [fr["greedy_accuracy"]], "s", color="#56B4E9", mec="white", label="greedy")
|
|
187
|
+
for i, p in enumerate(points):
|
|
188
|
+
ax.plot([p["mean_tokens"]], [p["accuracy"]], "D", color=OI_CYCLE[i % len(OI_CYCLE)], mec="white", ms=7, label=p["system"])
|
|
189
|
+
ax.set_xscale("log")
|
|
190
|
+
ax.xaxis.set_minor_formatter(matplotlib.ticker.NullFormatter())
|
|
191
|
+
ax.set_xlabel("Mean tokens per item (prompt + completion)")
|
|
192
|
+
ax.set_ylabel("Accuracy")
|
|
193
|
+
ax.legend(fontsize=7, frameon=False, loc="lower right")
|
|
194
|
+
ax.spines[["top", "right"]].set_visible(False)
|
|
195
|
+
fig.tight_layout()
|
|
196
|
+
fig.savefig(out_path, bbox_inches="tight", dpi=200)
|
|
197
|
+
plt.close(fig)
|
pace/rollout_card.py
ADDED
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
"""Export a PACE compute block in a form that sits inside a rollout card (Masters, Liu and Albrecht 2026, arXiv:2605.12131).
|
|
2
|
+
|
|
3
|
+
What the rollout-card paper specifies (its Appendix E): a card is a logical directory with manifest.json, five append-only JSONL streams (events, nodes, edges,
|
|
4
|
+
annotations, mutations) and optional blobs; a card lists the reporting rules behind every reported number, each with a name, an implementation or version,
|
|
5
|
+
a configuration, input streams, an output target and a drops manifest; and new metadata goes into annotations.jsonl under a namespace. The paper has no dedicated
|
|
6
|
+
compute section.
|
|
7
|
+
|
|
8
|
+
What this module does: it maps the PACE resource vector R and the frontier gap onto those two extension points, without claiming more.
|
|
9
|
+
* `annotations`: rows in the annotations-stream schema (target_type, target_id, namespace, sequence, payload, created_at) with namespace "pace.compute",
|
|
10
|
+
one row for the MAS resource vector, one for the SAS vector and ratios (if a baseline is present), one for the frontier-gap result;
|
|
11
|
+
* `reporting_rules`: one entry for the frontier-gap rule with its configuration and a drops manifest (fields read, structures collapsed, loss classes).
|
|
12
|
+
Limits, repeated in every export: this is a compute block, not a complete rollout card (no events, nodes or edges are written); the manifest key names are not
|
|
13
|
+
listed in the rollout-card paper, so none are invented here; and the output has not been checked with the reference validator (ERGON), which is not
|
|
14
|
+
installed here. Nothing is estimated: a dimension that the log does not report stays null.
|
|
15
|
+
Formats: JSON, YAML (written by a small emitter, no dependency) and a one-page LaTeX block.
|
|
16
|
+
"""
|
|
17
|
+
from __future__ import annotations
|
|
18
|
+
|
|
19
|
+
import datetime
|
|
20
|
+
import json
|
|
21
|
+
from typing import Any, Dict, List, Optional
|
|
22
|
+
|
|
23
|
+
SPEC = "arXiv:2605.12131"
|
|
24
|
+
NAMESPACE = "pace.compute"
|
|
25
|
+
DIMENSIONS = ("r_reason", "r_token", "r_cost", "r_lat", "r_sample", "r_sys")
|
|
26
|
+
DIM_LABEL = {"r_reason": "thinking tokens", "r_token": "total tokens", "r_cost": "cost (USD)", "r_lat": "latency (s)", "r_sample": "independent samples", "r_sys": "tool calls"}
|
|
27
|
+
LIMITS = ("A compute block, not a complete rollout card: no events, nodes or edges are included. Manifest key names are not specified in the rollout-card paper and "
|
|
28
|
+
"are not invented here. Not checked with the reference validator.")
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def build_block(rep: dict, run_id: str = "run", created_at: Optional[str] = None, version: str = "0.1") -> Dict[str, Any]:
|
|
32
|
+
"""Build the compute block from a `pace.build_report` result. `created_at` defaults to the current UTC time (ISO 8601)."""
|
|
33
|
+
when = created_at or datetime.datetime.now(datetime.timezone.utc).replace(microsecond=0).isoformat()
|
|
34
|
+
rc = rep.get("resource_card") or {}
|
|
35
|
+
g = rep["frontier_gap"]
|
|
36
|
+
rows: List[dict] = []
|
|
37
|
+
|
|
38
|
+
def add(payload: dict) -> None:
|
|
39
|
+
rows.append({"target_type": "run", "target_id": run_id, "namespace": NAMESPACE, "sequence": len(rows) + 1, "payload": payload, "created_at": when})
|
|
40
|
+
add({"kind": "resource_vector", "system": rep.get("system", "mas"), "unit": "per item (mean)", "dimensions": {d: (rc.get("mas_per_item") or {}).get(d) for d in DIMENSIONS}})
|
|
41
|
+
if rc.get("sas_per_item") is not None:
|
|
42
|
+
add({"kind": "resource_vector", "system": "sas", "unit": "per item (mean)", "dimensions": {d: rc["sas_per_item"].get(d) for d in DIMENSIONS}})
|
|
43
|
+
add({"kind": "ratios_mas_over_sas", "dimensions": {d: rc["ratios_mas_over_sas"].get(d) for d in DIMENSIONS},
|
|
44
|
+
"control_states": rc.get("control_states"), "note": "null: not reported by a side, or 0/0"})
|
|
45
|
+
add({"kind": "frontier_gap", "estimate": g["gap"], "ci95": [g["ci_low"], g["ci_high"]], "interval": "paired bootstrap over items", "n_items": g["n_items"],
|
|
46
|
+
"system_accuracy": g["system_accuracy"], "frontier_accuracy_at_system_tokens": g["frontier_at_system_tokens"], "system_mean_tokens": g["system_mean_tokens"],
|
|
47
|
+
"range_status": g["range_status"], "greedy_accuracy": rep.get("greedy_accuracy"), "pool_k": rep["pool_k"]})
|
|
48
|
+
from . import __version__
|
|
49
|
+
rule = {"name": "pace.frontier_gap", "implementation": "pace %s (pace.frontier.frontier_gap)" % __version__,
|
|
50
|
+
"configuration": {"n_boot": g["n_boot"], "pool_k": rep["pool_k"], "tokens": "prompt plus completion tokens of every call of the item",
|
|
51
|
+
"frontier": "isotonic (pool-adjacent-violators) self-consistency accuracy at the system's mean tokens"},
|
|
52
|
+
"input_streams": ["per-call usage log (pace run log)", "greedy and sampled answer pool (pace baseline)"],
|
|
53
|
+
"output_target": "annotations.jsonl, namespace %s" % NAMESPACE,
|
|
54
|
+
"drops_manifest": {"fields_read": ["item id", "correct", "prompt_tokens", "completion_tokens", "latency"],
|
|
55
|
+
"collapsed": ["per-call records summed to per-item totals", "agents and rounds pooled"],
|
|
56
|
+
"loss_classes": ["message content not read", "agent identity erased", "round structure erased", "items without a complete pool dropped (%d)" % rep.get("pool_items_dropped_incomplete", 0),
|
|
57
|
+
"calls without a usage block not counted (%d)" % rep.get("system_calls_missing_usage", 0)]}}
|
|
58
|
+
return {"format": "cme-compute-block", "version": version, "targets": "rollout card (%s), annotations namespace and reporting-rule registry" % SPEC, "limits": LIMITS,
|
|
59
|
+
"annotations": rows, "reporting_rules": [rule]}
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def _scalar(v: Any) -> str:
|
|
63
|
+
if v is None:
|
|
64
|
+
return "null"
|
|
65
|
+
if isinstance(v, bool):
|
|
66
|
+
return "true" if v else "false"
|
|
67
|
+
if isinstance(v, (int, float)):
|
|
68
|
+
return repr(v)
|
|
69
|
+
s = str(v)
|
|
70
|
+
return json.dumps(s) if (not s or s != s.strip() or any(c in s for c in ':#{}[],&*!|>\'"%@`\n') or s.lower() in ("null", "true", "false", "yes", "no")
|
|
71
|
+
or s[0] in "-?0123456789") else s
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def to_yaml(obj: Any, indent: int = 0) -> str:
|
|
75
|
+
"""Minimal YAML emitter for nested dicts, lists and scalars (block style)."""
|
|
76
|
+
pad = " " * indent
|
|
77
|
+
if isinstance(obj, dict):
|
|
78
|
+
if not obj:
|
|
79
|
+
return pad + "{}\n"
|
|
80
|
+
out = ""
|
|
81
|
+
for k, v in obj.items():
|
|
82
|
+
if isinstance(v, (dict, list)) and v:
|
|
83
|
+
out += "%s%s:\n%s" % (pad, k, to_yaml(v, indent + 1))
|
|
84
|
+
else:
|
|
85
|
+
out += "%s%s: %s\n" % (pad, k, "{}" if isinstance(v, dict) else "[]" if isinstance(v, list) else _scalar(v))
|
|
86
|
+
return out
|
|
87
|
+
if isinstance(obj, list):
|
|
88
|
+
out = ""
|
|
89
|
+
for v in obj:
|
|
90
|
+
if isinstance(v, (dict, list)) and v:
|
|
91
|
+
body = to_yaml(v, indent + 1)
|
|
92
|
+
out += "%s- %s" % (pad, body[len(pad) + 2:])
|
|
93
|
+
else:
|
|
94
|
+
out += "%s- %s\n" % (pad, _scalar(v))
|
|
95
|
+
return out
|
|
96
|
+
return pad + _scalar(obj) + "\n"
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def _tex(s: str) -> str:
|
|
100
|
+
bs = chr(92)
|
|
101
|
+
return s.replace(bs, "").replace("_", bs + "_").replace("%", bs + "%").replace("&", bs + "&").replace("#", bs + "#")
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def to_latex(block: dict, title: str = "PACE compute block") -> str:
|
|
105
|
+
"""One-page LaTeX block: resource vector (MAS, SAS, ratio) and the frontier-gap line. Null cells print n/a."""
|
|
106
|
+
bs = chr(92)
|
|
107
|
+
ann = {a["payload"]["kind"] + ":" + str(a["payload"].get("system", "")): a["payload"] for a in block["annotations"]}
|
|
108
|
+
mas = next(p for p in (a["payload"] for a in block["annotations"]) if p["kind"] == "resource_vector")
|
|
109
|
+
sas = next((a["payload"] for a in block["annotations"] if a["payload"]["kind"] == "resource_vector" and a["payload"]["system"] == "sas"), None)
|
|
110
|
+
rat = next((a["payload"] for a in block["annotations"] if a["payload"]["kind"] == "ratios_mas_over_sas"), None)
|
|
111
|
+
gap = next(a["payload"] for a in block["annotations"] if a["payload"]["kind"] == "frontier_gap")
|
|
112
|
+
|
|
113
|
+
def f(v, nd=3):
|
|
114
|
+
return "n/a" if v is None else ("inf" if v == float("inf") else ("%.0f" % v if abs(v) >= 100 else "%.*g" % (nd, v)))
|
|
115
|
+
lines = [bs + "begin{table}[t]", bs + "centering", bs + "small",
|
|
116
|
+
bs + "caption{" + _tex(title) + " (namespace " + bs + "texttt{" + _tex(NAMESPACE) + "}). Per-item means from the run log; n/a: not reported by the log. "
|
|
117
|
+
"Compute block for a rollout card (" + SPEC.replace("arXiv:", "arXiv:") + "), not a complete card.}",
|
|
118
|
+
bs + "label{tab:cme_compute_block}", bs + "begin{tabular}{lrrr}", bs + "toprule", "Dimension & " + _tex(mas["system"].upper()) + " & SAS & Ratio " + bs + bs, bs + "midrule"]
|
|
119
|
+
for d in DIMENSIONS:
|
|
120
|
+
lines.append("%s (%s) & %s & %s & %s %s" % (_tex(d), _tex(DIM_LABEL[d]), f(mas["dimensions"][d]), f(sas["dimensions"][d]) if sas else "n/a", f(rat["dimensions"][d]) if rat else "n/a", bs + bs))
|
|
121
|
+
lines += [bs + "midrule",
|
|
122
|
+
bs + "multicolumn{4}{l}{Frontier gap $%+.3f$, 95%% paired bootstrap interval [$%+.3f$, $%+.3f$] over %d items; frontier at the system's mean tokens %.3f (%s).} %s" % (
|
|
123
|
+
gap["estimate"], gap["ci95"][0], gap["ci95"][1], gap["n_items"], gap["frontier_accuracy_at_system_tokens"], _tex(gap["range_status"]), bs + bs),
|
|
124
|
+
bs + "bottomrule", bs + "end{tabular}", bs + "end{table}"]
|
|
125
|
+
return "\n".join(lines).replace("95%", "95" + bs + "%") + "\n"
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
def write_block(rep: dict, prefix: str, run_id: str = "run") -> List[str]:
|
|
129
|
+
"""Write PREFIX.json, PREFIX.yaml and PREFIX.tex; return the paths."""
|
|
130
|
+
block = build_block(rep, run_id)
|
|
131
|
+
paths = []
|
|
132
|
+
for ext, text in (("json", json.dumps(block, indent=2) + "\n"), ("yaml", to_yaml(block)), ("tex", to_latex(block))):
|
|
133
|
+
p = "%s.%s" % (prefix, ext)
|
|
134
|
+
with open(p, "w", encoding="utf-8") as fh:
|
|
135
|
+
fh.write(text)
|
|
136
|
+
paths.append(p)
|
|
137
|
+
return paths
|
pace/schema.py
ADDED
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
"""Small, dependency-free schemas for portable PACE logs."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from dataclasses import asdict, dataclass
|
|
6
|
+
from typing import Any
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def _bool(value: Any) -> bool | None:
|
|
10
|
+
if value is None or value == "":
|
|
11
|
+
return None
|
|
12
|
+
if isinstance(value, bool):
|
|
13
|
+
return value
|
|
14
|
+
if isinstance(value, (int, float)):
|
|
15
|
+
return bool(value)
|
|
16
|
+
text = str(value).strip().lower()
|
|
17
|
+
if text in {"true", "yes", "y", "1", "correct", "right"}:
|
|
18
|
+
return True
|
|
19
|
+
if text in {"false", "no", "n", "0", "incorrect", "wrong"}:
|
|
20
|
+
return False
|
|
21
|
+
raise ValueError("cannot parse boolean value %r" % (value,))
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
@dataclass
|
|
25
|
+
class ItemLog:
|
|
26
|
+
"""One scored agent answer at one debate round."""
|
|
27
|
+
|
|
28
|
+
item: str
|
|
29
|
+
agent: str
|
|
30
|
+
round: int
|
|
31
|
+
correct: bool
|
|
32
|
+
answer: str | None = None
|
|
33
|
+
prompt_tokens: float | None = None
|
|
34
|
+
completion_tokens: float | None = None
|
|
35
|
+
|
|
36
|
+
def __post_init__(self) -> None:
|
|
37
|
+
self.item = str(self.item)
|
|
38
|
+
self.agent = str(self.agent)
|
|
39
|
+
self.round = int(self.round)
|
|
40
|
+
parsed = _bool(self.correct)
|
|
41
|
+
if parsed is None:
|
|
42
|
+
raise ValueError("correct is required")
|
|
43
|
+
self.correct = parsed
|
|
44
|
+
for field in ("prompt_tokens", "completion_tokens"):
|
|
45
|
+
value = getattr(self, field)
|
|
46
|
+
if value is not None and value != "":
|
|
47
|
+
setattr(self, field, float(value))
|
|
48
|
+
else:
|
|
49
|
+
setattr(self, field, None)
|
|
50
|
+
|
|
51
|
+
def to_dict(self) -> dict[str, Any]:
|
|
52
|
+
result = asdict(self)
|
|
53
|
+
return {key: value for key, value in result.items() if value is not None}
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def item_log_from_mapping(row: dict[str, Any]) -> ItemLog:
|
|
57
|
+
"""Validate common aliases used by external trace exporters."""
|
|
58
|
+
item = row.get("item", row.get("item_id", row.get("id")))
|
|
59
|
+
agent = row.get("agent", row.get("agent_id", row.get("name", "agent_0")))
|
|
60
|
+
round_no = row.get("round", row.get("round_id", 0))
|
|
61
|
+
correct = row.get("correct", row.get("is_correct"))
|
|
62
|
+
if item is None or correct is None:
|
|
63
|
+
raise ValueError(
|
|
64
|
+
"each record needs item_id/item, agent_id/agent, round, and correct"
|
|
65
|
+
)
|
|
66
|
+
return ItemLog(
|
|
67
|
+
item,
|
|
68
|
+
agent,
|
|
69
|
+
round_no,
|
|
70
|
+
correct,
|
|
71
|
+
row.get("answer"),
|
|
72
|
+
row.get("prompt_tokens"),
|
|
73
|
+
row.get("completion_tokens"),
|
|
74
|
+
)
|