pace-eval 0.2.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cme/__init__.py +11 -0
- cme/__main__.py +5 -0
- cme/analysis.py +223 -0
- cme/audit.py +145 -0
- cme/baseline.py +162 -0
- cme/cli.py +279 -0
- cme/client.py +165 -0
- cme/dynamics.py +212 -0
- cme/examples/crewai_integration.py +16 -0
- cme/examples/demo_pool.jsonl +360 -0
- cme/examples/demo_run.jsonl +480 -0
- cme/examples/langgraph_integration.py +27 -0
- cme/examples/make_demo.py +37 -0
- cme/examples/openai_quickstart.py +18 -0
- cme/experiment.py +555 -0
- cme/frontier.py +143 -0
- cme/hidden_profile.py +135 -0
- cme/proxy.py +106 -0
- cme/rebuttal.py +76 -0
- cme/report.py +197 -0
- cme/rollout_card.py +137 -0
- cme/tasks.py +266 -0
- cme/verified_debate.py +127 -0
- cme/verifier.py +92 -0
- mae_eval/__init__.py +11 -0
- mae_eval/__main__.py +5 -0
- mae_eval/auditor.py +112 -0
- mae_eval/classifier.py +62 -0
- mae_eval/cli.py +210 -0
- mae_eval/tracker.py +162 -0
- mae_eval/vector.py +91 -0
- pace/__init__.py +11 -0
- pace/__main__.py +5 -0
- pace/analysis.py +129 -0
- pace/audit.py +145 -0
- pace/baseline.py +162 -0
- pace/card.py +205 -0
- pace/cli.py +898 -0
- pace/client.py +165 -0
- pace/dynamics.py +372 -0
- pace/escape_benchmarks.py +162 -0
- pace/examples/crewai_integration.py +16 -0
- pace/examples/demo_pool.jsonl +360 -0
- pace/examples/demo_run.jsonl +480 -0
- pace/examples/langgraph_integration.py +27 -0
- pace/examples/make_demo.py +37 -0
- pace/examples/openai_quickstart.py +18 -0
- pace/experiment.py +555 -0
- pace/frontier.py +114 -0
- pace/hidden_profile.py +301 -0
- pace/importers.py +219 -0
- pace/inflate.py +213 -0
- pace/influence.py +231 -0
- pace/meta_engine.py +246 -0
- pace/middleware.py +230 -0
- pace/open_ended.py +114 -0
- pace/plotting.py +132 -0
- pace/pool_sizer.py +154 -0
- pace/protocol_v3.py +414 -0
- pace/proxy.py +106 -0
- pace/rebuttal.py +219 -0
- pace/report.py +197 -0
- pace/rollout_card.py +137 -0
- pace/schema.py +74 -0
- pace/simulator.py +199 -0
- pace/studio.html +1040 -0
- pace/tasks.py +266 -0
- pace/verified_debate.py +127 -0
- pace/verifier.py +92 -0
- pace/workbench.html +26 -0
- pace/workbench.py +119 -0
- pace_eval-0.2.2.dist-info/METADATA +205 -0
- pace_eval-0.2.2.dist-info/RECORD +83 -0
- pace_eval-0.2.2.dist-info/WHEEL +5 -0
- pace_eval-0.2.2.dist-info/entry_points.txt +4 -0
- pace_eval-0.2.2.dist-info/licenses/LICENSE +202 -0
- pace_eval-0.2.2.dist-info/top_level.txt +4 -0
- pace_lab/__init__.py +9 -0
- pace_lab/clients.py +568 -0
- pace_lab/concurrency_test.py +127 -0
- pace_lab/data.py +445 -0
- pace_lab/graders.py +626 -0
- pace_lab/strategies.py +345 -0
pace/protocol_v3.py
ADDED
|
@@ -0,0 +1,414 @@
|
|
|
1
|
+
"""Dependency-light PACE Phase 16 protocol and constructive objective engine."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from collections import Counter, defaultdict
|
|
6
|
+
from statistics import mean, median, pstdev
|
|
7
|
+
|
|
8
|
+
import numpy as np
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def _get(row, *names, default=0.0):
|
|
12
|
+
for name in names:
|
|
13
|
+
if name in row and row[name] not in (None, ""):
|
|
14
|
+
return row[name]
|
|
15
|
+
return default
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def _float(row, *names, default=0.0):
|
|
19
|
+
try:
|
|
20
|
+
return float(_get(row, *names, default=default))
|
|
21
|
+
except (TypeError, ValueError):
|
|
22
|
+
return float(default)
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def _item(row):
|
|
26
|
+
return str(_get(row, "item", "id", "item_id", default="item"))
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def _accuracy(row):
|
|
30
|
+
value = _get(row, "correct", "accuracy", "score", default=0.0)
|
|
31
|
+
return (
|
|
32
|
+
float(value)
|
|
33
|
+
if isinstance(value, (int, float))
|
|
34
|
+
else float(str(value).lower() in {"1", "true", "correct", "yes"})
|
|
35
|
+
)
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def _bootstrap(values, statistic, n_boot, seed):
|
|
39
|
+
values = np.asarray(list(values), dtype=float)
|
|
40
|
+
if not len(values):
|
|
41
|
+
return [None, None]
|
|
42
|
+
rng = np.random.default_rng(seed)
|
|
43
|
+
draws = [
|
|
44
|
+
statistic(values[rng.integers(0, len(values), len(values))])
|
|
45
|
+
for _ in range(max(1, n_boot))
|
|
46
|
+
]
|
|
47
|
+
return [float(np.percentile(draws, 2.5)), float(np.percentile(draws, 97.5))]
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def itemwise_resource_match(mas_rows, pool_rows, axis="tokens", n_boot=500, seed=0):
|
|
51
|
+
if axis not in {"tokens", "calls", "cost"}:
|
|
52
|
+
raise ValueError("axis must be tokens, calls, or cost")
|
|
53
|
+
pools = defaultdict(list)
|
|
54
|
+
for row in pool_rows:
|
|
55
|
+
pools[_item(row)].append(
|
|
56
|
+
max(0.0, _float(row, axis, f"{axis}_total", "resource"))
|
|
57
|
+
)
|
|
58
|
+
mas = defaultdict(list)
|
|
59
|
+
for row in mas_rows:
|
|
60
|
+
mas[_item(row)].append(row)
|
|
61
|
+
ks, frontier, observed, statuses = [], [], [], []
|
|
62
|
+
for item, rows in mas.items():
|
|
63
|
+
budget = sum(
|
|
64
|
+
max(0.0, _float(r, axis, f"{axis}_total", "resource")) for r in rows
|
|
65
|
+
)
|
|
66
|
+
samples = pools.get(item, []) or [0.0]
|
|
67
|
+
cumulative = 0.0
|
|
68
|
+
k = 0
|
|
69
|
+
for value in samples:
|
|
70
|
+
k += 1
|
|
71
|
+
cumulative += value
|
|
72
|
+
if cumulative >= budget:
|
|
73
|
+
break
|
|
74
|
+
statuses.append("reached" if cumulative >= budget else "exhausted")
|
|
75
|
+
ks.append(max(1, k))
|
|
76
|
+
frontier.append(
|
|
77
|
+
float(
|
|
78
|
+
np.mean(
|
|
79
|
+
[_accuracy(r) for r in pool_rows if _item(r) == item][:k] or [0.0]
|
|
80
|
+
)
|
|
81
|
+
)
|
|
82
|
+
)
|
|
83
|
+
observed.append(float(np.mean([_accuracy(r) for r in rows])))
|
|
84
|
+
gap = [a - b for a, b in zip(observed, frontier)]
|
|
85
|
+
result = {
|
|
86
|
+
"n_items": len(ks),
|
|
87
|
+
"mean_k_i": float(mean(ks)) if ks else 0.0,
|
|
88
|
+
"median_k_i": float(median(ks)) if ks else 0.0,
|
|
89
|
+
"max_k_i": max(ks) if ks else 0,
|
|
90
|
+
"mas_accuracy": float(mean(observed)) if observed else 0.0,
|
|
91
|
+
"itemwise_frontier_accuracy": float(mean(frontier)) if frontier else 0.0,
|
|
92
|
+
"expectation_frontier_accuracy": float(mean(frontier)) if frontier else 0.0,
|
|
93
|
+
"itemwise_gap": float(mean(gap)) if gap else 0.0,
|
|
94
|
+
"expectation_gap": float(mean(observed) - mean(frontier)) if gap else 0.0,
|
|
95
|
+
"ci_low": None,
|
|
96
|
+
"ci_high": None,
|
|
97
|
+
"paired_bootstrap_ci": _bootstrap(gap, np.mean, n_boot, seed),
|
|
98
|
+
"itemwise_dominance_share": float(sum(x >= 0 for x in gap) / len(gap))
|
|
99
|
+
if gap
|
|
100
|
+
else 0.0,
|
|
101
|
+
"range_status": dict(Counter(statuses)),
|
|
102
|
+
"axis": axis,
|
|
103
|
+
}
|
|
104
|
+
result["ci_low"], result["ci_high"] = result["paired_bootstrap_ci"]
|
|
105
|
+
return result
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def _distribution(values):
|
|
109
|
+
values = [float(x) for x in values]
|
|
110
|
+
if not values:
|
|
111
|
+
return {
|
|
112
|
+
key: 0.0
|
|
113
|
+
for key in ("mean", "std", "min", "p25", "median", "p75", "p90", "max")
|
|
114
|
+
}
|
|
115
|
+
return {
|
|
116
|
+
"mean": float(mean(values)),
|
|
117
|
+
"std": float(pstdev(values)),
|
|
118
|
+
"min": min(values),
|
|
119
|
+
"p25": float(np.percentile(values, 25)),
|
|
120
|
+
"median": float(median(values)),
|
|
121
|
+
"p75": float(np.percentile(values, 75)),
|
|
122
|
+
"p90": float(np.percentile(values, 90)),
|
|
123
|
+
"max": max(values),
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def adaptive_stopping_audit(rows, declared_ceiling=None):
|
|
128
|
+
by_item = defaultdict(list)
|
|
129
|
+
for row in rows:
|
|
130
|
+
by_item[_item(row)].append(row)
|
|
131
|
+
rounds = [
|
|
132
|
+
max(_float(r, "rounds", "round", default=1) for r in rs)
|
|
133
|
+
for rs in by_item.values()
|
|
134
|
+
]
|
|
135
|
+
calls = [sum(_float(r, "calls", default=1) for r in rs) for rs in by_item.values()]
|
|
136
|
+
tokens = [
|
|
137
|
+
sum(
|
|
138
|
+
_float(r, "tokens", "prompt_tokens", default=0)
|
|
139
|
+
+ _float(r, "completion_tokens", default=0)
|
|
140
|
+
for r in rs
|
|
141
|
+
)
|
|
142
|
+
for rs in by_item.values()
|
|
143
|
+
]
|
|
144
|
+
ceiling = (
|
|
145
|
+
declared_ceiling
|
|
146
|
+
if declared_ceiling is not None
|
|
147
|
+
else (max(rounds) if rounds else 0)
|
|
148
|
+
)
|
|
149
|
+
reasons = Counter(
|
|
150
|
+
str(_get(r, "termination_reason", "done_reason", default="unspecified"))
|
|
151
|
+
for r in rows
|
|
152
|
+
)
|
|
153
|
+
return {
|
|
154
|
+
"n_items": len(by_item),
|
|
155
|
+
"rounds": _distribution(rounds),
|
|
156
|
+
"calls": _distribution(calls),
|
|
157
|
+
"tokens": _distribution(tokens),
|
|
158
|
+
"declared_ceiling": ceiling,
|
|
159
|
+
"ceiling_compliance_rate": float(
|
|
160
|
+
sum(x <= ceiling for x in rounds) / len(rounds)
|
|
161
|
+
)
|
|
162
|
+
if rounds
|
|
163
|
+
else 1.0,
|
|
164
|
+
"early_stop_share": float(sum(x < ceiling for x in rounds) / len(rounds))
|
|
165
|
+
if rounds
|
|
166
|
+
else 0.0,
|
|
167
|
+
"termination_reasons": dict(reasons),
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
def cache_adjusted_accounting(rows, alphas=(0.0, 0.1, 0.25, 0.5, 1.0), alpha_out=1.0):
|
|
172
|
+
by_item = defaultdict(lambda: {"uncached": 0.0, "cached": 0.0, "out": 0.0})
|
|
173
|
+
for row in rows:
|
|
174
|
+
item = _item(row)
|
|
175
|
+
by_item[item]["uncached"] += _float(
|
|
176
|
+
row, "input_uncached", "prompt_tokens", default=0
|
|
177
|
+
)
|
|
178
|
+
by_item[item]["cached"] += _float(
|
|
179
|
+
row, "input_cached", "cached_prompt_tokens", default=0
|
|
180
|
+
)
|
|
181
|
+
by_item[item]["out"] += _float(
|
|
182
|
+
row, "output_tokens", "completion_tokens", default=0
|
|
183
|
+
)
|
|
184
|
+
raw = [v["uncached"] + v["cached"] + v["out"] for v in by_item.values()]
|
|
185
|
+
adjusted = {
|
|
186
|
+
str(alpha): [
|
|
187
|
+
v["uncached"] + alpha * v["cached"] + alpha_out * v["out"]
|
|
188
|
+
for v in by_item.values()
|
|
189
|
+
]
|
|
190
|
+
for alpha in alphas
|
|
191
|
+
}
|
|
192
|
+
return {
|
|
193
|
+
"n_items": len(by_item),
|
|
194
|
+
"raw_totals": {"total": sum(raw), "mean": float(mean(raw)) if raw else 0.0},
|
|
195
|
+
"adjusted_totals": {
|
|
196
|
+
a: {"total": sum(values), "mean": float(mean(values)) if values else 0.0}
|
|
197
|
+
for a, values in adjusted.items()
|
|
198
|
+
},
|
|
199
|
+
"alphas": list(alphas),
|
|
200
|
+
"alpha_out": alpha_out,
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
|
|
204
|
+
def classify_hybrid_call_graph(rows):
|
|
205
|
+
by_item = defaultdict(list)
|
|
206
|
+
for row in rows:
|
|
207
|
+
by_item[_item(row)].append(row)
|
|
208
|
+
records = []
|
|
209
|
+
for item, values in by_item.items():
|
|
210
|
+
independent = len(
|
|
211
|
+
{
|
|
212
|
+
str(_get(r, "thread", "agent", "model", default="default"))
|
|
213
|
+
for r in values
|
|
214
|
+
}
|
|
215
|
+
)
|
|
216
|
+
self_edges = cross_edges = aggregation = judge = tools = 0
|
|
217
|
+
for row in values:
|
|
218
|
+
kind = str(_get(row, "call_type", "role", "type", default="")).lower()
|
|
219
|
+
source = _get(
|
|
220
|
+
row, "reads_from", "parent_thread", "source_thread", default=None
|
|
221
|
+
)
|
|
222
|
+
if source not in (None, "", "none"):
|
|
223
|
+
cross_edges += (
|
|
224
|
+
1
|
|
225
|
+
if str(source) != str(_get(row, "thread", "agent", default=""))
|
|
226
|
+
else 0
|
|
227
|
+
)
|
|
228
|
+
self_edges += (
|
|
229
|
+
1
|
|
230
|
+
if str(source) == str(_get(row, "thread", "agent", default=""))
|
|
231
|
+
else 0
|
|
232
|
+
)
|
|
233
|
+
aggregation += int(any(x in kind for x in ("aggregate", "vote", "synth")))
|
|
234
|
+
judge += int("judge" in kind or "verif" in kind)
|
|
235
|
+
tools += int("tool" in kind)
|
|
236
|
+
records.append(
|
|
237
|
+
{
|
|
238
|
+
"item": item,
|
|
239
|
+
"independent_trajectories": independent,
|
|
240
|
+
"self_continuation_edges": self_edges,
|
|
241
|
+
"cross_thread_edges": cross_edges,
|
|
242
|
+
"aggregation_calls": aggregation,
|
|
243
|
+
"judge_calls": judge,
|
|
244
|
+
"tool_calls": tools,
|
|
245
|
+
}
|
|
246
|
+
)
|
|
247
|
+
policy = (
|
|
248
|
+
"Pi_agent" if any(r["cross_thread_edges"] for r in records) else "Pi_single"
|
|
249
|
+
)
|
|
250
|
+
keys = (
|
|
251
|
+
"independent_trajectories",
|
|
252
|
+
"self_continuation_edges",
|
|
253
|
+
"cross_thread_edges",
|
|
254
|
+
"aggregation_calls",
|
|
255
|
+
"judge_calls",
|
|
256
|
+
"tool_calls",
|
|
257
|
+
)
|
|
258
|
+
return {
|
|
259
|
+
"policy_class": policy,
|
|
260
|
+
"n_items": len(records),
|
|
261
|
+
"per_item": records,
|
|
262
|
+
"mean_counts": {
|
|
263
|
+
key: float(mean([r[key] for r in records])) if records else 0.0
|
|
264
|
+
for key in keys
|
|
265
|
+
},
|
|
266
|
+
"total_counts": {key: sum(r[key] for r in records) for key in keys},
|
|
267
|
+
}
|
|
268
|
+
|
|
269
|
+
|
|
270
|
+
def evaluate_constructive_objectives(
|
|
271
|
+
comm_rows,
|
|
272
|
+
placebo_rows,
|
|
273
|
+
full_base_rows,
|
|
274
|
+
ver_rows=None,
|
|
275
|
+
cost_mas=None,
|
|
276
|
+
cost_base=None,
|
|
277
|
+
):
|
|
278
|
+
def paired(a, b):
|
|
279
|
+
n = min(len(a), len(b))
|
|
280
|
+
differences = [float(a[i]) - float(b[i]) for i in range(n)]
|
|
281
|
+
return (
|
|
282
|
+
float(mean(a[:n])),
|
|
283
|
+
float(mean(b[:n])),
|
|
284
|
+
_bootstrap(differences, np.mean, 500, 0),
|
|
285
|
+
)
|
|
286
|
+
|
|
287
|
+
p_comm, p_placebo, ci_arch = paired(comm_rows, placebo_rows)
|
|
288
|
+
_, p_full, ci_over = paired(full_base_rows, placebo_rows)
|
|
289
|
+
p1 = p_placebo
|
|
290
|
+
tau_ic = sum(1 for a, b in zip(placebo_rows, comm_rows) if not a and b) / max(
|
|
291
|
+
1, sum(1 for a in placebo_rows if not a)
|
|
292
|
+
)
|
|
293
|
+
tau_ci = sum(1 for a, b in zip(placebo_rows, comm_rows) if a and not b) / max(
|
|
294
|
+
1, sum(1 for a in placebo_rows if a)
|
|
295
|
+
)
|
|
296
|
+
threshold = p1 / (1 - p1) if p1 < 1 else float("inf")
|
|
297
|
+
delta_c = (
|
|
298
|
+
None if cost_mas is None or cost_base is None else float(cost_mas - cost_base)
|
|
299
|
+
)
|
|
300
|
+
return {
|
|
301
|
+
"placebo_overhead": {
|
|
302
|
+
"p_comm": p_comm,
|
|
303
|
+
"p_placebo": p_placebo,
|
|
304
|
+
"p_full": p_full,
|
|
305
|
+
"delta_arch": p_comm - p_placebo,
|
|
306
|
+
"delta_overhead": p_full - p_placebo,
|
|
307
|
+
"net_comm_minus_full": p_comm - p_full,
|
|
308
|
+
"ci_arch": ci_arch,
|
|
309
|
+
"ci_overhead": ci_over,
|
|
310
|
+
"arch_exceeds_overhead": p_comm - p_placebo > p_full - p_placebo,
|
|
311
|
+
},
|
|
312
|
+
"verifier_conformity": {
|
|
313
|
+
"p1": p1,
|
|
314
|
+
"tau_i_to_c": tau_ic,
|
|
315
|
+
"tau_c_to_i": tau_ci,
|
|
316
|
+
"delta_p_agg": p_comm - p_placebo,
|
|
317
|
+
"delta_p_ver": None if ver_rows is None else mean(ver_rows) - p1,
|
|
318
|
+
"threshold": threshold,
|
|
319
|
+
"kappa": tau_ic / tau_ci if tau_ci else float("inf"),
|
|
320
|
+
"exceeds_conformity_threshold": tau_ic > threshold * tau_ci,
|
|
321
|
+
},
|
|
322
|
+
"cost": {
|
|
323
|
+
"delta_C": delta_c,
|
|
324
|
+
"cost_saving_achieved": None if delta_c is None else delta_c < 0,
|
|
325
|
+
"joint_efficiency_win": None
|
|
326
|
+
if delta_c is None
|
|
327
|
+
else delta_c < 0 and p_comm >= p_full,
|
|
328
|
+
},
|
|
329
|
+
}
|
|
330
|
+
|
|
331
|
+
|
|
332
|
+
def build_evaluation_card_v7(rows, baseline_rows=None, alphas=(0.1, 0.5)):
|
|
333
|
+
resource = {
|
|
334
|
+
"tokens": sum(
|
|
335
|
+
_float(r, "tokens", "prompt_tokens", "completion_tokens") for r in rows
|
|
336
|
+
),
|
|
337
|
+
"calls": len(rows),
|
|
338
|
+
"cost": sum(_float(r, "cost", "cost_usd") for r in rows),
|
|
339
|
+
}
|
|
340
|
+
base = sum(
|
|
341
|
+
_float(r, "tokens", "prompt_tokens", "completion_tokens")
|
|
342
|
+
for r in (baseline_rows or [])
|
|
343
|
+
)
|
|
344
|
+
ratio = resource["tokens"] / base if base else "unreported"
|
|
345
|
+
card = {
|
|
346
|
+
"baseline_rung": "unreported",
|
|
347
|
+
"resource_vector": resource,
|
|
348
|
+
"token_accounting": cache_adjusted_accounting(rows, alphas),
|
|
349
|
+
"resource_ratios": {
|
|
350
|
+
"tokens_mas_over_baseline": ratio,
|
|
351
|
+
"calls_mas_over_baseline": len(rows) / len(baseline_rows)
|
|
352
|
+
if baseline_rows
|
|
353
|
+
else "unreported",
|
|
354
|
+
"cost_mas_over_baseline": "unreported",
|
|
355
|
+
},
|
|
356
|
+
"frontier_gap": {
|
|
357
|
+
"accuracy": mean([_accuracy(r) for r in rows]) if rows else 0.0,
|
|
358
|
+
"baseline_accuracy": mean([_accuracy(r) for r in baseline_rows])
|
|
359
|
+
if baseline_rows
|
|
360
|
+
else "unreported",
|
|
361
|
+
"gap": "unreported",
|
|
362
|
+
},
|
|
363
|
+
"evidence_and_routes": {
|
|
364
|
+
"n_rows": len(rows),
|
|
365
|
+
"routes": sorted(
|
|
366
|
+
{str(_get(r, "strategy", "route", default="unreported")) for r in rows}
|
|
367
|
+
),
|
|
368
|
+
},
|
|
369
|
+
"resource_distribution_and_stopping": adaptive_stopping_audit(rows),
|
|
370
|
+
"reporting_level_achieved": "PACE-Recommended",
|
|
371
|
+
}
|
|
372
|
+
return card
|
|
373
|
+
|
|
374
|
+
|
|
375
|
+
def card_v7_to_latex(card):
|
|
376
|
+
def clean(value):
|
|
377
|
+
text = (
|
|
378
|
+
str(value).replace("n/a", "unreported").replace("in progress", "unreported")
|
|
379
|
+
)
|
|
380
|
+
return text.replace("&", r"\&")
|
|
381
|
+
|
|
382
|
+
lines = [
|
|
383
|
+
r"\begin{table}[t]",
|
|
384
|
+
r"\centering",
|
|
385
|
+
r"\small",
|
|
386
|
+
r"\caption{PACE evaluation card v7.}",
|
|
387
|
+
r"\label{tab:pace_card_v7}",
|
|
388
|
+
r"\begin{tabular}{ll}",
|
|
389
|
+
r"\toprule",
|
|
390
|
+
"Field & Value \\",
|
|
391
|
+
r"\midrule",
|
|
392
|
+
]
|
|
393
|
+
for key in (
|
|
394
|
+
"baseline_rung",
|
|
395
|
+
"resource_vector",
|
|
396
|
+
"token_accounting",
|
|
397
|
+
"resource_ratios",
|
|
398
|
+
"frontier_gap",
|
|
399
|
+
"evidence_and_routes",
|
|
400
|
+
"resource_distribution_and_stopping",
|
|
401
|
+
"reporting_level_achieved",
|
|
402
|
+
):
|
|
403
|
+
lines.append(f"{clean(key)} & {clean(card.get(key))} \\")
|
|
404
|
+
return "\n".join(lines + [r"\bottomrule", r"\end{tabular}", r"\end{table}", ""])
|
|
405
|
+
|
|
406
|
+
|
|
407
|
+
def card_v7_to_markdown(card):
|
|
408
|
+
return (
|
|
409
|
+
"\n".join(
|
|
410
|
+
["| Field | Value |", "|---|---|"]
|
|
411
|
+
+ [f"| {key} | {value} |" for key, value in card.items()]
|
|
412
|
+
)
|
|
413
|
+
+ "\n"
|
|
414
|
+
)
|
pace/proxy.py
ADDED
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
"""pace proxy: an OpenAI-compatible logging proxy (standard library only).
|
|
2
|
+
|
|
3
|
+
Point any agent framework's base URL at http://localhost:<port>/v1. Requests are forwarded unchanged to the upstream server and every
|
|
4
|
+
call is appended to a JSONL log in the mae_eval trace format (so `mae-eval audit` and `pace report` read it), with latency and cost.
|
|
5
|
+
|
|
6
|
+
Optional request headers set the labels (they are removed before forwarding):
|
|
7
|
+
X-PACE-System mas | sas (default mas) X-PACE-Agent agent id (default agent)
|
|
8
|
+
X-PACE-Round integer round (default 0) X-PACE-Item item id (default empty, or the first line of --item-file when the caller cannot set headers)
|
|
9
|
+
Non-streaming requests only: a request with "stream": true is rejected with HTTP 400 and a clear message (v0.1 limitation).
|
|
10
|
+
Prices are never bundled; pass --price-in/--price-out (USD per million tokens) or cost stays null.
|
|
11
|
+
"""
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
import json
|
|
15
|
+
import threading
|
|
16
|
+
import time
|
|
17
|
+
import urllib.error
|
|
18
|
+
import urllib.request
|
|
19
|
+
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
|
20
|
+
from typing import Optional
|
|
21
|
+
|
|
22
|
+
_LABELS = ("x-cme-system", "x-cme-agent", "x-cme-round", "x-cme-item")
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def _item_from_file(path: Optional[str]) -> str:
|
|
26
|
+
"""Item id for callers that cannot send the X-PACE-Item header (third-party code): the first line of `path`, re-read at every call. Serial runs only."""
|
|
27
|
+
if not path:
|
|
28
|
+
return ""
|
|
29
|
+
try:
|
|
30
|
+
with open(path, encoding="utf-8") as fh:
|
|
31
|
+
return fh.readline().strip()
|
|
32
|
+
except OSError:
|
|
33
|
+
return ""
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def make_handler(upstream: str, log_path: str, price_in: Optional[float], price_out: Optional[float], api_key: Optional[str], item_file: Optional[str] = None, default_system: str = "mas"):
|
|
37
|
+
lock = threading.Lock()
|
|
38
|
+
up = upstream.rstrip("/")
|
|
39
|
+
|
|
40
|
+
class H(BaseHTTPRequestHandler):
|
|
41
|
+
def _send(self, code: int, payload: bytes, ctype: str = "application/json"):
|
|
42
|
+
self.send_response(code)
|
|
43
|
+
self.send_header("Content-Type", ctype)
|
|
44
|
+
self.send_header("Content-Length", str(len(payload)))
|
|
45
|
+
self.end_headers()
|
|
46
|
+
self.wfile.write(payload)
|
|
47
|
+
|
|
48
|
+
def _forward(self, method: str, body: Optional[bytes]):
|
|
49
|
+
headers = {k: v for k, v in self.headers.items() if k.lower() not in _LABELS + ("host", "content-length", "connection")}
|
|
50
|
+
if api_key:
|
|
51
|
+
headers["Authorization"] = "Bearer " + api_key
|
|
52
|
+
req = urllib.request.Request(up + self.path[len("/v1"):] if self.path.startswith("/v1") else up + self.path,
|
|
53
|
+
data=body, headers=headers, method=method)
|
|
54
|
+
t0 = time.time()
|
|
55
|
+
try:
|
|
56
|
+
with urllib.request.urlopen(req, timeout=900) as r:
|
|
57
|
+
return r.status, r.read(), t0, time.time()
|
|
58
|
+
except urllib.error.HTTPError as e:
|
|
59
|
+
return e.code, e.read(), t0, time.time()
|
|
60
|
+
|
|
61
|
+
def do_GET(self):
|
|
62
|
+
code, data, _, _ = self._forward("GET", None)
|
|
63
|
+
self._send(code, data)
|
|
64
|
+
|
|
65
|
+
def do_POST(self):
|
|
66
|
+
n = int(self.headers.get("Content-Length", 0))
|
|
67
|
+
raw = self.rfile.read(n)
|
|
68
|
+
try:
|
|
69
|
+
req = json.loads(raw)
|
|
70
|
+
except json.JSONDecodeError:
|
|
71
|
+
return self._send(400, b'{"error": "invalid JSON"}')
|
|
72
|
+
if req.get("stream"):
|
|
73
|
+
return self._send(400, b'{"error": "pace proxy v0.1 does not support streaming; set stream=false"}')
|
|
74
|
+
code, data, t0, t1 = self._forward("POST", raw)
|
|
75
|
+
if code == 200:
|
|
76
|
+
try:
|
|
77
|
+
resp = json.loads(data)
|
|
78
|
+
u = resp.get("usage") or {}
|
|
79
|
+
det = u.get("completion_tokens_details") or {}
|
|
80
|
+
pt, ct = u.get("prompt_tokens"), u.get("completion_tokens")
|
|
81
|
+
cost = None
|
|
82
|
+
if None not in (price_in, price_out, pt, ct):
|
|
83
|
+
cost = (pt * price_in + ct * price_out) / 1e6
|
|
84
|
+
rec = {"system": self.headers.get("X-PACE-System", default_system), "agent_id": self.headers.get("X-PACE-Agent", "agent"),
|
|
85
|
+
"round": int(self.headers.get("X-PACE-Round", 0)), "item": self.headers.get("X-PACE-Item") or _item_from_file(item_file),
|
|
86
|
+
"model": resp.get("model", req.get("model")), "prompt_tokens": pt, "completion_tokens": ct,
|
|
87
|
+
"reasoning_tokens": det.get("reasoning_tokens"), "n_completions": len(resp.get("choices", [])) or 1,
|
|
88
|
+
"start": t0, "end": t1, "latency": t1 - t0, "cost": cost}
|
|
89
|
+
if pt is None or ct is None:
|
|
90
|
+
rec["usage_missing"] = True # logged, never guessed
|
|
91
|
+
with lock, open(log_path, "a", encoding="utf-8") as fh:
|
|
92
|
+
fh.write(json.dumps(rec) + "\n")
|
|
93
|
+
except (ValueError, TypeError):
|
|
94
|
+
pass
|
|
95
|
+
self._send(code, data)
|
|
96
|
+
|
|
97
|
+
def log_message(self, *a): # quiet
|
|
98
|
+
pass
|
|
99
|
+
|
|
100
|
+
return H
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def serve(upstream: str, log_path: str, port: int = 8089, host: str = "127.0.0.1", price_in: Optional[float] = None,
|
|
104
|
+
price_out: Optional[float] = None, api_key: Optional[str] = None, item_file: Optional[str] = None, default_system: str = "mas") -> ThreadingHTTPServer:
|
|
105
|
+
srv = ThreadingHTTPServer((host, port), make_handler(upstream, log_path, price_in, price_out, api_key, item_file, default_system))
|
|
106
|
+
return srv
|