pace-eval 0.2.2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (83) hide show
  1. cme/__init__.py +11 -0
  2. cme/__main__.py +5 -0
  3. cme/analysis.py +223 -0
  4. cme/audit.py +145 -0
  5. cme/baseline.py +162 -0
  6. cme/cli.py +279 -0
  7. cme/client.py +165 -0
  8. cme/dynamics.py +212 -0
  9. cme/examples/crewai_integration.py +16 -0
  10. cme/examples/demo_pool.jsonl +360 -0
  11. cme/examples/demo_run.jsonl +480 -0
  12. cme/examples/langgraph_integration.py +27 -0
  13. cme/examples/make_demo.py +37 -0
  14. cme/examples/openai_quickstart.py +18 -0
  15. cme/experiment.py +555 -0
  16. cme/frontier.py +143 -0
  17. cme/hidden_profile.py +135 -0
  18. cme/proxy.py +106 -0
  19. cme/rebuttal.py +76 -0
  20. cme/report.py +197 -0
  21. cme/rollout_card.py +137 -0
  22. cme/tasks.py +266 -0
  23. cme/verified_debate.py +127 -0
  24. cme/verifier.py +92 -0
  25. mae_eval/__init__.py +11 -0
  26. mae_eval/__main__.py +5 -0
  27. mae_eval/auditor.py +112 -0
  28. mae_eval/classifier.py +62 -0
  29. mae_eval/cli.py +210 -0
  30. mae_eval/tracker.py +162 -0
  31. mae_eval/vector.py +91 -0
  32. pace/__init__.py +11 -0
  33. pace/__main__.py +5 -0
  34. pace/analysis.py +129 -0
  35. pace/audit.py +145 -0
  36. pace/baseline.py +162 -0
  37. pace/card.py +205 -0
  38. pace/cli.py +898 -0
  39. pace/client.py +165 -0
  40. pace/dynamics.py +372 -0
  41. pace/escape_benchmarks.py +162 -0
  42. pace/examples/crewai_integration.py +16 -0
  43. pace/examples/demo_pool.jsonl +360 -0
  44. pace/examples/demo_run.jsonl +480 -0
  45. pace/examples/langgraph_integration.py +27 -0
  46. pace/examples/make_demo.py +37 -0
  47. pace/examples/openai_quickstart.py +18 -0
  48. pace/experiment.py +555 -0
  49. pace/frontier.py +114 -0
  50. pace/hidden_profile.py +301 -0
  51. pace/importers.py +219 -0
  52. pace/inflate.py +213 -0
  53. pace/influence.py +231 -0
  54. pace/meta_engine.py +246 -0
  55. pace/middleware.py +230 -0
  56. pace/open_ended.py +114 -0
  57. pace/plotting.py +132 -0
  58. pace/pool_sizer.py +154 -0
  59. pace/protocol_v3.py +414 -0
  60. pace/proxy.py +106 -0
  61. pace/rebuttal.py +219 -0
  62. pace/report.py +197 -0
  63. pace/rollout_card.py +137 -0
  64. pace/schema.py +74 -0
  65. pace/simulator.py +199 -0
  66. pace/studio.html +1040 -0
  67. pace/tasks.py +266 -0
  68. pace/verified_debate.py +127 -0
  69. pace/verifier.py +92 -0
  70. pace/workbench.html +26 -0
  71. pace/workbench.py +119 -0
  72. pace_eval-0.2.2.dist-info/METADATA +205 -0
  73. pace_eval-0.2.2.dist-info/RECORD +83 -0
  74. pace_eval-0.2.2.dist-info/WHEEL +5 -0
  75. pace_eval-0.2.2.dist-info/entry_points.txt +4 -0
  76. pace_eval-0.2.2.dist-info/licenses/LICENSE +202 -0
  77. pace_eval-0.2.2.dist-info/top_level.txt +4 -0
  78. pace_lab/__init__.py +9 -0
  79. pace_lab/clients.py +568 -0
  80. pace_lab/concurrency_test.py +127 -0
  81. pace_lab/data.py +445 -0
  82. pace_lab/graders.py +626 -0
  83. pace_lab/strategies.py +345 -0
pace/protocol_v3.py ADDED
@@ -0,0 +1,414 @@
1
+ """Dependency-light PACE Phase 16 protocol and constructive objective engine."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from collections import Counter, defaultdict
6
+ from statistics import mean, median, pstdev
7
+
8
+ import numpy as np
9
+
10
+
11
+ def _get(row, *names, default=0.0):
12
+ for name in names:
13
+ if name in row and row[name] not in (None, ""):
14
+ return row[name]
15
+ return default
16
+
17
+
18
+ def _float(row, *names, default=0.0):
19
+ try:
20
+ return float(_get(row, *names, default=default))
21
+ except (TypeError, ValueError):
22
+ return float(default)
23
+
24
+
25
+ def _item(row):
26
+ return str(_get(row, "item", "id", "item_id", default="item"))
27
+
28
+
29
+ def _accuracy(row):
30
+ value = _get(row, "correct", "accuracy", "score", default=0.0)
31
+ return (
32
+ float(value)
33
+ if isinstance(value, (int, float))
34
+ else float(str(value).lower() in {"1", "true", "correct", "yes"})
35
+ )
36
+
37
+
38
+ def _bootstrap(values, statistic, n_boot, seed):
39
+ values = np.asarray(list(values), dtype=float)
40
+ if not len(values):
41
+ return [None, None]
42
+ rng = np.random.default_rng(seed)
43
+ draws = [
44
+ statistic(values[rng.integers(0, len(values), len(values))])
45
+ for _ in range(max(1, n_boot))
46
+ ]
47
+ return [float(np.percentile(draws, 2.5)), float(np.percentile(draws, 97.5))]
48
+
49
+
50
+ def itemwise_resource_match(mas_rows, pool_rows, axis="tokens", n_boot=500, seed=0):
51
+ if axis not in {"tokens", "calls", "cost"}:
52
+ raise ValueError("axis must be tokens, calls, or cost")
53
+ pools = defaultdict(list)
54
+ for row in pool_rows:
55
+ pools[_item(row)].append(
56
+ max(0.0, _float(row, axis, f"{axis}_total", "resource"))
57
+ )
58
+ mas = defaultdict(list)
59
+ for row in mas_rows:
60
+ mas[_item(row)].append(row)
61
+ ks, frontier, observed, statuses = [], [], [], []
62
+ for item, rows in mas.items():
63
+ budget = sum(
64
+ max(0.0, _float(r, axis, f"{axis}_total", "resource")) for r in rows
65
+ )
66
+ samples = pools.get(item, []) or [0.0]
67
+ cumulative = 0.0
68
+ k = 0
69
+ for value in samples:
70
+ k += 1
71
+ cumulative += value
72
+ if cumulative >= budget:
73
+ break
74
+ statuses.append("reached" if cumulative >= budget else "exhausted")
75
+ ks.append(max(1, k))
76
+ frontier.append(
77
+ float(
78
+ np.mean(
79
+ [_accuracy(r) for r in pool_rows if _item(r) == item][:k] or [0.0]
80
+ )
81
+ )
82
+ )
83
+ observed.append(float(np.mean([_accuracy(r) for r in rows])))
84
+ gap = [a - b for a, b in zip(observed, frontier)]
85
+ result = {
86
+ "n_items": len(ks),
87
+ "mean_k_i": float(mean(ks)) if ks else 0.0,
88
+ "median_k_i": float(median(ks)) if ks else 0.0,
89
+ "max_k_i": max(ks) if ks else 0,
90
+ "mas_accuracy": float(mean(observed)) if observed else 0.0,
91
+ "itemwise_frontier_accuracy": float(mean(frontier)) if frontier else 0.0,
92
+ "expectation_frontier_accuracy": float(mean(frontier)) if frontier else 0.0,
93
+ "itemwise_gap": float(mean(gap)) if gap else 0.0,
94
+ "expectation_gap": float(mean(observed) - mean(frontier)) if gap else 0.0,
95
+ "ci_low": None,
96
+ "ci_high": None,
97
+ "paired_bootstrap_ci": _bootstrap(gap, np.mean, n_boot, seed),
98
+ "itemwise_dominance_share": float(sum(x >= 0 for x in gap) / len(gap))
99
+ if gap
100
+ else 0.0,
101
+ "range_status": dict(Counter(statuses)),
102
+ "axis": axis,
103
+ }
104
+ result["ci_low"], result["ci_high"] = result["paired_bootstrap_ci"]
105
+ return result
106
+
107
+
108
+ def _distribution(values):
109
+ values = [float(x) for x in values]
110
+ if not values:
111
+ return {
112
+ key: 0.0
113
+ for key in ("mean", "std", "min", "p25", "median", "p75", "p90", "max")
114
+ }
115
+ return {
116
+ "mean": float(mean(values)),
117
+ "std": float(pstdev(values)),
118
+ "min": min(values),
119
+ "p25": float(np.percentile(values, 25)),
120
+ "median": float(median(values)),
121
+ "p75": float(np.percentile(values, 75)),
122
+ "p90": float(np.percentile(values, 90)),
123
+ "max": max(values),
124
+ }
125
+
126
+
127
+ def adaptive_stopping_audit(rows, declared_ceiling=None):
128
+ by_item = defaultdict(list)
129
+ for row in rows:
130
+ by_item[_item(row)].append(row)
131
+ rounds = [
132
+ max(_float(r, "rounds", "round", default=1) for r in rs)
133
+ for rs in by_item.values()
134
+ ]
135
+ calls = [sum(_float(r, "calls", default=1) for r in rs) for rs in by_item.values()]
136
+ tokens = [
137
+ sum(
138
+ _float(r, "tokens", "prompt_tokens", default=0)
139
+ + _float(r, "completion_tokens", default=0)
140
+ for r in rs
141
+ )
142
+ for rs in by_item.values()
143
+ ]
144
+ ceiling = (
145
+ declared_ceiling
146
+ if declared_ceiling is not None
147
+ else (max(rounds) if rounds else 0)
148
+ )
149
+ reasons = Counter(
150
+ str(_get(r, "termination_reason", "done_reason", default="unspecified"))
151
+ for r in rows
152
+ )
153
+ return {
154
+ "n_items": len(by_item),
155
+ "rounds": _distribution(rounds),
156
+ "calls": _distribution(calls),
157
+ "tokens": _distribution(tokens),
158
+ "declared_ceiling": ceiling,
159
+ "ceiling_compliance_rate": float(
160
+ sum(x <= ceiling for x in rounds) / len(rounds)
161
+ )
162
+ if rounds
163
+ else 1.0,
164
+ "early_stop_share": float(sum(x < ceiling for x in rounds) / len(rounds))
165
+ if rounds
166
+ else 0.0,
167
+ "termination_reasons": dict(reasons),
168
+ }
169
+
170
+
171
+ def cache_adjusted_accounting(rows, alphas=(0.0, 0.1, 0.25, 0.5, 1.0), alpha_out=1.0):
172
+ by_item = defaultdict(lambda: {"uncached": 0.0, "cached": 0.0, "out": 0.0})
173
+ for row in rows:
174
+ item = _item(row)
175
+ by_item[item]["uncached"] += _float(
176
+ row, "input_uncached", "prompt_tokens", default=0
177
+ )
178
+ by_item[item]["cached"] += _float(
179
+ row, "input_cached", "cached_prompt_tokens", default=0
180
+ )
181
+ by_item[item]["out"] += _float(
182
+ row, "output_tokens", "completion_tokens", default=0
183
+ )
184
+ raw = [v["uncached"] + v["cached"] + v["out"] for v in by_item.values()]
185
+ adjusted = {
186
+ str(alpha): [
187
+ v["uncached"] + alpha * v["cached"] + alpha_out * v["out"]
188
+ for v in by_item.values()
189
+ ]
190
+ for alpha in alphas
191
+ }
192
+ return {
193
+ "n_items": len(by_item),
194
+ "raw_totals": {"total": sum(raw), "mean": float(mean(raw)) if raw else 0.0},
195
+ "adjusted_totals": {
196
+ a: {"total": sum(values), "mean": float(mean(values)) if values else 0.0}
197
+ for a, values in adjusted.items()
198
+ },
199
+ "alphas": list(alphas),
200
+ "alpha_out": alpha_out,
201
+ }
202
+
203
+
204
+ def classify_hybrid_call_graph(rows):
205
+ by_item = defaultdict(list)
206
+ for row in rows:
207
+ by_item[_item(row)].append(row)
208
+ records = []
209
+ for item, values in by_item.items():
210
+ independent = len(
211
+ {
212
+ str(_get(r, "thread", "agent", "model", default="default"))
213
+ for r in values
214
+ }
215
+ )
216
+ self_edges = cross_edges = aggregation = judge = tools = 0
217
+ for row in values:
218
+ kind = str(_get(row, "call_type", "role", "type", default="")).lower()
219
+ source = _get(
220
+ row, "reads_from", "parent_thread", "source_thread", default=None
221
+ )
222
+ if source not in (None, "", "none"):
223
+ cross_edges += (
224
+ 1
225
+ if str(source) != str(_get(row, "thread", "agent", default=""))
226
+ else 0
227
+ )
228
+ self_edges += (
229
+ 1
230
+ if str(source) == str(_get(row, "thread", "agent", default=""))
231
+ else 0
232
+ )
233
+ aggregation += int(any(x in kind for x in ("aggregate", "vote", "synth")))
234
+ judge += int("judge" in kind or "verif" in kind)
235
+ tools += int("tool" in kind)
236
+ records.append(
237
+ {
238
+ "item": item,
239
+ "independent_trajectories": independent,
240
+ "self_continuation_edges": self_edges,
241
+ "cross_thread_edges": cross_edges,
242
+ "aggregation_calls": aggregation,
243
+ "judge_calls": judge,
244
+ "tool_calls": tools,
245
+ }
246
+ )
247
+ policy = (
248
+ "Pi_agent" if any(r["cross_thread_edges"] for r in records) else "Pi_single"
249
+ )
250
+ keys = (
251
+ "independent_trajectories",
252
+ "self_continuation_edges",
253
+ "cross_thread_edges",
254
+ "aggregation_calls",
255
+ "judge_calls",
256
+ "tool_calls",
257
+ )
258
+ return {
259
+ "policy_class": policy,
260
+ "n_items": len(records),
261
+ "per_item": records,
262
+ "mean_counts": {
263
+ key: float(mean([r[key] for r in records])) if records else 0.0
264
+ for key in keys
265
+ },
266
+ "total_counts": {key: sum(r[key] for r in records) for key in keys},
267
+ }
268
+
269
+
270
+ def evaluate_constructive_objectives(
271
+ comm_rows,
272
+ placebo_rows,
273
+ full_base_rows,
274
+ ver_rows=None,
275
+ cost_mas=None,
276
+ cost_base=None,
277
+ ):
278
+ def paired(a, b):
279
+ n = min(len(a), len(b))
280
+ differences = [float(a[i]) - float(b[i]) for i in range(n)]
281
+ return (
282
+ float(mean(a[:n])),
283
+ float(mean(b[:n])),
284
+ _bootstrap(differences, np.mean, 500, 0),
285
+ )
286
+
287
+ p_comm, p_placebo, ci_arch = paired(comm_rows, placebo_rows)
288
+ _, p_full, ci_over = paired(full_base_rows, placebo_rows)
289
+ p1 = p_placebo
290
+ tau_ic = sum(1 for a, b in zip(placebo_rows, comm_rows) if not a and b) / max(
291
+ 1, sum(1 for a in placebo_rows if not a)
292
+ )
293
+ tau_ci = sum(1 for a, b in zip(placebo_rows, comm_rows) if a and not b) / max(
294
+ 1, sum(1 for a in placebo_rows if a)
295
+ )
296
+ threshold = p1 / (1 - p1) if p1 < 1 else float("inf")
297
+ delta_c = (
298
+ None if cost_mas is None or cost_base is None else float(cost_mas - cost_base)
299
+ )
300
+ return {
301
+ "placebo_overhead": {
302
+ "p_comm": p_comm,
303
+ "p_placebo": p_placebo,
304
+ "p_full": p_full,
305
+ "delta_arch": p_comm - p_placebo,
306
+ "delta_overhead": p_full - p_placebo,
307
+ "net_comm_minus_full": p_comm - p_full,
308
+ "ci_arch": ci_arch,
309
+ "ci_overhead": ci_over,
310
+ "arch_exceeds_overhead": p_comm - p_placebo > p_full - p_placebo,
311
+ },
312
+ "verifier_conformity": {
313
+ "p1": p1,
314
+ "tau_i_to_c": tau_ic,
315
+ "tau_c_to_i": tau_ci,
316
+ "delta_p_agg": p_comm - p_placebo,
317
+ "delta_p_ver": None if ver_rows is None else mean(ver_rows) - p1,
318
+ "threshold": threshold,
319
+ "kappa": tau_ic / tau_ci if tau_ci else float("inf"),
320
+ "exceeds_conformity_threshold": tau_ic > threshold * tau_ci,
321
+ },
322
+ "cost": {
323
+ "delta_C": delta_c,
324
+ "cost_saving_achieved": None if delta_c is None else delta_c < 0,
325
+ "joint_efficiency_win": None
326
+ if delta_c is None
327
+ else delta_c < 0 and p_comm >= p_full,
328
+ },
329
+ }
330
+
331
+
332
+ def build_evaluation_card_v7(rows, baseline_rows=None, alphas=(0.1, 0.5)):
333
+ resource = {
334
+ "tokens": sum(
335
+ _float(r, "tokens", "prompt_tokens", "completion_tokens") for r in rows
336
+ ),
337
+ "calls": len(rows),
338
+ "cost": sum(_float(r, "cost", "cost_usd") for r in rows),
339
+ }
340
+ base = sum(
341
+ _float(r, "tokens", "prompt_tokens", "completion_tokens")
342
+ for r in (baseline_rows or [])
343
+ )
344
+ ratio = resource["tokens"] / base if base else "unreported"
345
+ card = {
346
+ "baseline_rung": "unreported",
347
+ "resource_vector": resource,
348
+ "token_accounting": cache_adjusted_accounting(rows, alphas),
349
+ "resource_ratios": {
350
+ "tokens_mas_over_baseline": ratio,
351
+ "calls_mas_over_baseline": len(rows) / len(baseline_rows)
352
+ if baseline_rows
353
+ else "unreported",
354
+ "cost_mas_over_baseline": "unreported",
355
+ },
356
+ "frontier_gap": {
357
+ "accuracy": mean([_accuracy(r) for r in rows]) if rows else 0.0,
358
+ "baseline_accuracy": mean([_accuracy(r) for r in baseline_rows])
359
+ if baseline_rows
360
+ else "unreported",
361
+ "gap": "unreported",
362
+ },
363
+ "evidence_and_routes": {
364
+ "n_rows": len(rows),
365
+ "routes": sorted(
366
+ {str(_get(r, "strategy", "route", default="unreported")) for r in rows}
367
+ ),
368
+ },
369
+ "resource_distribution_and_stopping": adaptive_stopping_audit(rows),
370
+ "reporting_level_achieved": "PACE-Recommended",
371
+ }
372
+ return card
373
+
374
+
375
+ def card_v7_to_latex(card):
376
+ def clean(value):
377
+ text = (
378
+ str(value).replace("n/a", "unreported").replace("in progress", "unreported")
379
+ )
380
+ return text.replace("&", r"\&")
381
+
382
+ lines = [
383
+ r"\begin{table}[t]",
384
+ r"\centering",
385
+ r"\small",
386
+ r"\caption{PACE evaluation card v7.}",
387
+ r"\label{tab:pace_card_v7}",
388
+ r"\begin{tabular}{ll}",
389
+ r"\toprule",
390
+ "Field & Value \\",
391
+ r"\midrule",
392
+ ]
393
+ for key in (
394
+ "baseline_rung",
395
+ "resource_vector",
396
+ "token_accounting",
397
+ "resource_ratios",
398
+ "frontier_gap",
399
+ "evidence_and_routes",
400
+ "resource_distribution_and_stopping",
401
+ "reporting_level_achieved",
402
+ ):
403
+ lines.append(f"{clean(key)} & {clean(card.get(key))} \\")
404
+ return "\n".join(lines + [r"\bottomrule", r"\end{tabular}", r"\end{table}", ""])
405
+
406
+
407
+ def card_v7_to_markdown(card):
408
+ return (
409
+ "\n".join(
410
+ ["| Field | Value |", "|---|---|"]
411
+ + [f"| {key} | {value} |" for key, value in card.items()]
412
+ )
413
+ + "\n"
414
+ )
pace/proxy.py ADDED
@@ -0,0 +1,106 @@
1
+ """pace proxy: an OpenAI-compatible logging proxy (standard library only).
2
+
3
+ Point any agent framework's base URL at http://localhost:<port>/v1. Requests are forwarded unchanged to the upstream server and every
4
+ call is appended to a JSONL log in the mae_eval trace format (so `mae-eval audit` and `pace report` read it), with latency and cost.
5
+
6
+ Optional request headers set the labels (they are removed before forwarding):
7
+ X-PACE-System mas | sas (default mas) X-PACE-Agent agent id (default agent)
8
+ X-PACE-Round integer round (default 0) X-PACE-Item item id (default empty, or the first line of --item-file when the caller cannot set headers)
9
+ Non-streaming requests only: a request with "stream": true is rejected with HTTP 400 and a clear message (v0.1 limitation).
10
+ Prices are never bundled; pass --price-in/--price-out (USD per million tokens) or cost stays null.
11
+ """
12
+ from __future__ import annotations
13
+
14
+ import json
15
+ import threading
16
+ import time
17
+ import urllib.error
18
+ import urllib.request
19
+ from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
20
+ from typing import Optional
21
+
22
+ _LABELS = ("x-cme-system", "x-cme-agent", "x-cme-round", "x-cme-item")
23
+
24
+
25
+ def _item_from_file(path: Optional[str]) -> str:
26
+ """Item id for callers that cannot send the X-PACE-Item header (third-party code): the first line of `path`, re-read at every call. Serial runs only."""
27
+ if not path:
28
+ return ""
29
+ try:
30
+ with open(path, encoding="utf-8") as fh:
31
+ return fh.readline().strip()
32
+ except OSError:
33
+ return ""
34
+
35
+
36
+ def make_handler(upstream: str, log_path: str, price_in: Optional[float], price_out: Optional[float], api_key: Optional[str], item_file: Optional[str] = None, default_system: str = "mas"):
37
+ lock = threading.Lock()
38
+ up = upstream.rstrip("/")
39
+
40
+ class H(BaseHTTPRequestHandler):
41
+ def _send(self, code: int, payload: bytes, ctype: str = "application/json"):
42
+ self.send_response(code)
43
+ self.send_header("Content-Type", ctype)
44
+ self.send_header("Content-Length", str(len(payload)))
45
+ self.end_headers()
46
+ self.wfile.write(payload)
47
+
48
+ def _forward(self, method: str, body: Optional[bytes]):
49
+ headers = {k: v for k, v in self.headers.items() if k.lower() not in _LABELS + ("host", "content-length", "connection")}
50
+ if api_key:
51
+ headers["Authorization"] = "Bearer " + api_key
52
+ req = urllib.request.Request(up + self.path[len("/v1"):] if self.path.startswith("/v1") else up + self.path,
53
+ data=body, headers=headers, method=method)
54
+ t0 = time.time()
55
+ try:
56
+ with urllib.request.urlopen(req, timeout=900) as r:
57
+ return r.status, r.read(), t0, time.time()
58
+ except urllib.error.HTTPError as e:
59
+ return e.code, e.read(), t0, time.time()
60
+
61
+ def do_GET(self):
62
+ code, data, _, _ = self._forward("GET", None)
63
+ self._send(code, data)
64
+
65
+ def do_POST(self):
66
+ n = int(self.headers.get("Content-Length", 0))
67
+ raw = self.rfile.read(n)
68
+ try:
69
+ req = json.loads(raw)
70
+ except json.JSONDecodeError:
71
+ return self._send(400, b'{"error": "invalid JSON"}')
72
+ if req.get("stream"):
73
+ return self._send(400, b'{"error": "pace proxy v0.1 does not support streaming; set stream=false"}')
74
+ code, data, t0, t1 = self._forward("POST", raw)
75
+ if code == 200:
76
+ try:
77
+ resp = json.loads(data)
78
+ u = resp.get("usage") or {}
79
+ det = u.get("completion_tokens_details") or {}
80
+ pt, ct = u.get("prompt_tokens"), u.get("completion_tokens")
81
+ cost = None
82
+ if None not in (price_in, price_out, pt, ct):
83
+ cost = (pt * price_in + ct * price_out) / 1e6
84
+ rec = {"system": self.headers.get("X-PACE-System", default_system), "agent_id": self.headers.get("X-PACE-Agent", "agent"),
85
+ "round": int(self.headers.get("X-PACE-Round", 0)), "item": self.headers.get("X-PACE-Item") or _item_from_file(item_file),
86
+ "model": resp.get("model", req.get("model")), "prompt_tokens": pt, "completion_tokens": ct,
87
+ "reasoning_tokens": det.get("reasoning_tokens"), "n_completions": len(resp.get("choices", [])) or 1,
88
+ "start": t0, "end": t1, "latency": t1 - t0, "cost": cost}
89
+ if pt is None or ct is None:
90
+ rec["usage_missing"] = True # logged, never guessed
91
+ with lock, open(log_path, "a", encoding="utf-8") as fh:
92
+ fh.write(json.dumps(rec) + "\n")
93
+ except (ValueError, TypeError):
94
+ pass
95
+ self._send(code, data)
96
+
97
+ def log_message(self, *a): # quiet
98
+ pass
99
+
100
+ return H
101
+
102
+
103
+ def serve(upstream: str, log_path: str, port: int = 8089, host: str = "127.0.0.1", price_in: Optional[float] = None,
104
+ price_out: Optional[float] = None, api_key: Optional[str] = None, item_file: Optional[str] = None, default_system: str = "mas") -> ThreadingHTTPServer:
105
+ srv = ThreadingHTTPServer((host, port), make_handler(upstream, log_path, price_in, price_out, api_key, item_file, default_system))
106
+ return srv