pace-eval 0.2.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cme/__init__.py +11 -0
- cme/__main__.py +5 -0
- cme/analysis.py +223 -0
- cme/audit.py +145 -0
- cme/baseline.py +162 -0
- cme/cli.py +279 -0
- cme/client.py +165 -0
- cme/dynamics.py +212 -0
- cme/examples/crewai_integration.py +16 -0
- cme/examples/demo_pool.jsonl +360 -0
- cme/examples/demo_run.jsonl +480 -0
- cme/examples/langgraph_integration.py +27 -0
- cme/examples/make_demo.py +37 -0
- cme/examples/openai_quickstart.py +18 -0
- cme/experiment.py +555 -0
- cme/frontier.py +143 -0
- cme/hidden_profile.py +135 -0
- cme/proxy.py +106 -0
- cme/rebuttal.py +76 -0
- cme/report.py +197 -0
- cme/rollout_card.py +137 -0
- cme/tasks.py +266 -0
- cme/verified_debate.py +127 -0
- cme/verifier.py +92 -0
- mae_eval/__init__.py +11 -0
- mae_eval/__main__.py +5 -0
- mae_eval/auditor.py +112 -0
- mae_eval/classifier.py +62 -0
- mae_eval/cli.py +210 -0
- mae_eval/tracker.py +162 -0
- mae_eval/vector.py +91 -0
- pace/__init__.py +11 -0
- pace/__main__.py +5 -0
- pace/analysis.py +129 -0
- pace/audit.py +145 -0
- pace/baseline.py +162 -0
- pace/card.py +205 -0
- pace/cli.py +898 -0
- pace/client.py +165 -0
- pace/dynamics.py +372 -0
- pace/escape_benchmarks.py +162 -0
- pace/examples/crewai_integration.py +16 -0
- pace/examples/demo_pool.jsonl +360 -0
- pace/examples/demo_run.jsonl +480 -0
- pace/examples/langgraph_integration.py +27 -0
- pace/examples/make_demo.py +37 -0
- pace/examples/openai_quickstart.py +18 -0
- pace/experiment.py +555 -0
- pace/frontier.py +114 -0
- pace/hidden_profile.py +301 -0
- pace/importers.py +219 -0
- pace/inflate.py +213 -0
- pace/influence.py +231 -0
- pace/meta_engine.py +246 -0
- pace/middleware.py +230 -0
- pace/open_ended.py +114 -0
- pace/plotting.py +132 -0
- pace/pool_sizer.py +154 -0
- pace/protocol_v3.py +414 -0
- pace/proxy.py +106 -0
- pace/rebuttal.py +219 -0
- pace/report.py +197 -0
- pace/rollout_card.py +137 -0
- pace/schema.py +74 -0
- pace/simulator.py +199 -0
- pace/studio.html +1040 -0
- pace/tasks.py +266 -0
- pace/verified_debate.py +127 -0
- pace/verifier.py +92 -0
- pace/workbench.html +26 -0
- pace/workbench.py +119 -0
- pace_eval-0.2.2.dist-info/METADATA +205 -0
- pace_eval-0.2.2.dist-info/RECORD +83 -0
- pace_eval-0.2.2.dist-info/WHEEL +5 -0
- pace_eval-0.2.2.dist-info/entry_points.txt +4 -0
- pace_eval-0.2.2.dist-info/licenses/LICENSE +202 -0
- pace_eval-0.2.2.dist-info/top_level.txt +4 -0
- pace_lab/__init__.py +9 -0
- pace_lab/clients.py +568 -0
- pace_lab/concurrency_test.py +127 -0
- pace_lab/data.py +445 -0
- pace_lab/graders.py +626 -0
- pace_lab/strategies.py +345 -0
cme/cli.py
ADDED
|
@@ -0,0 +1,279 @@
|
|
|
1
|
+
"""cme command line: proxy | baseline | report."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import argparse
|
|
5
|
+
import json
|
|
6
|
+
import sys
|
|
7
|
+
from typing import Optional, Sequence
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def _proxy(a) -> int:
|
|
11
|
+
from .proxy import serve
|
|
12
|
+
srv = serve(a.upstream, a.log, a.port, a.host, a.price_in, a.price_out, a.api_key, a.item_file, a.system_label)
|
|
13
|
+
print("cme proxy on http://%s:%d/v1 -> %s ; logging to %s (Ctrl+C to stop)" % (a.host, a.port, a.upstream, a.log), file=sys.stderr)
|
|
14
|
+
try:
|
|
15
|
+
srv.serve_forever()
|
|
16
|
+
except KeyboardInterrupt:
|
|
17
|
+
pass
|
|
18
|
+
return 0
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def _baseline(a) -> int:
|
|
22
|
+
from .baseline import OpenAICompatClient, load_bench, ollama_digest, run_baseline
|
|
23
|
+
items = load_bench(a.bench, a.max_items)
|
|
24
|
+
digest = ollama_digest(a.model) if "11434" in a.base_url else None
|
|
25
|
+
res = run_baseline(OpenAICompatClient(a.base_url, a.api_key), a.model, items, a.out, a.k, a.temperature, a.seed, a.max_tokens, digest,
|
|
26
|
+
progress=(lambda m: print(m, file=sys.stderr)) if a.verbose else None)
|
|
27
|
+
print(json.dumps(res))
|
|
28
|
+
return 0 if res["errors"] == 0 else 1
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def _report(a) -> int:
|
|
32
|
+
from .report import build_report, plot, to_latex
|
|
33
|
+
try:
|
|
34
|
+
rep = build_report(a.pool, a.logs, a.system, a.price_in, a.price_out, a.n_boot, a.perms, a.seed, a.k)
|
|
35
|
+
except (OSError, ValueError) as e:
|
|
36
|
+
print("error: %s" % e, file=sys.stderr)
|
|
37
|
+
return 1
|
|
38
|
+
g = rep["frontier_gap"]
|
|
39
|
+
print("items %d | pool k=%d | system accuracy %.3f at %.0f tokens/item | frontier %.3f (%s) | gap %+.3f [%+.3f, %+.3f]" % (
|
|
40
|
+
rep["n_items"], rep["pool_k"], g["system_accuracy"], g["system_mean_tokens"], g["frontier_at_system_tokens"], g["range_status"],
|
|
41
|
+
g["gap"], g["ci_low"], g["ci_high"]))
|
|
42
|
+
rc = rep["resource_card"]
|
|
43
|
+
if rc:
|
|
44
|
+
print("MAS/SAS ratios: " + ", ".join("%s=%s" % (k, "n/a" if v is None else "%.3g" % v) for k, v in rc["ratios_mas_over_sas"].items()))
|
|
45
|
+
if a.json:
|
|
46
|
+
json.dump(rep, open(a.json, "w", encoding="utf-8"), indent=2, default=str)
|
|
47
|
+
if a.latex:
|
|
48
|
+
open(a.latex, "w", encoding="utf-8").write(to_latex(rep))
|
|
49
|
+
if a.plot:
|
|
50
|
+
plot(rep, a.plot)
|
|
51
|
+
if a.export_latex or a.export_markdown:
|
|
52
|
+
from .rebuttal import row_from_report, to_latex as rb_tex, to_markdown
|
|
53
|
+
rows = [row_from_report(rep, a.benchmark, a.model_label)]
|
|
54
|
+
if a.export_latex:
|
|
55
|
+
open(a.export_latex, "w", encoding="utf-8").write(rb_tex(rows))
|
|
56
|
+
if a.export_markdown:
|
|
57
|
+
open(a.export_markdown, "w", encoding="utf-8").write(to_markdown(rows))
|
|
58
|
+
if a.export_rollout_card:
|
|
59
|
+
from .rollout_card import write_block
|
|
60
|
+
for path in write_block(rep, a.export_rollout_card, a.run_id):
|
|
61
|
+
print("wrote " + path)
|
|
62
|
+
return 0
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def _classify(a) -> int:
|
|
66
|
+
from mae_eval.classifier import classify_with_reason
|
|
67
|
+
try:
|
|
68
|
+
cfg = json.load(open(a.config, encoding="utf-8"))
|
|
69
|
+
rung, rule, reason = classify_with_reason(cfg)
|
|
70
|
+
except (OSError, ValueError, TypeError) as e:
|
|
71
|
+
print("error: %s" % e, file=sys.stderr)
|
|
72
|
+
return 1
|
|
73
|
+
print(json.dumps({"rung": rung, "rule": rule, "reason": reason}))
|
|
74
|
+
return 0
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def _audit(a) -> int:
|
|
78
|
+
from .audit import audit, render
|
|
79
|
+
try:
|
|
80
|
+
card = audit(a.input, a.baseline_config, a.tolerance, a.price_in, a.price_out)
|
|
81
|
+
except (OSError, ValueError) as e:
|
|
82
|
+
print("error: %s" % e, file=sys.stderr)
|
|
83
|
+
return 1
|
|
84
|
+
if a.json:
|
|
85
|
+
print(json.dumps(card, indent=2, default=lambda x: None if x != x else str(x)))
|
|
86
|
+
else:
|
|
87
|
+
text = render(card)
|
|
88
|
+
try:
|
|
89
|
+
text.encode(sys.stdout.encoding or "utf-8")
|
|
90
|
+
except (UnicodeEncodeError, LookupError): # a console without box-drawing characters (for example a Windows code page): fall back to ASCII
|
|
91
|
+
text = render(card, ascii_only=True)
|
|
92
|
+
print(text)
|
|
93
|
+
return 0 if card["ok"] or not a.strict else 2
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def _dynamics(a) -> int:
|
|
97
|
+
from .dynamics import analyse, excess_over_null, render
|
|
98
|
+
try:
|
|
99
|
+
res = analyse(a.log, a.n_boot, a.seed, a.first_round, a.last_round)
|
|
100
|
+
if getattr(a, "null", None):
|
|
101
|
+
null_res = analyse(a.null, a.n_boot, a.seed, a.first_round, a.last_round)
|
|
102
|
+
res["excess_over_null"] = excess_over_null(res, null_res)
|
|
103
|
+
except (OSError, ValueError) as e:
|
|
104
|
+
print("error: %s" % e, file=sys.stderr)
|
|
105
|
+
return 1
|
|
106
|
+
if a.json:
|
|
107
|
+
print(json.dumps(res, indent=2, default=str))
|
|
108
|
+
else:
|
|
109
|
+
print(render(res))
|
|
110
|
+
return 0
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def _hidden_profile(a) -> int:
|
|
114
|
+
from .hidden_profile import generate, valid, validate
|
|
115
|
+
try:
|
|
116
|
+
items = generate(a.n, a.agents, a.criteria, a.split, a.margin, a.seed, a.format)
|
|
117
|
+
except ValueError as e:
|
|
118
|
+
print("error: %s" % e, file=sys.stderr)
|
|
119
|
+
return 1
|
|
120
|
+
counts = validate(items)
|
|
121
|
+
msg = "generated %d items (agents %d, criteria %d, split %d, margin %d, seed %d, format %s); decoy leads a view in violation: %d, target not leading pooled: %d, uncovered split criteria: %d" % (
|
|
122
|
+
counts["items"], a.agents, a.criteria, a.split, a.margin, a.seed, a.format, counts["decoy_not_leading_a_view"], counts["target_not_leading_pooled"], counts["uncovered_split_criteria"])
|
|
123
|
+
print(msg, file=sys.stderr)
|
|
124
|
+
if not valid(counts):
|
|
125
|
+
print("error: validity check failed; no output written", file=sys.stderr)
|
|
126
|
+
return 1
|
|
127
|
+
text = chr(10).join(json.dumps(it) for it in items) + chr(10)
|
|
128
|
+
if a.out:
|
|
129
|
+
with open(a.out, "w", encoding="utf-8") as fh:
|
|
130
|
+
fh.write(text)
|
|
131
|
+
else:
|
|
132
|
+
sys.stdout.write(text)
|
|
133
|
+
return 0
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def _frontier(a) -> int:
|
|
137
|
+
from .report import frontier_report, plot_frontier, system_points
|
|
138
|
+
try:
|
|
139
|
+
fr = frontier_report(a.samples, a.k, a.perms, a.seed)
|
|
140
|
+
pts = system_points(a.logs) if a.logs else []
|
|
141
|
+
except (OSError, ValueError, KeyError) as e:
|
|
142
|
+
print("error: %s" % e, file=sys.stderr)
|
|
143
|
+
return 1
|
|
144
|
+
print("items %d | pool k=%d | greedy accuracy %s" % (fr["n_items"], fr["pool_k"], "n/a" if fr["greedy_accuracy"] is None else "%.3f" % fr["greedy_accuracy"]))
|
|
145
|
+
print("k mean tokens SC accuracy isotonic")
|
|
146
|
+
for i, (t, x, y) in enumerate(zip(fr["tokens"], fr["sc_accuracy"], fr["isotonic"]), 1):
|
|
147
|
+
print("%-2d %11.0f %11.3f %8.3f" % (i, t, x, y))
|
|
148
|
+
for p in pts:
|
|
149
|
+
print("system %-12s accuracy %.3f at %.0f tokens/item (n=%d)" % (p["system"], p["accuracy"], p["mean_tokens"], p["n_items"]))
|
|
150
|
+
if a.json:
|
|
151
|
+
json.dump({"frontier": fr, "systems": pts}, open(a.json, "w", encoding="utf-8"), indent=2)
|
|
152
|
+
if a.out:
|
|
153
|
+
plot_frontier(fr, pts, a.out)
|
|
154
|
+
return 0
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
def _rebuttal(a) -> int:
|
|
158
|
+
from .rebuttal import rows_from_contrasts, to_latex as rb_tex, to_markdown
|
|
159
|
+
try:
|
|
160
|
+
rows = rows_from_contrasts(a.contrasts, a.study)
|
|
161
|
+
except (OSError, KeyError, ValueError) as e:
|
|
162
|
+
print("error: %s" % e, file=sys.stderr)
|
|
163
|
+
return 1
|
|
164
|
+
if not rows:
|
|
165
|
+
print("error: no rows", file=sys.stderr)
|
|
166
|
+
return 1
|
|
167
|
+
if a.export_latex:
|
|
168
|
+
open(a.export_latex, "w", encoding="utf-8").write(rb_tex(rows))
|
|
169
|
+
if a.export_markdown:
|
|
170
|
+
open(a.export_markdown, "w", encoding="utf-8").write(to_markdown(rows))
|
|
171
|
+
if not (a.export_latex or a.export_markdown):
|
|
172
|
+
print(to_markdown(rows))
|
|
173
|
+
return 0
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
def build_parser() -> argparse.ArgumentParser:
|
|
177
|
+
p = argparse.ArgumentParser(prog="cme", description="Compute-matched evaluation: measure systems against the self-consistency frontier.")
|
|
178
|
+
sub = p.add_subparsers(dest="cmd", required=True)
|
|
179
|
+
x = sub.add_parser("proxy", help="OpenAI-compatible logging proxy")
|
|
180
|
+
x.add_argument("--upstream", default="http://localhost:11434/v1")
|
|
181
|
+
x.add_argument("--port", type=int, default=8089)
|
|
182
|
+
x.add_argument("--host", default="127.0.0.1")
|
|
183
|
+
x.add_argument("--log", default="run.jsonl")
|
|
184
|
+
x.add_argument("--price-in", type=float, default=None, help="USD per million input tokens (never bundled)")
|
|
185
|
+
x.add_argument("--price-out", type=float, default=None)
|
|
186
|
+
x.add_argument("--api-key", default=None)
|
|
187
|
+
x.add_argument("--system-label", default="mas", help="value of the log's system field when a request has no X-CME-System header")
|
|
188
|
+
x.add_argument("--item-file", default=None, help="file whose first line is the current item id, used when a request has no X-CME-Item header (serial runs of third-party code)")
|
|
189
|
+
x.set_defaults(func=_proxy)
|
|
190
|
+
b = sub.add_parser("baseline", help="greedy + k-sample pool, cached and resumable")
|
|
191
|
+
b.add_argument("--model", required=True)
|
|
192
|
+
b.add_argument("--bench", required=True, help="path to JSONL {id, question, answer}, or 'toy' (synthetic smoke test)")
|
|
193
|
+
b.add_argument("--base-url", default="http://localhost:11434/v1")
|
|
194
|
+
b.add_argument("--api-key", default="none")
|
|
195
|
+
b.add_argument("--k", type=int, default=16)
|
|
196
|
+
b.add_argument("--temperature", type=float, default=0.7)
|
|
197
|
+
b.add_argument("--seed", type=int, default=0)
|
|
198
|
+
b.add_argument("--max-tokens", type=int, default=2048)
|
|
199
|
+
b.add_argument("--max-items", type=int, default=None)
|
|
200
|
+
b.add_argument("--out", default="baseline_cache.jsonl")
|
|
201
|
+
b.add_argument("--verbose", action="store_true")
|
|
202
|
+
b.set_defaults(func=_baseline)
|
|
203
|
+
r = sub.add_parser("report", help="frontier gap, resource card, LaTeX table and figure")
|
|
204
|
+
r.add_argument("--pool", required=True, help="baseline cache from `cme baseline`")
|
|
205
|
+
r.add_argument("--logs", required=True, help="system log (calls and results)")
|
|
206
|
+
r.add_argument("--system", default="mas")
|
|
207
|
+
r.add_argument("--price-in", type=float, default=None)
|
|
208
|
+
r.add_argument("--price-out", type=float, default=None)
|
|
209
|
+
r.add_argument("--k", type=int, default=None, help="use the first k samples per item")
|
|
210
|
+
r.add_argument("--n-boot", type=int, default=2000)
|
|
211
|
+
r.add_argument("--perms", type=int, default=100)
|
|
212
|
+
r.add_argument("--seed", type=int, default=0)
|
|
213
|
+
r.add_argument("--json", default=None)
|
|
214
|
+
r.add_argument("--latex", default=None)
|
|
215
|
+
r.add_argument("--plot", default=None, help="vector figure path (.pdf)")
|
|
216
|
+
r.add_argument("--export-latex", default=None, help="write the nine-column rebuttal table for this run (LaTeX)")
|
|
217
|
+
r.add_argument("--export-markdown", default=None, help="write the nine-column rebuttal table for this run (Markdown)")
|
|
218
|
+
r.add_argument("--benchmark", default="n/a", help="benchmark label for the exported row")
|
|
219
|
+
r.add_argument("--model-label", default="n/a", help="model label for the exported row")
|
|
220
|
+
r.add_argument("--export-rollout-card", default=None, metavar="PREFIX", help="write PREFIX.json, PREFIX.yaml and PREFIX.tex: a CME compute block for a rollout card (arXiv:2605.12131)")
|
|
221
|
+
r.add_argument("--run-id", default="run", help="target id used in the rollout-card annotation rows")
|
|
222
|
+
r.set_defaults(func=_report)
|
|
223
|
+
c = sub.add_parser("classify", help="baseline rung (A to D) of a single-agent configuration given as JSON")
|
|
224
|
+
c.add_argument("--config", required=True, help="JSON file; keys are listed in code/mae_eval/classifier.py")
|
|
225
|
+
c.set_defaults(func=_classify)
|
|
226
|
+
u = sub.add_parser("audit", help="one-screen diagnostic card for a run log (mas and sas calls) or a baseline configuration (JSON)")
|
|
227
|
+
u.add_argument("input", help="JSONL run log with 'mas' and 'sas' calls, or a JSON baseline configuration")
|
|
228
|
+
u.add_argument("--baseline-config", default=None, help="JSON configuration of the single-agent baseline, to classify its rung when INPUT is a log")
|
|
229
|
+
u.add_argument("--tolerance", type=float, default=0.05)
|
|
230
|
+
u.add_argument("--price-in", type=float, default=None)
|
|
231
|
+
u.add_argument("--price-out", type=float, default=None)
|
|
232
|
+
u.add_argument("--json", action="store_true")
|
|
233
|
+
u.add_argument("--strict", action="store_true", help="exit with status 2 when the badge is a warning")
|
|
234
|
+
u.set_defaults(func=_audit)
|
|
235
|
+
d = sub.add_parser("dynamics", help="flip counts, tau rates, implied threshold, peer correlation, K_eff and tokens by round from a debate log")
|
|
236
|
+
d.add_argument("log", help="JSONL with item, agent, round, correct (and optionally prompt_tokens, completion_tokens) per agent answer")
|
|
237
|
+
d.add_argument("--null", default=None, help="optional no-communication control JSONL log to compute excess transition rates and excess peer correlation")
|
|
238
|
+
d.add_argument("--n-boot", type=int, default=1000)
|
|
239
|
+
d.add_argument("--seed", type=int, default=0)
|
|
240
|
+
d.add_argument("--first-round", type=int, default=None)
|
|
241
|
+
d.add_argument("--last-round", type=int, default=None)
|
|
242
|
+
d.add_argument("--json", action="store_true")
|
|
243
|
+
d.set_defaults(func=_dynamics)
|
|
244
|
+
h = sub.add_parser("hidden-profile", help="generate synthetic split-evidence (hidden-profile) items and check their validity")
|
|
245
|
+
h.add_argument("--n", type=int, default=100, help="number of items")
|
|
246
|
+
h.add_argument("--agents", type=int, default=5)
|
|
247
|
+
h.add_argument("--criteria", type=int, default=10, help="total number of criteria")
|
|
248
|
+
h.add_argument("--split", type=int, default=6, help="number of criteria divided among the agents (the rest are shared)")
|
|
249
|
+
h.add_argument("--margin", type=int, default=3, help="score gap per criterion between the favoured and the disfavoured candidate (at least 2)")
|
|
250
|
+
h.add_argument("--seed", type=int, default=42)
|
|
251
|
+
h.add_argument("--format", choices=["text", "numeric"], default="text")
|
|
252
|
+
h.add_argument("--out", default=None, help="output JSONL (default: standard output)")
|
|
253
|
+
h.set_defaults(func=_hidden_profile)
|
|
254
|
+
f = sub.add_parser("frontier", help="self-consistency frontier of a sample pool, optionally with system points from a run log")
|
|
255
|
+
f.add_argument("--samples", required=True, help="baseline cache from `cme baseline` (greedy + sampled completions per item)")
|
|
256
|
+
f.add_argument("--logs", default=None, help="run log; every value of its 'system' field is drawn as a point")
|
|
257
|
+
f.add_argument("--k", type=int, default=None)
|
|
258
|
+
f.add_argument("--perms", type=int, default=100)
|
|
259
|
+
f.add_argument("--seed", type=int, default=0)
|
|
260
|
+
f.add_argument("--json", default=None)
|
|
261
|
+
f.add_argument("--out", default=None, help="figure path (.pdf vector, or .png)")
|
|
262
|
+
f.set_defaults(func=_frontier)
|
|
263
|
+
t = sub.add_parser("rebuttal", help="nine-column rebuttal table from a contrasts CSV (default: outputs/baseline_tier_contrasts.csv)")
|
|
264
|
+
t.add_argument("--contrasts", "--data", dest="contrasts", default="outputs/baseline_tier_contrasts.csv",
|
|
265
|
+
help="contrasts CSV with columns study, category (rung), baseline, key 'Study|bench|model|arm', p_base, p_mas")
|
|
266
|
+
t.add_argument("--study", default=None)
|
|
267
|
+
t.add_argument("--export-latex", default=None)
|
|
268
|
+
t.add_argument("--export-markdown", default=None)
|
|
269
|
+
t.set_defaults(func=_rebuttal)
|
|
270
|
+
return p
|
|
271
|
+
|
|
272
|
+
|
|
273
|
+
def main(argv: Optional[Sequence[str]] = None) -> int:
|
|
274
|
+
a = build_parser().parse_args(argv)
|
|
275
|
+
return a.func(a)
|
|
276
|
+
|
|
277
|
+
|
|
278
|
+
if __name__ == "__main__":
|
|
279
|
+
sys.exit(main())
|
cme/client.py
ADDED
|
@@ -0,0 +1,165 @@
|
|
|
1
|
+
"""cme.wrap: log the token usage of an existing client without a proxy process (standard library only).
|
|
2
|
+
|
|
3
|
+
import cme
|
|
4
|
+
from openai import OpenAI
|
|
5
|
+
client = cme.wrap(OpenAI(), log_path="cme_trace.jsonl")
|
|
6
|
+
with cme.track(item_id="item_042", system="mas", agent="agent_1", round=2):
|
|
7
|
+
response = client.chat.completions.create(model=..., messages=...)
|
|
8
|
+
|
|
9
|
+
The wrapper forwards every attribute to the wrapped client and intercepts only the calls that return token usage: `chat.completions.create`,
|
|
10
|
+
`completions.create` (OpenAI-style clients, sync or async) and a top-level `completion` (a litellm-style module or client). Each successful call
|
|
11
|
+
appends one JSON line in the same format as `cme proxy`, so `cme report --logs` and `mae-eval audit` read it. The response is returned unchanged.
|
|
12
|
+
|
|
13
|
+
Limits (v0.1): streaming calls raise ValueError, because their usage is not known when the call returns and cme never estimates tokens; a response
|
|
14
|
+
without a usage block is logged with `usage_missing: true`; prices are never bundled (pass price_in/price_out in USD per million tokens, else cost is null).
|
|
15
|
+
"""
|
|
16
|
+
from __future__ import annotations
|
|
17
|
+
|
|
18
|
+
import contextlib
|
|
19
|
+
import contextvars
|
|
20
|
+
import inspect
|
|
21
|
+
import json
|
|
22
|
+
import threading
|
|
23
|
+
import time
|
|
24
|
+
from typing import Any, Dict, Optional
|
|
25
|
+
|
|
26
|
+
_LABELS: contextvars.ContextVar = contextvars.ContextVar("cme_labels", default={})
|
|
27
|
+
_LOCKS: Dict[str, threading.Lock] = {}
|
|
28
|
+
_LOCKS_GUARD = threading.Lock()
|
|
29
|
+
_LOGGED = {("chat", "completions", "create"), ("completions", "create"), ("completion",)}
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
@contextlib.contextmanager
|
|
33
|
+
def track(item_id: Optional[str] = None, system: Optional[str] = None, agent: Optional[str] = None, round: Optional[int] = None, **extra):
|
|
34
|
+
"""Label every wrapped call made inside the block. Blocks nest; an inner label overrides the outer one. Unset labels keep the outer value."""
|
|
35
|
+
labels = dict(_LABELS.get())
|
|
36
|
+
for key, val in (("item", item_id), ("system", system), ("agent_id", agent), ("round", round)):
|
|
37
|
+
if val is not None:
|
|
38
|
+
labels[key] = val
|
|
39
|
+
labels.update(extra)
|
|
40
|
+
token = _LABELS.set(labels)
|
|
41
|
+
try:
|
|
42
|
+
yield labels
|
|
43
|
+
finally:
|
|
44
|
+
_LABELS.reset(token)
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def _get(obj: Any, name: str):
|
|
48
|
+
if obj is None:
|
|
49
|
+
return None
|
|
50
|
+
if isinstance(obj, dict):
|
|
51
|
+
return obj.get(name)
|
|
52
|
+
return getattr(obj, name, None)
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def _record(kwargs: dict, resp: Any, t0: float, t1: float, price_in: Optional[float], price_out: Optional[float]) -> dict:
|
|
56
|
+
usage = _get(resp, "usage")
|
|
57
|
+
pt, ct = _get(usage, "prompt_tokens"), _get(usage, "completion_tokens")
|
|
58
|
+
det = _get(usage, "completion_tokens_details")
|
|
59
|
+
cost = None
|
|
60
|
+
if None not in (price_in, price_out, pt, ct):
|
|
61
|
+
cost = (pt * price_in + ct * price_out) / 1e6
|
|
62
|
+
labels = _LABELS.get()
|
|
63
|
+
choices = _get(resp, "choices") or []
|
|
64
|
+
rec = {"system": labels.get("system", "mas"), "agent_id": labels.get("agent_id", "agent"), "round": int(labels.get("round", 0)),
|
|
65
|
+
"item": str(labels.get("item", "")), "model": _get(resp, "model") or kwargs.get("model"), "prompt_tokens": pt, "completion_tokens": ct,
|
|
66
|
+
"reasoning_tokens": _get(det, "reasoning_tokens"), "n_completions": len(choices) or 1, "start": t0, "end": t1, "latency": t1 - t0, "cost": cost}
|
|
67
|
+
for key, val in labels.items():
|
|
68
|
+
if key not in ("system", "agent_id", "round", "item"):
|
|
69
|
+
rec[key] = val
|
|
70
|
+
if pt is None or ct is None:
|
|
71
|
+
rec["usage_missing"] = True # logged, never guessed
|
|
72
|
+
return rec
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
class _Logger:
|
|
76
|
+
def __init__(self, log_path: str, price_in: Optional[float], price_out: Optional[float]):
|
|
77
|
+
self.path, self.price_in, self.price_out = str(log_path), price_in, price_out
|
|
78
|
+
with _LOCKS_GUARD:
|
|
79
|
+
self.lock = _LOCKS.setdefault(self.path, threading.Lock())
|
|
80
|
+
|
|
81
|
+
def write(self, kwargs: dict, resp: Any, t0: float, t1: float) -> None:
|
|
82
|
+
line = json.dumps(_record(kwargs, resp, t0, t1, self.price_in, self.price_out), default=str)
|
|
83
|
+
with self.lock, open(self.path, "a", encoding="utf-8") as fh:
|
|
84
|
+
fh.write(line + "\n")
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
class _Wrapped:
|
|
88
|
+
"""Transparent proxy over a client (or one of its sub-objects); only the usage-returning calls are intercepted."""
|
|
89
|
+
|
|
90
|
+
def __init__(self, target: Any, logger: _Logger, path: tuple = ()):
|
|
91
|
+
object.__setattr__(self, "_cme_target", target)
|
|
92
|
+
object.__setattr__(self, "_cme_logger", logger)
|
|
93
|
+
object.__setattr__(self, "_cme_path", path)
|
|
94
|
+
|
|
95
|
+
def __getattr__(self, name: str):
|
|
96
|
+
target = object.__getattribute__(self, "_cme_target")
|
|
97
|
+
logger = object.__getattribute__(self, "_cme_logger")
|
|
98
|
+
path = object.__getattribute__(self, "_cme_path") + (name,)
|
|
99
|
+
attr = getattr(target, name)
|
|
100
|
+
if path in _LOGGED and callable(attr):
|
|
101
|
+
return _intercept(attr, logger)
|
|
102
|
+
if callable(attr) or name.startswith("_") or isinstance(attr, (str, int, float, bool, bytes, type(None))):
|
|
103
|
+
return attr
|
|
104
|
+
if any(p[:len(path)] == path for p in _LOGGED):
|
|
105
|
+
return _Wrapped(attr, logger, path)
|
|
106
|
+
return attr
|
|
107
|
+
|
|
108
|
+
def __setattr__(self, name, value):
|
|
109
|
+
setattr(object.__getattribute__(self, "_cme_target"), name, value)
|
|
110
|
+
|
|
111
|
+
def __repr__(self):
|
|
112
|
+
return "cme.wrap(%r)" % (object.__getattribute__(self, "_cme_target"),)
|
|
113
|
+
|
|
114
|
+
def __enter__(self):
|
|
115
|
+
target = object.__getattribute__(self, "_cme_target")
|
|
116
|
+
if hasattr(target, "__enter__"):
|
|
117
|
+
target.__enter__()
|
|
118
|
+
return self
|
|
119
|
+
|
|
120
|
+
def __exit__(self, *exc):
|
|
121
|
+
target = object.__getattribute__(self, "_cme_target")
|
|
122
|
+
return target.__exit__(*exc) if hasattr(target, "__exit__") else None
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
def _intercept(fn, logger: _Logger):
|
|
126
|
+
def guard(kwargs: dict) -> None:
|
|
127
|
+
if kwargs.get("stream"):
|
|
128
|
+
raise ValueError("cme.wrap v0.1 does not support streaming calls: usage is not known when the call returns. Use stream=False.")
|
|
129
|
+
|
|
130
|
+
if inspect.iscoroutinefunction(fn):
|
|
131
|
+
async def acall(*args, **kwargs):
|
|
132
|
+
guard(kwargs)
|
|
133
|
+
t0 = time.time()
|
|
134
|
+
resp = await fn(*args, **kwargs)
|
|
135
|
+
logger.write(kwargs, resp, t0, time.time())
|
|
136
|
+
return resp
|
|
137
|
+
return acall
|
|
138
|
+
|
|
139
|
+
def call(*args, **kwargs):
|
|
140
|
+
guard(kwargs)
|
|
141
|
+
t0 = time.time()
|
|
142
|
+
resp = fn(*args, **kwargs)
|
|
143
|
+
if inspect.isawaitable(resp): # an async client whose method is not a plain coroutine function
|
|
144
|
+
async def finish():
|
|
145
|
+
out = await resp
|
|
146
|
+
logger.write(kwargs, out, t0, time.time())
|
|
147
|
+
return out
|
|
148
|
+
return finish()
|
|
149
|
+
logger.write(kwargs, resp, t0, time.time())
|
|
150
|
+
return resp
|
|
151
|
+
return call
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
def wrap(client: Any, log_path: str = "cme_trace.jsonl", price_in: Optional[float] = None, price_out: Optional[float] = None) -> _Wrapped:
|
|
155
|
+
"""Return `client` with usage logging on its completion calls. Prices are USD per million tokens; omit them and cost is null."""
|
|
156
|
+
return _Wrapped(client, _Logger(log_path, price_in, price_out))
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
def log_result(log_path: str, item_id: str, correct: bool, system: Optional[str] = None) -> None:
|
|
160
|
+
"""Append the correctness of one item's final answer (the line `cme report` pairs with the logged calls). Call it once per item, after scoring."""
|
|
161
|
+
sysname = system if system is not None else _LABELS.get().get("system", "mas")
|
|
162
|
+
with _LOCKS_GUARD:
|
|
163
|
+
lock = _LOCKS.setdefault(str(log_path), threading.Lock())
|
|
164
|
+
with lock, open(str(log_path), "a", encoding="utf-8") as fh:
|
|
165
|
+
fh.write(json.dumps({"system": sysname, "item": str(item_id), "correct": bool(correct)}) + "\n")
|
cme/dynamics.py
ADDED
|
@@ -0,0 +1,212 @@
|
|
|
1
|
+
"""cme dynamics: how agents' answers change across debate rounds (standard library plus numpy).
|
|
2
|
+
|
|
3
|
+
Input: a JSONL file with one record per agent answer, {"item": str, "agent": str or int, "round": int, "correct": bool} and optionally "prompt_tokens" and
|
|
4
|
+
"completion_tokens". Nothing is estimated for missing fields: records without "correct" are ignored, and token statistics need both token fields.
|
|
5
|
+
|
|
6
|
+
Output (point estimates and 95% percentile intervals from a paired bootstrap that resamples items):
|
|
7
|
+
flips between the first and the last round: wrong to right, right to wrong, right to right, wrong to wrong (counts, over item-agent pairs observed in both rounds);
|
|
8
|
+
tau_ic = wrong-to-right / (all wrong in the first round) and tau_ci = right-to-wrong / (all right in the first round), the two rates of the conformity-to-truth model;
|
|
9
|
+
p_star_hat = tau_ic / (tau_ic + tau_ci), the threshold implied by those rates (undefined if both rates are zero);
|
|
10
|
+
p1 and p_last, the mean accuracy of an agent in the first and the last round;
|
|
11
|
+
rho_total, the intraclass correlation of correctness across the agents of an item in each round (reported in the JSON also as rho_peer, its earlier name), and K_eff = K / (1 + (K - 1) rho_total)
|
|
12
|
+
with rho_total clipped to [0, 1]; delta_rho_debate = rho_total(last round) - rho_total(first round), the excess correlation induced by debate, with its own bootstrap interval;
|
|
13
|
+
mean tokens per call by round.
|
|
14
|
+
Limits: rho_total is the correlation of correctness among agents on the same item, so it includes shared item difficulty as well as correlated errors; the difference across rounds is the
|
|
15
|
+
quantity that isolates what the exchange of messages adds, and it is not a measure of error correlation between models. The tau
|
|
16
|
+
rates assume the constant-rate model of the paper; this command reports what the logs show under that model and does not test the model.
|
|
17
|
+
"""
|
|
18
|
+
from __future__ import annotations
|
|
19
|
+
|
|
20
|
+
import json
|
|
21
|
+
from collections import defaultdict
|
|
22
|
+
from typing import Dict, List, Optional
|
|
23
|
+
|
|
24
|
+
import numpy as np
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def load(path: str):
|
|
28
|
+
"""Return {item: {agent: {round: correct}}} and {item: {round: [tokens per call]}}."""
|
|
29
|
+
ans: Dict[str, Dict[str, Dict[int, int]]] = defaultdict(lambda: defaultdict(dict))
|
|
30
|
+
tok: Dict[str, Dict[int, List[float]]] = defaultdict(lambda: defaultdict(list))
|
|
31
|
+
with open(path, encoding="utf-8") as fh:
|
|
32
|
+
for no, line in enumerate(fh, 1):
|
|
33
|
+
if not line.strip():
|
|
34
|
+
continue
|
|
35
|
+
r = json.loads(line)
|
|
36
|
+
if not isinstance(r, dict) or "correct" not in r or "round" not in r or "item" not in r or "agent" not in r:
|
|
37
|
+
continue
|
|
38
|
+
ans[str(r["item"])][str(r["agent"])][int(r["round"])] = int(bool(r["correct"]))
|
|
39
|
+
if r.get("prompt_tokens") is not None and r.get("completion_tokens") is not None:
|
|
40
|
+
tok[str(r["item"])][int(r["round"])].append(float(r["prompt_tokens"]) + float(r["completion_tokens"]))
|
|
41
|
+
if not ans:
|
|
42
|
+
raise ValueError("no usable records: each line needs item, agent, round and correct")
|
|
43
|
+
return ans, tok
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def icc(Z: np.ndarray) -> Optional[float]:
|
|
47
|
+
"""One-way intraclass correlation of an items x agents 0/1 matrix (balanced)."""
|
|
48
|
+
n, k = Z.shape
|
|
49
|
+
if n < 2 or k < 2:
|
|
50
|
+
return None
|
|
51
|
+
m = Z.mean(axis=1)
|
|
52
|
+
g = Z.mean()
|
|
53
|
+
msb = k * np.sum((m - g) ** 2) / (n - 1)
|
|
54
|
+
msw = np.sum((Z - m[:, None]) ** 2) / (n * (k - 1))
|
|
55
|
+
den = msb + (k - 1) * msw
|
|
56
|
+
return None if den == 0 else float((msb - msw) / den)
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def stats(ans, items: List[str], r0: int, r1: int, rounds: List[int]) -> dict:
|
|
60
|
+
wr = rw = rr = ww = 0
|
|
61
|
+
a0 = []
|
|
62
|
+
a1 = []
|
|
63
|
+
for it in items:
|
|
64
|
+
for ag, d in ans[it].items():
|
|
65
|
+
if r0 in d and r1 in d:
|
|
66
|
+
a, b = d[r0], d[r1]
|
|
67
|
+
a0.append(a)
|
|
68
|
+
a1.append(b)
|
|
69
|
+
wr += (a == 0 and b == 1)
|
|
70
|
+
rw += (a == 1 and b == 0)
|
|
71
|
+
rr += (a == 1 and b == 1)
|
|
72
|
+
ww += (a == 0 and b == 0)
|
|
73
|
+
out = dict(wrong_to_right=wr, right_to_wrong=rw, right_to_right=rr, wrong_to_wrong=ww)
|
|
74
|
+
out["tau_ic"] = wr / (wr + ww) if wr + ww else None
|
|
75
|
+
out["tau_ci"] = rw / (rw + rr) if rw + rr else None
|
|
76
|
+
ti, tc = out["tau_ic"], out["tau_ci"]
|
|
77
|
+
out["p_star_hat"] = ti / (ti + tc) if ti is not None and tc is not None and ti + tc > 0 else None
|
|
78
|
+
adj_wr = adj_rw = adj_rr = adj_ww = 0
|
|
79
|
+
active_rounds = [r for r in rounds if r0 <= r <= r1]
|
|
80
|
+
for it in items:
|
|
81
|
+
for ag, d in ans[it].items():
|
|
82
|
+
for r_a, r_b in zip(active_rounds[:-1], active_rounds[1:]):
|
|
83
|
+
if r_a in d and r_b in d:
|
|
84
|
+
a, b = d[r_a], d[r_b]
|
|
85
|
+
adj_wr += (a == 0 and b == 1)
|
|
86
|
+
adj_rw += (a == 1 and b == 0)
|
|
87
|
+
adj_rr += (a == 1 and b == 1)
|
|
88
|
+
adj_ww += (a == 0 and b == 0)
|
|
89
|
+
out["adjacent_wrong_to_right"] = adj_wr
|
|
90
|
+
out["adjacent_right_to_wrong"] = adj_rw
|
|
91
|
+
out["adjacent_right_to_right"] = adj_rr
|
|
92
|
+
out["adjacent_wrong_to_wrong"] = adj_ww
|
|
93
|
+
out["adjacent_tau_ic"] = adj_wr / (adj_wr + adj_ww) if adj_wr + adj_ww else None
|
|
94
|
+
out["adjacent_tau_ci"] = adj_rw / (adj_rw + adj_rr) if adj_rw + adj_rr else None
|
|
95
|
+
ati, atc = out["adjacent_tau_ic"], out["adjacent_tau_ci"]
|
|
96
|
+
out["adjacent_p_star_hat"] = ati / (ati + atc) if ati is not None and atc is not None and ati + atc > 0 else None
|
|
97
|
+
out["p1"] = float(np.mean(a0)) if a0 else None
|
|
98
|
+
out["p_last"] = float(np.mean(a1)) if a1 else None
|
|
99
|
+
rho, keff = {}, {}
|
|
100
|
+
for r in rounds:
|
|
101
|
+
rows = [[d[r] for d in ans[it].values() if r in d] for it in items]
|
|
102
|
+
k = min((len(x) for x in rows), default=0)
|
|
103
|
+
Z = np.array([x[:k] for x in rows], dtype=float) if k >= 2 else None
|
|
104
|
+
v = icc(Z) if Z is not None else None
|
|
105
|
+
rho[r] = v
|
|
106
|
+
keff[r] = (k / (1 + (k - 1) * min(1.0, max(0.0, v)))) if v is not None else None
|
|
107
|
+
out["rho_peer"], out["rho_total"], out["k_eff"] = rho, rho, keff # rho_peer is the earlier name of rho_total
|
|
108
|
+
out["delta_rho_debate"] = (rho[r1] - rho[r0]) if rho.get(r0) is not None and rho.get(r1) is not None else None
|
|
109
|
+
return out
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def analyse(path: str, n_boot: int = 1000, seed: int = 0, first: Optional[int] = None, last: Optional[int] = None) -> dict:
|
|
113
|
+
ans, tok = load(path)
|
|
114
|
+
items = sorted(ans)
|
|
115
|
+
rounds = sorted({r for it in items for d in ans[it].values() for r in d})
|
|
116
|
+
if len(rounds) < 2:
|
|
117
|
+
raise ValueError("need answers from at least two rounds; found %s" % rounds)
|
|
118
|
+
r0, r1 = (first if first is not None else rounds[0]), (last if last is not None else rounds[-1])
|
|
119
|
+
if r0 not in rounds or r1 not in rounds or r0 >= r1:
|
|
120
|
+
raise ValueError("--first-round and --last-round must be distinct rounds present in the log, first < last; found rounds %s" % rounds)
|
|
121
|
+
point = stats(ans, items, r0, r1, rounds)
|
|
122
|
+
rng = np.random.default_rng(seed)
|
|
123
|
+
boots = []
|
|
124
|
+
for _ in range(n_boot):
|
|
125
|
+
pick = [items[i] for i in rng.integers(0, len(items), len(items))]
|
|
126
|
+
# a resampled item can repeat: give each copy its own key so it is counted as many times as drawn
|
|
127
|
+
ans_b = {"%s#%d" % (it, c): ans[it] for c, it in enumerate(pick)}
|
|
128
|
+
boots.append(stats(ans_b, list(ans_b), r0, r1, rounds))
|
|
129
|
+
|
|
130
|
+
def ci(get):
|
|
131
|
+
v = [x for x in (get(b) for b in boots) if x is not None]
|
|
132
|
+
return [float(np.percentile(v, 2.5)), float(np.percentile(v, 97.5))] if len(v) >= max(20, n_boot // 10) else None
|
|
133
|
+
cis = {k: ci(lambda b, k=k: b[k]) for k in ("tau_ic", "tau_ci", "p_star_hat", "adjacent_tau_ic", "adjacent_tau_ci", "adjacent_p_star_hat", "p1", "p_last", "delta_rho_debate")}
|
|
134
|
+
cis["rho_peer"] = cis["rho_total"] = {r: ci(lambda b, r=r: b["rho_peer"][r]) for r in rounds}
|
|
135
|
+
cis["k_eff"] = {r: ci(lambda b, r=r: b["k_eff"][r]) for r in rounds}
|
|
136
|
+
tokens = {}
|
|
137
|
+
for r in rounds:
|
|
138
|
+
per_item = [np.mean(tok[it][r]) for it in items if tok[it].get(r)]
|
|
139
|
+
if per_item:
|
|
140
|
+
tokens[r] = float(np.mean(per_item))
|
|
141
|
+
return dict(n_items=len(items), rounds=rounds, first_round=r0, last_round=r1, n_boot=n_boot, point=point, ci95=cis, mean_tokens_per_call_by_round=tokens or None)
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
def _f(x, nd=3):
|
|
145
|
+
return "n/a" if x is None else ("%.*f" % (nd, x))
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
def render(res: dict) -> str:
|
|
149
|
+
p, c = res["point"], res["ci95"]
|
|
150
|
+
|
|
151
|
+
def with_ci(key, nd=3):
|
|
152
|
+
return "%s [%s, %s]" % (_f(p[key], nd), _f(c[key][0], nd), _f(c[key][1], nd)) if c.get(key) else _f(p[key], nd)
|
|
153
|
+
out = ["Debate dynamics: %d items, rounds %s (first %d to last %d), %d bootstrap resamples over items" % (res["n_items"], res["rounds"], res["first_round"], res["last_round"], res["n_boot"]),
|
|
154
|
+
"flips (item-agent pairs): wrong->right %d, right->wrong %d, right->right %d, wrong->wrong %d" % (p["wrong_to_right"], p["right_to_wrong"], p["right_to_right"], p["wrong_to_wrong"]),
|
|
155
|
+
"tau_ic (wrong -> right): " + with_ci("tau_ic"), "tau_ci (right -> wrong): " + with_ci("tau_ci"),
|
|
156
|
+
"p_star_hat = tau_ic / (tau_ic + tau_ci): " + with_ci("p_star_hat"),
|
|
157
|
+
"p1 (first round accuracy): " + with_ci("p1") + " p_last: " + with_ci("p_last"),
|
|
158
|
+
"delta_rho_debate = rho_total(round %d) - rho_total(round %d): %s" % (res["last_round"], res["first_round"], with_ci("delta_rho_debate"))]
|
|
159
|
+
for r in res["rounds"]:
|
|
160
|
+
rc, kc = c["rho_peer"][r], c["k_eff"][r]
|
|
161
|
+
out.append("round %d: rho_total %s%s, K_eff %s%s" % (r, _f(p["rho_peer"][r]), " [%s, %s]" % (_f(rc[0]), _f(rc[1])) if rc else "", _f(p["k_eff"][r], 2),
|
|
162
|
+
" [%s, %s]" % (_f(kc[0], 2), _f(kc[1], 2)) if kc else ""))
|
|
163
|
+
if res["mean_tokens_per_call_by_round"]:
|
|
164
|
+
out.append("mean tokens per call by round: " + ", ".join("r%d %.0f" % (r, t) for r, t in res["mean_tokens_per_call_by_round"].items()))
|
|
165
|
+
else:
|
|
166
|
+
out.append("mean tokens per call by round: n/a (no token fields in the log)")
|
|
167
|
+
if res.get("excess_over_null"):
|
|
168
|
+
ex = res["excess_over_null"]
|
|
169
|
+
out.append(
|
|
170
|
+
"excess over null: adjacent_tau_ic %s, adjacent_tau_ci %s, delta_rho_debate %s"
|
|
171
|
+
% (_f(ex.get("excess_adjacent_tau_ic")), _f(ex.get("excess_adjacent_tau_ci")), _f(ex.get("excess_delta_rho_debate")))
|
|
172
|
+
)
|
|
173
|
+
out.append("rho_total is the correlation of correctness across agents of an item and includes shared item difficulty; delta_rho_debate is its change from the first to the last round, in which shared difficulty largely cancels, so it estimates the correlation added by communication. The tau rates assume the constant-rate model.")
|
|
174
|
+
return "\n".join(out)
|
|
175
|
+
|
|
176
|
+
|
|
177
|
+
def peer_transition_table(ans: dict, items: list, rounds: list) -> dict:
|
|
178
|
+
"""Compute P(correct at t+1 | own_state_t, n_correct_peers_t) across adjacent rounds."""
|
|
179
|
+
counts = {}
|
|
180
|
+
for it in items:
|
|
181
|
+
agents = ans[it]
|
|
182
|
+
for r_t, r_next in zip(rounds[:-1], rounds[1:]):
|
|
183
|
+
present = [a for a, d in agents.items() if r_t in d and r_next in d]
|
|
184
|
+
if len(present) < 2:
|
|
185
|
+
continue
|
|
186
|
+
n_right_total = sum(agents[a][r_t] for a in present)
|
|
187
|
+
for a in present:
|
|
188
|
+
own = agents[a][r_t]
|
|
189
|
+
n_peers_right = n_right_total - own
|
|
190
|
+
key = (own, n_peers_right)
|
|
191
|
+
c = counts.setdefault(key, [0, 0])
|
|
192
|
+
c[0] += agents[a][r_next]
|
|
193
|
+
c[1] += 1
|
|
194
|
+
return {
|
|
195
|
+
f"own={own},peers_right={pr}": {"p_next": n_ok / tot, "n": tot}
|
|
196
|
+
for (own, pr), (n_ok, tot) in sorted(counts.items())
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
|
|
200
|
+
def excess_over_null(res: dict, null_res: dict) -> dict:
|
|
201
|
+
"""Compute excess adjacent transition rates and excess delta_rho_debate over a no-communication null log."""
|
|
202
|
+
p, np_ = res["point"], null_res["point"]
|
|
203
|
+
def _diff(k):
|
|
204
|
+
a, b = p.get(k), np_.get(k)
|
|
205
|
+
return None if (a is None or b is None) else (a - b)
|
|
206
|
+
return {
|
|
207
|
+
"excess_adjacent_tau_ic": _diff("adjacent_tau_ic"),
|
|
208
|
+
"excess_adjacent_tau_ci": _diff("adjacent_tau_ci"),
|
|
209
|
+
"excess_delta_rho_debate": _diff("delta_rho_debate"),
|
|
210
|
+
"null_adjacent_p_star_hat": np_.get("adjacent_p_star_hat"),
|
|
211
|
+
"null_p1": np_.get("p1"),
|
|
212
|
+
}
|