pace-eval 0.2.2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (83) hide show
  1. cme/__init__.py +11 -0
  2. cme/__main__.py +5 -0
  3. cme/analysis.py +223 -0
  4. cme/audit.py +145 -0
  5. cme/baseline.py +162 -0
  6. cme/cli.py +279 -0
  7. cme/client.py +165 -0
  8. cme/dynamics.py +212 -0
  9. cme/examples/crewai_integration.py +16 -0
  10. cme/examples/demo_pool.jsonl +360 -0
  11. cme/examples/demo_run.jsonl +480 -0
  12. cme/examples/langgraph_integration.py +27 -0
  13. cme/examples/make_demo.py +37 -0
  14. cme/examples/openai_quickstart.py +18 -0
  15. cme/experiment.py +555 -0
  16. cme/frontier.py +143 -0
  17. cme/hidden_profile.py +135 -0
  18. cme/proxy.py +106 -0
  19. cme/rebuttal.py +76 -0
  20. cme/report.py +197 -0
  21. cme/rollout_card.py +137 -0
  22. cme/tasks.py +266 -0
  23. cme/verified_debate.py +127 -0
  24. cme/verifier.py +92 -0
  25. mae_eval/__init__.py +11 -0
  26. mae_eval/__main__.py +5 -0
  27. mae_eval/auditor.py +112 -0
  28. mae_eval/classifier.py +62 -0
  29. mae_eval/cli.py +210 -0
  30. mae_eval/tracker.py +162 -0
  31. mae_eval/vector.py +91 -0
  32. pace/__init__.py +11 -0
  33. pace/__main__.py +5 -0
  34. pace/analysis.py +129 -0
  35. pace/audit.py +145 -0
  36. pace/baseline.py +162 -0
  37. pace/card.py +205 -0
  38. pace/cli.py +898 -0
  39. pace/client.py +165 -0
  40. pace/dynamics.py +372 -0
  41. pace/escape_benchmarks.py +162 -0
  42. pace/examples/crewai_integration.py +16 -0
  43. pace/examples/demo_pool.jsonl +360 -0
  44. pace/examples/demo_run.jsonl +480 -0
  45. pace/examples/langgraph_integration.py +27 -0
  46. pace/examples/make_demo.py +37 -0
  47. pace/examples/openai_quickstart.py +18 -0
  48. pace/experiment.py +555 -0
  49. pace/frontier.py +114 -0
  50. pace/hidden_profile.py +301 -0
  51. pace/importers.py +219 -0
  52. pace/inflate.py +213 -0
  53. pace/influence.py +231 -0
  54. pace/meta_engine.py +246 -0
  55. pace/middleware.py +230 -0
  56. pace/open_ended.py +114 -0
  57. pace/plotting.py +132 -0
  58. pace/pool_sizer.py +154 -0
  59. pace/protocol_v3.py +414 -0
  60. pace/proxy.py +106 -0
  61. pace/rebuttal.py +219 -0
  62. pace/report.py +197 -0
  63. pace/rollout_card.py +137 -0
  64. pace/schema.py +74 -0
  65. pace/simulator.py +199 -0
  66. pace/studio.html +1040 -0
  67. pace/tasks.py +266 -0
  68. pace/verified_debate.py +127 -0
  69. pace/verifier.py +92 -0
  70. pace/workbench.html +26 -0
  71. pace/workbench.py +119 -0
  72. pace_eval-0.2.2.dist-info/METADATA +205 -0
  73. pace_eval-0.2.2.dist-info/RECORD +83 -0
  74. pace_eval-0.2.2.dist-info/WHEEL +5 -0
  75. pace_eval-0.2.2.dist-info/entry_points.txt +4 -0
  76. pace_eval-0.2.2.dist-info/licenses/LICENSE +202 -0
  77. pace_eval-0.2.2.dist-info/top_level.txt +4 -0
  78. pace_lab/__init__.py +9 -0
  79. pace_lab/clients.py +568 -0
  80. pace_lab/concurrency_test.py +127 -0
  81. pace_lab/data.py +445 -0
  82. pace_lab/graders.py +626 -0
  83. pace_lab/strategies.py +345 -0
cme/cli.py ADDED
@@ -0,0 +1,279 @@
1
+ """cme command line: proxy | baseline | report."""
2
+ from __future__ import annotations
3
+
4
+ import argparse
5
+ import json
6
+ import sys
7
+ from typing import Optional, Sequence
8
+
9
+
10
+ def _proxy(a) -> int:
11
+ from .proxy import serve
12
+ srv = serve(a.upstream, a.log, a.port, a.host, a.price_in, a.price_out, a.api_key, a.item_file, a.system_label)
13
+ print("cme proxy on http://%s:%d/v1 -> %s ; logging to %s (Ctrl+C to stop)" % (a.host, a.port, a.upstream, a.log), file=sys.stderr)
14
+ try:
15
+ srv.serve_forever()
16
+ except KeyboardInterrupt:
17
+ pass
18
+ return 0
19
+
20
+
21
+ def _baseline(a) -> int:
22
+ from .baseline import OpenAICompatClient, load_bench, ollama_digest, run_baseline
23
+ items = load_bench(a.bench, a.max_items)
24
+ digest = ollama_digest(a.model) if "11434" in a.base_url else None
25
+ res = run_baseline(OpenAICompatClient(a.base_url, a.api_key), a.model, items, a.out, a.k, a.temperature, a.seed, a.max_tokens, digest,
26
+ progress=(lambda m: print(m, file=sys.stderr)) if a.verbose else None)
27
+ print(json.dumps(res))
28
+ return 0 if res["errors"] == 0 else 1
29
+
30
+
31
+ def _report(a) -> int:
32
+ from .report import build_report, plot, to_latex
33
+ try:
34
+ rep = build_report(a.pool, a.logs, a.system, a.price_in, a.price_out, a.n_boot, a.perms, a.seed, a.k)
35
+ except (OSError, ValueError) as e:
36
+ print("error: %s" % e, file=sys.stderr)
37
+ return 1
38
+ g = rep["frontier_gap"]
39
+ print("items %d | pool k=%d | system accuracy %.3f at %.0f tokens/item | frontier %.3f (%s) | gap %+.3f [%+.3f, %+.3f]" % (
40
+ rep["n_items"], rep["pool_k"], g["system_accuracy"], g["system_mean_tokens"], g["frontier_at_system_tokens"], g["range_status"],
41
+ g["gap"], g["ci_low"], g["ci_high"]))
42
+ rc = rep["resource_card"]
43
+ if rc:
44
+ print("MAS/SAS ratios: " + ", ".join("%s=%s" % (k, "n/a" if v is None else "%.3g" % v) for k, v in rc["ratios_mas_over_sas"].items()))
45
+ if a.json:
46
+ json.dump(rep, open(a.json, "w", encoding="utf-8"), indent=2, default=str)
47
+ if a.latex:
48
+ open(a.latex, "w", encoding="utf-8").write(to_latex(rep))
49
+ if a.plot:
50
+ plot(rep, a.plot)
51
+ if a.export_latex or a.export_markdown:
52
+ from .rebuttal import row_from_report, to_latex as rb_tex, to_markdown
53
+ rows = [row_from_report(rep, a.benchmark, a.model_label)]
54
+ if a.export_latex:
55
+ open(a.export_latex, "w", encoding="utf-8").write(rb_tex(rows))
56
+ if a.export_markdown:
57
+ open(a.export_markdown, "w", encoding="utf-8").write(to_markdown(rows))
58
+ if a.export_rollout_card:
59
+ from .rollout_card import write_block
60
+ for path in write_block(rep, a.export_rollout_card, a.run_id):
61
+ print("wrote " + path)
62
+ return 0
63
+
64
+
65
+ def _classify(a) -> int:
66
+ from mae_eval.classifier import classify_with_reason
67
+ try:
68
+ cfg = json.load(open(a.config, encoding="utf-8"))
69
+ rung, rule, reason = classify_with_reason(cfg)
70
+ except (OSError, ValueError, TypeError) as e:
71
+ print("error: %s" % e, file=sys.stderr)
72
+ return 1
73
+ print(json.dumps({"rung": rung, "rule": rule, "reason": reason}))
74
+ return 0
75
+
76
+
77
+ def _audit(a) -> int:
78
+ from .audit import audit, render
79
+ try:
80
+ card = audit(a.input, a.baseline_config, a.tolerance, a.price_in, a.price_out)
81
+ except (OSError, ValueError) as e:
82
+ print("error: %s" % e, file=sys.stderr)
83
+ return 1
84
+ if a.json:
85
+ print(json.dumps(card, indent=2, default=lambda x: None if x != x else str(x)))
86
+ else:
87
+ text = render(card)
88
+ try:
89
+ text.encode(sys.stdout.encoding or "utf-8")
90
+ except (UnicodeEncodeError, LookupError): # a console without box-drawing characters (for example a Windows code page): fall back to ASCII
91
+ text = render(card, ascii_only=True)
92
+ print(text)
93
+ return 0 if card["ok"] or not a.strict else 2
94
+
95
+
96
+ def _dynamics(a) -> int:
97
+ from .dynamics import analyse, excess_over_null, render
98
+ try:
99
+ res = analyse(a.log, a.n_boot, a.seed, a.first_round, a.last_round)
100
+ if getattr(a, "null", None):
101
+ null_res = analyse(a.null, a.n_boot, a.seed, a.first_round, a.last_round)
102
+ res["excess_over_null"] = excess_over_null(res, null_res)
103
+ except (OSError, ValueError) as e:
104
+ print("error: %s" % e, file=sys.stderr)
105
+ return 1
106
+ if a.json:
107
+ print(json.dumps(res, indent=2, default=str))
108
+ else:
109
+ print(render(res))
110
+ return 0
111
+
112
+
113
+ def _hidden_profile(a) -> int:
114
+ from .hidden_profile import generate, valid, validate
115
+ try:
116
+ items = generate(a.n, a.agents, a.criteria, a.split, a.margin, a.seed, a.format)
117
+ except ValueError as e:
118
+ print("error: %s" % e, file=sys.stderr)
119
+ return 1
120
+ counts = validate(items)
121
+ msg = "generated %d items (agents %d, criteria %d, split %d, margin %d, seed %d, format %s); decoy leads a view in violation: %d, target not leading pooled: %d, uncovered split criteria: %d" % (
122
+ counts["items"], a.agents, a.criteria, a.split, a.margin, a.seed, a.format, counts["decoy_not_leading_a_view"], counts["target_not_leading_pooled"], counts["uncovered_split_criteria"])
123
+ print(msg, file=sys.stderr)
124
+ if not valid(counts):
125
+ print("error: validity check failed; no output written", file=sys.stderr)
126
+ return 1
127
+ text = chr(10).join(json.dumps(it) for it in items) + chr(10)
128
+ if a.out:
129
+ with open(a.out, "w", encoding="utf-8") as fh:
130
+ fh.write(text)
131
+ else:
132
+ sys.stdout.write(text)
133
+ return 0
134
+
135
+
136
+ def _frontier(a) -> int:
137
+ from .report import frontier_report, plot_frontier, system_points
138
+ try:
139
+ fr = frontier_report(a.samples, a.k, a.perms, a.seed)
140
+ pts = system_points(a.logs) if a.logs else []
141
+ except (OSError, ValueError, KeyError) as e:
142
+ print("error: %s" % e, file=sys.stderr)
143
+ return 1
144
+ print("items %d | pool k=%d | greedy accuracy %s" % (fr["n_items"], fr["pool_k"], "n/a" if fr["greedy_accuracy"] is None else "%.3f" % fr["greedy_accuracy"]))
145
+ print("k mean tokens SC accuracy isotonic")
146
+ for i, (t, x, y) in enumerate(zip(fr["tokens"], fr["sc_accuracy"], fr["isotonic"]), 1):
147
+ print("%-2d %11.0f %11.3f %8.3f" % (i, t, x, y))
148
+ for p in pts:
149
+ print("system %-12s accuracy %.3f at %.0f tokens/item (n=%d)" % (p["system"], p["accuracy"], p["mean_tokens"], p["n_items"]))
150
+ if a.json:
151
+ json.dump({"frontier": fr, "systems": pts}, open(a.json, "w", encoding="utf-8"), indent=2)
152
+ if a.out:
153
+ plot_frontier(fr, pts, a.out)
154
+ return 0
155
+
156
+
157
+ def _rebuttal(a) -> int:
158
+ from .rebuttal import rows_from_contrasts, to_latex as rb_tex, to_markdown
159
+ try:
160
+ rows = rows_from_contrasts(a.contrasts, a.study)
161
+ except (OSError, KeyError, ValueError) as e:
162
+ print("error: %s" % e, file=sys.stderr)
163
+ return 1
164
+ if not rows:
165
+ print("error: no rows", file=sys.stderr)
166
+ return 1
167
+ if a.export_latex:
168
+ open(a.export_latex, "w", encoding="utf-8").write(rb_tex(rows))
169
+ if a.export_markdown:
170
+ open(a.export_markdown, "w", encoding="utf-8").write(to_markdown(rows))
171
+ if not (a.export_latex or a.export_markdown):
172
+ print(to_markdown(rows))
173
+ return 0
174
+
175
+
176
+ def build_parser() -> argparse.ArgumentParser:
177
+ p = argparse.ArgumentParser(prog="cme", description="Compute-matched evaluation: measure systems against the self-consistency frontier.")
178
+ sub = p.add_subparsers(dest="cmd", required=True)
179
+ x = sub.add_parser("proxy", help="OpenAI-compatible logging proxy")
180
+ x.add_argument("--upstream", default="http://localhost:11434/v1")
181
+ x.add_argument("--port", type=int, default=8089)
182
+ x.add_argument("--host", default="127.0.0.1")
183
+ x.add_argument("--log", default="run.jsonl")
184
+ x.add_argument("--price-in", type=float, default=None, help="USD per million input tokens (never bundled)")
185
+ x.add_argument("--price-out", type=float, default=None)
186
+ x.add_argument("--api-key", default=None)
187
+ x.add_argument("--system-label", default="mas", help="value of the log's system field when a request has no X-CME-System header")
188
+ x.add_argument("--item-file", default=None, help="file whose first line is the current item id, used when a request has no X-CME-Item header (serial runs of third-party code)")
189
+ x.set_defaults(func=_proxy)
190
+ b = sub.add_parser("baseline", help="greedy + k-sample pool, cached and resumable")
191
+ b.add_argument("--model", required=True)
192
+ b.add_argument("--bench", required=True, help="path to JSONL {id, question, answer}, or 'toy' (synthetic smoke test)")
193
+ b.add_argument("--base-url", default="http://localhost:11434/v1")
194
+ b.add_argument("--api-key", default="none")
195
+ b.add_argument("--k", type=int, default=16)
196
+ b.add_argument("--temperature", type=float, default=0.7)
197
+ b.add_argument("--seed", type=int, default=0)
198
+ b.add_argument("--max-tokens", type=int, default=2048)
199
+ b.add_argument("--max-items", type=int, default=None)
200
+ b.add_argument("--out", default="baseline_cache.jsonl")
201
+ b.add_argument("--verbose", action="store_true")
202
+ b.set_defaults(func=_baseline)
203
+ r = sub.add_parser("report", help="frontier gap, resource card, LaTeX table and figure")
204
+ r.add_argument("--pool", required=True, help="baseline cache from `cme baseline`")
205
+ r.add_argument("--logs", required=True, help="system log (calls and results)")
206
+ r.add_argument("--system", default="mas")
207
+ r.add_argument("--price-in", type=float, default=None)
208
+ r.add_argument("--price-out", type=float, default=None)
209
+ r.add_argument("--k", type=int, default=None, help="use the first k samples per item")
210
+ r.add_argument("--n-boot", type=int, default=2000)
211
+ r.add_argument("--perms", type=int, default=100)
212
+ r.add_argument("--seed", type=int, default=0)
213
+ r.add_argument("--json", default=None)
214
+ r.add_argument("--latex", default=None)
215
+ r.add_argument("--plot", default=None, help="vector figure path (.pdf)")
216
+ r.add_argument("--export-latex", default=None, help="write the nine-column rebuttal table for this run (LaTeX)")
217
+ r.add_argument("--export-markdown", default=None, help="write the nine-column rebuttal table for this run (Markdown)")
218
+ r.add_argument("--benchmark", default="n/a", help="benchmark label for the exported row")
219
+ r.add_argument("--model-label", default="n/a", help="model label for the exported row")
220
+ r.add_argument("--export-rollout-card", default=None, metavar="PREFIX", help="write PREFIX.json, PREFIX.yaml and PREFIX.tex: a CME compute block for a rollout card (arXiv:2605.12131)")
221
+ r.add_argument("--run-id", default="run", help="target id used in the rollout-card annotation rows")
222
+ r.set_defaults(func=_report)
223
+ c = sub.add_parser("classify", help="baseline rung (A to D) of a single-agent configuration given as JSON")
224
+ c.add_argument("--config", required=True, help="JSON file; keys are listed in code/mae_eval/classifier.py")
225
+ c.set_defaults(func=_classify)
226
+ u = sub.add_parser("audit", help="one-screen diagnostic card for a run log (mas and sas calls) or a baseline configuration (JSON)")
227
+ u.add_argument("input", help="JSONL run log with 'mas' and 'sas' calls, or a JSON baseline configuration")
228
+ u.add_argument("--baseline-config", default=None, help="JSON configuration of the single-agent baseline, to classify its rung when INPUT is a log")
229
+ u.add_argument("--tolerance", type=float, default=0.05)
230
+ u.add_argument("--price-in", type=float, default=None)
231
+ u.add_argument("--price-out", type=float, default=None)
232
+ u.add_argument("--json", action="store_true")
233
+ u.add_argument("--strict", action="store_true", help="exit with status 2 when the badge is a warning")
234
+ u.set_defaults(func=_audit)
235
+ d = sub.add_parser("dynamics", help="flip counts, tau rates, implied threshold, peer correlation, K_eff and tokens by round from a debate log")
236
+ d.add_argument("log", help="JSONL with item, agent, round, correct (and optionally prompt_tokens, completion_tokens) per agent answer")
237
+ d.add_argument("--null", default=None, help="optional no-communication control JSONL log to compute excess transition rates and excess peer correlation")
238
+ d.add_argument("--n-boot", type=int, default=1000)
239
+ d.add_argument("--seed", type=int, default=0)
240
+ d.add_argument("--first-round", type=int, default=None)
241
+ d.add_argument("--last-round", type=int, default=None)
242
+ d.add_argument("--json", action="store_true")
243
+ d.set_defaults(func=_dynamics)
244
+ h = sub.add_parser("hidden-profile", help="generate synthetic split-evidence (hidden-profile) items and check their validity")
245
+ h.add_argument("--n", type=int, default=100, help="number of items")
246
+ h.add_argument("--agents", type=int, default=5)
247
+ h.add_argument("--criteria", type=int, default=10, help="total number of criteria")
248
+ h.add_argument("--split", type=int, default=6, help="number of criteria divided among the agents (the rest are shared)")
249
+ h.add_argument("--margin", type=int, default=3, help="score gap per criterion between the favoured and the disfavoured candidate (at least 2)")
250
+ h.add_argument("--seed", type=int, default=42)
251
+ h.add_argument("--format", choices=["text", "numeric"], default="text")
252
+ h.add_argument("--out", default=None, help="output JSONL (default: standard output)")
253
+ h.set_defaults(func=_hidden_profile)
254
+ f = sub.add_parser("frontier", help="self-consistency frontier of a sample pool, optionally with system points from a run log")
255
+ f.add_argument("--samples", required=True, help="baseline cache from `cme baseline` (greedy + sampled completions per item)")
256
+ f.add_argument("--logs", default=None, help="run log; every value of its 'system' field is drawn as a point")
257
+ f.add_argument("--k", type=int, default=None)
258
+ f.add_argument("--perms", type=int, default=100)
259
+ f.add_argument("--seed", type=int, default=0)
260
+ f.add_argument("--json", default=None)
261
+ f.add_argument("--out", default=None, help="figure path (.pdf vector, or .png)")
262
+ f.set_defaults(func=_frontier)
263
+ t = sub.add_parser("rebuttal", help="nine-column rebuttal table from a contrasts CSV (default: outputs/baseline_tier_contrasts.csv)")
264
+ t.add_argument("--contrasts", "--data", dest="contrasts", default="outputs/baseline_tier_contrasts.csv",
265
+ help="contrasts CSV with columns study, category (rung), baseline, key 'Study|bench|model|arm', p_base, p_mas")
266
+ t.add_argument("--study", default=None)
267
+ t.add_argument("--export-latex", default=None)
268
+ t.add_argument("--export-markdown", default=None)
269
+ t.set_defaults(func=_rebuttal)
270
+ return p
271
+
272
+
273
+ def main(argv: Optional[Sequence[str]] = None) -> int:
274
+ a = build_parser().parse_args(argv)
275
+ return a.func(a)
276
+
277
+
278
+ if __name__ == "__main__":
279
+ sys.exit(main())
cme/client.py ADDED
@@ -0,0 +1,165 @@
1
+ """cme.wrap: log the token usage of an existing client without a proxy process (standard library only).
2
+
3
+ import cme
4
+ from openai import OpenAI
5
+ client = cme.wrap(OpenAI(), log_path="cme_trace.jsonl")
6
+ with cme.track(item_id="item_042", system="mas", agent="agent_1", round=2):
7
+ response = client.chat.completions.create(model=..., messages=...)
8
+
9
+ The wrapper forwards every attribute to the wrapped client and intercepts only the calls that return token usage: `chat.completions.create`,
10
+ `completions.create` (OpenAI-style clients, sync or async) and a top-level `completion` (a litellm-style module or client). Each successful call
11
+ appends one JSON line in the same format as `cme proxy`, so `cme report --logs` and `mae-eval audit` read it. The response is returned unchanged.
12
+
13
+ Limits (v0.1): streaming calls raise ValueError, because their usage is not known when the call returns and cme never estimates tokens; a response
14
+ without a usage block is logged with `usage_missing: true`; prices are never bundled (pass price_in/price_out in USD per million tokens, else cost is null).
15
+ """
16
+ from __future__ import annotations
17
+
18
+ import contextlib
19
+ import contextvars
20
+ import inspect
21
+ import json
22
+ import threading
23
+ import time
24
+ from typing import Any, Dict, Optional
25
+
26
+ _LABELS: contextvars.ContextVar = contextvars.ContextVar("cme_labels", default={})
27
+ _LOCKS: Dict[str, threading.Lock] = {}
28
+ _LOCKS_GUARD = threading.Lock()
29
+ _LOGGED = {("chat", "completions", "create"), ("completions", "create"), ("completion",)}
30
+
31
+
32
+ @contextlib.contextmanager
33
+ def track(item_id: Optional[str] = None, system: Optional[str] = None, agent: Optional[str] = None, round: Optional[int] = None, **extra):
34
+ """Label every wrapped call made inside the block. Blocks nest; an inner label overrides the outer one. Unset labels keep the outer value."""
35
+ labels = dict(_LABELS.get())
36
+ for key, val in (("item", item_id), ("system", system), ("agent_id", agent), ("round", round)):
37
+ if val is not None:
38
+ labels[key] = val
39
+ labels.update(extra)
40
+ token = _LABELS.set(labels)
41
+ try:
42
+ yield labels
43
+ finally:
44
+ _LABELS.reset(token)
45
+
46
+
47
+ def _get(obj: Any, name: str):
48
+ if obj is None:
49
+ return None
50
+ if isinstance(obj, dict):
51
+ return obj.get(name)
52
+ return getattr(obj, name, None)
53
+
54
+
55
+ def _record(kwargs: dict, resp: Any, t0: float, t1: float, price_in: Optional[float], price_out: Optional[float]) -> dict:
56
+ usage = _get(resp, "usage")
57
+ pt, ct = _get(usage, "prompt_tokens"), _get(usage, "completion_tokens")
58
+ det = _get(usage, "completion_tokens_details")
59
+ cost = None
60
+ if None not in (price_in, price_out, pt, ct):
61
+ cost = (pt * price_in + ct * price_out) / 1e6
62
+ labels = _LABELS.get()
63
+ choices = _get(resp, "choices") or []
64
+ rec = {"system": labels.get("system", "mas"), "agent_id": labels.get("agent_id", "agent"), "round": int(labels.get("round", 0)),
65
+ "item": str(labels.get("item", "")), "model": _get(resp, "model") or kwargs.get("model"), "prompt_tokens": pt, "completion_tokens": ct,
66
+ "reasoning_tokens": _get(det, "reasoning_tokens"), "n_completions": len(choices) or 1, "start": t0, "end": t1, "latency": t1 - t0, "cost": cost}
67
+ for key, val in labels.items():
68
+ if key not in ("system", "agent_id", "round", "item"):
69
+ rec[key] = val
70
+ if pt is None or ct is None:
71
+ rec["usage_missing"] = True # logged, never guessed
72
+ return rec
73
+
74
+
75
+ class _Logger:
76
+ def __init__(self, log_path: str, price_in: Optional[float], price_out: Optional[float]):
77
+ self.path, self.price_in, self.price_out = str(log_path), price_in, price_out
78
+ with _LOCKS_GUARD:
79
+ self.lock = _LOCKS.setdefault(self.path, threading.Lock())
80
+
81
+ def write(self, kwargs: dict, resp: Any, t0: float, t1: float) -> None:
82
+ line = json.dumps(_record(kwargs, resp, t0, t1, self.price_in, self.price_out), default=str)
83
+ with self.lock, open(self.path, "a", encoding="utf-8") as fh:
84
+ fh.write(line + "\n")
85
+
86
+
87
+ class _Wrapped:
88
+ """Transparent proxy over a client (or one of its sub-objects); only the usage-returning calls are intercepted."""
89
+
90
+ def __init__(self, target: Any, logger: _Logger, path: tuple = ()):
91
+ object.__setattr__(self, "_cme_target", target)
92
+ object.__setattr__(self, "_cme_logger", logger)
93
+ object.__setattr__(self, "_cme_path", path)
94
+
95
+ def __getattr__(self, name: str):
96
+ target = object.__getattribute__(self, "_cme_target")
97
+ logger = object.__getattribute__(self, "_cme_logger")
98
+ path = object.__getattribute__(self, "_cme_path") + (name,)
99
+ attr = getattr(target, name)
100
+ if path in _LOGGED and callable(attr):
101
+ return _intercept(attr, logger)
102
+ if callable(attr) or name.startswith("_") or isinstance(attr, (str, int, float, bool, bytes, type(None))):
103
+ return attr
104
+ if any(p[:len(path)] == path for p in _LOGGED):
105
+ return _Wrapped(attr, logger, path)
106
+ return attr
107
+
108
+ def __setattr__(self, name, value):
109
+ setattr(object.__getattribute__(self, "_cme_target"), name, value)
110
+
111
+ def __repr__(self):
112
+ return "cme.wrap(%r)" % (object.__getattribute__(self, "_cme_target"),)
113
+
114
+ def __enter__(self):
115
+ target = object.__getattribute__(self, "_cme_target")
116
+ if hasattr(target, "__enter__"):
117
+ target.__enter__()
118
+ return self
119
+
120
+ def __exit__(self, *exc):
121
+ target = object.__getattribute__(self, "_cme_target")
122
+ return target.__exit__(*exc) if hasattr(target, "__exit__") else None
123
+
124
+
125
+ def _intercept(fn, logger: _Logger):
126
+ def guard(kwargs: dict) -> None:
127
+ if kwargs.get("stream"):
128
+ raise ValueError("cme.wrap v0.1 does not support streaming calls: usage is not known when the call returns. Use stream=False.")
129
+
130
+ if inspect.iscoroutinefunction(fn):
131
+ async def acall(*args, **kwargs):
132
+ guard(kwargs)
133
+ t0 = time.time()
134
+ resp = await fn(*args, **kwargs)
135
+ logger.write(kwargs, resp, t0, time.time())
136
+ return resp
137
+ return acall
138
+
139
+ def call(*args, **kwargs):
140
+ guard(kwargs)
141
+ t0 = time.time()
142
+ resp = fn(*args, **kwargs)
143
+ if inspect.isawaitable(resp): # an async client whose method is not a plain coroutine function
144
+ async def finish():
145
+ out = await resp
146
+ logger.write(kwargs, out, t0, time.time())
147
+ return out
148
+ return finish()
149
+ logger.write(kwargs, resp, t0, time.time())
150
+ return resp
151
+ return call
152
+
153
+
154
+ def wrap(client: Any, log_path: str = "cme_trace.jsonl", price_in: Optional[float] = None, price_out: Optional[float] = None) -> _Wrapped:
155
+ """Return `client` with usage logging on its completion calls. Prices are USD per million tokens; omit them and cost is null."""
156
+ return _Wrapped(client, _Logger(log_path, price_in, price_out))
157
+
158
+
159
+ def log_result(log_path: str, item_id: str, correct: bool, system: Optional[str] = None) -> None:
160
+ """Append the correctness of one item's final answer (the line `cme report` pairs with the logged calls). Call it once per item, after scoring."""
161
+ sysname = system if system is not None else _LABELS.get().get("system", "mas")
162
+ with _LOCKS_GUARD:
163
+ lock = _LOCKS.setdefault(str(log_path), threading.Lock())
164
+ with lock, open(str(log_path), "a", encoding="utf-8") as fh:
165
+ fh.write(json.dumps({"system": sysname, "item": str(item_id), "correct": bool(correct)}) + "\n")
cme/dynamics.py ADDED
@@ -0,0 +1,212 @@
1
+ """cme dynamics: how agents' answers change across debate rounds (standard library plus numpy).
2
+
3
+ Input: a JSONL file with one record per agent answer, {"item": str, "agent": str or int, "round": int, "correct": bool} and optionally "prompt_tokens" and
4
+ "completion_tokens". Nothing is estimated for missing fields: records without "correct" are ignored, and token statistics need both token fields.
5
+
6
+ Output (point estimates and 95% percentile intervals from a paired bootstrap that resamples items):
7
+ flips between the first and the last round: wrong to right, right to wrong, right to right, wrong to wrong (counts, over item-agent pairs observed in both rounds);
8
+ tau_ic = wrong-to-right / (all wrong in the first round) and tau_ci = right-to-wrong / (all right in the first round), the two rates of the conformity-to-truth model;
9
+ p_star_hat = tau_ic / (tau_ic + tau_ci), the threshold implied by those rates (undefined if both rates are zero);
10
+ p1 and p_last, the mean accuracy of an agent in the first and the last round;
11
+ rho_total, the intraclass correlation of correctness across the agents of an item in each round (reported in the JSON also as rho_peer, its earlier name), and K_eff = K / (1 + (K - 1) rho_total)
12
+ with rho_total clipped to [0, 1]; delta_rho_debate = rho_total(last round) - rho_total(first round), the excess correlation induced by debate, with its own bootstrap interval;
13
+ mean tokens per call by round.
14
+ Limits: rho_total is the correlation of correctness among agents on the same item, so it includes shared item difficulty as well as correlated errors; the difference across rounds is the
15
+ quantity that isolates what the exchange of messages adds, and it is not a measure of error correlation between models. The tau
16
+ rates assume the constant-rate model of the paper; this command reports what the logs show under that model and does not test the model.
17
+ """
18
+ from __future__ import annotations
19
+
20
+ import json
21
+ from collections import defaultdict
22
+ from typing import Dict, List, Optional
23
+
24
+ import numpy as np
25
+
26
+
27
+ def load(path: str):
28
+ """Return {item: {agent: {round: correct}}} and {item: {round: [tokens per call]}}."""
29
+ ans: Dict[str, Dict[str, Dict[int, int]]] = defaultdict(lambda: defaultdict(dict))
30
+ tok: Dict[str, Dict[int, List[float]]] = defaultdict(lambda: defaultdict(list))
31
+ with open(path, encoding="utf-8") as fh:
32
+ for no, line in enumerate(fh, 1):
33
+ if not line.strip():
34
+ continue
35
+ r = json.loads(line)
36
+ if not isinstance(r, dict) or "correct" not in r or "round" not in r or "item" not in r or "agent" not in r:
37
+ continue
38
+ ans[str(r["item"])][str(r["agent"])][int(r["round"])] = int(bool(r["correct"]))
39
+ if r.get("prompt_tokens") is not None and r.get("completion_tokens") is not None:
40
+ tok[str(r["item"])][int(r["round"])].append(float(r["prompt_tokens"]) + float(r["completion_tokens"]))
41
+ if not ans:
42
+ raise ValueError("no usable records: each line needs item, agent, round and correct")
43
+ return ans, tok
44
+
45
+
46
+ def icc(Z: np.ndarray) -> Optional[float]:
47
+ """One-way intraclass correlation of an items x agents 0/1 matrix (balanced)."""
48
+ n, k = Z.shape
49
+ if n < 2 or k < 2:
50
+ return None
51
+ m = Z.mean(axis=1)
52
+ g = Z.mean()
53
+ msb = k * np.sum((m - g) ** 2) / (n - 1)
54
+ msw = np.sum((Z - m[:, None]) ** 2) / (n * (k - 1))
55
+ den = msb + (k - 1) * msw
56
+ return None if den == 0 else float((msb - msw) / den)
57
+
58
+
59
+ def stats(ans, items: List[str], r0: int, r1: int, rounds: List[int]) -> dict:
60
+ wr = rw = rr = ww = 0
61
+ a0 = []
62
+ a1 = []
63
+ for it in items:
64
+ for ag, d in ans[it].items():
65
+ if r0 in d and r1 in d:
66
+ a, b = d[r0], d[r1]
67
+ a0.append(a)
68
+ a1.append(b)
69
+ wr += (a == 0 and b == 1)
70
+ rw += (a == 1 and b == 0)
71
+ rr += (a == 1 and b == 1)
72
+ ww += (a == 0 and b == 0)
73
+ out = dict(wrong_to_right=wr, right_to_wrong=rw, right_to_right=rr, wrong_to_wrong=ww)
74
+ out["tau_ic"] = wr / (wr + ww) if wr + ww else None
75
+ out["tau_ci"] = rw / (rw + rr) if rw + rr else None
76
+ ti, tc = out["tau_ic"], out["tau_ci"]
77
+ out["p_star_hat"] = ti / (ti + tc) if ti is not None and tc is not None and ti + tc > 0 else None
78
+ adj_wr = adj_rw = adj_rr = adj_ww = 0
79
+ active_rounds = [r for r in rounds if r0 <= r <= r1]
80
+ for it in items:
81
+ for ag, d in ans[it].items():
82
+ for r_a, r_b in zip(active_rounds[:-1], active_rounds[1:]):
83
+ if r_a in d and r_b in d:
84
+ a, b = d[r_a], d[r_b]
85
+ adj_wr += (a == 0 and b == 1)
86
+ adj_rw += (a == 1 and b == 0)
87
+ adj_rr += (a == 1 and b == 1)
88
+ adj_ww += (a == 0 and b == 0)
89
+ out["adjacent_wrong_to_right"] = adj_wr
90
+ out["adjacent_right_to_wrong"] = adj_rw
91
+ out["adjacent_right_to_right"] = adj_rr
92
+ out["adjacent_wrong_to_wrong"] = adj_ww
93
+ out["adjacent_tau_ic"] = adj_wr / (adj_wr + adj_ww) if adj_wr + adj_ww else None
94
+ out["adjacent_tau_ci"] = adj_rw / (adj_rw + adj_rr) if adj_rw + adj_rr else None
95
+ ati, atc = out["adjacent_tau_ic"], out["adjacent_tau_ci"]
96
+ out["adjacent_p_star_hat"] = ati / (ati + atc) if ati is not None and atc is not None and ati + atc > 0 else None
97
+ out["p1"] = float(np.mean(a0)) if a0 else None
98
+ out["p_last"] = float(np.mean(a1)) if a1 else None
99
+ rho, keff = {}, {}
100
+ for r in rounds:
101
+ rows = [[d[r] for d in ans[it].values() if r in d] for it in items]
102
+ k = min((len(x) for x in rows), default=0)
103
+ Z = np.array([x[:k] for x in rows], dtype=float) if k >= 2 else None
104
+ v = icc(Z) if Z is not None else None
105
+ rho[r] = v
106
+ keff[r] = (k / (1 + (k - 1) * min(1.0, max(0.0, v)))) if v is not None else None
107
+ out["rho_peer"], out["rho_total"], out["k_eff"] = rho, rho, keff # rho_peer is the earlier name of rho_total
108
+ out["delta_rho_debate"] = (rho[r1] - rho[r0]) if rho.get(r0) is not None and rho.get(r1) is not None else None
109
+ return out
110
+
111
+
112
+ def analyse(path: str, n_boot: int = 1000, seed: int = 0, first: Optional[int] = None, last: Optional[int] = None) -> dict:
113
+ ans, tok = load(path)
114
+ items = sorted(ans)
115
+ rounds = sorted({r for it in items for d in ans[it].values() for r in d})
116
+ if len(rounds) < 2:
117
+ raise ValueError("need answers from at least two rounds; found %s" % rounds)
118
+ r0, r1 = (first if first is not None else rounds[0]), (last if last is not None else rounds[-1])
119
+ if r0 not in rounds or r1 not in rounds or r0 >= r1:
120
+ raise ValueError("--first-round and --last-round must be distinct rounds present in the log, first < last; found rounds %s" % rounds)
121
+ point = stats(ans, items, r0, r1, rounds)
122
+ rng = np.random.default_rng(seed)
123
+ boots = []
124
+ for _ in range(n_boot):
125
+ pick = [items[i] for i in rng.integers(0, len(items), len(items))]
126
+ # a resampled item can repeat: give each copy its own key so it is counted as many times as drawn
127
+ ans_b = {"%s#%d" % (it, c): ans[it] for c, it in enumerate(pick)}
128
+ boots.append(stats(ans_b, list(ans_b), r0, r1, rounds))
129
+
130
+ def ci(get):
131
+ v = [x for x in (get(b) for b in boots) if x is not None]
132
+ return [float(np.percentile(v, 2.5)), float(np.percentile(v, 97.5))] if len(v) >= max(20, n_boot // 10) else None
133
+ cis = {k: ci(lambda b, k=k: b[k]) for k in ("tau_ic", "tau_ci", "p_star_hat", "adjacent_tau_ic", "adjacent_tau_ci", "adjacent_p_star_hat", "p1", "p_last", "delta_rho_debate")}
134
+ cis["rho_peer"] = cis["rho_total"] = {r: ci(lambda b, r=r: b["rho_peer"][r]) for r in rounds}
135
+ cis["k_eff"] = {r: ci(lambda b, r=r: b["k_eff"][r]) for r in rounds}
136
+ tokens = {}
137
+ for r in rounds:
138
+ per_item = [np.mean(tok[it][r]) for it in items if tok[it].get(r)]
139
+ if per_item:
140
+ tokens[r] = float(np.mean(per_item))
141
+ return dict(n_items=len(items), rounds=rounds, first_round=r0, last_round=r1, n_boot=n_boot, point=point, ci95=cis, mean_tokens_per_call_by_round=tokens or None)
142
+
143
+
144
+ def _f(x, nd=3):
145
+ return "n/a" if x is None else ("%.*f" % (nd, x))
146
+
147
+
148
+ def render(res: dict) -> str:
149
+ p, c = res["point"], res["ci95"]
150
+
151
+ def with_ci(key, nd=3):
152
+ return "%s [%s, %s]" % (_f(p[key], nd), _f(c[key][0], nd), _f(c[key][1], nd)) if c.get(key) else _f(p[key], nd)
153
+ out = ["Debate dynamics: %d items, rounds %s (first %d to last %d), %d bootstrap resamples over items" % (res["n_items"], res["rounds"], res["first_round"], res["last_round"], res["n_boot"]),
154
+ "flips (item-agent pairs): wrong->right %d, right->wrong %d, right->right %d, wrong->wrong %d" % (p["wrong_to_right"], p["right_to_wrong"], p["right_to_right"], p["wrong_to_wrong"]),
155
+ "tau_ic (wrong -> right): " + with_ci("tau_ic"), "tau_ci (right -> wrong): " + with_ci("tau_ci"),
156
+ "p_star_hat = tau_ic / (tau_ic + tau_ci): " + with_ci("p_star_hat"),
157
+ "p1 (first round accuracy): " + with_ci("p1") + " p_last: " + with_ci("p_last"),
158
+ "delta_rho_debate = rho_total(round %d) - rho_total(round %d): %s" % (res["last_round"], res["first_round"], with_ci("delta_rho_debate"))]
159
+ for r in res["rounds"]:
160
+ rc, kc = c["rho_peer"][r], c["k_eff"][r]
161
+ out.append("round %d: rho_total %s%s, K_eff %s%s" % (r, _f(p["rho_peer"][r]), " [%s, %s]" % (_f(rc[0]), _f(rc[1])) if rc else "", _f(p["k_eff"][r], 2),
162
+ " [%s, %s]" % (_f(kc[0], 2), _f(kc[1], 2)) if kc else ""))
163
+ if res["mean_tokens_per_call_by_round"]:
164
+ out.append("mean tokens per call by round: " + ", ".join("r%d %.0f" % (r, t) for r, t in res["mean_tokens_per_call_by_round"].items()))
165
+ else:
166
+ out.append("mean tokens per call by round: n/a (no token fields in the log)")
167
+ if res.get("excess_over_null"):
168
+ ex = res["excess_over_null"]
169
+ out.append(
170
+ "excess over null: adjacent_tau_ic %s, adjacent_tau_ci %s, delta_rho_debate %s"
171
+ % (_f(ex.get("excess_adjacent_tau_ic")), _f(ex.get("excess_adjacent_tau_ci")), _f(ex.get("excess_delta_rho_debate")))
172
+ )
173
+ out.append("rho_total is the correlation of correctness across agents of an item and includes shared item difficulty; delta_rho_debate is its change from the first to the last round, in which shared difficulty largely cancels, so it estimates the correlation added by communication. The tau rates assume the constant-rate model.")
174
+ return "\n".join(out)
175
+
176
+
177
+ def peer_transition_table(ans: dict, items: list, rounds: list) -> dict:
178
+ """Compute P(correct at t+1 | own_state_t, n_correct_peers_t) across adjacent rounds."""
179
+ counts = {}
180
+ for it in items:
181
+ agents = ans[it]
182
+ for r_t, r_next in zip(rounds[:-1], rounds[1:]):
183
+ present = [a for a, d in agents.items() if r_t in d and r_next in d]
184
+ if len(present) < 2:
185
+ continue
186
+ n_right_total = sum(agents[a][r_t] for a in present)
187
+ for a in present:
188
+ own = agents[a][r_t]
189
+ n_peers_right = n_right_total - own
190
+ key = (own, n_peers_right)
191
+ c = counts.setdefault(key, [0, 0])
192
+ c[0] += agents[a][r_next]
193
+ c[1] += 1
194
+ return {
195
+ f"own={own},peers_right={pr}": {"p_next": n_ok / tot, "n": tot}
196
+ for (own, pr), (n_ok, tot) in sorted(counts.items())
197
+ }
198
+
199
+
200
+ def excess_over_null(res: dict, null_res: dict) -> dict:
201
+ """Compute excess adjacent transition rates and excess delta_rho_debate over a no-communication null log."""
202
+ p, np_ = res["point"], null_res["point"]
203
+ def _diff(k):
204
+ a, b = p.get(k), np_.get(k)
205
+ return None if (a is None or b is None) else (a - b)
206
+ return {
207
+ "excess_adjacent_tau_ic": _diff("adjacent_tau_ic"),
208
+ "excess_adjacent_tau_ci": _diff("adjacent_tau_ci"),
209
+ "excess_delta_rho_debate": _diff("delta_rho_debate"),
210
+ "null_adjacent_p_star_hat": np_.get("adjacent_p_star_hat"),
211
+ "null_p1": np_.get("p1"),
212
+ }