pace-eval 0.2.2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (83) hide show
  1. cme/__init__.py +11 -0
  2. cme/__main__.py +5 -0
  3. cme/analysis.py +223 -0
  4. cme/audit.py +145 -0
  5. cme/baseline.py +162 -0
  6. cme/cli.py +279 -0
  7. cme/client.py +165 -0
  8. cme/dynamics.py +212 -0
  9. cme/examples/crewai_integration.py +16 -0
  10. cme/examples/demo_pool.jsonl +360 -0
  11. cme/examples/demo_run.jsonl +480 -0
  12. cme/examples/langgraph_integration.py +27 -0
  13. cme/examples/make_demo.py +37 -0
  14. cme/examples/openai_quickstart.py +18 -0
  15. cme/experiment.py +555 -0
  16. cme/frontier.py +143 -0
  17. cme/hidden_profile.py +135 -0
  18. cme/proxy.py +106 -0
  19. cme/rebuttal.py +76 -0
  20. cme/report.py +197 -0
  21. cme/rollout_card.py +137 -0
  22. cme/tasks.py +266 -0
  23. cme/verified_debate.py +127 -0
  24. cme/verifier.py +92 -0
  25. mae_eval/__init__.py +11 -0
  26. mae_eval/__main__.py +5 -0
  27. mae_eval/auditor.py +112 -0
  28. mae_eval/classifier.py +62 -0
  29. mae_eval/cli.py +210 -0
  30. mae_eval/tracker.py +162 -0
  31. mae_eval/vector.py +91 -0
  32. pace/__init__.py +11 -0
  33. pace/__main__.py +5 -0
  34. pace/analysis.py +129 -0
  35. pace/audit.py +145 -0
  36. pace/baseline.py +162 -0
  37. pace/card.py +205 -0
  38. pace/cli.py +898 -0
  39. pace/client.py +165 -0
  40. pace/dynamics.py +372 -0
  41. pace/escape_benchmarks.py +162 -0
  42. pace/examples/crewai_integration.py +16 -0
  43. pace/examples/demo_pool.jsonl +360 -0
  44. pace/examples/demo_run.jsonl +480 -0
  45. pace/examples/langgraph_integration.py +27 -0
  46. pace/examples/make_demo.py +37 -0
  47. pace/examples/openai_quickstart.py +18 -0
  48. pace/experiment.py +555 -0
  49. pace/frontier.py +114 -0
  50. pace/hidden_profile.py +301 -0
  51. pace/importers.py +219 -0
  52. pace/inflate.py +213 -0
  53. pace/influence.py +231 -0
  54. pace/meta_engine.py +246 -0
  55. pace/middleware.py +230 -0
  56. pace/open_ended.py +114 -0
  57. pace/plotting.py +132 -0
  58. pace/pool_sizer.py +154 -0
  59. pace/protocol_v3.py +414 -0
  60. pace/proxy.py +106 -0
  61. pace/rebuttal.py +219 -0
  62. pace/report.py +197 -0
  63. pace/rollout_card.py +137 -0
  64. pace/schema.py +74 -0
  65. pace/simulator.py +199 -0
  66. pace/studio.html +1040 -0
  67. pace/tasks.py +266 -0
  68. pace/verified_debate.py +127 -0
  69. pace/verifier.py +92 -0
  70. pace/workbench.html +26 -0
  71. pace/workbench.py +119 -0
  72. pace_eval-0.2.2.dist-info/METADATA +205 -0
  73. pace_eval-0.2.2.dist-info/RECORD +83 -0
  74. pace_eval-0.2.2.dist-info/WHEEL +5 -0
  75. pace_eval-0.2.2.dist-info/entry_points.txt +4 -0
  76. pace_eval-0.2.2.dist-info/licenses/LICENSE +202 -0
  77. pace_eval-0.2.2.dist-info/top_level.txt +4 -0
  78. pace_lab/__init__.py +9 -0
  79. pace_lab/clients.py +568 -0
  80. pace_lab/concurrency_test.py +127 -0
  81. pace_lab/data.py +445 -0
  82. pace_lab/graders.py +626 -0
  83. pace_lab/strategies.py +345 -0
pace/client.py ADDED
@@ -0,0 +1,165 @@
1
+ """pace.wrap: log the token usage of an existing client without a proxy process (standard library only).
2
+
3
+ import pace
4
+ from openai import OpenAI
5
+ client = pace.wrap(OpenAI(), log_path="pace_trace.jsonl")
6
+ with pace.track(item_id="item_042", system="mas", agent="agent_1", round=2):
7
+ response = client.chat.completions.create(model=..., messages=...)
8
+
9
+ The wrapper forwards every attribute to the wrapped client and intercepts only the calls that return token usage: `chat.completions.create`,
10
+ `completions.create` (OpenAI-style clients, sync or async) and a top-level `completion` (a litellm-style module or client). Each successful call
11
+ appends one JSON line in the same format as `pace proxy`, so `pace report --logs` and `mae-eval audit` read it. The response is returned unchanged.
12
+
13
+ Limits (v0.1): streaming calls raise ValueError, because their usage is not known when the call returns and pace never estimates tokens; a response
14
+ without a usage block is logged with `usage_missing: true`; prices are never bundled (pass price_in/price_out in USD per million tokens, else cost is null).
15
+ """
16
+ from __future__ import annotations
17
+
18
+ import contextlib
19
+ import contextvars
20
+ import inspect
21
+ import json
22
+ import threading
23
+ import time
24
+ from typing import Any, Dict, Optional
25
+
26
+ _LABELS: contextvars.ContextVar = contextvars.ContextVar("cme_labels", default={})
27
+ _LOCKS: Dict[str, threading.Lock] = {}
28
+ _LOCKS_GUARD = threading.Lock()
29
+ _LOGGED = {("chat", "completions", "create"), ("completions", "create"), ("completion",)}
30
+
31
+
32
+ @contextlib.contextmanager
33
+ def track(item_id: Optional[str] = None, system: Optional[str] = None, agent: Optional[str] = None, round: Optional[int] = None, **extra):
34
+ """Label every wrapped call made inside the block. Blocks nest; an inner label overrides the outer one. Unset labels keep the outer value."""
35
+ labels = dict(_LABELS.get())
36
+ for key, val in (("item", item_id), ("system", system), ("agent_id", agent), ("round", round)):
37
+ if val is not None:
38
+ labels[key] = val
39
+ labels.update(extra)
40
+ token = _LABELS.set(labels)
41
+ try:
42
+ yield labels
43
+ finally:
44
+ _LABELS.reset(token)
45
+
46
+
47
+ def _get(obj: Any, name: str):
48
+ if obj is None:
49
+ return None
50
+ if isinstance(obj, dict):
51
+ return obj.get(name)
52
+ return getattr(obj, name, None)
53
+
54
+
55
+ def _record(kwargs: dict, resp: Any, t0: float, t1: float, price_in: Optional[float], price_out: Optional[float]) -> dict:
56
+ usage = _get(resp, "usage")
57
+ pt, ct = _get(usage, "prompt_tokens"), _get(usage, "completion_tokens")
58
+ det = _get(usage, "completion_tokens_details")
59
+ cost = None
60
+ if None not in (price_in, price_out, pt, ct):
61
+ cost = (pt * price_in + ct * price_out) / 1e6
62
+ labels = _LABELS.get()
63
+ choices = _get(resp, "choices") or []
64
+ rec = {"system": labels.get("system", "mas"), "agent_id": labels.get("agent_id", "agent"), "round": int(labels.get("round", 0)),
65
+ "item": str(labels.get("item", "")), "model": _get(resp, "model") or kwargs.get("model"), "prompt_tokens": pt, "completion_tokens": ct,
66
+ "reasoning_tokens": _get(det, "reasoning_tokens"), "n_completions": len(choices) or 1, "start": t0, "end": t1, "latency": t1 - t0, "cost": cost}
67
+ for key, val in labels.items():
68
+ if key not in ("system", "agent_id", "round", "item"):
69
+ rec[key] = val
70
+ if pt is None or ct is None:
71
+ rec["usage_missing"] = True # logged, never guessed
72
+ return rec
73
+
74
+
75
+ class _Logger:
76
+ def __init__(self, log_path: str, price_in: Optional[float], price_out: Optional[float]):
77
+ self.path, self.price_in, self.price_out = str(log_path), price_in, price_out
78
+ with _LOCKS_GUARD:
79
+ self.lock = _LOCKS.setdefault(self.path, threading.Lock())
80
+
81
+ def write(self, kwargs: dict, resp: Any, t0: float, t1: float) -> None:
82
+ line = json.dumps(_record(kwargs, resp, t0, t1, self.price_in, self.price_out), default=str)
83
+ with self.lock, open(self.path, "a", encoding="utf-8") as fh:
84
+ fh.write(line + "\n")
85
+
86
+
87
+ class _Wrapped:
88
+ """Transparent proxy over a client (or one of its sub-objects); only the usage-returning calls are intercepted."""
89
+
90
+ def __init__(self, target: Any, logger: _Logger, path: tuple = ()):
91
+ object.__setattr__(self, "_cme_target", target)
92
+ object.__setattr__(self, "_cme_logger", logger)
93
+ object.__setattr__(self, "_cme_path", path)
94
+
95
+ def __getattr__(self, name: str):
96
+ target = object.__getattribute__(self, "_cme_target")
97
+ logger = object.__getattribute__(self, "_cme_logger")
98
+ path = object.__getattribute__(self, "_cme_path") + (name,)
99
+ attr = getattr(target, name)
100
+ if path in _LOGGED and callable(attr):
101
+ return _intercept(attr, logger)
102
+ if callable(attr) or name.startswith("_") or isinstance(attr, (str, int, float, bool, bytes, type(None))):
103
+ return attr
104
+ if any(p[:len(path)] == path for p in _LOGGED):
105
+ return _Wrapped(attr, logger, path)
106
+ return attr
107
+
108
+ def __setattr__(self, name, value):
109
+ setattr(object.__getattribute__(self, "_cme_target"), name, value)
110
+
111
+ def __repr__(self):
112
+ return "pace.wrap(%r)" % (object.__getattribute__(self, "_cme_target"),)
113
+
114
+ def __enter__(self):
115
+ target = object.__getattribute__(self, "_cme_target")
116
+ if hasattr(target, "__enter__"):
117
+ target.__enter__()
118
+ return self
119
+
120
+ def __exit__(self, *exc):
121
+ target = object.__getattribute__(self, "_cme_target")
122
+ return target.__exit__(*exc) if hasattr(target, "__exit__") else None
123
+
124
+
125
+ def _intercept(fn, logger: _Logger):
126
+ def guard(kwargs: dict) -> None:
127
+ if kwargs.get("stream"):
128
+ raise ValueError("pace.wrap v0.1 does not support streaming calls: usage is not known when the call returns. Use stream=False.")
129
+
130
+ if inspect.iscoroutinefunction(fn):
131
+ async def acall(*args, **kwargs):
132
+ guard(kwargs)
133
+ t0 = time.time()
134
+ resp = await fn(*args, **kwargs)
135
+ logger.write(kwargs, resp, t0, time.time())
136
+ return resp
137
+ return acall
138
+
139
+ def call(*args, **kwargs):
140
+ guard(kwargs)
141
+ t0 = time.time()
142
+ resp = fn(*args, **kwargs)
143
+ if inspect.isawaitable(resp): # an async client whose method is not a plain coroutine function
144
+ async def finish():
145
+ out = await resp
146
+ logger.write(kwargs, out, t0, time.time())
147
+ return out
148
+ return finish()
149
+ logger.write(kwargs, resp, t0, time.time())
150
+ return resp
151
+ return call
152
+
153
+
154
+ def wrap(client: Any, log_path: str = "pace_trace.jsonl", price_in: Optional[float] = None, price_out: Optional[float] = None) -> _Wrapped:
155
+ """Return `client` with usage logging on its completion calls. Prices are USD per million tokens; omit them and cost is null."""
156
+ return _Wrapped(client, _Logger(log_path, price_in, price_out))
157
+
158
+
159
+ def log_result(log_path: str, item_id: str, correct: bool, system: Optional[str] = None) -> None:
160
+ """Append the correctness of one item's final answer (the line `pace report` pairs with the logged calls). Call it once per item, after scoring."""
161
+ sysname = system if system is not None else _LABELS.get().get("system", "mas")
162
+ with _LOCKS_GUARD:
163
+ lock = _LOCKS.setdefault(str(log_path), threading.Lock())
164
+ with lock, open(str(log_path), "a", encoding="utf-8") as fh:
165
+ fh.write(json.dumps({"system": sysname, "item": str(item_id), "correct": bool(correct)}) + "\n")
pace/dynamics.py ADDED
@@ -0,0 +1,372 @@
1
+ """pace dynamics: how agents' answers change across debate rounds (standard library plus numpy).
2
+
3
+ Input: a JSONL file with one record per agent answer, {"item": str, "agent": str or int, "round": int, "correct": bool} and optionally "prompt_tokens" and
4
+ "completion_tokens". Nothing is estimated for missing fields: records without "correct" are ignored, and token statistics need both token fields.
5
+
6
+ Output (point estimates and 95% percentile intervals from a paired bootstrap that resamples items):
7
+ flips between the first and the last round: wrong to right, right to wrong, right to right, wrong to wrong (counts, over item-agent pairs observed in both rounds);
8
+ tau_ic = wrong-to-right / (all wrong in the first round) and tau_ci = right-to-wrong / (all right in the first round), the two rates of the conformity-to-truth model;
9
+ p_star_hat = tau_ic / (tau_ic + tau_ci), the threshold implied by those rates (undefined if both rates are zero);
10
+ p1 and p_last, the mean accuracy of an agent in the first and the last round;
11
+ rho_total, the intraclass correlation of correctness across the agents of an item in each round (reported in the JSON also as rho_peer, its earlier name), and K_eff = K / (1 + (K - 1) rho_total)
12
+ with rho_total clipped to [0, 1]; delta_rho_debate = rho_total(last round) - rho_total(first round), the excess correlation induced by debate, with its own bootstrap interval;
13
+ mean tokens per call by round.
14
+ Limits: rho_total is the correlation of correctness among agents on the same item, so it includes shared item difficulty as well as correlated errors; the difference across rounds is the
15
+ quantity that isolates what the exchange of messages adds, and it is not a measure of error correlation between models. The tau
16
+ rates assume the constant-rate model of the paper; this command reports what the logs show under that model and does not test the model.
17
+ """
18
+
19
+ from __future__ import annotations
20
+
21
+ import json
22
+ from collections import defaultdict
23
+
24
+ import numpy as np
25
+
26
+
27
+ def load(path: str):
28
+ """Return {item: {agent: {round: correct}}} and {item: {round: [tokens per call]}}."""
29
+ ans: dict[str, dict[str, dict[int, int]]] = defaultdict(lambda: defaultdict(dict))
30
+ tok: dict[str, dict[int, list[float]]] = defaultdict(lambda: defaultdict(list))
31
+ with open(path, encoding="utf-8") as fh:
32
+ for no, line in enumerate(fh, 1):
33
+ if not line.strip():
34
+ continue
35
+ r = json.loads(line)
36
+ if (
37
+ not isinstance(r, dict)
38
+ or "correct" not in r
39
+ or "round" not in r
40
+ or "item" not in r
41
+ or "agent" not in r
42
+ ):
43
+ continue
44
+ ans[str(r["item"])][str(r["agent"])][int(r["round"])] = int(
45
+ bool(r["correct"])
46
+ )
47
+ if (
48
+ r.get("prompt_tokens") is not None
49
+ and r.get("completion_tokens") is not None
50
+ ):
51
+ tok[str(r["item"])][int(r["round"])].append(
52
+ float(r["prompt_tokens"]) + float(r["completion_tokens"])
53
+ )
54
+ if not ans:
55
+ raise ValueError(
56
+ "no usable records: each line needs item, agent, round and correct"
57
+ )
58
+ return ans, tok
59
+
60
+
61
+ def icc(Z: np.ndarray) -> float | None:
62
+ """One-way intraclass correlation of an items x agents 0/1 matrix (balanced)."""
63
+ n, k = Z.shape
64
+ if n < 2 or k < 2:
65
+ return None
66
+ m = Z.mean(axis=1)
67
+ g = Z.mean()
68
+ msb = k * np.sum((m - g) ** 2) / (n - 1)
69
+ msw = np.sum((Z - m[:, None]) ** 2) / (n * (k - 1))
70
+ den = msb + (k - 1) * msw
71
+ return None if den == 0 else float((msb - msw) / den)
72
+
73
+
74
+ def stats(ans, items: list[str], r0: int, r1: int, rounds: list[int]) -> dict:
75
+ wr = rw = rr = ww = 0
76
+ a0 = []
77
+ a1 = []
78
+ for it in items:
79
+ for ag, d in ans[it].items():
80
+ if r0 in d and r1 in d:
81
+ a, b = d[r0], d[r1]
82
+ a0.append(a)
83
+ a1.append(b)
84
+ wr += a == 0 and b == 1
85
+ rw += a == 1 and b == 0
86
+ rr += a == 1 and b == 1
87
+ ww += a == 0 and b == 0
88
+ out = dict(
89
+ wrong_to_right=wr, right_to_wrong=rw, right_to_right=rr, wrong_to_wrong=ww
90
+ )
91
+ out["tau_ic"] = wr / (wr + ww) if wr + ww else None
92
+ out["tau_ci"] = rw / (rw + rr) if rw + rr else None
93
+ ti, tc = out["tau_ic"], out["tau_ci"]
94
+ out["p_star_hat"] = (
95
+ ti / (ti + tc) if ti is not None and tc is not None and ti + tc > 0 else None
96
+ )
97
+ adj_wr = adj_rw = adj_rr = adj_ww = 0
98
+ active_rounds = [r for r in rounds if r0 <= r <= r1]
99
+ for it in items:
100
+ for ag, d in ans[it].items():
101
+ for r_a, r_b in zip(active_rounds[:-1], active_rounds[1:]):
102
+ if r_a in d and r_b in d:
103
+ a, b = d[r_a], d[r_b]
104
+ adj_wr += a == 0 and b == 1
105
+ adj_rw += a == 1 and b == 0
106
+ adj_rr += a == 1 and b == 1
107
+ adj_ww += a == 0 and b == 0
108
+ out["adjacent_wrong_to_right"] = adj_wr
109
+ out["adjacent_right_to_wrong"] = adj_rw
110
+ out["adjacent_right_to_right"] = adj_rr
111
+ out["adjacent_wrong_to_wrong"] = adj_ww
112
+ out["adjacent_tau_ic"] = adj_wr / (adj_wr + adj_ww) if adj_wr + adj_ww else None
113
+ out["adjacent_tau_ci"] = adj_rw / (adj_rw + adj_rr) if adj_rw + adj_rr else None
114
+ ati, atc = out["adjacent_tau_ic"], out["adjacent_tau_ci"]
115
+ out["adjacent_p_star_hat"] = (
116
+ ati / (ati + atc)
117
+ if ati is not None and atc is not None and ati + atc > 0
118
+ else None
119
+ )
120
+ out["p1"] = float(np.mean(a0)) if a0 else None
121
+ out["p_last"] = float(np.mean(a1)) if a1 else None
122
+ rho, keff = {}, {}
123
+ for r in rounds:
124
+ rows = [[d[r] for d in ans[it].values() if r in d] for it in items]
125
+ k = min((len(x) for x in rows), default=0)
126
+ Z = np.array([x[:k] for x in rows], dtype=float) if k >= 2 else None
127
+ v = icc(Z) if Z is not None else None
128
+ rho[r] = v
129
+ keff[r] = (k / (1 + (k - 1) * min(1.0, max(0.0, v)))) if v is not None else None
130
+ out["rho_peer"], out["rho_total"], out["k_eff"] = (
131
+ rho,
132
+ rho,
133
+ keff,
134
+ ) # rho_peer is the earlier name of rho_total
135
+ out["delta_rho_debate"] = (
136
+ (rho[r1] - rho[r0])
137
+ if rho.get(r0) is not None and rho.get(r1) is not None
138
+ else None
139
+ )
140
+ return out
141
+
142
+
143
+ def analyse(
144
+ path: str,
145
+ n_boot: int = 1000,
146
+ seed: int = 0,
147
+ first: int | None = None,
148
+ last: int | None = None,
149
+ ) -> dict:
150
+ ans, tok = load(path)
151
+ items = sorted(ans)
152
+ rounds = sorted({r for it in items for d in ans[it].values() for r in d})
153
+ if len(rounds) < 2:
154
+ raise ValueError("need answers from at least two rounds; found %s" % rounds)
155
+ r0, r1 = (
156
+ (first if first is not None else rounds[0]),
157
+ (last if last is not None else rounds[-1]),
158
+ )
159
+ if r0 not in rounds or r1 not in rounds or r0 >= r1:
160
+ raise ValueError(
161
+ "--first-round and --last-round must be distinct rounds present in the log, first < last; found rounds %s"
162
+ % rounds
163
+ )
164
+ point = stats(ans, items, r0, r1, rounds)
165
+ rng = np.random.default_rng(seed)
166
+ boots = []
167
+ for _ in range(n_boot):
168
+ pick = [items[i] for i in rng.integers(0, len(items), len(items))]
169
+ # a resampled item can repeat: give each copy its own key so it is counted as many times as drawn
170
+ ans_b = {"%s#%d" % (it, c): ans[it] for c, it in enumerate(pick)}
171
+ boots.append(stats(ans_b, list(ans_b), r0, r1, rounds))
172
+
173
+ def ci(get):
174
+ v = [x for x in (get(b) for b in boots) if x is not None]
175
+ return (
176
+ [float(np.percentile(v, 2.5)), float(np.percentile(v, 97.5))]
177
+ if len(v) >= max(20, n_boot // 10)
178
+ else None
179
+ )
180
+
181
+ cis = {
182
+ k: ci(lambda b, k=k: b[k])
183
+ for k in (
184
+ "tau_ic",
185
+ "tau_ci",
186
+ "p_star_hat",
187
+ "adjacent_tau_ic",
188
+ "adjacent_tau_ci",
189
+ "adjacent_p_star_hat",
190
+ "p1",
191
+ "p_last",
192
+ "delta_rho_debate",
193
+ )
194
+ }
195
+ cis["rho_peer"] = cis["rho_total"] = {
196
+ r: ci(lambda b, r=r: b["rho_peer"][r]) for r in rounds
197
+ }
198
+ cis["k_eff"] = {r: ci(lambda b, r=r: b["k_eff"][r]) for r in rounds}
199
+ tokens = {}
200
+ for r in rounds:
201
+ per_item = [np.mean(tok[it][r]) for it in items if tok[it].get(r)]
202
+ if per_item:
203
+ tokens[r] = float(np.mean(per_item))
204
+ return dict(
205
+ n_items=len(items),
206
+ rounds=rounds,
207
+ first_round=r0,
208
+ last_round=r1,
209
+ n_boot=n_boot,
210
+ point=point,
211
+ ci95=cis,
212
+ mean_tokens_per_call_by_round=tokens or None,
213
+ )
214
+
215
+
216
+ def _f(x, nd=3):
217
+ return "n/a" if x is None else ("%.*f" % (nd, x))
218
+
219
+
220
+ def render(res: dict) -> str:
221
+ p, c = res["point"], res["ci95"]
222
+
223
+ def with_ci(key, nd=3):
224
+ return (
225
+ "%s [%s, %s]" % (_f(p[key], nd), _f(c[key][0], nd), _f(c[key][1], nd))
226
+ if c.get(key)
227
+ else _f(p[key], nd)
228
+ )
229
+
230
+ out = [
231
+ "Debate dynamics: %d items, rounds %s (first %d to last %d), %d bootstrap resamples over items"
232
+ % (
233
+ res["n_items"],
234
+ res["rounds"],
235
+ res["first_round"],
236
+ res["last_round"],
237
+ res["n_boot"],
238
+ ),
239
+ "flips (item-agent pairs): wrong->right %d, right->wrong %d, right->right %d, wrong->wrong %d"
240
+ % (
241
+ p["wrong_to_right"],
242
+ p["right_to_wrong"],
243
+ p["right_to_right"],
244
+ p["wrong_to_wrong"],
245
+ ),
246
+ "tau_ic (wrong -> right): " + with_ci("tau_ic"),
247
+ "tau_ci (right -> wrong): " + with_ci("tau_ci"),
248
+ "p_star_hat = tau_ic / (tau_ic + tau_ci): " + with_ci("p_star_hat"),
249
+ "p1 (first round accuracy): "
250
+ + with_ci("p1")
251
+ + " p_last: "
252
+ + with_ci("p_last"),
253
+ "delta_rho_debate = rho_total(round %d) - rho_total(round %d): %s"
254
+ % (res["last_round"], res["first_round"], with_ci("delta_rho_debate")),
255
+ ]
256
+ for r in res["rounds"]:
257
+ rc, kc = c["rho_peer"][r], c["k_eff"][r]
258
+ out.append(
259
+ "round %d: rho_total %s%s, K_eff %s%s"
260
+ % (
261
+ r,
262
+ _f(p["rho_peer"][r]),
263
+ " [%s, %s]" % (_f(rc[0]), _f(rc[1])) if rc else "",
264
+ _f(p["k_eff"][r], 2),
265
+ " [%s, %s]" % (_f(kc[0], 2), _f(kc[1], 2)) if kc else "",
266
+ )
267
+ )
268
+ if res["mean_tokens_per_call_by_round"]:
269
+ out.append(
270
+ "mean tokens per call by round: "
271
+ + ", ".join(
272
+ "r%d %.0f" % (r, t)
273
+ for r, t in res["mean_tokens_per_call_by_round"].items()
274
+ )
275
+ )
276
+ else:
277
+ out.append("mean tokens per call by round: n/a (no token fields in the log)")
278
+ if res.get("excess_over_null"):
279
+ ex = res["excess_over_null"]
280
+ out.append(
281
+ "excess over null: adjacent_tau_ic %s, adjacent_tau_ci %s, delta_rho_debate %s"
282
+ % (
283
+ _f(ex.get("excess_adjacent_tau_ic")),
284
+ _f(ex.get("excess_adjacent_tau_ci")),
285
+ _f(ex.get("excess_delta_rho_debate")),
286
+ )
287
+ )
288
+ out.append(
289
+ "rho_total is the correlation of correctness across agents of an item and includes shared item difficulty; delta_rho_debate is its change from the first to the last round, in which shared difficulty largely cancels, so it estimates the correlation added by communication. The tau rates assume the constant-rate model."
290
+ )
291
+ return "\n".join(out)
292
+
293
+
294
+ def peer_transition_table(ans: dict, items: list, rounds: list) -> dict:
295
+ """Compute P(correct at t+1 | own_state_t, n_correct_peers_t) across adjacent rounds."""
296
+ counts = {}
297
+ for it in items:
298
+ agents = ans[it]
299
+ for r_t, r_next in zip(rounds[:-1], rounds[1:]):
300
+ present = [a for a, d in agents.items() if r_t in d and r_next in d]
301
+ if len(present) < 2:
302
+ continue
303
+ n_right_total = sum(agents[a][r_t] for a in present)
304
+ for a in present:
305
+ own = agents[a][r_t]
306
+ n_peers_right = n_right_total - own
307
+ key = (own, n_peers_right)
308
+ c = counts.setdefault(key, [0, 0])
309
+ c[0] += agents[a][r_next]
310
+ c[1] += 1
311
+ return {
312
+ f"own={own},peers_right={pr}": {"p_next": n_ok / tot, "n": tot}
313
+ for (own, pr), (n_ok, tot) in sorted(counts.items())
314
+ }
315
+
316
+
317
+ def excess_over_null(res: dict, null_res: dict) -> dict:
318
+ """Compute excess adjacent transition rates and excess delta_rho_debate over a no-communication null log."""
319
+ p, np_ = res["point"], null_res["point"]
320
+
321
+ def _diff(k):
322
+ a, b = p.get(k), np_.get(k)
323
+ return None if (a is None or b is None) else (a - b)
324
+
325
+ return {
326
+ "excess_adjacent_tau_ic": _diff("adjacent_tau_ic"),
327
+ "excess_adjacent_tau_ci": _diff("adjacent_tau_ci"),
328
+ "excess_delta_rho_debate": _diff("delta_rho_debate"),
329
+ "null_adjacent_p_star_hat": np_.get("adjacent_p_star_hat"),
330
+ "null_p1": np_.get("p1"),
331
+ }
332
+
333
+
334
+ def influence_diagnostics(path: str, k_agents: int | None = None) -> dict:
335
+ """Load a debate log and attach the Phase 2 peer-influence diagnostics."""
336
+ from .influence import (
337
+ dual_transition_tables,
338
+ fit_influence_models,
339
+ flow_identity_check,
340
+ )
341
+
342
+ records = []
343
+ with open(path, encoding="utf-8") as handle:
344
+ records = [json.loads(line) for line in handle if line.strip()]
345
+ tables = dual_transition_tables(records)
346
+ if k_agents is None:
347
+ k_agents = len({str(row["agent"]) for row in records if "agent" in row})
348
+ first = [
349
+ row
350
+ for row in records
351
+ if int(row.get("round", -1)) == min(int(item["round"]) for item in records)
352
+ ]
353
+ wrong = sum(not bool(row["correct"]) for row in first)
354
+ right = len(first) - wrong
355
+ transitions = []
356
+ for row in records:
357
+ transitions.append(row)
358
+ by_agent = defaultdict(dict)
359
+ for row in records:
360
+ by_agent[(str(row.get("item")), str(row.get("agent")))][
361
+ int(row.get("round"))
362
+ ] = bool(row.get("correct"))
363
+ fic = fci = 0
364
+ for state in by_agent.values():
365
+ for left, right_state in zip(sorted(state), sorted(state)[1:]):
366
+ fic += int(not state[left] and state[right_state])
367
+ fci += int(state[left] and not state[right_state])
368
+ return {
369
+ "transitions": tables,
370
+ "influence_models": fit_influence_models(tables["by_correct_peers"], k_agents),
371
+ "flow_identity": flow_identity_check(wrong, right, fic, fci),
372
+ }