pace-eval 0.2.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cme/__init__.py +11 -0
- cme/__main__.py +5 -0
- cme/analysis.py +223 -0
- cme/audit.py +145 -0
- cme/baseline.py +162 -0
- cme/cli.py +279 -0
- cme/client.py +165 -0
- cme/dynamics.py +212 -0
- cme/examples/crewai_integration.py +16 -0
- cme/examples/demo_pool.jsonl +360 -0
- cme/examples/demo_run.jsonl +480 -0
- cme/examples/langgraph_integration.py +27 -0
- cme/examples/make_demo.py +37 -0
- cme/examples/openai_quickstart.py +18 -0
- cme/experiment.py +555 -0
- cme/frontier.py +143 -0
- cme/hidden_profile.py +135 -0
- cme/proxy.py +106 -0
- cme/rebuttal.py +76 -0
- cme/report.py +197 -0
- cme/rollout_card.py +137 -0
- cme/tasks.py +266 -0
- cme/verified_debate.py +127 -0
- cme/verifier.py +92 -0
- mae_eval/__init__.py +11 -0
- mae_eval/__main__.py +5 -0
- mae_eval/auditor.py +112 -0
- mae_eval/classifier.py +62 -0
- mae_eval/cli.py +210 -0
- mae_eval/tracker.py +162 -0
- mae_eval/vector.py +91 -0
- pace/__init__.py +11 -0
- pace/__main__.py +5 -0
- pace/analysis.py +129 -0
- pace/audit.py +145 -0
- pace/baseline.py +162 -0
- pace/card.py +205 -0
- pace/cli.py +898 -0
- pace/client.py +165 -0
- pace/dynamics.py +372 -0
- pace/escape_benchmarks.py +162 -0
- pace/examples/crewai_integration.py +16 -0
- pace/examples/demo_pool.jsonl +360 -0
- pace/examples/demo_run.jsonl +480 -0
- pace/examples/langgraph_integration.py +27 -0
- pace/examples/make_demo.py +37 -0
- pace/examples/openai_quickstart.py +18 -0
- pace/experiment.py +555 -0
- pace/frontier.py +114 -0
- pace/hidden_profile.py +301 -0
- pace/importers.py +219 -0
- pace/inflate.py +213 -0
- pace/influence.py +231 -0
- pace/meta_engine.py +246 -0
- pace/middleware.py +230 -0
- pace/open_ended.py +114 -0
- pace/plotting.py +132 -0
- pace/pool_sizer.py +154 -0
- pace/protocol_v3.py +414 -0
- pace/proxy.py +106 -0
- pace/rebuttal.py +219 -0
- pace/report.py +197 -0
- pace/rollout_card.py +137 -0
- pace/schema.py +74 -0
- pace/simulator.py +199 -0
- pace/studio.html +1040 -0
- pace/tasks.py +266 -0
- pace/verified_debate.py +127 -0
- pace/verifier.py +92 -0
- pace/workbench.html +26 -0
- pace/workbench.py +119 -0
- pace_eval-0.2.2.dist-info/METADATA +205 -0
- pace_eval-0.2.2.dist-info/RECORD +83 -0
- pace_eval-0.2.2.dist-info/WHEEL +5 -0
- pace_eval-0.2.2.dist-info/entry_points.txt +4 -0
- pace_eval-0.2.2.dist-info/licenses/LICENSE +202 -0
- pace_eval-0.2.2.dist-info/top_level.txt +4 -0
- pace_lab/__init__.py +9 -0
- pace_lab/clients.py +568 -0
- pace_lab/concurrency_test.py +127 -0
- pace_lab/data.py +445 -0
- pace_lab/graders.py +626 -0
- pace_lab/strategies.py +345 -0
pace/client.py
ADDED
|
@@ -0,0 +1,165 @@
|
|
|
1
|
+
"""pace.wrap: log the token usage of an existing client without a proxy process (standard library only).
|
|
2
|
+
|
|
3
|
+
import pace
|
|
4
|
+
from openai import OpenAI
|
|
5
|
+
client = pace.wrap(OpenAI(), log_path="pace_trace.jsonl")
|
|
6
|
+
with pace.track(item_id="item_042", system="mas", agent="agent_1", round=2):
|
|
7
|
+
response = client.chat.completions.create(model=..., messages=...)
|
|
8
|
+
|
|
9
|
+
The wrapper forwards every attribute to the wrapped client and intercepts only the calls that return token usage: `chat.completions.create`,
|
|
10
|
+
`completions.create` (OpenAI-style clients, sync or async) and a top-level `completion` (a litellm-style module or client). Each successful call
|
|
11
|
+
appends one JSON line in the same format as `pace proxy`, so `pace report --logs` and `mae-eval audit` read it. The response is returned unchanged.
|
|
12
|
+
|
|
13
|
+
Limits (v0.1): streaming calls raise ValueError, because their usage is not known when the call returns and pace never estimates tokens; a response
|
|
14
|
+
without a usage block is logged with `usage_missing: true`; prices are never bundled (pass price_in/price_out in USD per million tokens, else cost is null).
|
|
15
|
+
"""
|
|
16
|
+
from __future__ import annotations
|
|
17
|
+
|
|
18
|
+
import contextlib
|
|
19
|
+
import contextvars
|
|
20
|
+
import inspect
|
|
21
|
+
import json
|
|
22
|
+
import threading
|
|
23
|
+
import time
|
|
24
|
+
from typing import Any, Dict, Optional
|
|
25
|
+
|
|
26
|
+
_LABELS: contextvars.ContextVar = contextvars.ContextVar("cme_labels", default={})
|
|
27
|
+
_LOCKS: Dict[str, threading.Lock] = {}
|
|
28
|
+
_LOCKS_GUARD = threading.Lock()
|
|
29
|
+
_LOGGED = {("chat", "completions", "create"), ("completions", "create"), ("completion",)}
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
@contextlib.contextmanager
|
|
33
|
+
def track(item_id: Optional[str] = None, system: Optional[str] = None, agent: Optional[str] = None, round: Optional[int] = None, **extra):
|
|
34
|
+
"""Label every wrapped call made inside the block. Blocks nest; an inner label overrides the outer one. Unset labels keep the outer value."""
|
|
35
|
+
labels = dict(_LABELS.get())
|
|
36
|
+
for key, val in (("item", item_id), ("system", system), ("agent_id", agent), ("round", round)):
|
|
37
|
+
if val is not None:
|
|
38
|
+
labels[key] = val
|
|
39
|
+
labels.update(extra)
|
|
40
|
+
token = _LABELS.set(labels)
|
|
41
|
+
try:
|
|
42
|
+
yield labels
|
|
43
|
+
finally:
|
|
44
|
+
_LABELS.reset(token)
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def _get(obj: Any, name: str):
|
|
48
|
+
if obj is None:
|
|
49
|
+
return None
|
|
50
|
+
if isinstance(obj, dict):
|
|
51
|
+
return obj.get(name)
|
|
52
|
+
return getattr(obj, name, None)
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def _record(kwargs: dict, resp: Any, t0: float, t1: float, price_in: Optional[float], price_out: Optional[float]) -> dict:
|
|
56
|
+
usage = _get(resp, "usage")
|
|
57
|
+
pt, ct = _get(usage, "prompt_tokens"), _get(usage, "completion_tokens")
|
|
58
|
+
det = _get(usage, "completion_tokens_details")
|
|
59
|
+
cost = None
|
|
60
|
+
if None not in (price_in, price_out, pt, ct):
|
|
61
|
+
cost = (pt * price_in + ct * price_out) / 1e6
|
|
62
|
+
labels = _LABELS.get()
|
|
63
|
+
choices = _get(resp, "choices") or []
|
|
64
|
+
rec = {"system": labels.get("system", "mas"), "agent_id": labels.get("agent_id", "agent"), "round": int(labels.get("round", 0)),
|
|
65
|
+
"item": str(labels.get("item", "")), "model": _get(resp, "model") or kwargs.get("model"), "prompt_tokens": pt, "completion_tokens": ct,
|
|
66
|
+
"reasoning_tokens": _get(det, "reasoning_tokens"), "n_completions": len(choices) or 1, "start": t0, "end": t1, "latency": t1 - t0, "cost": cost}
|
|
67
|
+
for key, val in labels.items():
|
|
68
|
+
if key not in ("system", "agent_id", "round", "item"):
|
|
69
|
+
rec[key] = val
|
|
70
|
+
if pt is None or ct is None:
|
|
71
|
+
rec["usage_missing"] = True # logged, never guessed
|
|
72
|
+
return rec
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
class _Logger:
|
|
76
|
+
def __init__(self, log_path: str, price_in: Optional[float], price_out: Optional[float]):
|
|
77
|
+
self.path, self.price_in, self.price_out = str(log_path), price_in, price_out
|
|
78
|
+
with _LOCKS_GUARD:
|
|
79
|
+
self.lock = _LOCKS.setdefault(self.path, threading.Lock())
|
|
80
|
+
|
|
81
|
+
def write(self, kwargs: dict, resp: Any, t0: float, t1: float) -> None:
|
|
82
|
+
line = json.dumps(_record(kwargs, resp, t0, t1, self.price_in, self.price_out), default=str)
|
|
83
|
+
with self.lock, open(self.path, "a", encoding="utf-8") as fh:
|
|
84
|
+
fh.write(line + "\n")
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
class _Wrapped:
|
|
88
|
+
"""Transparent proxy over a client (or one of its sub-objects); only the usage-returning calls are intercepted."""
|
|
89
|
+
|
|
90
|
+
def __init__(self, target: Any, logger: _Logger, path: tuple = ()):
|
|
91
|
+
object.__setattr__(self, "_cme_target", target)
|
|
92
|
+
object.__setattr__(self, "_cme_logger", logger)
|
|
93
|
+
object.__setattr__(self, "_cme_path", path)
|
|
94
|
+
|
|
95
|
+
def __getattr__(self, name: str):
|
|
96
|
+
target = object.__getattribute__(self, "_cme_target")
|
|
97
|
+
logger = object.__getattribute__(self, "_cme_logger")
|
|
98
|
+
path = object.__getattribute__(self, "_cme_path") + (name,)
|
|
99
|
+
attr = getattr(target, name)
|
|
100
|
+
if path in _LOGGED and callable(attr):
|
|
101
|
+
return _intercept(attr, logger)
|
|
102
|
+
if callable(attr) or name.startswith("_") or isinstance(attr, (str, int, float, bool, bytes, type(None))):
|
|
103
|
+
return attr
|
|
104
|
+
if any(p[:len(path)] == path for p in _LOGGED):
|
|
105
|
+
return _Wrapped(attr, logger, path)
|
|
106
|
+
return attr
|
|
107
|
+
|
|
108
|
+
def __setattr__(self, name, value):
|
|
109
|
+
setattr(object.__getattribute__(self, "_cme_target"), name, value)
|
|
110
|
+
|
|
111
|
+
def __repr__(self):
|
|
112
|
+
return "pace.wrap(%r)" % (object.__getattribute__(self, "_cme_target"),)
|
|
113
|
+
|
|
114
|
+
def __enter__(self):
|
|
115
|
+
target = object.__getattribute__(self, "_cme_target")
|
|
116
|
+
if hasattr(target, "__enter__"):
|
|
117
|
+
target.__enter__()
|
|
118
|
+
return self
|
|
119
|
+
|
|
120
|
+
def __exit__(self, *exc):
|
|
121
|
+
target = object.__getattribute__(self, "_cme_target")
|
|
122
|
+
return target.__exit__(*exc) if hasattr(target, "__exit__") else None
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
def _intercept(fn, logger: _Logger):
|
|
126
|
+
def guard(kwargs: dict) -> None:
|
|
127
|
+
if kwargs.get("stream"):
|
|
128
|
+
raise ValueError("pace.wrap v0.1 does not support streaming calls: usage is not known when the call returns. Use stream=False.")
|
|
129
|
+
|
|
130
|
+
if inspect.iscoroutinefunction(fn):
|
|
131
|
+
async def acall(*args, **kwargs):
|
|
132
|
+
guard(kwargs)
|
|
133
|
+
t0 = time.time()
|
|
134
|
+
resp = await fn(*args, **kwargs)
|
|
135
|
+
logger.write(kwargs, resp, t0, time.time())
|
|
136
|
+
return resp
|
|
137
|
+
return acall
|
|
138
|
+
|
|
139
|
+
def call(*args, **kwargs):
|
|
140
|
+
guard(kwargs)
|
|
141
|
+
t0 = time.time()
|
|
142
|
+
resp = fn(*args, **kwargs)
|
|
143
|
+
if inspect.isawaitable(resp): # an async client whose method is not a plain coroutine function
|
|
144
|
+
async def finish():
|
|
145
|
+
out = await resp
|
|
146
|
+
logger.write(kwargs, out, t0, time.time())
|
|
147
|
+
return out
|
|
148
|
+
return finish()
|
|
149
|
+
logger.write(kwargs, resp, t0, time.time())
|
|
150
|
+
return resp
|
|
151
|
+
return call
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
def wrap(client: Any, log_path: str = "pace_trace.jsonl", price_in: Optional[float] = None, price_out: Optional[float] = None) -> _Wrapped:
|
|
155
|
+
"""Return `client` with usage logging on its completion calls. Prices are USD per million tokens; omit them and cost is null."""
|
|
156
|
+
return _Wrapped(client, _Logger(log_path, price_in, price_out))
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
def log_result(log_path: str, item_id: str, correct: bool, system: Optional[str] = None) -> None:
|
|
160
|
+
"""Append the correctness of one item's final answer (the line `pace report` pairs with the logged calls). Call it once per item, after scoring."""
|
|
161
|
+
sysname = system if system is not None else _LABELS.get().get("system", "mas")
|
|
162
|
+
with _LOCKS_GUARD:
|
|
163
|
+
lock = _LOCKS.setdefault(str(log_path), threading.Lock())
|
|
164
|
+
with lock, open(str(log_path), "a", encoding="utf-8") as fh:
|
|
165
|
+
fh.write(json.dumps({"system": sysname, "item": str(item_id), "correct": bool(correct)}) + "\n")
|
pace/dynamics.py
ADDED
|
@@ -0,0 +1,372 @@
|
|
|
1
|
+
"""pace dynamics: how agents' answers change across debate rounds (standard library plus numpy).
|
|
2
|
+
|
|
3
|
+
Input: a JSONL file with one record per agent answer, {"item": str, "agent": str or int, "round": int, "correct": bool} and optionally "prompt_tokens" and
|
|
4
|
+
"completion_tokens". Nothing is estimated for missing fields: records without "correct" are ignored, and token statistics need both token fields.
|
|
5
|
+
|
|
6
|
+
Output (point estimates and 95% percentile intervals from a paired bootstrap that resamples items):
|
|
7
|
+
flips between the first and the last round: wrong to right, right to wrong, right to right, wrong to wrong (counts, over item-agent pairs observed in both rounds);
|
|
8
|
+
tau_ic = wrong-to-right / (all wrong in the first round) and tau_ci = right-to-wrong / (all right in the first round), the two rates of the conformity-to-truth model;
|
|
9
|
+
p_star_hat = tau_ic / (tau_ic + tau_ci), the threshold implied by those rates (undefined if both rates are zero);
|
|
10
|
+
p1 and p_last, the mean accuracy of an agent in the first and the last round;
|
|
11
|
+
rho_total, the intraclass correlation of correctness across the agents of an item in each round (reported in the JSON also as rho_peer, its earlier name), and K_eff = K / (1 + (K - 1) rho_total)
|
|
12
|
+
with rho_total clipped to [0, 1]; delta_rho_debate = rho_total(last round) - rho_total(first round), the excess correlation induced by debate, with its own bootstrap interval;
|
|
13
|
+
mean tokens per call by round.
|
|
14
|
+
Limits: rho_total is the correlation of correctness among agents on the same item, so it includes shared item difficulty as well as correlated errors; the difference across rounds is the
|
|
15
|
+
quantity that isolates what the exchange of messages adds, and it is not a measure of error correlation between models. The tau
|
|
16
|
+
rates assume the constant-rate model of the paper; this command reports what the logs show under that model and does not test the model.
|
|
17
|
+
"""
|
|
18
|
+
|
|
19
|
+
from __future__ import annotations
|
|
20
|
+
|
|
21
|
+
import json
|
|
22
|
+
from collections import defaultdict
|
|
23
|
+
|
|
24
|
+
import numpy as np
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def load(path: str):
|
|
28
|
+
"""Return {item: {agent: {round: correct}}} and {item: {round: [tokens per call]}}."""
|
|
29
|
+
ans: dict[str, dict[str, dict[int, int]]] = defaultdict(lambda: defaultdict(dict))
|
|
30
|
+
tok: dict[str, dict[int, list[float]]] = defaultdict(lambda: defaultdict(list))
|
|
31
|
+
with open(path, encoding="utf-8") as fh:
|
|
32
|
+
for no, line in enumerate(fh, 1):
|
|
33
|
+
if not line.strip():
|
|
34
|
+
continue
|
|
35
|
+
r = json.loads(line)
|
|
36
|
+
if (
|
|
37
|
+
not isinstance(r, dict)
|
|
38
|
+
or "correct" not in r
|
|
39
|
+
or "round" not in r
|
|
40
|
+
or "item" not in r
|
|
41
|
+
or "agent" not in r
|
|
42
|
+
):
|
|
43
|
+
continue
|
|
44
|
+
ans[str(r["item"])][str(r["agent"])][int(r["round"])] = int(
|
|
45
|
+
bool(r["correct"])
|
|
46
|
+
)
|
|
47
|
+
if (
|
|
48
|
+
r.get("prompt_tokens") is not None
|
|
49
|
+
and r.get("completion_tokens") is not None
|
|
50
|
+
):
|
|
51
|
+
tok[str(r["item"])][int(r["round"])].append(
|
|
52
|
+
float(r["prompt_tokens"]) + float(r["completion_tokens"])
|
|
53
|
+
)
|
|
54
|
+
if not ans:
|
|
55
|
+
raise ValueError(
|
|
56
|
+
"no usable records: each line needs item, agent, round and correct"
|
|
57
|
+
)
|
|
58
|
+
return ans, tok
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def icc(Z: np.ndarray) -> float | None:
|
|
62
|
+
"""One-way intraclass correlation of an items x agents 0/1 matrix (balanced)."""
|
|
63
|
+
n, k = Z.shape
|
|
64
|
+
if n < 2 or k < 2:
|
|
65
|
+
return None
|
|
66
|
+
m = Z.mean(axis=1)
|
|
67
|
+
g = Z.mean()
|
|
68
|
+
msb = k * np.sum((m - g) ** 2) / (n - 1)
|
|
69
|
+
msw = np.sum((Z - m[:, None]) ** 2) / (n * (k - 1))
|
|
70
|
+
den = msb + (k - 1) * msw
|
|
71
|
+
return None if den == 0 else float((msb - msw) / den)
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def stats(ans, items: list[str], r0: int, r1: int, rounds: list[int]) -> dict:
|
|
75
|
+
wr = rw = rr = ww = 0
|
|
76
|
+
a0 = []
|
|
77
|
+
a1 = []
|
|
78
|
+
for it in items:
|
|
79
|
+
for ag, d in ans[it].items():
|
|
80
|
+
if r0 in d and r1 in d:
|
|
81
|
+
a, b = d[r0], d[r1]
|
|
82
|
+
a0.append(a)
|
|
83
|
+
a1.append(b)
|
|
84
|
+
wr += a == 0 and b == 1
|
|
85
|
+
rw += a == 1 and b == 0
|
|
86
|
+
rr += a == 1 and b == 1
|
|
87
|
+
ww += a == 0 and b == 0
|
|
88
|
+
out = dict(
|
|
89
|
+
wrong_to_right=wr, right_to_wrong=rw, right_to_right=rr, wrong_to_wrong=ww
|
|
90
|
+
)
|
|
91
|
+
out["tau_ic"] = wr / (wr + ww) if wr + ww else None
|
|
92
|
+
out["tau_ci"] = rw / (rw + rr) if rw + rr else None
|
|
93
|
+
ti, tc = out["tau_ic"], out["tau_ci"]
|
|
94
|
+
out["p_star_hat"] = (
|
|
95
|
+
ti / (ti + tc) if ti is not None and tc is not None and ti + tc > 0 else None
|
|
96
|
+
)
|
|
97
|
+
adj_wr = adj_rw = adj_rr = adj_ww = 0
|
|
98
|
+
active_rounds = [r for r in rounds if r0 <= r <= r1]
|
|
99
|
+
for it in items:
|
|
100
|
+
for ag, d in ans[it].items():
|
|
101
|
+
for r_a, r_b in zip(active_rounds[:-1], active_rounds[1:]):
|
|
102
|
+
if r_a in d and r_b in d:
|
|
103
|
+
a, b = d[r_a], d[r_b]
|
|
104
|
+
adj_wr += a == 0 and b == 1
|
|
105
|
+
adj_rw += a == 1 and b == 0
|
|
106
|
+
adj_rr += a == 1 and b == 1
|
|
107
|
+
adj_ww += a == 0 and b == 0
|
|
108
|
+
out["adjacent_wrong_to_right"] = adj_wr
|
|
109
|
+
out["adjacent_right_to_wrong"] = adj_rw
|
|
110
|
+
out["adjacent_right_to_right"] = adj_rr
|
|
111
|
+
out["adjacent_wrong_to_wrong"] = adj_ww
|
|
112
|
+
out["adjacent_tau_ic"] = adj_wr / (adj_wr + adj_ww) if adj_wr + adj_ww else None
|
|
113
|
+
out["adjacent_tau_ci"] = adj_rw / (adj_rw + adj_rr) if adj_rw + adj_rr else None
|
|
114
|
+
ati, atc = out["adjacent_tau_ic"], out["adjacent_tau_ci"]
|
|
115
|
+
out["adjacent_p_star_hat"] = (
|
|
116
|
+
ati / (ati + atc)
|
|
117
|
+
if ati is not None and atc is not None and ati + atc > 0
|
|
118
|
+
else None
|
|
119
|
+
)
|
|
120
|
+
out["p1"] = float(np.mean(a0)) if a0 else None
|
|
121
|
+
out["p_last"] = float(np.mean(a1)) if a1 else None
|
|
122
|
+
rho, keff = {}, {}
|
|
123
|
+
for r in rounds:
|
|
124
|
+
rows = [[d[r] for d in ans[it].values() if r in d] for it in items]
|
|
125
|
+
k = min((len(x) for x in rows), default=0)
|
|
126
|
+
Z = np.array([x[:k] for x in rows], dtype=float) if k >= 2 else None
|
|
127
|
+
v = icc(Z) if Z is not None else None
|
|
128
|
+
rho[r] = v
|
|
129
|
+
keff[r] = (k / (1 + (k - 1) * min(1.0, max(0.0, v)))) if v is not None else None
|
|
130
|
+
out["rho_peer"], out["rho_total"], out["k_eff"] = (
|
|
131
|
+
rho,
|
|
132
|
+
rho,
|
|
133
|
+
keff,
|
|
134
|
+
) # rho_peer is the earlier name of rho_total
|
|
135
|
+
out["delta_rho_debate"] = (
|
|
136
|
+
(rho[r1] - rho[r0])
|
|
137
|
+
if rho.get(r0) is not None and rho.get(r1) is not None
|
|
138
|
+
else None
|
|
139
|
+
)
|
|
140
|
+
return out
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
def analyse(
|
|
144
|
+
path: str,
|
|
145
|
+
n_boot: int = 1000,
|
|
146
|
+
seed: int = 0,
|
|
147
|
+
first: int | None = None,
|
|
148
|
+
last: int | None = None,
|
|
149
|
+
) -> dict:
|
|
150
|
+
ans, tok = load(path)
|
|
151
|
+
items = sorted(ans)
|
|
152
|
+
rounds = sorted({r for it in items for d in ans[it].values() for r in d})
|
|
153
|
+
if len(rounds) < 2:
|
|
154
|
+
raise ValueError("need answers from at least two rounds; found %s" % rounds)
|
|
155
|
+
r0, r1 = (
|
|
156
|
+
(first if first is not None else rounds[0]),
|
|
157
|
+
(last if last is not None else rounds[-1]),
|
|
158
|
+
)
|
|
159
|
+
if r0 not in rounds or r1 not in rounds or r0 >= r1:
|
|
160
|
+
raise ValueError(
|
|
161
|
+
"--first-round and --last-round must be distinct rounds present in the log, first < last; found rounds %s"
|
|
162
|
+
% rounds
|
|
163
|
+
)
|
|
164
|
+
point = stats(ans, items, r0, r1, rounds)
|
|
165
|
+
rng = np.random.default_rng(seed)
|
|
166
|
+
boots = []
|
|
167
|
+
for _ in range(n_boot):
|
|
168
|
+
pick = [items[i] for i in rng.integers(0, len(items), len(items))]
|
|
169
|
+
# a resampled item can repeat: give each copy its own key so it is counted as many times as drawn
|
|
170
|
+
ans_b = {"%s#%d" % (it, c): ans[it] for c, it in enumerate(pick)}
|
|
171
|
+
boots.append(stats(ans_b, list(ans_b), r0, r1, rounds))
|
|
172
|
+
|
|
173
|
+
def ci(get):
|
|
174
|
+
v = [x for x in (get(b) for b in boots) if x is not None]
|
|
175
|
+
return (
|
|
176
|
+
[float(np.percentile(v, 2.5)), float(np.percentile(v, 97.5))]
|
|
177
|
+
if len(v) >= max(20, n_boot // 10)
|
|
178
|
+
else None
|
|
179
|
+
)
|
|
180
|
+
|
|
181
|
+
cis = {
|
|
182
|
+
k: ci(lambda b, k=k: b[k])
|
|
183
|
+
for k in (
|
|
184
|
+
"tau_ic",
|
|
185
|
+
"tau_ci",
|
|
186
|
+
"p_star_hat",
|
|
187
|
+
"adjacent_tau_ic",
|
|
188
|
+
"adjacent_tau_ci",
|
|
189
|
+
"adjacent_p_star_hat",
|
|
190
|
+
"p1",
|
|
191
|
+
"p_last",
|
|
192
|
+
"delta_rho_debate",
|
|
193
|
+
)
|
|
194
|
+
}
|
|
195
|
+
cis["rho_peer"] = cis["rho_total"] = {
|
|
196
|
+
r: ci(lambda b, r=r: b["rho_peer"][r]) for r in rounds
|
|
197
|
+
}
|
|
198
|
+
cis["k_eff"] = {r: ci(lambda b, r=r: b["k_eff"][r]) for r in rounds}
|
|
199
|
+
tokens = {}
|
|
200
|
+
for r in rounds:
|
|
201
|
+
per_item = [np.mean(tok[it][r]) for it in items if tok[it].get(r)]
|
|
202
|
+
if per_item:
|
|
203
|
+
tokens[r] = float(np.mean(per_item))
|
|
204
|
+
return dict(
|
|
205
|
+
n_items=len(items),
|
|
206
|
+
rounds=rounds,
|
|
207
|
+
first_round=r0,
|
|
208
|
+
last_round=r1,
|
|
209
|
+
n_boot=n_boot,
|
|
210
|
+
point=point,
|
|
211
|
+
ci95=cis,
|
|
212
|
+
mean_tokens_per_call_by_round=tokens or None,
|
|
213
|
+
)
|
|
214
|
+
|
|
215
|
+
|
|
216
|
+
def _f(x, nd=3):
|
|
217
|
+
return "n/a" if x is None else ("%.*f" % (nd, x))
|
|
218
|
+
|
|
219
|
+
|
|
220
|
+
def render(res: dict) -> str:
|
|
221
|
+
p, c = res["point"], res["ci95"]
|
|
222
|
+
|
|
223
|
+
def with_ci(key, nd=3):
|
|
224
|
+
return (
|
|
225
|
+
"%s [%s, %s]" % (_f(p[key], nd), _f(c[key][0], nd), _f(c[key][1], nd))
|
|
226
|
+
if c.get(key)
|
|
227
|
+
else _f(p[key], nd)
|
|
228
|
+
)
|
|
229
|
+
|
|
230
|
+
out = [
|
|
231
|
+
"Debate dynamics: %d items, rounds %s (first %d to last %d), %d bootstrap resamples over items"
|
|
232
|
+
% (
|
|
233
|
+
res["n_items"],
|
|
234
|
+
res["rounds"],
|
|
235
|
+
res["first_round"],
|
|
236
|
+
res["last_round"],
|
|
237
|
+
res["n_boot"],
|
|
238
|
+
),
|
|
239
|
+
"flips (item-agent pairs): wrong->right %d, right->wrong %d, right->right %d, wrong->wrong %d"
|
|
240
|
+
% (
|
|
241
|
+
p["wrong_to_right"],
|
|
242
|
+
p["right_to_wrong"],
|
|
243
|
+
p["right_to_right"],
|
|
244
|
+
p["wrong_to_wrong"],
|
|
245
|
+
),
|
|
246
|
+
"tau_ic (wrong -> right): " + with_ci("tau_ic"),
|
|
247
|
+
"tau_ci (right -> wrong): " + with_ci("tau_ci"),
|
|
248
|
+
"p_star_hat = tau_ic / (tau_ic + tau_ci): " + with_ci("p_star_hat"),
|
|
249
|
+
"p1 (first round accuracy): "
|
|
250
|
+
+ with_ci("p1")
|
|
251
|
+
+ " p_last: "
|
|
252
|
+
+ with_ci("p_last"),
|
|
253
|
+
"delta_rho_debate = rho_total(round %d) - rho_total(round %d): %s"
|
|
254
|
+
% (res["last_round"], res["first_round"], with_ci("delta_rho_debate")),
|
|
255
|
+
]
|
|
256
|
+
for r in res["rounds"]:
|
|
257
|
+
rc, kc = c["rho_peer"][r], c["k_eff"][r]
|
|
258
|
+
out.append(
|
|
259
|
+
"round %d: rho_total %s%s, K_eff %s%s"
|
|
260
|
+
% (
|
|
261
|
+
r,
|
|
262
|
+
_f(p["rho_peer"][r]),
|
|
263
|
+
" [%s, %s]" % (_f(rc[0]), _f(rc[1])) if rc else "",
|
|
264
|
+
_f(p["k_eff"][r], 2),
|
|
265
|
+
" [%s, %s]" % (_f(kc[0], 2), _f(kc[1], 2)) if kc else "",
|
|
266
|
+
)
|
|
267
|
+
)
|
|
268
|
+
if res["mean_tokens_per_call_by_round"]:
|
|
269
|
+
out.append(
|
|
270
|
+
"mean tokens per call by round: "
|
|
271
|
+
+ ", ".join(
|
|
272
|
+
"r%d %.0f" % (r, t)
|
|
273
|
+
for r, t in res["mean_tokens_per_call_by_round"].items()
|
|
274
|
+
)
|
|
275
|
+
)
|
|
276
|
+
else:
|
|
277
|
+
out.append("mean tokens per call by round: n/a (no token fields in the log)")
|
|
278
|
+
if res.get("excess_over_null"):
|
|
279
|
+
ex = res["excess_over_null"]
|
|
280
|
+
out.append(
|
|
281
|
+
"excess over null: adjacent_tau_ic %s, adjacent_tau_ci %s, delta_rho_debate %s"
|
|
282
|
+
% (
|
|
283
|
+
_f(ex.get("excess_adjacent_tau_ic")),
|
|
284
|
+
_f(ex.get("excess_adjacent_tau_ci")),
|
|
285
|
+
_f(ex.get("excess_delta_rho_debate")),
|
|
286
|
+
)
|
|
287
|
+
)
|
|
288
|
+
out.append(
|
|
289
|
+
"rho_total is the correlation of correctness across agents of an item and includes shared item difficulty; delta_rho_debate is its change from the first to the last round, in which shared difficulty largely cancels, so it estimates the correlation added by communication. The tau rates assume the constant-rate model."
|
|
290
|
+
)
|
|
291
|
+
return "\n".join(out)
|
|
292
|
+
|
|
293
|
+
|
|
294
|
+
def peer_transition_table(ans: dict, items: list, rounds: list) -> dict:
|
|
295
|
+
"""Compute P(correct at t+1 | own_state_t, n_correct_peers_t) across adjacent rounds."""
|
|
296
|
+
counts = {}
|
|
297
|
+
for it in items:
|
|
298
|
+
agents = ans[it]
|
|
299
|
+
for r_t, r_next in zip(rounds[:-1], rounds[1:]):
|
|
300
|
+
present = [a for a, d in agents.items() if r_t in d and r_next in d]
|
|
301
|
+
if len(present) < 2:
|
|
302
|
+
continue
|
|
303
|
+
n_right_total = sum(agents[a][r_t] for a in present)
|
|
304
|
+
for a in present:
|
|
305
|
+
own = agents[a][r_t]
|
|
306
|
+
n_peers_right = n_right_total - own
|
|
307
|
+
key = (own, n_peers_right)
|
|
308
|
+
c = counts.setdefault(key, [0, 0])
|
|
309
|
+
c[0] += agents[a][r_next]
|
|
310
|
+
c[1] += 1
|
|
311
|
+
return {
|
|
312
|
+
f"own={own},peers_right={pr}": {"p_next": n_ok / tot, "n": tot}
|
|
313
|
+
for (own, pr), (n_ok, tot) in sorted(counts.items())
|
|
314
|
+
}
|
|
315
|
+
|
|
316
|
+
|
|
317
|
+
def excess_over_null(res: dict, null_res: dict) -> dict:
|
|
318
|
+
"""Compute excess adjacent transition rates and excess delta_rho_debate over a no-communication null log."""
|
|
319
|
+
p, np_ = res["point"], null_res["point"]
|
|
320
|
+
|
|
321
|
+
def _diff(k):
|
|
322
|
+
a, b = p.get(k), np_.get(k)
|
|
323
|
+
return None if (a is None or b is None) else (a - b)
|
|
324
|
+
|
|
325
|
+
return {
|
|
326
|
+
"excess_adjacent_tau_ic": _diff("adjacent_tau_ic"),
|
|
327
|
+
"excess_adjacent_tau_ci": _diff("adjacent_tau_ci"),
|
|
328
|
+
"excess_delta_rho_debate": _diff("delta_rho_debate"),
|
|
329
|
+
"null_adjacent_p_star_hat": np_.get("adjacent_p_star_hat"),
|
|
330
|
+
"null_p1": np_.get("p1"),
|
|
331
|
+
}
|
|
332
|
+
|
|
333
|
+
|
|
334
|
+
def influence_diagnostics(path: str, k_agents: int | None = None) -> dict:
|
|
335
|
+
"""Load a debate log and attach the Phase 2 peer-influence diagnostics."""
|
|
336
|
+
from .influence import (
|
|
337
|
+
dual_transition_tables,
|
|
338
|
+
fit_influence_models,
|
|
339
|
+
flow_identity_check,
|
|
340
|
+
)
|
|
341
|
+
|
|
342
|
+
records = []
|
|
343
|
+
with open(path, encoding="utf-8") as handle:
|
|
344
|
+
records = [json.loads(line) for line in handle if line.strip()]
|
|
345
|
+
tables = dual_transition_tables(records)
|
|
346
|
+
if k_agents is None:
|
|
347
|
+
k_agents = len({str(row["agent"]) for row in records if "agent" in row})
|
|
348
|
+
first = [
|
|
349
|
+
row
|
|
350
|
+
for row in records
|
|
351
|
+
if int(row.get("round", -1)) == min(int(item["round"]) for item in records)
|
|
352
|
+
]
|
|
353
|
+
wrong = sum(not bool(row["correct"]) for row in first)
|
|
354
|
+
right = len(first) - wrong
|
|
355
|
+
transitions = []
|
|
356
|
+
for row in records:
|
|
357
|
+
transitions.append(row)
|
|
358
|
+
by_agent = defaultdict(dict)
|
|
359
|
+
for row in records:
|
|
360
|
+
by_agent[(str(row.get("item")), str(row.get("agent")))][
|
|
361
|
+
int(row.get("round"))
|
|
362
|
+
] = bool(row.get("correct"))
|
|
363
|
+
fic = fci = 0
|
|
364
|
+
for state in by_agent.values():
|
|
365
|
+
for left, right_state in zip(sorted(state), sorted(state)[1:]):
|
|
366
|
+
fic += int(not state[left] and state[right_state])
|
|
367
|
+
fci += int(state[left] and not state[right_state])
|
|
368
|
+
return {
|
|
369
|
+
"transitions": tables,
|
|
370
|
+
"influence_models": fit_influence_models(tables["by_correct_peers"], k_agents),
|
|
371
|
+
"flow_identity": flow_identity_check(wrong, right, fic, fci),
|
|
372
|
+
}
|