pace-eval 0.2.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cme/__init__.py +11 -0
- cme/__main__.py +5 -0
- cme/analysis.py +223 -0
- cme/audit.py +145 -0
- cme/baseline.py +162 -0
- cme/cli.py +279 -0
- cme/client.py +165 -0
- cme/dynamics.py +212 -0
- cme/examples/crewai_integration.py +16 -0
- cme/examples/demo_pool.jsonl +360 -0
- cme/examples/demo_run.jsonl +480 -0
- cme/examples/langgraph_integration.py +27 -0
- cme/examples/make_demo.py +37 -0
- cme/examples/openai_quickstart.py +18 -0
- cme/experiment.py +555 -0
- cme/frontier.py +143 -0
- cme/hidden_profile.py +135 -0
- cme/proxy.py +106 -0
- cme/rebuttal.py +76 -0
- cme/report.py +197 -0
- cme/rollout_card.py +137 -0
- cme/tasks.py +266 -0
- cme/verified_debate.py +127 -0
- cme/verifier.py +92 -0
- mae_eval/__init__.py +11 -0
- mae_eval/__main__.py +5 -0
- mae_eval/auditor.py +112 -0
- mae_eval/classifier.py +62 -0
- mae_eval/cli.py +210 -0
- mae_eval/tracker.py +162 -0
- mae_eval/vector.py +91 -0
- pace/__init__.py +11 -0
- pace/__main__.py +5 -0
- pace/analysis.py +129 -0
- pace/audit.py +145 -0
- pace/baseline.py +162 -0
- pace/card.py +205 -0
- pace/cli.py +898 -0
- pace/client.py +165 -0
- pace/dynamics.py +372 -0
- pace/escape_benchmarks.py +162 -0
- pace/examples/crewai_integration.py +16 -0
- pace/examples/demo_pool.jsonl +360 -0
- pace/examples/demo_run.jsonl +480 -0
- pace/examples/langgraph_integration.py +27 -0
- pace/examples/make_demo.py +37 -0
- pace/examples/openai_quickstart.py +18 -0
- pace/experiment.py +555 -0
- pace/frontier.py +114 -0
- pace/hidden_profile.py +301 -0
- pace/importers.py +219 -0
- pace/inflate.py +213 -0
- pace/influence.py +231 -0
- pace/meta_engine.py +246 -0
- pace/middleware.py +230 -0
- pace/open_ended.py +114 -0
- pace/plotting.py +132 -0
- pace/pool_sizer.py +154 -0
- pace/protocol_v3.py +414 -0
- pace/proxy.py +106 -0
- pace/rebuttal.py +219 -0
- pace/report.py +197 -0
- pace/rollout_card.py +137 -0
- pace/schema.py +74 -0
- pace/simulator.py +199 -0
- pace/studio.html +1040 -0
- pace/tasks.py +266 -0
- pace/verified_debate.py +127 -0
- pace/verifier.py +92 -0
- pace/workbench.html +26 -0
- pace/workbench.py +119 -0
- pace_eval-0.2.2.dist-info/METADATA +205 -0
- pace_eval-0.2.2.dist-info/RECORD +83 -0
- pace_eval-0.2.2.dist-info/WHEEL +5 -0
- pace_eval-0.2.2.dist-info/entry_points.txt +4 -0
- pace_eval-0.2.2.dist-info/licenses/LICENSE +202 -0
- pace_eval-0.2.2.dist-info/top_level.txt +4 -0
- pace_lab/__init__.py +9 -0
- pace_lab/clients.py +568 -0
- pace_lab/concurrency_test.py +127 -0
- pace_lab/data.py +445 -0
- pace_lab/graders.py +626 -0
- pace_lab/strategies.py +345 -0
pace/frontier.py
ADDED
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
"""Self-consistency frontier and frontier gap with a paired bootstrap.
|
|
2
|
+
|
|
3
|
+
A sample pool holds, for each item, up to K independent sampled answers with token counts. For k = 1..K the expected accuracy of
|
|
4
|
+
plurality voting over k samples is estimated over random k-subsets (ties share credit equally). The cost of k samples is k times the
|
|
5
|
+
mean tokens per sample. The frontier is the nondecreasing (isotonic) fit of accuracy against tokens, interpolated linearly.
|
|
6
|
+
|
|
7
|
+
frontier gap = accuracy of a system - frontier(mean tokens the system used). A positive gap means the system sits above what
|
|
8
|
+
sampling alone would reach at that measured token cost. The bootstrap resamples items and recomputes everything (pool curve,
|
|
9
|
+
isotonic fit, interpolation, system accuracy), so the interval accounts for sampling noise in both sides.
|
|
10
|
+
"""
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
from collections import Counter
|
|
14
|
+
from typing import Dict, List, Optional, Sequence, Tuple
|
|
15
|
+
|
|
16
|
+
import numpy as np
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def plurality_credit(answers: Sequence[str], gold: str) -> float:
|
|
20
|
+
"""1 if the plurality answer equals gold; 1/m if gold is one of m tied leaders; 0 otherwise."""
|
|
21
|
+
c = Counter(answers)
|
|
22
|
+
top = max(c.values())
|
|
23
|
+
leaders = [a for a, v in c.items() if v == top]
|
|
24
|
+
return (1.0 / len(leaders)) if gold in leaders else 0.0
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def item_vote_curve(answers: Sequence[str], gold: str, K: int, n_perm: int, rng: np.random.Generator) -> np.ndarray:
|
|
28
|
+
"""Expected plurality accuracy of k = 1..K samples for one item. Each random permutation gives nested prefixes; the value at
|
|
29
|
+
k equal to the number of samples is exact (order does not matter)."""
|
|
30
|
+
n = len(answers)
|
|
31
|
+
if K > n:
|
|
32
|
+
raise ValueError("K exceeds the number of samples for an item")
|
|
33
|
+
out = np.zeros(K)
|
|
34
|
+
for _ in range(n_perm):
|
|
35
|
+
c: Counter = Counter()
|
|
36
|
+
for k, i in enumerate(rng.permutation(n)[:K], 1):
|
|
37
|
+
c[answers[i]] += 1
|
|
38
|
+
top = max(c.values())
|
|
39
|
+
leaders = [a for a, v in c.items() if v == top]
|
|
40
|
+
out[k - 1] += (1.0 / len(leaders)) if gold in leaders else 0.0
|
|
41
|
+
return out / n_perm
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def pav_increasing(y: np.ndarray, w: Optional[np.ndarray] = None) -> np.ndarray:
|
|
45
|
+
"""Weighted pool-adjacent-violators: the nondecreasing least-squares fit to y."""
|
|
46
|
+
y = np.asarray(y, dtype=float)
|
|
47
|
+
w = np.ones_like(y) if w is None else np.asarray(w, dtype=float)
|
|
48
|
+
blocks: List[List[float]] = [] # [value, weight, count]
|
|
49
|
+
for yi, wi in zip(y, w):
|
|
50
|
+
blocks.append([yi, wi, 1])
|
|
51
|
+
while len(blocks) > 1 and blocks[-2][0] > blocks[-1][0]:
|
|
52
|
+
v2, w2, c2 = blocks.pop()
|
|
53
|
+
v1, w1, c1 = blocks.pop()
|
|
54
|
+
blocks.append([(v1 * w1 + v2 * w2) / (w1 + w2), w1 + w2, c1 + c2])
|
|
55
|
+
return np.concatenate([[b[0]] * int(b[2]) for b in blocks])
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def frontier_at(tokens_k: np.ndarray, acc_k: np.ndarray, t: float) -> Tuple[float, str]:
|
|
59
|
+
"""Isotonic frontier interpolated at t. Returns (value, status) with status in {'inside', 'below range', 'beyond range'}."""
|
|
60
|
+
fit = pav_increasing(acc_k)
|
|
61
|
+
if t < tokens_k[0]:
|
|
62
|
+
return float(fit[0]), "below range"
|
|
63
|
+
if t > tokens_k[-1]:
|
|
64
|
+
return float(fit[-1]), "beyond range"
|
|
65
|
+
return float(np.interp(t, tokens_k, fit)), "inside"
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
class Pool:
|
|
69
|
+
"""Per-item sampled answers and token counts. items: id -> dict(gold, answers[list[str]], tokens[list[float]])."""
|
|
70
|
+
|
|
71
|
+
def __init__(self, items: Dict[str, dict], K: Optional[int] = None):
|
|
72
|
+
self.items = items
|
|
73
|
+
self.K = K or min(len(v["answers"]) for v in items.values())
|
|
74
|
+
if self.K < 1:
|
|
75
|
+
raise ValueError("pool has an item with no samples")
|
|
76
|
+
|
|
77
|
+
def curves(self, n_perm: int = 100, seed: int = 0) -> Tuple[List[str], np.ndarray, np.ndarray]:
|
|
78
|
+
"""Return (item ids, acc[item, k], tokens_per_sample[item]) with k = 1..K."""
|
|
79
|
+
rng = np.random.default_rng(seed)
|
|
80
|
+
ids = sorted(self.items)
|
|
81
|
+
acc = np.zeros((len(ids), self.K))
|
|
82
|
+
tok = np.zeros(len(ids))
|
|
83
|
+
for i, it in enumerate(ids):
|
|
84
|
+
v = self.items[it]
|
|
85
|
+
acc[i] = item_vote_curve(v["answers"][: len(v["answers"])], v["gold"], self.K, n_perm, rng)
|
|
86
|
+
tok[i] = float(np.mean(v["tokens"]))
|
|
87
|
+
return ids, acc, tok
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def frontier_curve(acc: np.ndarray, tok: np.ndarray) -> Tuple[np.ndarray, np.ndarray]:
|
|
91
|
+
"""Population-level points: tokens_k = k * mean tokens per sample, acc_k = mean over items."""
|
|
92
|
+
K = acc.shape[1]
|
|
93
|
+
return np.arange(1, K + 1) * float(tok.mean()), acc.mean(axis=0)
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def frontier_gap(acc: np.ndarray, tok: np.ndarray, sys_correct: np.ndarray, sys_tokens: np.ndarray,
|
|
97
|
+
n_boot: int = 2000, seed: int = 0, alpha: float = 0.05) -> dict:
|
|
98
|
+
"""Paired bootstrap of (system accuracy - frontier at the system's mean tokens). Arrays are aligned by item."""
|
|
99
|
+
n = acc.shape[0]
|
|
100
|
+
if not (len(sys_correct) == len(sys_tokens) == n):
|
|
101
|
+
raise ValueError("pool and system arrays must cover the same items")
|
|
102
|
+
tk, ak = frontier_curve(acc, tok)
|
|
103
|
+
f0, status = frontier_at(tk, ak, float(sys_tokens.mean()))
|
|
104
|
+
est = float(sys_correct.mean() - f0)
|
|
105
|
+
rng = np.random.default_rng(seed)
|
|
106
|
+
gaps = np.empty(n_boot)
|
|
107
|
+
for b in range(n_boot):
|
|
108
|
+
ix = rng.integers(0, n, n)
|
|
109
|
+
tkb, akb = frontier_curve(acc[ix], tok[ix])
|
|
110
|
+
fb, _ = frontier_at(tkb, akb, float(sys_tokens[ix].mean()))
|
|
111
|
+
gaps[b] = sys_correct[ix].mean() - fb
|
|
112
|
+
lo, hi = np.quantile(gaps, [alpha / 2, 1 - alpha / 2])
|
|
113
|
+
return {"gap": est, "ci_low": float(lo), "ci_high": float(hi), "p_gap_le_0": float((np.sum(gaps <= 0) + 1) / (n_boot + 1)), "frontier_at_system_tokens": f0, "range_status": status,
|
|
114
|
+
"system_accuracy": float(sys_correct.mean()), "system_mean_tokens": float(sys_tokens.mean()), "n_items": int(n), "n_boot": n_boot}
|
pace/hidden_profile.py
ADDED
|
@@ -0,0 +1,301 @@
|
|
|
1
|
+
"""pace hidden-profile: a configurable generator for the synthetic split-evidence (hidden-profile) task, with a built-in validity checker.
|
|
2
|
+
|
|
3
|
+
Design (Stasser and Titus 1985): four candidates are scored on `criteria` equally weighted criteria. `criteria - split` criteria are shared by all agents and favour a
|
|
4
|
+
decoy candidate; the `split` other criteria favour the target and are divided among `agents` agents, each of whom sees the shared criteria and `ceil(split / agents)`
|
|
5
|
+
of the split ones. Every single view then favours the decoy, and only the pooled evidence favours the target. Scores run from `low = 2` to `low + margin`.
|
|
6
|
+
|
|
7
|
+
With the default knobs (agents 5, criteria 10, split 6, margin 3, text format) the prompts, views and gold answers are identical to `pace.tasks.hidden_profile` (same seed, same random stream; `meta` has extra keys),
|
|
8
|
+
which is the task behind the released 500-item file.
|
|
9
|
+
|
|
10
|
+
Validity checks, computed from the generated scores (not from the construction): in every individual view the decoy has the strictly highest total of the four
|
|
11
|
+
candidates; on the pooled evidence the target has the strictly highest total; every split criterion appears in at least one view. Infeasible knobs are rejected
|
|
12
|
+
before generation: the shared criteria must outnumber the split criteria in one view (`criteria - split > ceil(split / agents)`) and the split criteria must outnumber
|
|
13
|
+
the shared ones (`split > criteria - split`), and `margin` must be at least 2.
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
from __future__ import annotations
|
|
17
|
+
|
|
18
|
+
import math
|
|
19
|
+
import random
|
|
20
|
+
|
|
21
|
+
NAMES = ["Avery", "Blake", "Casey", "Drew"]
|
|
22
|
+
LOW = 2
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def check_knobs(agents: int, criteria: int, split: int, margin: int) -> None:
|
|
26
|
+
if agents < 2:
|
|
27
|
+
raise ValueError("agents must be at least 2")
|
|
28
|
+
if margin < 2:
|
|
29
|
+
raise ValueError("margin must be at least 2")
|
|
30
|
+
if not 1 <= split < criteria:
|
|
31
|
+
raise ValueError("split must satisfy 1 <= split < criteria")
|
|
32
|
+
shared, view = criteria - split, math.ceil(split / agents)
|
|
33
|
+
if shared <= view:
|
|
34
|
+
raise ValueError(
|
|
35
|
+
"infeasible: the decoy cannot lead a single view; need criteria - split (%d) > ceil(split / agents) (%d)"
|
|
36
|
+
% (shared, view)
|
|
37
|
+
)
|
|
38
|
+
if split <= shared:
|
|
39
|
+
raise ValueError(
|
|
40
|
+
"infeasible: the target cannot lead the pooled evidence; need split (%d) > criteria - split (%d)"
|
|
41
|
+
% (split, shared)
|
|
42
|
+
)
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def _scores_text(sc, js):
|
|
46
|
+
return "\n".join(
|
|
47
|
+
"%s on %s: %d" % (NAMES[c], "criterion %d" % (j + 1), sc[c][j])
|
|
48
|
+
for j in js
|
|
49
|
+
for c in range(4)
|
|
50
|
+
)
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def _scores_numeric(sc, js):
|
|
54
|
+
rows = ["criterion," + ",".join(NAMES)]
|
|
55
|
+
rows += ["%d," % (j + 1) + ",".join(str(sc[c][j]) for c in range(4)) for j in js]
|
|
56
|
+
return "\n".join(rows)
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def generate(
|
|
60
|
+
n: int,
|
|
61
|
+
agents: int = 5,
|
|
62
|
+
criteria: int = 10,
|
|
63
|
+
split: int = 6,
|
|
64
|
+
margin: int = 3,
|
|
65
|
+
seed: int = 42,
|
|
66
|
+
fmt: str = "text",
|
|
67
|
+
) -> list[dict]:
|
|
68
|
+
"""Return n items {id, task, prompt, gold, kind, meta, views}. See the module docstring."""
|
|
69
|
+
if fmt not in ("text", "numeric"):
|
|
70
|
+
raise ValueError("fmt must be 'text' or 'numeric'")
|
|
71
|
+
check_knobs(agents, criteria, split, margin)
|
|
72
|
+
shared, view = criteria - split, math.ceil(split / agents)
|
|
73
|
+
high, cap = LOW + margin, min(LOW + margin - 2, 4)
|
|
74
|
+
rng = random.Random(seed)
|
|
75
|
+
out: list[dict] = []
|
|
76
|
+
while len(out) < n:
|
|
77
|
+
target, decoy = rng.sample(range(4), 2)
|
|
78
|
+
sc = [[rng.randint(2, 4) for _ in range(criteria)] for _ in range(4)]
|
|
79
|
+
for j in range(shared): # shared criteria: the decoy leads clearly
|
|
80
|
+
for c in range(4):
|
|
81
|
+
sc[c][j] = min(sc[c][j], cap)
|
|
82
|
+
sc[decoy][j] = high
|
|
83
|
+
sc[target][j] = LOW
|
|
84
|
+
for j in range(shared, criteria): # split criteria: the target leads
|
|
85
|
+
for c in range(4):
|
|
86
|
+
sc[c][j] = min(sc[c][j], cap)
|
|
87
|
+
sc[target][j] = high
|
|
88
|
+
sc[decoy][j] = LOW
|
|
89
|
+
tot = [sum(r) for r in sc]
|
|
90
|
+
if sorted(tot)[-1] == sorted(tot)[-2] or tot.index(max(tot)) != target:
|
|
91
|
+
continue
|
|
92
|
+
shown = _scores_text if fmt == "text" else _scores_numeric
|
|
93
|
+
head = (
|
|
94
|
+
"A committee must pick one of four candidates (%s) using %d equally weighted criteria, each scored %d to %d. The best candidate has the highest total score.\n\n"
|
|
95
|
+
% (", ".join(NAMES), criteria, 1, high)
|
|
96
|
+
)
|
|
97
|
+
tail = (
|
|
98
|
+
"\n\nWhich candidate has the highest total over all %d criteria? Give the name."
|
|
99
|
+
% criteria
|
|
100
|
+
)
|
|
101
|
+
full = head + "All scores:\n" + shown(sc, range(criteria)) + tail
|
|
102
|
+
order = list(range(shared, criteria))
|
|
103
|
+
rng.shuffle(order)
|
|
104
|
+
views = []
|
|
105
|
+
for a in range(agents):
|
|
106
|
+
mine = sorted(order[(view * a + k) % split] for k in range(view))
|
|
107
|
+
views.append(
|
|
108
|
+
head
|
|
109
|
+
+ "You see only part of the evidence. Scores you can see:\n"
|
|
110
|
+
+ shown(sc, list(range(shared)) + mine)
|
|
111
|
+
+ "\n\nOther committee members see the other criteria. Which candidate has the highest total over all %d criteria? Give the name."
|
|
112
|
+
% criteria
|
|
113
|
+
)
|
|
114
|
+
out.append(
|
|
115
|
+
{
|
|
116
|
+
"id": "hp%04d" % len(out),
|
|
117
|
+
"task": "hidden_profile",
|
|
118
|
+
"prompt": full,
|
|
119
|
+
"gold": NAMES[target],
|
|
120
|
+
"kind": "name",
|
|
121
|
+
"meta": {
|
|
122
|
+
"synthetic": True,
|
|
123
|
+
"totals": tot,
|
|
124
|
+
"decoy": NAMES[decoy],
|
|
125
|
+
"knobs": dict(
|
|
126
|
+
agents=agents,
|
|
127
|
+
criteria=criteria,
|
|
128
|
+
split=split,
|
|
129
|
+
margin=margin,
|
|
130
|
+
seed=seed,
|
|
131
|
+
fmt=fmt,
|
|
132
|
+
),
|
|
133
|
+
},
|
|
134
|
+
"views": views,
|
|
135
|
+
}
|
|
136
|
+
)
|
|
137
|
+
return out
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
def _parse_view(text: str, fmt: str, criteria: int):
|
|
141
|
+
"""Recover {criterion_index: {name: score}} from a prompt or view, in either format."""
|
|
142
|
+
got: dict[int, dict[str, int]] = {}
|
|
143
|
+
if fmt == "text":
|
|
144
|
+
import re
|
|
145
|
+
|
|
146
|
+
for name, j, s in re.findall(r"(\w+) on criterion (\d+): (\d+)", text):
|
|
147
|
+
got.setdefault(int(j) - 1, {})[name] = int(s)
|
|
148
|
+
else:
|
|
149
|
+
for line in text.splitlines():
|
|
150
|
+
parts = line.split(",")
|
|
151
|
+
if len(parts) == 5 and parts[0].isdigit():
|
|
152
|
+
got[int(parts[0]) - 1] = dict(zip(NAMES, map(int, parts[1:])))
|
|
153
|
+
return got
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
def validate(items: list[dict]) -> dict[str, int]:
|
|
157
|
+
"""Check each item from its prompt text: decoy strictly leads every view, target strictly leads the pooled evidence, every split criterion is seen. Returns counts."""
|
|
158
|
+
bad = dict(
|
|
159
|
+
items=len(items),
|
|
160
|
+
decoy_not_leading_a_view=0,
|
|
161
|
+
target_not_leading_pooled=0,
|
|
162
|
+
uncovered_split_criteria=0,
|
|
163
|
+
)
|
|
164
|
+
for it in items:
|
|
165
|
+
k = it["meta"]["knobs"]
|
|
166
|
+
fmt = k["fmt"] if "fmt" in k else "text"
|
|
167
|
+
full = _parse_view(it["prompt"], fmt, k["criteria"])
|
|
168
|
+
decoy, target = it["meta"]["decoy"], it["gold"]
|
|
169
|
+
tot = {nm: sum(full[j][nm] for j in full) for nm in NAMES}
|
|
170
|
+
best = sorted(tot.values())[-1]
|
|
171
|
+
if not (tot[target] == best and list(tot.values()).count(best) == 1):
|
|
172
|
+
bad["target_not_leading_pooled"] += 1
|
|
173
|
+
seen = set()
|
|
174
|
+
shared = k["criteria"] - k["split"]
|
|
175
|
+
view_bad = False
|
|
176
|
+
for v in it["views"]:
|
|
177
|
+
part = _parse_view(v, fmt, k["criteria"])
|
|
178
|
+
seen |= {j for j in part if j >= shared}
|
|
179
|
+
vt = {nm: sum(part[j][nm] for j in part) for nm in NAMES}
|
|
180
|
+
vb = sorted(vt.values())[-1]
|
|
181
|
+
if not (vt[decoy] == vb and list(vt.values()).count(vb) == 1):
|
|
182
|
+
view_bad = True
|
|
183
|
+
bad["decoy_not_leading_a_view"] += view_bad
|
|
184
|
+
bad["uncovered_split_criteria"] += (
|
|
185
|
+
len(set(range(shared, k["criteria"])) - seen) > 0
|
|
186
|
+
)
|
|
187
|
+
return bad
|
|
188
|
+
|
|
189
|
+
|
|
190
|
+
def valid(counts: dict[str, int]) -> bool:
|
|
191
|
+
return not any(v for k, v in counts.items() if k != "items")
|
|
192
|
+
|
|
193
|
+
|
|
194
|
+
def _round_accuracy(
|
|
195
|
+
records: list[dict], round_num: int, majority: bool = True
|
|
196
|
+
) -> float:
|
|
197
|
+
by_item = {}
|
|
198
|
+
for row in records:
|
|
199
|
+
if int(row.get("round", -1)) == round_num:
|
|
200
|
+
by_item.setdefault(str(row["item"]), []).append(bool(row["correct"]))
|
|
201
|
+
if not by_item:
|
|
202
|
+
return 0.0
|
|
203
|
+
if majority:
|
|
204
|
+
return sum(
|
|
205
|
+
(sum(values) / len(values))
|
|
206
|
+
if sum(values) * 2 == len(values)
|
|
207
|
+
else float(sum(values) * 2 > len(values))
|
|
208
|
+
for values in by_item.values()
|
|
209
|
+
) / len(by_item)
|
|
210
|
+
return sum(sum(values) / len(values) for values in by_item.values()) / len(by_item)
|
|
211
|
+
|
|
212
|
+
|
|
213
|
+
def evaluate_hidden_profile_logs(
|
|
214
|
+
split_debate_log: list[dict],
|
|
215
|
+
shared_debate_log: list[dict] | None = None,
|
|
216
|
+
full_info_accuracy: float = 1.0,
|
|
217
|
+
) -> dict:
|
|
218
|
+
rounds = [int(row["round"]) for row in split_debate_log]
|
|
219
|
+
first = min(rounds) if rounds else 1
|
|
220
|
+
last = max(rounds) if rounds else first
|
|
221
|
+
acc_vote = _round_accuracy(split_debate_log, first)
|
|
222
|
+
acc_debate = _round_accuracy(split_debate_log, last)
|
|
223
|
+
efficiency = (
|
|
224
|
+
(acc_debate - acc_vote) / (full_info_accuracy - acc_vote)
|
|
225
|
+
if full_info_accuracy > acc_vote
|
|
226
|
+
else None
|
|
227
|
+
)
|
|
228
|
+
result = {
|
|
229
|
+
"acc_vote": acc_vote,
|
|
230
|
+
"acc_debate": acc_debate,
|
|
231
|
+
"pooling_efficiency": efficiency,
|
|
232
|
+
"first_round": first,
|
|
233
|
+
"final_round": last,
|
|
234
|
+
}
|
|
235
|
+
if shared_debate_log is not None:
|
|
236
|
+
shared_rounds = [int(row["round"]) for row in shared_debate_log]
|
|
237
|
+
shared_first, shared_last = min(shared_rounds), max(shared_rounds)
|
|
238
|
+
shared_vote = _round_accuracy(shared_debate_log, shared_first)
|
|
239
|
+
shared_final = _round_accuracy(shared_debate_log, shared_last)
|
|
240
|
+
result.update(
|
|
241
|
+
{
|
|
242
|
+
"acc_vote_shared": shared_vote,
|
|
243
|
+
"acc_debate_shared": shared_final,
|
|
244
|
+
"p1_did": (acc_debate - acc_vote) - (shared_final - shared_vote),
|
|
245
|
+
}
|
|
246
|
+
)
|
|
247
|
+
return result
|
|
248
|
+
|
|
249
|
+
|
|
250
|
+
def stratified_p4_conformity(
|
|
251
|
+
debate_records: list[dict], placebo_records: list[dict]
|
|
252
|
+
) -> dict:
|
|
253
|
+
def item_stats(records):
|
|
254
|
+
grouped = {}
|
|
255
|
+
for row in records:
|
|
256
|
+
grouped.setdefault(str(row["item"]), []).append(row)
|
|
257
|
+
output = {}
|
|
258
|
+
for item, rows in grouped.items():
|
|
259
|
+
first = min(int(row["round"]) for row in rows)
|
|
260
|
+
last = max(int(row["round"]) for row in rows)
|
|
261
|
+
initial = [
|
|
262
|
+
bool(row["correct"]) for row in rows if int(row["round"]) == first
|
|
263
|
+
]
|
|
264
|
+
final = [bool(row["correct"]) for row in rows if int(row["round"]) == last]
|
|
265
|
+
output[item] = (
|
|
266
|
+
sum(initial),
|
|
267
|
+
len(initial),
|
|
268
|
+
sum(final) / len(final) if final else 0.0,
|
|
269
|
+
)
|
|
270
|
+
return output
|
|
271
|
+
|
|
272
|
+
debate, placebo = item_stats(debate_records), item_stats(placebo_records)
|
|
273
|
+
strata = {}
|
|
274
|
+
for stratum in sorted(
|
|
275
|
+
set(value[0] for value in debate.values())
|
|
276
|
+
& set(value[0] for value in placebo.values())
|
|
277
|
+
):
|
|
278
|
+
d = [value[2] for value in debate.values() if value[0] == stratum]
|
|
279
|
+
p = [value[2] for value in placebo.values() if value[0] == stratum]
|
|
280
|
+
if d and p:
|
|
281
|
+
strata[stratum] = {
|
|
282
|
+
"acc_final_debate": sum(d) / len(d),
|
|
283
|
+
"acc_final_placebo": sum(p) / len(p),
|
|
284
|
+
"difference": sum(d) / len(d) - sum(p) / len(p),
|
|
285
|
+
"n_debate": len(d),
|
|
286
|
+
"n_placebo": len(p),
|
|
287
|
+
}
|
|
288
|
+
total = sum(value["n_debate"] for value in strata.values())
|
|
289
|
+
weighted = (
|
|
290
|
+
sum(
|
|
291
|
+
value["difference"] * value["n_debate"] / total for value in strata.values()
|
|
292
|
+
)
|
|
293
|
+
if total
|
|
294
|
+
else None
|
|
295
|
+
)
|
|
296
|
+
return {
|
|
297
|
+
"strata": strata,
|
|
298
|
+
"weighted_difference": weighted,
|
|
299
|
+
"p4_difference": weighted,
|
|
300
|
+
"n_items": total,
|
|
301
|
+
}
|
pace/importers.py
ADDED
|
@@ -0,0 +1,219 @@
|
|
|
1
|
+
"""Import common multi-agent trace exports into portable PACE JSONL."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import csv
|
|
6
|
+
import json
|
|
7
|
+
from collections.abc import Iterable
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
from typing import Any
|
|
10
|
+
|
|
11
|
+
from .schema import ItemLog, item_log_from_mapping
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def _records(path: str) -> list[Any]:
|
|
15
|
+
text = Path(path).read_text(encoding="utf-8").strip()
|
|
16
|
+
if not text:
|
|
17
|
+
return []
|
|
18
|
+
try:
|
|
19
|
+
value = json.loads(text)
|
|
20
|
+
return value if isinstance(value, list) else [value]
|
|
21
|
+
except json.JSONDecodeError:
|
|
22
|
+
return [json.loads(line) for line in text.splitlines() if line.strip()]
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def _usage(value: Any) -> dict[str, Any]:
|
|
26
|
+
if not isinstance(value, dict):
|
|
27
|
+
return {}
|
|
28
|
+
return {
|
|
29
|
+
"prompt_tokens": value.get("prompt_tokens", value.get("input_tokens")),
|
|
30
|
+
"completion_tokens": value.get("completion_tokens", value.get("output_tokens")),
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def _message_rows(
|
|
35
|
+
messages: Iterable[Any], item: Any, default_round: int = 0
|
|
36
|
+
) -> Iterable[dict[str, Any]]:
|
|
37
|
+
for index, message in enumerate(messages):
|
|
38
|
+
if not isinstance(message, dict):
|
|
39
|
+
continue
|
|
40
|
+
row = dict(message)
|
|
41
|
+
usage = _usage(row.get("usage", row.get("token_usage")))
|
|
42
|
+
row.update({key: value for key, value in usage.items() if value is not None})
|
|
43
|
+
row["item"] = row.get("item", row.get("item_id", item))
|
|
44
|
+
row["agent"] = row.get(
|
|
45
|
+
"agent",
|
|
46
|
+
row.get("agent_id", row.get("name", row.get("role", "agent_%d" % index))),
|
|
47
|
+
)
|
|
48
|
+
row["round"] = row.get("round", row.get("round_id", default_round + index))
|
|
49
|
+
if "correct" not in row and "is_correct" not in row:
|
|
50
|
+
continue
|
|
51
|
+
yield row
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def import_autogen(records: list[Any]) -> list[ItemLog]:
|
|
55
|
+
result = []
|
|
56
|
+
for record in records:
|
|
57
|
+
if isinstance(record, dict) and isinstance(record.get("messages"), list):
|
|
58
|
+
rows = list(
|
|
59
|
+
_message_rows(
|
|
60
|
+
record["messages"],
|
|
61
|
+
record.get("item_id", record.get("id", "item_0")),
|
|
62
|
+
record.get("round", 0),
|
|
63
|
+
)
|
|
64
|
+
)
|
|
65
|
+
score = record.get("correct", record.get("is_correct"))
|
|
66
|
+
if score is not None:
|
|
67
|
+
for row in rows:
|
|
68
|
+
row.setdefault("correct", score)
|
|
69
|
+
result.extend(item_log_from_mapping(row) for row in rows)
|
|
70
|
+
elif isinstance(record, dict):
|
|
71
|
+
result.extend(
|
|
72
|
+
item_log_from_mapping(row)
|
|
73
|
+
for row in _message_rows(
|
|
74
|
+
[record], record.get("item_id", record.get("id", "item_0"))
|
|
75
|
+
)
|
|
76
|
+
)
|
|
77
|
+
return result
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def _nested_messages(value: Any) -> Iterable[dict[str, Any]]:
|
|
81
|
+
if isinstance(value, dict):
|
|
82
|
+
if isinstance(value.get("messages"), list):
|
|
83
|
+
yield from (x for x in value["messages"] if isinstance(x, dict))
|
|
84
|
+
else:
|
|
85
|
+
for child in value.values():
|
|
86
|
+
yield from _nested_messages(child)
|
|
87
|
+
elif isinstance(value, list):
|
|
88
|
+
for child in value:
|
|
89
|
+
yield from _nested_messages(child)
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def import_langgraph(records: list[Any]) -> list[ItemLog]:
|
|
93
|
+
result = []
|
|
94
|
+
for record in records:
|
|
95
|
+
if not isinstance(record, dict):
|
|
96
|
+
continue
|
|
97
|
+
item = record.get("item_id", record.get("id", "item_0"))
|
|
98
|
+
rows = list(
|
|
99
|
+
_message_rows(
|
|
100
|
+
list(_nested_messages(record.get("state", record))),
|
|
101
|
+
item,
|
|
102
|
+
record.get("round", 0),
|
|
103
|
+
)
|
|
104
|
+
)
|
|
105
|
+
score = record.get("correct", record.get("is_correct"))
|
|
106
|
+
if score is not None:
|
|
107
|
+
for row in rows:
|
|
108
|
+
row.setdefault("correct", score)
|
|
109
|
+
for row in rows:
|
|
110
|
+
result.append(item_log_from_mapping(row))
|
|
111
|
+
return result
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def import_crewai(records: list[Any]) -> list[ItemLog]:
|
|
115
|
+
result = []
|
|
116
|
+
for record in records:
|
|
117
|
+
if not isinstance(record, dict):
|
|
118
|
+
continue
|
|
119
|
+
tasks = record.get("tasks", [record])
|
|
120
|
+
for index, task in enumerate(tasks):
|
|
121
|
+
if not isinstance(task, dict):
|
|
122
|
+
continue
|
|
123
|
+
row = dict(task)
|
|
124
|
+
row["item"] = row.get(
|
|
125
|
+
"item",
|
|
126
|
+
row.get("item_id", record.get("item_id", record.get("id", "item_0"))),
|
|
127
|
+
)
|
|
128
|
+
row["agent"] = row.get(
|
|
129
|
+
"agent", row.get("agent_id", row.get("agent_name", "agent_%d" % index))
|
|
130
|
+
)
|
|
131
|
+
row["round"] = row.get("round", row.get("task_index", index))
|
|
132
|
+
row["answer"] = row.get("output", row.get("answer"))
|
|
133
|
+
usage = _usage(row.get("usage", row.get("token_usage", row.get("metrics"))))
|
|
134
|
+
row.update(
|
|
135
|
+
{key: value for key, value in usage.items() if value is not None}
|
|
136
|
+
)
|
|
137
|
+
if row.get("correct") is not None or row.get("is_correct") is not None:
|
|
138
|
+
result.append(item_log_from_mapping(row))
|
|
139
|
+
return result
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
def import_csv(records: list[Any]) -> list[ItemLog]:
|
|
143
|
+
return [item_log_from_mapping(row) for row in records if isinstance(row, dict)]
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
def import_inspect(records: list[Any]) -> list[ItemLog]:
|
|
147
|
+
"""Convert Inspect AI EvalLog JSON payloads into canonical PACE ItemLog records."""
|
|
148
|
+
result = []
|
|
149
|
+
for record in records:
|
|
150
|
+
if not isinstance(record, dict):
|
|
151
|
+
continue
|
|
152
|
+
samples = record.get("samples", [record])
|
|
153
|
+
for sample in samples:
|
|
154
|
+
if not isinstance(sample, dict):
|
|
155
|
+
continue
|
|
156
|
+
item_id = str(sample.get("id", "item_0"))
|
|
157
|
+
scores = sample.get("scores", {})
|
|
158
|
+
correct = False
|
|
159
|
+
if isinstance(scores, dict):
|
|
160
|
+
for scorer_obj in scores.values():
|
|
161
|
+
val = scorer_obj.get("value") if isinstance(scorer_obj, dict) else scorer_obj
|
|
162
|
+
if val in ("C", "correct", 1, 1.0, True):
|
|
163
|
+
correct = True
|
|
164
|
+
break
|
|
165
|
+
messages = sample.get("messages", [])
|
|
166
|
+
assistant_turns = [m for m in messages if isinstance(m, dict) and m.get("role") == "assistant"]
|
|
167
|
+
usage = sample.get("model_usage", {})
|
|
168
|
+
total_in = sum(u.get("input_tokens", 0) for u in usage.values() if isinstance(u, dict)) if isinstance(usage, dict) else 0
|
|
169
|
+
total_out = sum(u.get("output_tokens", 0) for u in usage.values() if isinstance(u, dict)) if isinstance(usage, dict) else 0
|
|
170
|
+
total_reason = sum(u.get("reasoning_tokens", 0) for u in usage.values() if isinstance(u, dict)) if isinstance(usage, dict) else 0
|
|
171
|
+
n_turns = max(1, len(assistant_turns))
|
|
172
|
+
for idx, msg in enumerate(assistant_turns or [{}]):
|
|
173
|
+
agent_id = str(msg.get("source") or msg.get("name") or f"agent_{idx}")
|
|
174
|
+
tool_calls = bool(msg.get("tool_calls"))
|
|
175
|
+
result.append(
|
|
176
|
+
item_log_from_mapping(
|
|
177
|
+
{
|
|
178
|
+
"item": item_id,
|
|
179
|
+
"agent": agent_id,
|
|
180
|
+
"round": idx + 1,
|
|
181
|
+
"correct": bool(correct),
|
|
182
|
+
"prompt_tokens": round(total_in / n_turns, 2) if total_in else None,
|
|
183
|
+
"completion_tokens": round(total_out / n_turns, 2) if total_out else None,
|
|
184
|
+
"reasoning_tokens": round(total_reason / n_turns, 2) if total_reason else None,
|
|
185
|
+
"tool_calls": 1 if tool_calls else 0,
|
|
186
|
+
}
|
|
187
|
+
)
|
|
188
|
+
)
|
|
189
|
+
return result
|
|
190
|
+
|
|
191
|
+
|
|
192
|
+
def load_and_import(path: str, format_name: str) -> list[ItemLog]:
|
|
193
|
+
if format_name == "csv":
|
|
194
|
+
with open(path, newline="", encoding="utf-8") as handle:
|
|
195
|
+
return import_csv(list(csv.DictReader(handle)))
|
|
196
|
+
records = _records(path)
|
|
197
|
+
importer = {
|
|
198
|
+
"autogen": import_autogen,
|
|
199
|
+
"langgraph": import_langgraph,
|
|
200
|
+
"crewai": import_crewai,
|
|
201
|
+
"inspect": import_inspect,
|
|
202
|
+
}.get(format_name)
|
|
203
|
+
if importer is None:
|
|
204
|
+
raise ValueError("unknown trace format: %s" % format_name)
|
|
205
|
+
logs = importer(records)
|
|
206
|
+
if not logs:
|
|
207
|
+
raise ValueError("no scored records found in %s trace" % format_name)
|
|
208
|
+
return logs
|
|
209
|
+
|
|
210
|
+
|
|
211
|
+
def write_jsonl(logs: Iterable[ItemLog], output: str) -> int:
|
|
212
|
+
logs = list(logs)
|
|
213
|
+
if not logs:
|
|
214
|
+
raise ValueError("no records to write")
|
|
215
|
+
with open(output, "w", encoding="utf-8", newline="\n") as handle:
|
|
216
|
+
handle.writelines(
|
|
217
|
+
json.dumps(log.to_dict(), sort_keys=True) + "\n" for log in logs
|
|
218
|
+
)
|
|
219
|
+
return len(logs)
|