pace-eval 0.2.2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (83) hide show
  1. cme/__init__.py +11 -0
  2. cme/__main__.py +5 -0
  3. cme/analysis.py +223 -0
  4. cme/audit.py +145 -0
  5. cme/baseline.py +162 -0
  6. cme/cli.py +279 -0
  7. cme/client.py +165 -0
  8. cme/dynamics.py +212 -0
  9. cme/examples/crewai_integration.py +16 -0
  10. cme/examples/demo_pool.jsonl +360 -0
  11. cme/examples/demo_run.jsonl +480 -0
  12. cme/examples/langgraph_integration.py +27 -0
  13. cme/examples/make_demo.py +37 -0
  14. cme/examples/openai_quickstart.py +18 -0
  15. cme/experiment.py +555 -0
  16. cme/frontier.py +143 -0
  17. cme/hidden_profile.py +135 -0
  18. cme/proxy.py +106 -0
  19. cme/rebuttal.py +76 -0
  20. cme/report.py +197 -0
  21. cme/rollout_card.py +137 -0
  22. cme/tasks.py +266 -0
  23. cme/verified_debate.py +127 -0
  24. cme/verifier.py +92 -0
  25. mae_eval/__init__.py +11 -0
  26. mae_eval/__main__.py +5 -0
  27. mae_eval/auditor.py +112 -0
  28. mae_eval/classifier.py +62 -0
  29. mae_eval/cli.py +210 -0
  30. mae_eval/tracker.py +162 -0
  31. mae_eval/vector.py +91 -0
  32. pace/__init__.py +11 -0
  33. pace/__main__.py +5 -0
  34. pace/analysis.py +129 -0
  35. pace/audit.py +145 -0
  36. pace/baseline.py +162 -0
  37. pace/card.py +205 -0
  38. pace/cli.py +898 -0
  39. pace/client.py +165 -0
  40. pace/dynamics.py +372 -0
  41. pace/escape_benchmarks.py +162 -0
  42. pace/examples/crewai_integration.py +16 -0
  43. pace/examples/demo_pool.jsonl +360 -0
  44. pace/examples/demo_run.jsonl +480 -0
  45. pace/examples/langgraph_integration.py +27 -0
  46. pace/examples/make_demo.py +37 -0
  47. pace/examples/openai_quickstart.py +18 -0
  48. pace/experiment.py +555 -0
  49. pace/frontier.py +114 -0
  50. pace/hidden_profile.py +301 -0
  51. pace/importers.py +219 -0
  52. pace/inflate.py +213 -0
  53. pace/influence.py +231 -0
  54. pace/meta_engine.py +246 -0
  55. pace/middleware.py +230 -0
  56. pace/open_ended.py +114 -0
  57. pace/plotting.py +132 -0
  58. pace/pool_sizer.py +154 -0
  59. pace/protocol_v3.py +414 -0
  60. pace/proxy.py +106 -0
  61. pace/rebuttal.py +219 -0
  62. pace/report.py +197 -0
  63. pace/rollout_card.py +137 -0
  64. pace/schema.py +74 -0
  65. pace/simulator.py +199 -0
  66. pace/studio.html +1040 -0
  67. pace/tasks.py +266 -0
  68. pace/verified_debate.py +127 -0
  69. pace/verifier.py +92 -0
  70. pace/workbench.html +26 -0
  71. pace/workbench.py +119 -0
  72. pace_eval-0.2.2.dist-info/METADATA +205 -0
  73. pace_eval-0.2.2.dist-info/RECORD +83 -0
  74. pace_eval-0.2.2.dist-info/WHEEL +5 -0
  75. pace_eval-0.2.2.dist-info/entry_points.txt +4 -0
  76. pace_eval-0.2.2.dist-info/licenses/LICENSE +202 -0
  77. pace_eval-0.2.2.dist-info/top_level.txt +4 -0
  78. pace_lab/__init__.py +9 -0
  79. pace_lab/clients.py +568 -0
  80. pace_lab/concurrency_test.py +127 -0
  81. pace_lab/data.py +445 -0
  82. pace_lab/graders.py +626 -0
  83. pace_lab/strategies.py +345 -0
pace/frontier.py ADDED
@@ -0,0 +1,114 @@
1
+ """Self-consistency frontier and frontier gap with a paired bootstrap.
2
+
3
+ A sample pool holds, for each item, up to K independent sampled answers with token counts. For k = 1..K the expected accuracy of
4
+ plurality voting over k samples is estimated over random k-subsets (ties share credit equally). The cost of k samples is k times the
5
+ mean tokens per sample. The frontier is the nondecreasing (isotonic) fit of accuracy against tokens, interpolated linearly.
6
+
7
+ frontier gap = accuracy of a system - frontier(mean tokens the system used). A positive gap means the system sits above what
8
+ sampling alone would reach at that measured token cost. The bootstrap resamples items and recomputes everything (pool curve,
9
+ isotonic fit, interpolation, system accuracy), so the interval accounts for sampling noise in both sides.
10
+ """
11
+ from __future__ import annotations
12
+
13
+ from collections import Counter
14
+ from typing import Dict, List, Optional, Sequence, Tuple
15
+
16
+ import numpy as np
17
+
18
+
19
+ def plurality_credit(answers: Sequence[str], gold: str) -> float:
20
+ """1 if the plurality answer equals gold; 1/m if gold is one of m tied leaders; 0 otherwise."""
21
+ c = Counter(answers)
22
+ top = max(c.values())
23
+ leaders = [a for a, v in c.items() if v == top]
24
+ return (1.0 / len(leaders)) if gold in leaders else 0.0
25
+
26
+
27
+ def item_vote_curve(answers: Sequence[str], gold: str, K: int, n_perm: int, rng: np.random.Generator) -> np.ndarray:
28
+ """Expected plurality accuracy of k = 1..K samples for one item. Each random permutation gives nested prefixes; the value at
29
+ k equal to the number of samples is exact (order does not matter)."""
30
+ n = len(answers)
31
+ if K > n:
32
+ raise ValueError("K exceeds the number of samples for an item")
33
+ out = np.zeros(K)
34
+ for _ in range(n_perm):
35
+ c: Counter = Counter()
36
+ for k, i in enumerate(rng.permutation(n)[:K], 1):
37
+ c[answers[i]] += 1
38
+ top = max(c.values())
39
+ leaders = [a for a, v in c.items() if v == top]
40
+ out[k - 1] += (1.0 / len(leaders)) if gold in leaders else 0.0
41
+ return out / n_perm
42
+
43
+
44
+ def pav_increasing(y: np.ndarray, w: Optional[np.ndarray] = None) -> np.ndarray:
45
+ """Weighted pool-adjacent-violators: the nondecreasing least-squares fit to y."""
46
+ y = np.asarray(y, dtype=float)
47
+ w = np.ones_like(y) if w is None else np.asarray(w, dtype=float)
48
+ blocks: List[List[float]] = [] # [value, weight, count]
49
+ for yi, wi in zip(y, w):
50
+ blocks.append([yi, wi, 1])
51
+ while len(blocks) > 1 and blocks[-2][0] > blocks[-1][0]:
52
+ v2, w2, c2 = blocks.pop()
53
+ v1, w1, c1 = blocks.pop()
54
+ blocks.append([(v1 * w1 + v2 * w2) / (w1 + w2), w1 + w2, c1 + c2])
55
+ return np.concatenate([[b[0]] * int(b[2]) for b in blocks])
56
+
57
+
58
+ def frontier_at(tokens_k: np.ndarray, acc_k: np.ndarray, t: float) -> Tuple[float, str]:
59
+ """Isotonic frontier interpolated at t. Returns (value, status) with status in {'inside', 'below range', 'beyond range'}."""
60
+ fit = pav_increasing(acc_k)
61
+ if t < tokens_k[0]:
62
+ return float(fit[0]), "below range"
63
+ if t > tokens_k[-1]:
64
+ return float(fit[-1]), "beyond range"
65
+ return float(np.interp(t, tokens_k, fit)), "inside"
66
+
67
+
68
+ class Pool:
69
+ """Per-item sampled answers and token counts. items: id -> dict(gold, answers[list[str]], tokens[list[float]])."""
70
+
71
+ def __init__(self, items: Dict[str, dict], K: Optional[int] = None):
72
+ self.items = items
73
+ self.K = K or min(len(v["answers"]) for v in items.values())
74
+ if self.K < 1:
75
+ raise ValueError("pool has an item with no samples")
76
+
77
+ def curves(self, n_perm: int = 100, seed: int = 0) -> Tuple[List[str], np.ndarray, np.ndarray]:
78
+ """Return (item ids, acc[item, k], tokens_per_sample[item]) with k = 1..K."""
79
+ rng = np.random.default_rng(seed)
80
+ ids = sorted(self.items)
81
+ acc = np.zeros((len(ids), self.K))
82
+ tok = np.zeros(len(ids))
83
+ for i, it in enumerate(ids):
84
+ v = self.items[it]
85
+ acc[i] = item_vote_curve(v["answers"][: len(v["answers"])], v["gold"], self.K, n_perm, rng)
86
+ tok[i] = float(np.mean(v["tokens"]))
87
+ return ids, acc, tok
88
+
89
+
90
+ def frontier_curve(acc: np.ndarray, tok: np.ndarray) -> Tuple[np.ndarray, np.ndarray]:
91
+ """Population-level points: tokens_k = k * mean tokens per sample, acc_k = mean over items."""
92
+ K = acc.shape[1]
93
+ return np.arange(1, K + 1) * float(tok.mean()), acc.mean(axis=0)
94
+
95
+
96
+ def frontier_gap(acc: np.ndarray, tok: np.ndarray, sys_correct: np.ndarray, sys_tokens: np.ndarray,
97
+ n_boot: int = 2000, seed: int = 0, alpha: float = 0.05) -> dict:
98
+ """Paired bootstrap of (system accuracy - frontier at the system's mean tokens). Arrays are aligned by item."""
99
+ n = acc.shape[0]
100
+ if not (len(sys_correct) == len(sys_tokens) == n):
101
+ raise ValueError("pool and system arrays must cover the same items")
102
+ tk, ak = frontier_curve(acc, tok)
103
+ f0, status = frontier_at(tk, ak, float(sys_tokens.mean()))
104
+ est = float(sys_correct.mean() - f0)
105
+ rng = np.random.default_rng(seed)
106
+ gaps = np.empty(n_boot)
107
+ for b in range(n_boot):
108
+ ix = rng.integers(0, n, n)
109
+ tkb, akb = frontier_curve(acc[ix], tok[ix])
110
+ fb, _ = frontier_at(tkb, akb, float(sys_tokens[ix].mean()))
111
+ gaps[b] = sys_correct[ix].mean() - fb
112
+ lo, hi = np.quantile(gaps, [alpha / 2, 1 - alpha / 2])
113
+ return {"gap": est, "ci_low": float(lo), "ci_high": float(hi), "p_gap_le_0": float((np.sum(gaps <= 0) + 1) / (n_boot + 1)), "frontier_at_system_tokens": f0, "range_status": status,
114
+ "system_accuracy": float(sys_correct.mean()), "system_mean_tokens": float(sys_tokens.mean()), "n_items": int(n), "n_boot": n_boot}
pace/hidden_profile.py ADDED
@@ -0,0 +1,301 @@
1
+ """pace hidden-profile: a configurable generator for the synthetic split-evidence (hidden-profile) task, with a built-in validity checker.
2
+
3
+ Design (Stasser and Titus 1985): four candidates are scored on `criteria` equally weighted criteria. `criteria - split` criteria are shared by all agents and favour a
4
+ decoy candidate; the `split` other criteria favour the target and are divided among `agents` agents, each of whom sees the shared criteria and `ceil(split / agents)`
5
+ of the split ones. Every single view then favours the decoy, and only the pooled evidence favours the target. Scores run from `low = 2` to `low + margin`.
6
+
7
+ With the default knobs (agents 5, criteria 10, split 6, margin 3, text format) the prompts, views and gold answers are identical to `pace.tasks.hidden_profile` (same seed, same random stream; `meta` has extra keys),
8
+ which is the task behind the released 500-item file.
9
+
10
+ Validity checks, computed from the generated scores (not from the construction): in every individual view the decoy has the strictly highest total of the four
11
+ candidates; on the pooled evidence the target has the strictly highest total; every split criterion appears in at least one view. Infeasible knobs are rejected
12
+ before generation: the shared criteria must outnumber the split criteria in one view (`criteria - split > ceil(split / agents)`) and the split criteria must outnumber
13
+ the shared ones (`split > criteria - split`), and `margin` must be at least 2.
14
+ """
15
+
16
+ from __future__ import annotations
17
+
18
+ import math
19
+ import random
20
+
21
+ NAMES = ["Avery", "Blake", "Casey", "Drew"]
22
+ LOW = 2
23
+
24
+
25
+ def check_knobs(agents: int, criteria: int, split: int, margin: int) -> None:
26
+ if agents < 2:
27
+ raise ValueError("agents must be at least 2")
28
+ if margin < 2:
29
+ raise ValueError("margin must be at least 2")
30
+ if not 1 <= split < criteria:
31
+ raise ValueError("split must satisfy 1 <= split < criteria")
32
+ shared, view = criteria - split, math.ceil(split / agents)
33
+ if shared <= view:
34
+ raise ValueError(
35
+ "infeasible: the decoy cannot lead a single view; need criteria - split (%d) > ceil(split / agents) (%d)"
36
+ % (shared, view)
37
+ )
38
+ if split <= shared:
39
+ raise ValueError(
40
+ "infeasible: the target cannot lead the pooled evidence; need split (%d) > criteria - split (%d)"
41
+ % (split, shared)
42
+ )
43
+
44
+
45
+ def _scores_text(sc, js):
46
+ return "\n".join(
47
+ "%s on %s: %d" % (NAMES[c], "criterion %d" % (j + 1), sc[c][j])
48
+ for j in js
49
+ for c in range(4)
50
+ )
51
+
52
+
53
+ def _scores_numeric(sc, js):
54
+ rows = ["criterion," + ",".join(NAMES)]
55
+ rows += ["%d," % (j + 1) + ",".join(str(sc[c][j]) for c in range(4)) for j in js]
56
+ return "\n".join(rows)
57
+
58
+
59
+ def generate(
60
+ n: int,
61
+ agents: int = 5,
62
+ criteria: int = 10,
63
+ split: int = 6,
64
+ margin: int = 3,
65
+ seed: int = 42,
66
+ fmt: str = "text",
67
+ ) -> list[dict]:
68
+ """Return n items {id, task, prompt, gold, kind, meta, views}. See the module docstring."""
69
+ if fmt not in ("text", "numeric"):
70
+ raise ValueError("fmt must be 'text' or 'numeric'")
71
+ check_knobs(agents, criteria, split, margin)
72
+ shared, view = criteria - split, math.ceil(split / agents)
73
+ high, cap = LOW + margin, min(LOW + margin - 2, 4)
74
+ rng = random.Random(seed)
75
+ out: list[dict] = []
76
+ while len(out) < n:
77
+ target, decoy = rng.sample(range(4), 2)
78
+ sc = [[rng.randint(2, 4) for _ in range(criteria)] for _ in range(4)]
79
+ for j in range(shared): # shared criteria: the decoy leads clearly
80
+ for c in range(4):
81
+ sc[c][j] = min(sc[c][j], cap)
82
+ sc[decoy][j] = high
83
+ sc[target][j] = LOW
84
+ for j in range(shared, criteria): # split criteria: the target leads
85
+ for c in range(4):
86
+ sc[c][j] = min(sc[c][j], cap)
87
+ sc[target][j] = high
88
+ sc[decoy][j] = LOW
89
+ tot = [sum(r) for r in sc]
90
+ if sorted(tot)[-1] == sorted(tot)[-2] or tot.index(max(tot)) != target:
91
+ continue
92
+ shown = _scores_text if fmt == "text" else _scores_numeric
93
+ head = (
94
+ "A committee must pick one of four candidates (%s) using %d equally weighted criteria, each scored %d to %d. The best candidate has the highest total score.\n\n"
95
+ % (", ".join(NAMES), criteria, 1, high)
96
+ )
97
+ tail = (
98
+ "\n\nWhich candidate has the highest total over all %d criteria? Give the name."
99
+ % criteria
100
+ )
101
+ full = head + "All scores:\n" + shown(sc, range(criteria)) + tail
102
+ order = list(range(shared, criteria))
103
+ rng.shuffle(order)
104
+ views = []
105
+ for a in range(agents):
106
+ mine = sorted(order[(view * a + k) % split] for k in range(view))
107
+ views.append(
108
+ head
109
+ + "You see only part of the evidence. Scores you can see:\n"
110
+ + shown(sc, list(range(shared)) + mine)
111
+ + "\n\nOther committee members see the other criteria. Which candidate has the highest total over all %d criteria? Give the name."
112
+ % criteria
113
+ )
114
+ out.append(
115
+ {
116
+ "id": "hp%04d" % len(out),
117
+ "task": "hidden_profile",
118
+ "prompt": full,
119
+ "gold": NAMES[target],
120
+ "kind": "name",
121
+ "meta": {
122
+ "synthetic": True,
123
+ "totals": tot,
124
+ "decoy": NAMES[decoy],
125
+ "knobs": dict(
126
+ agents=agents,
127
+ criteria=criteria,
128
+ split=split,
129
+ margin=margin,
130
+ seed=seed,
131
+ fmt=fmt,
132
+ ),
133
+ },
134
+ "views": views,
135
+ }
136
+ )
137
+ return out
138
+
139
+
140
+ def _parse_view(text: str, fmt: str, criteria: int):
141
+ """Recover {criterion_index: {name: score}} from a prompt or view, in either format."""
142
+ got: dict[int, dict[str, int]] = {}
143
+ if fmt == "text":
144
+ import re
145
+
146
+ for name, j, s in re.findall(r"(\w+) on criterion (\d+): (\d+)", text):
147
+ got.setdefault(int(j) - 1, {})[name] = int(s)
148
+ else:
149
+ for line in text.splitlines():
150
+ parts = line.split(",")
151
+ if len(parts) == 5 and parts[0].isdigit():
152
+ got[int(parts[0]) - 1] = dict(zip(NAMES, map(int, parts[1:])))
153
+ return got
154
+
155
+
156
+ def validate(items: list[dict]) -> dict[str, int]:
157
+ """Check each item from its prompt text: decoy strictly leads every view, target strictly leads the pooled evidence, every split criterion is seen. Returns counts."""
158
+ bad = dict(
159
+ items=len(items),
160
+ decoy_not_leading_a_view=0,
161
+ target_not_leading_pooled=0,
162
+ uncovered_split_criteria=0,
163
+ )
164
+ for it in items:
165
+ k = it["meta"]["knobs"]
166
+ fmt = k["fmt"] if "fmt" in k else "text"
167
+ full = _parse_view(it["prompt"], fmt, k["criteria"])
168
+ decoy, target = it["meta"]["decoy"], it["gold"]
169
+ tot = {nm: sum(full[j][nm] for j in full) for nm in NAMES}
170
+ best = sorted(tot.values())[-1]
171
+ if not (tot[target] == best and list(tot.values()).count(best) == 1):
172
+ bad["target_not_leading_pooled"] += 1
173
+ seen = set()
174
+ shared = k["criteria"] - k["split"]
175
+ view_bad = False
176
+ for v in it["views"]:
177
+ part = _parse_view(v, fmt, k["criteria"])
178
+ seen |= {j for j in part if j >= shared}
179
+ vt = {nm: sum(part[j][nm] for j in part) for nm in NAMES}
180
+ vb = sorted(vt.values())[-1]
181
+ if not (vt[decoy] == vb and list(vt.values()).count(vb) == 1):
182
+ view_bad = True
183
+ bad["decoy_not_leading_a_view"] += view_bad
184
+ bad["uncovered_split_criteria"] += (
185
+ len(set(range(shared, k["criteria"])) - seen) > 0
186
+ )
187
+ return bad
188
+
189
+
190
+ def valid(counts: dict[str, int]) -> bool:
191
+ return not any(v for k, v in counts.items() if k != "items")
192
+
193
+
194
+ def _round_accuracy(
195
+ records: list[dict], round_num: int, majority: bool = True
196
+ ) -> float:
197
+ by_item = {}
198
+ for row in records:
199
+ if int(row.get("round", -1)) == round_num:
200
+ by_item.setdefault(str(row["item"]), []).append(bool(row["correct"]))
201
+ if not by_item:
202
+ return 0.0
203
+ if majority:
204
+ return sum(
205
+ (sum(values) / len(values))
206
+ if sum(values) * 2 == len(values)
207
+ else float(sum(values) * 2 > len(values))
208
+ for values in by_item.values()
209
+ ) / len(by_item)
210
+ return sum(sum(values) / len(values) for values in by_item.values()) / len(by_item)
211
+
212
+
213
+ def evaluate_hidden_profile_logs(
214
+ split_debate_log: list[dict],
215
+ shared_debate_log: list[dict] | None = None,
216
+ full_info_accuracy: float = 1.0,
217
+ ) -> dict:
218
+ rounds = [int(row["round"]) for row in split_debate_log]
219
+ first = min(rounds) if rounds else 1
220
+ last = max(rounds) if rounds else first
221
+ acc_vote = _round_accuracy(split_debate_log, first)
222
+ acc_debate = _round_accuracy(split_debate_log, last)
223
+ efficiency = (
224
+ (acc_debate - acc_vote) / (full_info_accuracy - acc_vote)
225
+ if full_info_accuracy > acc_vote
226
+ else None
227
+ )
228
+ result = {
229
+ "acc_vote": acc_vote,
230
+ "acc_debate": acc_debate,
231
+ "pooling_efficiency": efficiency,
232
+ "first_round": first,
233
+ "final_round": last,
234
+ }
235
+ if shared_debate_log is not None:
236
+ shared_rounds = [int(row["round"]) for row in shared_debate_log]
237
+ shared_first, shared_last = min(shared_rounds), max(shared_rounds)
238
+ shared_vote = _round_accuracy(shared_debate_log, shared_first)
239
+ shared_final = _round_accuracy(shared_debate_log, shared_last)
240
+ result.update(
241
+ {
242
+ "acc_vote_shared": shared_vote,
243
+ "acc_debate_shared": shared_final,
244
+ "p1_did": (acc_debate - acc_vote) - (shared_final - shared_vote),
245
+ }
246
+ )
247
+ return result
248
+
249
+
250
+ def stratified_p4_conformity(
251
+ debate_records: list[dict], placebo_records: list[dict]
252
+ ) -> dict:
253
+ def item_stats(records):
254
+ grouped = {}
255
+ for row in records:
256
+ grouped.setdefault(str(row["item"]), []).append(row)
257
+ output = {}
258
+ for item, rows in grouped.items():
259
+ first = min(int(row["round"]) for row in rows)
260
+ last = max(int(row["round"]) for row in rows)
261
+ initial = [
262
+ bool(row["correct"]) for row in rows if int(row["round"]) == first
263
+ ]
264
+ final = [bool(row["correct"]) for row in rows if int(row["round"]) == last]
265
+ output[item] = (
266
+ sum(initial),
267
+ len(initial),
268
+ sum(final) / len(final) if final else 0.0,
269
+ )
270
+ return output
271
+
272
+ debate, placebo = item_stats(debate_records), item_stats(placebo_records)
273
+ strata = {}
274
+ for stratum in sorted(
275
+ set(value[0] for value in debate.values())
276
+ & set(value[0] for value in placebo.values())
277
+ ):
278
+ d = [value[2] for value in debate.values() if value[0] == stratum]
279
+ p = [value[2] for value in placebo.values() if value[0] == stratum]
280
+ if d and p:
281
+ strata[stratum] = {
282
+ "acc_final_debate": sum(d) / len(d),
283
+ "acc_final_placebo": sum(p) / len(p),
284
+ "difference": sum(d) / len(d) - sum(p) / len(p),
285
+ "n_debate": len(d),
286
+ "n_placebo": len(p),
287
+ }
288
+ total = sum(value["n_debate"] for value in strata.values())
289
+ weighted = (
290
+ sum(
291
+ value["difference"] * value["n_debate"] / total for value in strata.values()
292
+ )
293
+ if total
294
+ else None
295
+ )
296
+ return {
297
+ "strata": strata,
298
+ "weighted_difference": weighted,
299
+ "p4_difference": weighted,
300
+ "n_items": total,
301
+ }
pace/importers.py ADDED
@@ -0,0 +1,219 @@
1
+ """Import common multi-agent trace exports into portable PACE JSONL."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import csv
6
+ import json
7
+ from collections.abc import Iterable
8
+ from pathlib import Path
9
+ from typing import Any
10
+
11
+ from .schema import ItemLog, item_log_from_mapping
12
+
13
+
14
+ def _records(path: str) -> list[Any]:
15
+ text = Path(path).read_text(encoding="utf-8").strip()
16
+ if not text:
17
+ return []
18
+ try:
19
+ value = json.loads(text)
20
+ return value if isinstance(value, list) else [value]
21
+ except json.JSONDecodeError:
22
+ return [json.loads(line) for line in text.splitlines() if line.strip()]
23
+
24
+
25
+ def _usage(value: Any) -> dict[str, Any]:
26
+ if not isinstance(value, dict):
27
+ return {}
28
+ return {
29
+ "prompt_tokens": value.get("prompt_tokens", value.get("input_tokens")),
30
+ "completion_tokens": value.get("completion_tokens", value.get("output_tokens")),
31
+ }
32
+
33
+
34
+ def _message_rows(
35
+ messages: Iterable[Any], item: Any, default_round: int = 0
36
+ ) -> Iterable[dict[str, Any]]:
37
+ for index, message in enumerate(messages):
38
+ if not isinstance(message, dict):
39
+ continue
40
+ row = dict(message)
41
+ usage = _usage(row.get("usage", row.get("token_usage")))
42
+ row.update({key: value for key, value in usage.items() if value is not None})
43
+ row["item"] = row.get("item", row.get("item_id", item))
44
+ row["agent"] = row.get(
45
+ "agent",
46
+ row.get("agent_id", row.get("name", row.get("role", "agent_%d" % index))),
47
+ )
48
+ row["round"] = row.get("round", row.get("round_id", default_round + index))
49
+ if "correct" not in row and "is_correct" not in row:
50
+ continue
51
+ yield row
52
+
53
+
54
+ def import_autogen(records: list[Any]) -> list[ItemLog]:
55
+ result = []
56
+ for record in records:
57
+ if isinstance(record, dict) and isinstance(record.get("messages"), list):
58
+ rows = list(
59
+ _message_rows(
60
+ record["messages"],
61
+ record.get("item_id", record.get("id", "item_0")),
62
+ record.get("round", 0),
63
+ )
64
+ )
65
+ score = record.get("correct", record.get("is_correct"))
66
+ if score is not None:
67
+ for row in rows:
68
+ row.setdefault("correct", score)
69
+ result.extend(item_log_from_mapping(row) for row in rows)
70
+ elif isinstance(record, dict):
71
+ result.extend(
72
+ item_log_from_mapping(row)
73
+ for row in _message_rows(
74
+ [record], record.get("item_id", record.get("id", "item_0"))
75
+ )
76
+ )
77
+ return result
78
+
79
+
80
+ def _nested_messages(value: Any) -> Iterable[dict[str, Any]]:
81
+ if isinstance(value, dict):
82
+ if isinstance(value.get("messages"), list):
83
+ yield from (x for x in value["messages"] if isinstance(x, dict))
84
+ else:
85
+ for child in value.values():
86
+ yield from _nested_messages(child)
87
+ elif isinstance(value, list):
88
+ for child in value:
89
+ yield from _nested_messages(child)
90
+
91
+
92
+ def import_langgraph(records: list[Any]) -> list[ItemLog]:
93
+ result = []
94
+ for record in records:
95
+ if not isinstance(record, dict):
96
+ continue
97
+ item = record.get("item_id", record.get("id", "item_0"))
98
+ rows = list(
99
+ _message_rows(
100
+ list(_nested_messages(record.get("state", record))),
101
+ item,
102
+ record.get("round", 0),
103
+ )
104
+ )
105
+ score = record.get("correct", record.get("is_correct"))
106
+ if score is not None:
107
+ for row in rows:
108
+ row.setdefault("correct", score)
109
+ for row in rows:
110
+ result.append(item_log_from_mapping(row))
111
+ return result
112
+
113
+
114
+ def import_crewai(records: list[Any]) -> list[ItemLog]:
115
+ result = []
116
+ for record in records:
117
+ if not isinstance(record, dict):
118
+ continue
119
+ tasks = record.get("tasks", [record])
120
+ for index, task in enumerate(tasks):
121
+ if not isinstance(task, dict):
122
+ continue
123
+ row = dict(task)
124
+ row["item"] = row.get(
125
+ "item",
126
+ row.get("item_id", record.get("item_id", record.get("id", "item_0"))),
127
+ )
128
+ row["agent"] = row.get(
129
+ "agent", row.get("agent_id", row.get("agent_name", "agent_%d" % index))
130
+ )
131
+ row["round"] = row.get("round", row.get("task_index", index))
132
+ row["answer"] = row.get("output", row.get("answer"))
133
+ usage = _usage(row.get("usage", row.get("token_usage", row.get("metrics"))))
134
+ row.update(
135
+ {key: value for key, value in usage.items() if value is not None}
136
+ )
137
+ if row.get("correct") is not None or row.get("is_correct") is not None:
138
+ result.append(item_log_from_mapping(row))
139
+ return result
140
+
141
+
142
+ def import_csv(records: list[Any]) -> list[ItemLog]:
143
+ return [item_log_from_mapping(row) for row in records if isinstance(row, dict)]
144
+
145
+
146
+ def import_inspect(records: list[Any]) -> list[ItemLog]:
147
+ """Convert Inspect AI EvalLog JSON payloads into canonical PACE ItemLog records."""
148
+ result = []
149
+ for record in records:
150
+ if not isinstance(record, dict):
151
+ continue
152
+ samples = record.get("samples", [record])
153
+ for sample in samples:
154
+ if not isinstance(sample, dict):
155
+ continue
156
+ item_id = str(sample.get("id", "item_0"))
157
+ scores = sample.get("scores", {})
158
+ correct = False
159
+ if isinstance(scores, dict):
160
+ for scorer_obj in scores.values():
161
+ val = scorer_obj.get("value") if isinstance(scorer_obj, dict) else scorer_obj
162
+ if val in ("C", "correct", 1, 1.0, True):
163
+ correct = True
164
+ break
165
+ messages = sample.get("messages", [])
166
+ assistant_turns = [m for m in messages if isinstance(m, dict) and m.get("role") == "assistant"]
167
+ usage = sample.get("model_usage", {})
168
+ total_in = sum(u.get("input_tokens", 0) for u in usage.values() if isinstance(u, dict)) if isinstance(usage, dict) else 0
169
+ total_out = sum(u.get("output_tokens", 0) for u in usage.values() if isinstance(u, dict)) if isinstance(usage, dict) else 0
170
+ total_reason = sum(u.get("reasoning_tokens", 0) for u in usage.values() if isinstance(u, dict)) if isinstance(usage, dict) else 0
171
+ n_turns = max(1, len(assistant_turns))
172
+ for idx, msg in enumerate(assistant_turns or [{}]):
173
+ agent_id = str(msg.get("source") or msg.get("name") or f"agent_{idx}")
174
+ tool_calls = bool(msg.get("tool_calls"))
175
+ result.append(
176
+ item_log_from_mapping(
177
+ {
178
+ "item": item_id,
179
+ "agent": agent_id,
180
+ "round": idx + 1,
181
+ "correct": bool(correct),
182
+ "prompt_tokens": round(total_in / n_turns, 2) if total_in else None,
183
+ "completion_tokens": round(total_out / n_turns, 2) if total_out else None,
184
+ "reasoning_tokens": round(total_reason / n_turns, 2) if total_reason else None,
185
+ "tool_calls": 1 if tool_calls else 0,
186
+ }
187
+ )
188
+ )
189
+ return result
190
+
191
+
192
+ def load_and_import(path: str, format_name: str) -> list[ItemLog]:
193
+ if format_name == "csv":
194
+ with open(path, newline="", encoding="utf-8") as handle:
195
+ return import_csv(list(csv.DictReader(handle)))
196
+ records = _records(path)
197
+ importer = {
198
+ "autogen": import_autogen,
199
+ "langgraph": import_langgraph,
200
+ "crewai": import_crewai,
201
+ "inspect": import_inspect,
202
+ }.get(format_name)
203
+ if importer is None:
204
+ raise ValueError("unknown trace format: %s" % format_name)
205
+ logs = importer(records)
206
+ if not logs:
207
+ raise ValueError("no scored records found in %s trace" % format_name)
208
+ return logs
209
+
210
+
211
+ def write_jsonl(logs: Iterable[ItemLog], output: str) -> int:
212
+ logs = list(logs)
213
+ if not logs:
214
+ raise ValueError("no records to write")
215
+ with open(output, "w", encoding="utf-8", newline="\n") as handle:
216
+ handle.writelines(
217
+ json.dumps(log.to_dict(), sort_keys=True) + "\n" for log in logs
218
+ )
219
+ return len(logs)