pace-eval 0.2.2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (83) hide show
  1. cme/__init__.py +11 -0
  2. cme/__main__.py +5 -0
  3. cme/analysis.py +223 -0
  4. cme/audit.py +145 -0
  5. cme/baseline.py +162 -0
  6. cme/cli.py +279 -0
  7. cme/client.py +165 -0
  8. cme/dynamics.py +212 -0
  9. cme/examples/crewai_integration.py +16 -0
  10. cme/examples/demo_pool.jsonl +360 -0
  11. cme/examples/demo_run.jsonl +480 -0
  12. cme/examples/langgraph_integration.py +27 -0
  13. cme/examples/make_demo.py +37 -0
  14. cme/examples/openai_quickstart.py +18 -0
  15. cme/experiment.py +555 -0
  16. cme/frontier.py +143 -0
  17. cme/hidden_profile.py +135 -0
  18. cme/proxy.py +106 -0
  19. cme/rebuttal.py +76 -0
  20. cme/report.py +197 -0
  21. cme/rollout_card.py +137 -0
  22. cme/tasks.py +266 -0
  23. cme/verified_debate.py +127 -0
  24. cme/verifier.py +92 -0
  25. mae_eval/__init__.py +11 -0
  26. mae_eval/__main__.py +5 -0
  27. mae_eval/auditor.py +112 -0
  28. mae_eval/classifier.py +62 -0
  29. mae_eval/cli.py +210 -0
  30. mae_eval/tracker.py +162 -0
  31. mae_eval/vector.py +91 -0
  32. pace/__init__.py +11 -0
  33. pace/__main__.py +5 -0
  34. pace/analysis.py +129 -0
  35. pace/audit.py +145 -0
  36. pace/baseline.py +162 -0
  37. pace/card.py +205 -0
  38. pace/cli.py +898 -0
  39. pace/client.py +165 -0
  40. pace/dynamics.py +372 -0
  41. pace/escape_benchmarks.py +162 -0
  42. pace/examples/crewai_integration.py +16 -0
  43. pace/examples/demo_pool.jsonl +360 -0
  44. pace/examples/demo_run.jsonl +480 -0
  45. pace/examples/langgraph_integration.py +27 -0
  46. pace/examples/make_demo.py +37 -0
  47. pace/examples/openai_quickstart.py +18 -0
  48. pace/experiment.py +555 -0
  49. pace/frontier.py +114 -0
  50. pace/hidden_profile.py +301 -0
  51. pace/importers.py +219 -0
  52. pace/inflate.py +213 -0
  53. pace/influence.py +231 -0
  54. pace/meta_engine.py +246 -0
  55. pace/middleware.py +230 -0
  56. pace/open_ended.py +114 -0
  57. pace/plotting.py +132 -0
  58. pace/pool_sizer.py +154 -0
  59. pace/protocol_v3.py +414 -0
  60. pace/proxy.py +106 -0
  61. pace/rebuttal.py +219 -0
  62. pace/report.py +197 -0
  63. pace/rollout_card.py +137 -0
  64. pace/schema.py +74 -0
  65. pace/simulator.py +199 -0
  66. pace/studio.html +1040 -0
  67. pace/tasks.py +266 -0
  68. pace/verified_debate.py +127 -0
  69. pace/verifier.py +92 -0
  70. pace/workbench.html +26 -0
  71. pace/workbench.py +119 -0
  72. pace_eval-0.2.2.dist-info/METADATA +205 -0
  73. pace_eval-0.2.2.dist-info/RECORD +83 -0
  74. pace_eval-0.2.2.dist-info/WHEEL +5 -0
  75. pace_eval-0.2.2.dist-info/entry_points.txt +4 -0
  76. pace_eval-0.2.2.dist-info/licenses/LICENSE +202 -0
  77. pace_eval-0.2.2.dist-info/top_level.txt +4 -0
  78. pace_lab/__init__.py +9 -0
  79. pace_lab/clients.py +568 -0
  80. pace_lab/concurrency_test.py +127 -0
  81. pace_lab/data.py +445 -0
  82. pace_lab/graders.py +626 -0
  83. pace_lab/strategies.py +345 -0
pace_lab/strategies.py ADDED
@@ -0,0 +1,345 @@
1
+ """Deterministic strategy runners over the pace_lab client and grader contracts."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import re
6
+ from collections import Counter
7
+ from collections.abc import Callable, Sequence
8
+ from typing import Any
9
+
10
+ from .clients import LLMClient, LLMRequest
11
+ from .graders import Grade, grade_item
12
+
13
+
14
+ _THINK_BLOCK = re.compile(r"<think>.*?</think>", re.S)
15
+ _EVIDENCE_TABLE = re.compile(r"<EVIDENCE_TABLE>(.*?)</EVIDENCE_TABLE>", re.S | re.I)
16
+ EVIDENCE_TABLE_SUFFIX = (
17
+ "\n\nBefore your answer, write a structured summary of every piece of evidence you can see, one line per item, inside "
18
+ "<EVIDENCE_TABLE> and </EVIDENCE_TABLE> tags. Then give your answer."
19
+ )
20
+
21
+
22
+ def strip_think(text: str) -> str:
23
+ """Remove <think>...</think> blocks (and an unclosed trailing <think> block) from a message before it is shared with peers.
24
+ The full text, including the thinking, stays in the logged row and the provider token counts still include it (R_token, R_reason)."""
25
+ out = _THINK_BLOCK.sub("", text or "")
26
+ if "<think>" in out:
27
+ out = out.split("<think>", 1)[0]
28
+ return out.strip()
29
+
30
+
31
+ def extract_evidence_table(text: str):
32
+ """Content of the <EVIDENCE_TABLE> block of a message (after thinking is stripped), or None if the agent wrote none."""
33
+ m = _EVIDENCE_TABLE.search(strip_think(text))
34
+ return m.group(1).strip() if m else None
35
+
36
+
37
+ def _call(
38
+ client: LLMClient,
39
+ item: dict,
40
+ prompt: str,
41
+ strategy: str,
42
+ round_num: int,
43
+ agent_id: str,
44
+ **extra: Any,
45
+ ) -> Any:
46
+ request = LLMRequest(
47
+ provider=extra.pop("provider", "ollama"),
48
+ model=extra.pop("model", "local"),
49
+ prompt=prompt,
50
+ temperature=float(extra.pop("temperature", 0.0)),
51
+ seed=extra.pop("seed", 0),
52
+ max_tokens=int(extra.pop("max_tokens", 512)),
53
+ item_id=str(item["id"]),
54
+ strategy=strategy,
55
+ round=round_num,
56
+ agent_id=agent_id,
57
+ extra=extra,
58
+ )
59
+ return client.call(request)
60
+
61
+
62
+ def _grade(item: dict, result: Any) -> Grade:
63
+ return grade_item(item, result.text or "")
64
+
65
+
66
+ def run_s1(client: LLMClient, items: Sequence[dict], **kwargs: Any) -> list[dict]:
67
+ """Run one greedy call per item."""
68
+ return _run_single(client, items, "S1", lambda item: item["prompt"], kwargs)
69
+
70
+
71
+ def _run_single(
72
+ client: LLMClient,
73
+ items: Sequence[dict],
74
+ strategy: str,
75
+ prompt_fn: Callable[[dict], str],
76
+ kwargs: dict,
77
+ ) -> list[dict]:
78
+ rows = []
79
+ for item in items:
80
+ result = _call(client, item, prompt_fn(item), strategy, 1, "agent_0", **kwargs)
81
+ grade = _grade(item, result)
82
+ rows.append(
83
+ {
84
+ "item": item["id"],
85
+ "strategy": strategy,
86
+ "agent": "agent_0",
87
+ "round": 1,
88
+ "text": result.text,
89
+ "correct": grade.correct,
90
+ "status": result.status,
91
+ }
92
+ )
93
+ return rows
94
+
95
+
96
+ def run_s2(
97
+ client: LLMClient, items: Sequence[dict], k: int = 4, **kwargs: Any
98
+ ) -> list[dict]:
99
+ """Run a noncommunicating k-sample pool and plurality vote by extracted answer."""
100
+ if k < 1:
101
+ raise ValueError("k must be positive")
102
+ rows = []
103
+ for item in items:
104
+ candidates = []
105
+ for sample in range(k):
106
+ result = _call(
107
+ client,
108
+ item,
109
+ item["prompt"],
110
+ "S2",
111
+ 1,
112
+ "sample_%d" % sample,
113
+ seed=sample,
114
+ **kwargs,
115
+ )
116
+ grade = _grade(item, result)
117
+ candidates.append(grade.normalized)
118
+ rows.append(
119
+ {
120
+ "item": item["id"],
121
+ "strategy": "S2",
122
+ "agent": "sample_%d" % sample,
123
+ "round": 1,
124
+ "text": result.text,
125
+ "correct": grade.correct,
126
+ "answer": grade.normalized,
127
+ "status": result.status,
128
+ }
129
+ )
130
+ winner = (
131
+ Counter(answer for answer in candidates if answer is not None).most_common(
132
+ 1
133
+ )[0][0]
134
+ if any(candidates)
135
+ else None
136
+ )
137
+ rows.append(
138
+ {
139
+ "item": item["id"],
140
+ "strategy": "S2",
141
+ "agent": "vote",
142
+ "round": 2,
143
+ "answer": winner,
144
+ "correct": winner == str(item.get("gold", "")).upper(),
145
+ }
146
+ )
147
+ return rows
148
+
149
+
150
+ def run_s2v(client: LLMClient, items: Sequence[dict], **kwargs: Any) -> list[dict]:
151
+ """Vote over independent split views without exposing other views."""
152
+ return _run_views(client, items, "S2v", 1, kwargs)
153
+
154
+
155
+ def run_s3(client: LLMClient, items: Sequence[dict], **kwargs: Any) -> list[dict]:
156
+ """Run one extended-thinking call per item."""
157
+ kwargs = dict(kwargs)
158
+ kwargs.setdefault("think", True)
159
+ return _run_single(client, items, "S3", lambda item: item["prompt"], kwargs)
160
+
161
+
162
+ def _run_views(
163
+ client: LLMClient, items: Sequence[dict], strategy: str, rounds: int, kwargs: dict
164
+ ) -> list[dict]:
165
+ rows = []
166
+ for item in items:
167
+ views = item.get("views") or [item["prompt"]]
168
+ for index, view in enumerate(views):
169
+ result = _call(
170
+ client, item, view, strategy, 1, "agent_%d" % index, **kwargs
171
+ )
172
+ grade = _grade(item, result)
173
+ rows.append(
174
+ {
175
+ "item": item["id"],
176
+ "strategy": strategy,
177
+ "agent": "agent_%d" % index,
178
+ "round": 1,
179
+ "text": result.text,
180
+ "correct": grade.correct,
181
+ "status": result.status,
182
+ }
183
+ )
184
+ return rows
185
+
186
+
187
+ def run_s4(
188
+ client: LLMClient,
189
+ items: Sequence[dict],
190
+ k_agents: int = 3,
191
+ rounds: int = 3,
192
+ **kwargs: Any,
193
+ ) -> list[dict]:
194
+ """Run homogeneous multi-round debate with explicit transcript context."""
195
+ return _debate(
196
+ client, items, "S4", k_agents, rounds, lambda item, _: item["prompt"], kwargs
197
+ )
198
+
199
+
200
+ def run_s4v(
201
+ client: LLMClient,
202
+ items: Sequence[dict],
203
+ k_agents: int = 3,
204
+ rounds: int = 3,
205
+ **kwargs: Any,
206
+ ) -> list[dict]:
207
+ """Run split-view debate; each prompt contains only the agent's own view and prior public messages."""
208
+ return _debate(
209
+ client,
210
+ items,
211
+ "S4v",
212
+ k_agents,
213
+ rounds,
214
+ lambda item, agent: (item.get("views") or [item["prompt"]])[agent],
215
+ kwargs,
216
+ )
217
+
218
+
219
+ def _debate(
220
+ client: LLMClient,
221
+ items: Sequence[dict],
222
+ strategy: str,
223
+ k_agents: int,
224
+ rounds: int,
225
+ view_fn: Callable[[dict, int], str],
226
+ kwargs: dict,
227
+ ) -> list[dict]:
228
+ if k_agents < 1 or rounds < 1:
229
+ raise ValueError("k_agents and rounds must be positive")
230
+ kwargs = dict(kwargs)
231
+ strip = bool(kwargs.pop("strip_think", True)) # peers never see <think> blocks unless strip_think=False
232
+ use_table = bool(kwargs.pop("evidence_table", False)) # optional structured evidence summary (Cell E1, Hidden Profile)
233
+ rows = []
234
+ for item in items:
235
+ public = []
236
+ for round_num in range(1, rounds + 1):
237
+ current = []
238
+ for agent in range(k_agents):
239
+ prompt = view_fn(item, agent) + (EVIDENCE_TABLE_SUFFIX if use_table else "") + (
240
+ "\nPublic messages:\n" + "\n".join(public) if public else ""
241
+ )
242
+ result = _call(
243
+ client,
244
+ item,
245
+ prompt,
246
+ strategy,
247
+ round_num,
248
+ "agent_%d" % agent,
249
+ **kwargs,
250
+ )
251
+ grade = _grade(item, result)
252
+ text = result.text or ""
253
+ shared = strip_think(text) if strip else text
254
+ if use_table:
255
+ table = extract_evidence_table(text)
256
+ shared = table if table is not None else shared
257
+ current.append(shared)
258
+ rows.append(
259
+ {
260
+ "item": item["id"],
261
+ "strategy": strategy,
262
+ "agent": "agent_%d" % agent,
263
+ "round": round_num,
264
+ "text": text,
265
+ "shared_text": shared,
266
+ "answer": grade.normalized,
267
+ "correct": grade.correct,
268
+ "status": result.status,
269
+ "prompt_tokens": getattr(result, "prompt_tokens", None),
270
+ "completion_tokens": getattr(result, "completion_tokens", None),
271
+ "reasoning_tokens": getattr(result, "reasoning_tokens", None),
272
+ "reasoning_chars": getattr(result, "reasoning_chars", None),
273
+ }
274
+ )
275
+ public = current
276
+ return rows
277
+
278
+
279
+ def run_s5(client: LLMClient, items: Sequence[dict], **kwargs: Any) -> list[dict]:
280
+ """Confidence-weighted debate, represented by a confidence request suffix."""
281
+ kwargs = dict(kwargs)
282
+ kwargs.setdefault("system", "Return an answer and confidence from 0 to 1.")
283
+ return (
284
+ run_s4(client, items, strategy="S5", **kwargs)
285
+ if False
286
+ else _debate(
287
+ client,
288
+ items,
289
+ "S5",
290
+ kwargs.pop("k_agents", 3),
291
+ kwargs.pop("rounds", 3),
292
+ lambda item, _: item["prompt"] + "\nState your confidence.",
293
+ kwargs,
294
+ )
295
+ )
296
+
297
+
298
+ def run_s7(client: LLMClient, items: Sequence[dict], **kwargs: Any) -> list[dict]:
299
+ """Length-matched placebo debate with fixed, noninformative messages."""
300
+ return _debate(
301
+ client,
302
+ items,
303
+ "S7",
304
+ kwargs.pop("k_agents", 3),
305
+ kwargs.pop("rounds", 3),
306
+ lambda item, _: (
307
+ item["prompt"] + "\nContinue independently; do not use peer content."
308
+ ),
309
+ kwargs,
310
+ )
311
+
312
+
313
+ def run_s8(client: LLMClient, items: Sequence[dict], **kwargs: Any) -> list[dict]:
314
+ """Two-layer mixture-of-agents runner: parallel first layer followed by an aggregator call."""
315
+ rows = _run_views(client, items, "S8", 1, kwargs)
316
+ for item in items:
317
+ messages = [strip_think(row["text"]) for row in rows if row["item"] == item["id"]] # peer-shared: thinking stripped
318
+ prompt = item["prompt"] + "\nCandidate answers:\n" + "\n".join(messages)
319
+ result = _call(client, item, prompt, "S8", 2, "aggregator", **kwargs)
320
+ grade = _grade(item, result)
321
+ rows.append(
322
+ {
323
+ "item": item["id"],
324
+ "strategy": "S8",
325
+ "agent": "aggregator",
326
+ "round": 2,
327
+ "text": result.text,
328
+ "correct": grade.correct,
329
+ "status": result.status,
330
+ }
331
+ )
332
+ return rows
333
+
334
+
335
+ STRATEGIES = {
336
+ "S1": run_s1,
337
+ "S2": run_s2,
338
+ "S2v": run_s2v,
339
+ "S3": run_s3,
340
+ "S4": run_s4,
341
+ "S4v": run_s4v,
342
+ "S5": run_s5,
343
+ "S7": run_s7,
344
+ "S8": run_s8,
345
+ }