pace-eval 0.2.2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (83) hide show
  1. cme/__init__.py +11 -0
  2. cme/__main__.py +5 -0
  3. cme/analysis.py +223 -0
  4. cme/audit.py +145 -0
  5. cme/baseline.py +162 -0
  6. cme/cli.py +279 -0
  7. cme/client.py +165 -0
  8. cme/dynamics.py +212 -0
  9. cme/examples/crewai_integration.py +16 -0
  10. cme/examples/demo_pool.jsonl +360 -0
  11. cme/examples/demo_run.jsonl +480 -0
  12. cme/examples/langgraph_integration.py +27 -0
  13. cme/examples/make_demo.py +37 -0
  14. cme/examples/openai_quickstart.py +18 -0
  15. cme/experiment.py +555 -0
  16. cme/frontier.py +143 -0
  17. cme/hidden_profile.py +135 -0
  18. cme/proxy.py +106 -0
  19. cme/rebuttal.py +76 -0
  20. cme/report.py +197 -0
  21. cme/rollout_card.py +137 -0
  22. cme/tasks.py +266 -0
  23. cme/verified_debate.py +127 -0
  24. cme/verifier.py +92 -0
  25. mae_eval/__init__.py +11 -0
  26. mae_eval/__main__.py +5 -0
  27. mae_eval/auditor.py +112 -0
  28. mae_eval/classifier.py +62 -0
  29. mae_eval/cli.py +210 -0
  30. mae_eval/tracker.py +162 -0
  31. mae_eval/vector.py +91 -0
  32. pace/__init__.py +11 -0
  33. pace/__main__.py +5 -0
  34. pace/analysis.py +129 -0
  35. pace/audit.py +145 -0
  36. pace/baseline.py +162 -0
  37. pace/card.py +205 -0
  38. pace/cli.py +898 -0
  39. pace/client.py +165 -0
  40. pace/dynamics.py +372 -0
  41. pace/escape_benchmarks.py +162 -0
  42. pace/examples/crewai_integration.py +16 -0
  43. pace/examples/demo_pool.jsonl +360 -0
  44. pace/examples/demo_run.jsonl +480 -0
  45. pace/examples/langgraph_integration.py +27 -0
  46. pace/examples/make_demo.py +37 -0
  47. pace/examples/openai_quickstart.py +18 -0
  48. pace/experiment.py +555 -0
  49. pace/frontier.py +114 -0
  50. pace/hidden_profile.py +301 -0
  51. pace/importers.py +219 -0
  52. pace/inflate.py +213 -0
  53. pace/influence.py +231 -0
  54. pace/meta_engine.py +246 -0
  55. pace/middleware.py +230 -0
  56. pace/open_ended.py +114 -0
  57. pace/plotting.py +132 -0
  58. pace/pool_sizer.py +154 -0
  59. pace/protocol_v3.py +414 -0
  60. pace/proxy.py +106 -0
  61. pace/rebuttal.py +219 -0
  62. pace/report.py +197 -0
  63. pace/rollout_card.py +137 -0
  64. pace/schema.py +74 -0
  65. pace/simulator.py +199 -0
  66. pace/studio.html +1040 -0
  67. pace/tasks.py +266 -0
  68. pace/verified_debate.py +127 -0
  69. pace/verifier.py +92 -0
  70. pace/workbench.html +26 -0
  71. pace/workbench.py +119 -0
  72. pace_eval-0.2.2.dist-info/METADATA +205 -0
  73. pace_eval-0.2.2.dist-info/RECORD +83 -0
  74. pace_eval-0.2.2.dist-info/WHEEL +5 -0
  75. pace_eval-0.2.2.dist-info/entry_points.txt +4 -0
  76. pace_eval-0.2.2.dist-info/licenses/LICENSE +202 -0
  77. pace_eval-0.2.2.dist-info/top_level.txt +4 -0
  78. pace_lab/__init__.py +9 -0
  79. pace_lab/clients.py +568 -0
  80. pace_lab/concurrency_test.py +127 -0
  81. pace_lab/data.py +445 -0
  82. pace_lab/graders.py +626 -0
  83. pace_lab/strategies.py +345 -0
@@ -0,0 +1,27 @@
1
+ """Instrument a LangGraph graph: wrap the client the nodes use and label each node's calls. Needs `pip install langgraph openai`.
2
+ Written from the public LangGraph API and not run in this repository's test environment (langgraph is not installed there)."""
3
+ from typing import TypedDict
4
+
5
+ import pace
6
+ from langgraph.graph import END, START, StateGraph
7
+ from openai import OpenAI
8
+
9
+ client = pace.wrap(OpenAI(base_url="http://localhost:11434/v1", api_key="ollama"), log_path="pace_trace.jsonl")
10
+ class State(TypedDict):
11
+ question: str
12
+ notes: list
13
+
14
+
15
+ def agent(name: str, rnd: int):
16
+ def node(state: State) -> dict:
17
+ with pace.track(agent=name, round=rnd): # labels apply to the calls inside this node only
18
+ r = client.chat.completions.create(model="llama3.1:8b", messages=[{"role": "user", "content": state["question"] + "\nNotes: " + str(state["notes"])}])
19
+ return {"notes": state["notes"] + [r.choices[0].message.content]}
20
+ return node
21
+
22
+
23
+ g = StateGraph(State)
24
+ g.add_node("a1", agent("a1", 1)); g.add_node("a2", agent("a2", 2))
25
+ g.add_edge(START, "a1"); g.add_edge("a1", "a2"); g.add_edge("a2", END)
26
+ with pace.track(item_id="item_001", system="mas"):
27
+ out = g.compile().invoke({"question": "What is 17 * 24?", "notes": []})
@@ -0,0 +1,37 @@
1
+ """Generate the SYNTHETIC demo data used by the quickstart notebook and tests: demo_pool.jsonl and demo_run.jsonl.
2
+
3
+ Nothing here comes from a model. Items have a hidden per-item success probability; sampled completions are correct with that probability;
4
+ three illustrative "systems" are drawn from fixed rules so the report has something to compare. The numbers say nothing about any real method.
5
+ Run: python code/cme/examples/make_demo.py (seed 7, 40 items, 8 samples per item)
6
+ """
7
+ import json
8
+ import os
9
+
10
+ import numpy as np
11
+
12
+ HERE = os.path.dirname(os.path.abspath(__file__))
13
+ rng = np.random.default_rng(7)
14
+ N, K = 40, 8
15
+ p_item = rng.beta(2.0, 2.0, N)
16
+ pool, run = [], []
17
+ for i in range(N):
18
+ item = "demo%02d" % i
19
+ tok_p, tok_c = int(rng.integers(60, 90)), int(rng.integers(180, 320))
20
+
21
+ def rec(arm, idx, ok):
22
+ return {"item": item, "arm": arm, "idx": idx, "gold": "G", "answer": "G" if ok else "w%d" % int(rng.integers(0, 3)), "correct": bool(ok), "prompt_tokens": tok_p,
23
+ "completion_tokens": int(tok_c * rng.uniform(0.8, 1.2)), "reasoning_tokens": None, "latency": float(rng.uniform(2, 6)), "model": "synthetic"}
24
+ pool.append(rec("greedy", 0, rng.random() < p_item[i] * 0.95))
25
+ pool += [rec("sample", j, rng.random() < p_item[i]) for j in range(K)]
26
+ # three illustrative systems: five-call debate with a small boost, three-call voting on fresh samples, one longer single call
27
+ for system, calls, boost, mult in (("debate", 5, 0.02, 1.9), ("vote3", 3, 0.0, 1.0), ("long_call", 1, 0.03, 4.0)):
28
+ ok = rng.random() < min(0.98, p_item[i] + boost)
29
+ for c in range(calls):
30
+ run.append({"system": system, "item": item, "prompt_tokens": int(tok_p * (1 + 0.5 * (system == "debate"))), "completion_tokens": int(tok_c * mult / calls * rng.uniform(0.9, 1.1)),
31
+ "n_completions": 1, "start": 0.0, "end": float(rng.uniform(3, 8))})
32
+ run.append({"system": system, "item": item, "correct": bool(ok)})
33
+ with open(os.path.join(HERE, "demo_pool.jsonl"), "w", encoding="utf-8", newline="\n") as fh:
34
+ fh.write("".join(json.dumps(r) + "\n" for r in pool))
35
+ with open(os.path.join(HERE, "demo_run.jsonl"), "w", encoding="utf-8", newline="\n") as fh:
36
+ fh.write("".join(json.dumps(r) + "\n" for r in run))
37
+ print("wrote demo_pool.jsonl (%d rows) and demo_run.jsonl (%d rows), synthetic" % (len(pool), len(run)))
@@ -0,0 +1,18 @@
1
+ """Log every call of an OpenAI-compatible client with pace.wrap, then audit the log. Needs `pip install openai` and a server (here Ollama)."""
2
+ import pace
3
+ from openai import OpenAI
4
+
5
+ LOG = "pace_trace.jsonl"
6
+ client = pace.wrap(OpenAI(base_url="http://localhost:11434/v1", api_key="ollama"), log_path=LOG) # prices omitted: cost stays null
7
+ ask = lambda q: client.chat.completions.create(model="llama3.1:8b", messages=[{"role": "user", "content": q}]).choices[0].message.content
8
+
9
+ with pace.track(item_id="item_001", system="sas", agent="solver", round=0): # the single-agent baseline: one call
10
+ answer = ask("What is 17 * 24? Answer with the number only.")
11
+ with pace.track(item_id="item_001", system="mas"): # the multi-agent system: three agents, two rounds
12
+ drafts = []
13
+ for rnd in (1, 2):
14
+ for agent in ("a1", "a2", "a3"):
15
+ with pace.track(agent=agent, round=rnd):
16
+ drafts.append(ask("Previous answers: %s\nWhat is 17 * 24? Answer with the number only." % drafts[-3:]))
17
+ pace.log_result(LOG, "item_001", correct=drafts[-1].strip() == "408", system="mas")
18
+ print("then: python -m pace audit %s --baseline-config sas.json" % LOG) # sas.json: {"n_samples": 1}
pace/experiment.py ADDED
@@ -0,0 +1,555 @@
1
+ """Controlled-experiment harness: strategies 1-9 on the same items with the same model (Phase 2).
2
+
3
+ Strategy numbers follow docs/PHASED_EXPERIMENT_AND_TOOL_ROADMAP.md section 2:
4
+ 1 greedy, one call 2 pool of k=16 independent samples (self-consistency curve for k = 1..16)
5
+ 3 one call with more thinking 4 vanilla debate (5 copies of one model, 3 rounds; round 1 = pool samples 0-4)
6
+ 5 debate with confidence-weighted aggregation 6 debate among different models (+ 6v: vote by the same mix, no communication)
7
+ 7 placebo debate (peer messages replaced by unrelated text of matched length) 8 mixture-of-agents aggregator (2 layers)
8
+ 9 verifier (code tasks): agents receive unit-test feedback; the independent arm gets the same verifier (best of the pool)
9
+ Round-1 answers of strategies 4, 5, 7 and 8 are the first five pool samples, so debate and voting differ only by the messages; the
10
+ tokens of reused calls are still charged to the strategy that uses them. Every call is cached in calls.jsonl (resumable), every
11
+ strategy outcome is written to results.jsonl. Nothing here is a result: results exist only for runs whose files exist.
12
+ Limitation: Ollama's prompt_eval_count can exclude prompt tokens served from its KV cache, so prompt tokens may be undercounted on
13
+ repeated prefixes; completion tokens and wall time are exact for the call.
14
+ """
15
+ from __future__ import annotations
16
+
17
+ import argparse
18
+ import hashlib
19
+ import json
20
+ import os
21
+ import random
22
+ import sys
23
+ import threading
24
+ import time
25
+ import urllib.error
26
+ import urllib.request
27
+ from collections import Counter
28
+ from typing import Dict, List, Optional
29
+
30
+ from . import tasks as T
31
+
32
+ THINK_MODELS = ("qwen3:", "deepseek-r1:", "qwen3.5:") # Ollama 'think' flag is passed only to these
33
+ SOLVE = "{q}\n\nThink step by step. Finish with two final lines: 'Answer: <answer>' and 'Confidence: <0-100>'."
34
+ SOLVE_CODE = "{q}\n\nAfter the code block add a final line 'Confidence: <0-100>'."
35
+ EXTENDED = ("{q}\n\nReason for much longer than usual: work the problem out fully, then re-check your work independently twice before answering. "
36
+ "Finish with two final lines: 'Answer: <answer>' and 'Confidence: <0-100>'.")
37
+ DEBATE = ("{q}\n\nYour previous response:\n{own}\n\nOther agents responded:\n{peers}\n\nUse their reasoning as additional information, but verify it yourself. "
38
+ "Give your updated solution. Finish with two final lines: 'Answer: <answer>' and 'Confidence: <0-100>'.")
39
+ MOA = ("{q}\n\nCandidate responses from other solvers:\n{cands}\n\nSynthesize the best final solution, correcting errors you find. "
40
+ "Finish with two final lines: 'Answer: <answer>' and 'Confidence: <0-100>'.")
41
+ FEEDBACK = "{q}\n\nYour previous code:\n```python\n{code}\n```\nIt was run against the unit tests and {status}.\nGive a corrected version in a ```python block, then a final line 'Confidence: <0-100>'."
42
+ TAIL = 800
43
+
44
+
45
+ class OllamaClient:
46
+ """Ollama native /api/chat client: real token counts (prompt_eval_count, eval_count) and wall time per call."""
47
+
48
+ def __init__(self, host: str = "http://localhost:11434", num_ctx: int = 8192, timeout: float = 1800.0):
49
+ self.host, self.num_ctx, self.timeout = host.rstrip("/"), num_ctx, timeout
50
+
51
+ def chat(self, model: str, prompt: str, temperature: float, seed: int, max_tokens: int = 2048, think: Optional[bool] = None) -> dict:
52
+ body = {"model": model, "messages": [{"role": "user", "content": prompt}], "stream": False,
53
+ "options": {"temperature": temperature, "seed": seed, "num_predict": max_tokens, "num_ctx": self.num_ctx}}
54
+ if think is not None and model.startswith(THINK_MODELS):
55
+ body["think"] = think
56
+ req = urllib.request.Request(self.host + "/api/chat", data=json.dumps(body).encode(), headers={"Content-Type": "application/json"})
57
+ t0 = time.time()
58
+ with urllib.request.urlopen(req, timeout=self.timeout) as r:
59
+ d = json.loads(r.read())
60
+ m = d.get("message", {})
61
+ return {"text": m.get("content") or "", "thinking_chars": len(m.get("thinking") or ""), "prompt_tokens": d.get("prompt_eval_count"),
62
+ "completion_tokens": d.get("eval_count"), "latency": time.time() - t0, "done_reason": d.get("done_reason")}
63
+
64
+
65
+ class Store:
66
+ """Append-only JSONL cache of calls keyed by a string; reloaded on start so interrupted runs resume."""
67
+
68
+ def __init__(self, path: str):
69
+ self.path, self.d = path, {}
70
+ self._lock = threading.Lock()
71
+ if os.path.exists(path):
72
+ for l in open(path, encoding="utf-8"):
73
+ if l.strip():
74
+ r = json.loads(l)
75
+ self.d[r["key"]] = r
76
+
77
+ def put(self, rec: dict) -> None:
78
+ with self._lock:
79
+ self.d[rec["key"]] = rec
80
+ with open(self.path, "a", encoding="utf-8") as fh:
81
+ fh.write(json.dumps(rec) + "\n")
82
+
83
+
84
+ class Ctx:
85
+ def __init__(self, client, store: Store, model: str, models: Optional[List[str]] = None, k: int = 16, agents: int = 5, rounds: int = 3,
86
+ seed: int = 0, temperature: float = 0.7, max_tokens: int = 2048, think_tokens: int = 8192,
87
+ price_in: Optional[float] = None, price_out: Optional[float] = None, retries: int = 3, think_default: bool = False):
88
+ self.client, self.store, self.model = client, store, model
89
+ self.models = models or [model]
90
+ self.k, self.agents, self.rounds, self.seed, self.temperature = k, agents, rounds, seed, temperature
91
+ self.max_tokens, self.think_tokens = max_tokens, think_tokens
92
+ self.price_in, self.price_out, self.retries = price_in, price_out, retries
93
+ self.think_default = think_default # value of the Ollama `think` flag for ordinary calls (True lets reasoning models report thinking separately)
94
+ self.used: List[dict] = []
95
+ self._digests: Dict[str, Optional[str]] = {}
96
+
97
+ def call(self, key: str, model: str, prompt: str, temperature: float, max_tokens: Optional[int] = None, think: Optional[bool] = None,
98
+ comm_chars: int = 0) -> dict:
99
+ rec = self.store.d.get(key)
100
+ if rec is None:
101
+ seed = int(hashlib.sha256((str(self.seed) + key).encode()).hexdigest()[:8], 16) if temperature > 0 else self.seed
102
+ r = None
103
+ for attempt in range(self.retries + 1):
104
+ try:
105
+ r = self.client.chat(model, prompt, temperature, seed, max_tokens or self.max_tokens, think)
106
+ break
107
+ except (urllib.error.URLError, TimeoutError, ConnectionError): # transient server errors only
108
+ if attempt == self.retries:
109
+ raise
110
+ time.sleep(5 * (attempt + 1))
111
+ cost = None
112
+ if None not in (self.price_in, self.price_out, r.get("prompt_tokens"), r.get("completion_tokens")):
113
+ cost = (r["prompt_tokens"] * self.price_in + r["completion_tokens"] * self.price_out) / 1e6 # local calls: prices 0.0
114
+ if model not in self._digests:
115
+ from .baseline import ollama_digest
116
+ self._digests[model] = ollama_digest(model, getattr(self.client, "host", "http://localhost:11434")) if hasattr(self.client, "host") else None
117
+ rec = {"key": key, "model": model, "model_digest": self._digests[model], "temperature": temperature, "seed": seed, "think": think,
118
+ "cost_usd": cost, "reasoning_tokens": None, "prompt_chars": len(prompt), "comm_chars": comm_chars, **r}
119
+ self.store.put(rec)
120
+ self.used.append(rec)
121
+ return rec
122
+
123
+ def begin(self):
124
+ self.used = []
125
+
126
+ def fork(self) -> "Ctx":
127
+ """A worker context sharing the client and the call store, with its own `used` list (thread safe)."""
128
+ c = Ctx(self.client, self.store, self.model, self.models, self.k, self.agents, self.rounds, self.seed, self.temperature, self.max_tokens,
129
+ self.think_tokens, self.price_in, self.price_out, self.retries, self.think_default)
130
+ c._digests = self._digests
131
+ return c
132
+
133
+
134
+ def _solve_prompt(item, q=None):
135
+ return (SOLVE_CODE if item["kind"] == "code" else SOLVE).format(q=q or item["prompt"])
136
+
137
+
138
+ def ans_of(item, rec) -> str:
139
+ """Normalised answer of a call (for code: pass/fail marker computed by running the tests)."""
140
+ if item["kind"] == "code":
141
+ return "pass" if T.score(item, rec["text"]) else "fail"
142
+ return T.normalize_answer(item, T.parse_answer(rec["text"]))
143
+
144
+
145
+ def vote(answers: List[str], gold_answer_fn, weights: Optional[List[float]] = None):
146
+ """Plurality (or weighted) vote. Returns (leaders, credit) where credit is 1/m if the gold is among m tied leaders."""
147
+ c: Counter = Counter()
148
+ for a, w in zip(answers, weights or [1.0] * len(answers)):
149
+ if a != "":
150
+ c[a] += w
151
+ if not c:
152
+ return [], 0.0
153
+ top = max(c.values())
154
+ leaders = sorted(a for a, v in c.items() if abs(v - top) < 1e-12)
155
+ return leaders, (1.0 / len(leaders)) if any(gold_answer_fn(a) for a in leaders) else 0.0
156
+
157
+
158
+ def _is_gold(item):
159
+ if item["kind"] == "text":
160
+ gs = {T._norm_text(x) for x in item["gold"]}
161
+ return lambda a: a in gs
162
+ if item["kind"] == "num":
163
+ return lambda a: a == T._norm_num(item["gold"])
164
+ if item["kind"] == "name":
165
+ return lambda a: a == str(item["gold"]).lower()
166
+ return lambda a: a == str(item["gold"]).upper()
167
+
168
+
169
+ def _tail(text: str, n: int = TAIL) -> str:
170
+ t = (text or "").strip()
171
+ return t[-n:] if len(t) > n else t
172
+
173
+
174
+ def pool_rec(ctx: Ctx, item: dict, idx: int, model: Optional[str] = None) -> dict:
175
+ m = model or ctx.model
176
+ return ctx.call("pool|%s|%s|%d" % (m, item["id"], idx), m, _solve_prompt(item), ctx.temperature, think=ctx.think_default)
177
+
178
+
179
+ def round1(ctx: Ctx, item: dict, n: int) -> List[dict]:
180
+ """Round-1 responses: pool samples 0..n-1, or one call per agent view when evidence is split."""
181
+ if item.get("views"):
182
+ return [ctx.call("r1v|%s|%s|%d" % (ctx.model, item["id"], i), ctx.model, _solve_prompt(item, item["views"][i % len(item["views"])]), ctx.temperature, think=ctx.think_default)
183
+ for i in range(n)]
184
+ return [pool_rec(ctx, item, i) for i in range(n)]
185
+
186
+
187
+ def _result(ctx, strategy, item, credit, final, extra=None):
188
+ toks = lambda f: sum((r.get(f) or 0) for r in ctx.used)
189
+ out = {"strategy": strategy, "model": ctx.model if len(ctx.models) == 1 else "+".join(ctx.models), "item": item["id"], "task": item["task"],
190
+ "correct": credit, "final": final, "calls": len(ctx.used), "prompt_tokens": toks("prompt_tokens"), "completion_tokens": toks("completion_tokens"),
191
+ "latency_sum": toks("latency"), "keys": [r["key"] for r in ctx.used]}
192
+ if extra:
193
+ out.update(extra)
194
+ return out
195
+
196
+
197
+ # ------------------------------------------------------------------------------------ strategies
198
+ def s1_greedy(ctx, item):
199
+ r = ctx.call("greedy|%s|%s" % (ctx.model, item["id"]), ctx.model, _solve_prompt(item), 0.0, think=ctx.think_default)
200
+ return _result(ctx, "S1", item, float(T.score(item, r["text"])), ans_of(item, r))
201
+
202
+
203
+ def s2_pool(ctx, item):
204
+ recs = [pool_rec(ctx, item, i) for i in range(ctx.k)]
205
+ if item["kind"] == "code":
206
+ passes = [ans_of(item, r) == "pass" for r in recs]
207
+ return _result(ctx, "S2", item, sum(passes) / len(passes), "pass@1 mean", {"pass_any": any(passes)})
208
+ leaders, cr = vote([ans_of(item, r) for r in recs], _is_gold(item))
209
+ res = _result(ctx, "S2", item, cr, "|".join(leaders))
210
+ if item.get("views"): # split evidence: independent agents each see one view, no communication, then vote
211
+ ctx.begin()
212
+ v = round1(ctx, item, ctx.agents)
213
+ lv, crv = vote([ans_of(item, r) for r in v], _is_gold(item))
214
+ return [res, _result(ctx, "S2v", item, crv, "|".join(lv))]
215
+ return res
216
+
217
+
218
+ def s3_think(ctx, item):
219
+ cap = ctx.model.startswith(THINK_MODELS)
220
+ prompt = _solve_prompt(item) if cap else EXTENDED.format(q=item["prompt"]) if item["kind"] != "code" else _solve_prompt(item)
221
+ r = ctx.call("think|%s|%s" % (ctx.model, item["id"]), ctx.model, prompt, 0.0, ctx.think_tokens if cap else 4096, think=True if cap else None)
222
+ return _result(ctx, "S3", item, float(T.score(item, r["text"])), ans_of(item, r), {"mode": "think-flag" if cap else "prompt-extended", "thinking_chars": r.get("thinking_chars")})
223
+
224
+
225
+ def _peers(ctx, texts, j, show_conf: bool, placebo_texts=None, transform=None):
226
+ lines = []
227
+ for q, t in enumerate(texts):
228
+ if q == j:
229
+ continue
230
+ body = _tail(t)
231
+ if transform is not None:
232
+ body = transform(body, q, j)
233
+ if placebo_texts is not None:
234
+ p = placebo_texts[q % len(placebo_texts)]
235
+ body = (p * (len(body) // max(1, len(p)) + 1))[: len(body)]
236
+ conf = T.parse_confidence(t)
237
+ lines.append("Agent %d%s: %s" % (q + 1, (" (confidence %.0f%%)" % (100 * conf)) if (show_conf and conf is not None) else "", body))
238
+ return "\n\n".join(lines)
239
+
240
+
241
+ def _debate(ctx, item, strategy, models, r1_recs, show_conf, weighted, placebo_pool=None, peer_transform=None):
242
+ cur = list(r1_recs)
243
+ rounds_ans = [[ans_of(item, r) for r in cur]]
244
+ rounds_tok = [[[r.get("prompt_tokens"), r.get("completion_tokens")] for r in cur]]
245
+ for rd in range(2, ctx.rounds + 1):
246
+ texts = [r["text"] for r in cur]
247
+ nxt = []
248
+ for j in range(len(cur)):
249
+ peers = _peers(ctx, texts, j, show_conf, placebo_pool, peer_transform)
250
+ # split evidence: an agent sees only its own view in every round (before 2026-11-12 the later rounds showed the full-evidence prompt to every agent; see docs/PREREGISTRATION_ADDENDUM_2.md)
251
+ q = item["views"][j % len(item["views"])] if item.get("views") else item["prompt"]
252
+ prompt = DEBATE.format(q=q, own=_tail(texts[j]), peers=peers)
253
+ nxt.append(ctx.call("%s|%s|%s|r%d|a%d" % (strategy, "+".join(models), item["id"], rd, j), models[j], prompt, ctx.temperature, think=ctx.think_default,
254
+ comm_chars=len(peers)))
255
+ cur = nxt
256
+ rounds_ans.append([ans_of(item, r) for r in cur])
257
+ rounds_tok.append([[r.get("prompt_tokens"), r.get("completion_tokens")] for r in cur])
258
+ answers = [ans_of(item, r) for r in cur]
259
+ w = [T.parse_confidence(r["text"]) or 0.5 for r in cur] if weighted else None
260
+ leaders, cr = vote(answers, _is_gold(item), w)
261
+ changed = sum(1 for a, b in zip([ans_of(item, r) for r in r1_recs], answers) if a != b)
262
+ return leaders, cr, {"answers_changed": changed, "round_answers": rounds_ans, "round_tokens": rounds_tok}
263
+
264
+
265
+ def s4_debate(ctx, item):
266
+ # Items with split evidence run as S4v (own keys, own label) so that results of the earlier implementation, which leaked the full evidence in rounds 2 and later, are never reused.
267
+ name = "S4v" if item.get("views") else "S4"
268
+ leaders, cr, ex = _debate(ctx, item, name, [ctx.model] * ctx.agents, round1(ctx, item, ctx.agents), False, False)
269
+ return _result(ctx, name, item, cr, "|".join(leaders), ex)
270
+
271
+
272
+ def s5_conf_debate(ctx, item):
273
+ leaders, cr, ex = _debate(ctx, item, "S5", [ctx.model] * ctx.agents, round1(ctx, item, ctx.agents), True, True)
274
+ return _result(ctx, "S5", item, cr, "|".join(leaders), ex)
275
+
276
+
277
+ def s6_hetero(ctx, item):
278
+ """Debate among different models, and (S6v) vote by the same mix with no communication. Returns two results."""
279
+ ms = [ctx.models[i % len(ctx.models)] for i in range(ctx.agents)]
280
+ r1 = [pool_rec(ctx, item, i // len(ctx.models), ms[i]) for i in range(ctx.agents)]
281
+ lv, crv = vote([ans_of(item, r) for r in r1], _is_gold(item))
282
+ vote_res = _result(ctx, "S6v", item, crv, "|".join(lv))
283
+ leaders, cr, ex = _debate(ctx, item, "S6", ms, r1, False, False)
284
+ deb = _result(ctx, "S6", item, cr, "|".join(leaders), ex)
285
+ return [vote_res, deb]
286
+
287
+
288
+ def s7_placebo(ctx, item, others: List[dict]):
289
+ r1 = round1(ctx, item, ctx.agents)
290
+ n_used = len(ctx.used)
291
+ foreign = [pool_rec(ctx, others[i % len(others)], i)["text"] for i in range(ctx.agents)]
292
+ del ctx.used[n_used:] # the borrowed text is input material for the control, not compute spent on this item
293
+ leaders, cr, ex = _debate(ctx, item, "S7", [ctx.model] * ctx.agents, r1, False, False, placebo_pool=[_tail(t) for t in foreign])
294
+ return _result(ctx, "S7", item, cr, "|".join(leaders), ex)
295
+
296
+
297
+ def s8_moa(ctx, item):
298
+ props = round1(ctx, item, ctx.agents)
299
+ cands = "\n\n".join("[%d] %s" % (i + 1, _tail(r["text"])) for i, r in enumerate(props))
300
+ l1 = [ctx.call("S8|L1|%s|%s|%d" % (ctx.model, item["id"], i), ctx.model, MOA.format(q=item["prompt"], cands=cands), ctx.temperature, think=ctx.think_default) for i in range(ctx.agents)]
301
+ cands2 = "\n\n".join("[%d] %s" % (i + 1, _tail(r["text"])) for i, r in enumerate(l1))
302
+ fin = ctx.call("S8|L2|%s|%s" % (ctx.model, item["id"]), ctx.model, MOA.format(q=item["prompt"], cands=cands2), 0.0, think=ctx.think_default)
303
+ return _result(ctx, "S8", item, float(T.score(item, fin["text"])), ans_of(item, fin))
304
+
305
+
306
+ def s9_verifier(ctx, item):
307
+ """Code only. Feedback arm: agent revises with unit-test results (up to rounds-1 revisions). Independent arm: best of the pool with the same verifier."""
308
+ if item["kind"] != "code":
309
+ raise ValueError("strategy 9 needs a code task")
310
+ ok = lambda rec: T.score(item, rec["text"])
311
+ pool = [pool_rec(ctx, item, i) for i in range(ctx.agents)]
312
+ ind = any(ok(r) for r in pool)
313
+ ind_res = _result(ctx, "S9i", item, float(ind), "verified best-of-%d" % ctx.agents)
314
+ ctx.begin()
315
+ cur, solved = pool[0], ok(pool[0])
316
+ for rd in range(2, ctx.rounds + 1):
317
+ if solved:
318
+ break
319
+ prompt = FEEDBACK.format(q=item["prompt"], code=T.extract_code(cur["text"]), status="failed" if not solved else "passed")
320
+ cur = ctx.call("S9|%s|%s|r%d" % (ctx.model, item["id"], rd), ctx.model, prompt, ctx.temperature, think=ctx.think_default)
321
+ solved = ok(cur)
322
+ ctx.used = [pool[0]] + [r for r in ctx.used if r is not pool[0]]
323
+ fb_res = _result(ctx, "S9", item, float(solved), "feedback loop")
324
+ return [ind_res, fb_res]
325
+
326
+
327
+ def s10_views_vote(ctx, item):
328
+ """Independent agents on split evidence, plurality vote (the S2v arm of strategy 2 without the 16-sample pool)."""
329
+ if not item.get("views"):
330
+ raise ValueError("strategy 10 needs a split-evidence task")
331
+ v = round1(ctx, item, ctx.agents)
332
+ lv, crv = vote([ans_of(item, r) for r in v], _is_gold(item))
333
+ return _result(ctx, "S2v", item, crv, "|".join(lv))
334
+
335
+
336
+ def load_items(task, n, stratify=False, evidence="split"):
337
+ """Task items; evidence='shared' removes the per-agent views of split-evidence items, so every agent sees all evidence (the redundant-evidence arm of the 2x2 design)."""
338
+ items = T.load_task(task, n, stratify=stratify)
339
+ if evidence == "shared":
340
+ items = [{k: v for k, v in it.items() if k != "views"} for it in items]
341
+ elif evidence != "split":
342
+ raise ValueError("evidence must be 'split' or 'shared'")
343
+ return items
344
+
345
+
346
+ STRATS = {1: s1_greedy, 2: s2_pool, 3: s3_think, 4: s4_debate, 5: s5_conf_debate, 6: s6_hetero, 7: s7_placebo, 8: s8_moa, 9: s9_verifier, 10: s10_views_vote}
347
+
348
+
349
+ def run(ctx: Ctx, items: List[dict], strategies: List[int], out_dir: str, progress=print, workers: int = 1) -> dict:
350
+ """Run strategies over items. Strategy 2 always runs first (the pool feeds debate round 1 and the placebo text). With workers > 1,
351
+ items of one strategy run concurrently; item failures are logged to errors.jsonl and the item is skipped, never filled in."""
352
+ from concurrent.futures import ThreadPoolExecutor
353
+ os.makedirs(out_dir, exist_ok=True)
354
+ rpath, epath = os.path.join(out_dir, "results.jsonl"), os.path.join(out_dir, "errors.jsonl")
355
+ order = sorted(set(strategies), key=lambda x: (x != 2, strategies.index(x)))
356
+ if 7 in order and 2 not in order:
357
+ order.insert(0, 2)
358
+ done = set()
359
+ if os.path.exists(rpath):
360
+ for l in open(rpath, encoding="utf-8"):
361
+ if l.strip():
362
+ r = json.loads(l)
363
+ done.add((r["strategy"], r["item"]))
364
+ lock = threading.Lock()
365
+ counts = {"new": 0, "errors": 0, "done_items": 0}
366
+ t0 = time.time()
367
+ label = {1: "S1", 2: "S2", 3: "S3", 4: "S4", 5: "S5", 6: "S6", 7: "S7", 8: "S8", 9: "S9", 10: "S2v", 11: "S4v", 12: "S12", 13: "S13"}
368
+
369
+ def work(args):
370
+ s, item = args
371
+ lab = "S4v" if (s == 4 and item.get("views")) else label[s]
372
+ if (lab, item["id"]) in done:
373
+ return
374
+ c = ctx.fork()
375
+ c.begin()
376
+ try:
377
+ res = STRATS[s](c, item, [x for x in items if x is not item]) if s == 7 else STRATS[s](c, item)
378
+ except Exception as e: # record and continue: no result is invented for a failed item
379
+ with lock:
380
+ counts["errors"] += 1
381
+ with open(epath, "a", encoding="utf-8") as fh:
382
+ fh.write(json.dumps({"strategy": label[s], "item": item["id"], "error": repr(e)[:300]}) + "\n")
383
+ return
384
+ with lock:
385
+ for r in (res if isinstance(res, list) else [res]):
386
+ with open(rpath, "a", encoding="utf-8") as fh:
387
+ fh.write(json.dumps(r) + "\n")
388
+ counts["new"] += 1
389
+ counts["done_items"] += 1
390
+ if progress and counts["done_items"] % 10 == 0:
391
+ progress("%s: %d items finished, %.0fs, errors %d" % (label[s], counts["done_items"], time.time() - t0, counts["errors"]))
392
+
393
+ for s in order:
394
+ if s == 9 and any(i["kind"] != "code" for i in items):
395
+ raise ValueError("strategy 9 needs a code task")
396
+ tasks = [(s, it) for it in items]
397
+ if workers > 1:
398
+ with ThreadPoolExecutor(workers) as ex:
399
+ list(ex.map(work, tasks))
400
+ else:
401
+ for t in tasks:
402
+ work(t)
403
+ return {"new_results": counts["new"], "errors": counts["errors"], "seconds": time.time() - t0}
404
+
405
+
406
+ def summarize(out_dir: str, project_n: Optional[int] = None, price_in: Optional[float] = None, price_out: Optional[float] = None) -> List[dict]:
407
+ rows = [json.loads(l) for l in open(os.path.join(out_dir, "results.jsonl"), encoding="utf-8") if l.strip()]
408
+ by: Dict[tuple, List[dict]] = {}
409
+ for r in rows:
410
+ by.setdefault((r["strategy"], r["model"], r["task"]), []).append(r)
411
+ out = []
412
+ for (s, m, t), rs in sorted(by.items()):
413
+ n = len(rs)
414
+ row = {"strategy": s, "model": m, "task": t, "n_items": n, "accuracy": sum(r["correct"] for r in rs) / n,
415
+ "calls_per_item": sum(r["calls"] for r in rs) / n, "prompt_tokens_per_item": sum(r["prompt_tokens"] for r in rs) / n,
416
+ "completion_tokens_per_item": sum(r["completion_tokens"] for r in rs) / n, "latency_sum_per_item_s": sum(r["latency_sum"] for r in rs) / n}
417
+ if project_n:
418
+ row["projected_calls"] = row["calls_per_item"] * project_n
419
+ row["projected_hours_serial"] = row["latency_sum_per_item_s"] * project_n / 3600
420
+ if price_in is not None and price_out is not None:
421
+ row["projected_cost_usd"] = (row["prompt_tokens_per_item"] * price_in + row["completion_tokens_per_item"] * price_out) * project_n / 1e6
422
+ out.append(row)
423
+ return out
424
+
425
+
426
+ def export_pool(store: Store, model: str, items: List[dict], path: str, k: int = 16) -> int:
427
+ """Write the S1/S2 calls in the `pace baseline` cache format so `pace report --pool` can read them. Non-code tasks only.
428
+ The gold label is the literal 'G'; a correct response gets answer 'G', a wrong one its own normalised answer plus '_wrong'."""
429
+ n = 0
430
+ with open(path, "w", encoding="utf-8") as fh:
431
+ for it in items:
432
+ if it["kind"] == "code":
433
+ continue
434
+ recs = [("greedy", 0, store.d.get("greedy|%s|%s" % (model, it["id"])))] + [("sample", i, store.d.get("pool|%s|%s|%d" % (model, it["id"], i))) for i in range(k)]
435
+ for arm, i, r in recs:
436
+ if r is None:
437
+ continue
438
+ ok = T.score(it, r["text"])
439
+ fh.write(json.dumps({"item": it["id"], "arm": arm, "idx": i, "gold": "G", "answer": "G" if ok else (ans_of(it, r) or "?") + "_wrong", "correct": ok,
440
+ "prompt_tokens": r["prompt_tokens"], "completion_tokens": r["completion_tokens"], "reasoning_tokens": None,
441
+ "latency": r["latency"], "model": model}) + "\n")
442
+ n += 1
443
+ return n
444
+
445
+
446
+ def write_manifest(path: str, a, items: List[dict]) -> None:
447
+ """Append one manifest entry: what was run, with which preregistration file (SHA-256), git commit, server version, and item ids."""
448
+ import subprocess
449
+ root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
450
+ prereg = os.path.join(root, "docs", "PREREGISTRATION.md")
451
+ sha = hashlib.sha256(open(prereg, "rb").read()).hexdigest() if os.path.exists(prereg) else None
452
+ try:
453
+ commit = subprocess.run(["git", "rev-parse", "HEAD"], cwd=root, capture_output=True, text=True, timeout=10).stdout.strip() or None
454
+ dirty = bool(subprocess.run(["git", "status", "--porcelain", "--", "docs/PREREGISTRATION.md", "code"], cwd=root, capture_output=True, text=True, timeout=10).stdout.strip())
455
+ except Exception:
456
+ commit, dirty = None, None
457
+ try:
458
+ with urllib.request.urlopen("http://localhost:11434/api/version", timeout=5) as r:
459
+ ver = json.loads(r.read()).get("version")
460
+ except Exception:
461
+ ver = None
462
+ from .baseline import ollama_digest
463
+ entry = {"time": time.strftime("%Y-%m-%d %H:%M:%S"), "preregistration_sha256": sha, "git_commit": commit, "code_or_prereg_uncommitted_changes": dirty,
464
+ "ollama_version": ver, "model": a.model, "model_digest": ollama_digest(a.model), "task": a.task, "n_items": len(items),
465
+ "item_ids_sha256": hashlib.sha256("|".join(i["id"] for i in items).encode()).hexdigest(), "strategies": a.strategies, "k": a.k, "agents": a.agents,
466
+ "rounds": a.rounds, "seed": a.seed, "max_tokens": a.max_tokens, "think_tokens": a.think_tokens, "workers": a.workers, "out": a.out}
467
+ os.makedirs(os.path.dirname(os.path.abspath(path)), exist_ok=True)
468
+ with open(path, "a", encoding="utf-8") as fh:
469
+ fh.write(json.dumps(entry) + "\n")
470
+
471
+
472
+ def main(argv=None) -> int:
473
+ ap = argparse.ArgumentParser(prog="pace-experiment", description=__doc__.split("\n")[0])
474
+ sub = ap.add_subparsers(dest="cmd", required=True)
475
+ r = sub.add_parser("run")
476
+ r.add_argument("--model", required=True)
477
+ r.add_argument("--models", default=None, help="comma-separated model list for strategy 6")
478
+ r.add_argument("--task", required=True, choices=list(T.SOURCES) + ["hidden_profile"])
479
+ r.add_argument("--n", type=int, default=50)
480
+ r.add_argument("--stratify", action="store_true", help="draw items round-robin over categories/subjects")
481
+ r.add_argument("--num-ctx", type=int, default=8192, help="Ollama context length; must exceed prompt + max tokens")
482
+ r.add_argument("--strategies", default="1")
483
+ r.add_argument("--out", required=True)
484
+ r.add_argument("--k", type=int, default=16)
485
+ r.add_argument("--agents", type=int, default=5)
486
+ r.add_argument("--rounds", type=int, default=3)
487
+ r.add_argument("--seed", type=int, default=0)
488
+ r.add_argument("--max-tokens", type=int, default=2048)
489
+ r.add_argument("--think-tokens", type=int, default=8192)
490
+ r.add_argument("--workers", type=int, default=1, help="concurrent items (Ollama serves them in parallel slots)")
491
+ r.add_argument("--price-in", type=float, default=0.0, help="USD per million input tokens (0.0 for local models)")
492
+ r.add_argument("--price-out", type=float, default=0.0)
493
+ r.add_argument("--think", action="store_true", help="pass think=true on ordinary calls (reasoning models then report thinking separately)")
494
+ r.add_argument("--manifest", default=None, help="write a run manifest (preregistration hash, git commit, versions) to this path")
495
+ r.add_argument("--evidence", choices=["split", "shared"], default="split", help="split-evidence tasks: each agent sees its own view (split) or all evidence (shared)")
496
+ tr = sub.add_parser("trace", help="write the round-by-round JSONL of a debate strategy for `pace dynamics` (reads the cache, never calls a model)")
497
+ tr.add_argument("--model", required=True)
498
+ tr.add_argument("--task", required=True, choices=list(T.SOURCES) + ["hidden_profile"])
499
+ tr.add_argument("--n", type=int, default=50)
500
+ tr.add_argument("--strategy", required=True, help="S4, S4v, S5, S12 or S12u")
501
+ tr.add_argument("--out", required=True, help="the cell directory of the run")
502
+ tr.add_argument("--agents", type=int, default=5)
503
+ tr.add_argument("--evidence", choices=["split", "shared"], default="split")
504
+ tr.add_argument("--dest", required=True)
505
+ rs = sub.add_parser("rescore", help="recompute results.jsonl from cached calls (never calls a model)")
506
+ rs.add_argument("--model", required=True)
507
+ rs.add_argument("--task", required=True, choices=list(T.SOURCES) + ["hidden_profile"])
508
+ rs.add_argument("--n", type=int, default=50)
509
+ rs.add_argument("--strategies", default="1")
510
+ rs.add_argument("--out", required=True)
511
+ rs.add_argument("--k", type=int, default=16)
512
+ rs.add_argument("--agents", type=int, default=5)
513
+ rs.add_argument("--rounds", type=int, default=3)
514
+ s = sub.add_parser("summary")
515
+ s.add_argument("--out", required=True)
516
+ s.add_argument("--project-n", type=int, default=None)
517
+ s.add_argument("--price-in", type=float, default=None)
518
+ s.add_argument("--price-out", type=float, default=None)
519
+ a = ap.parse_args(argv)
520
+ if a.cmd == "trace":
521
+ from .verified_debate import export_trace
522
+ n_rows = export_trace(a.out, load_items(a.task, a.n, evidence=a.evidence), a.strategy, a.model, a.agents, a.dest)
523
+ print("wrote %d agent-round records to %s" % (n_rows, a.dest))
524
+ return 0
525
+ if a.cmd == "summary":
526
+ for row in summarize(a.out, a.project_n, a.price_in, a.price_out):
527
+ print(json.dumps({k: (round(v, 4) if isinstance(v, float) else v) for k, v in row.items()}))
528
+ return 0
529
+ if a.cmd == "rescore":
530
+ class NoCall:
531
+ def chat(self, *x, **y):
532
+ raise RuntimeError("rescore must not call a model: a call is missing from the cache")
533
+ rp = os.path.join(a.out, "results.jsonl")
534
+ if os.path.exists(rp):
535
+ os.replace(rp, rp + ".bak")
536
+ ctx = Ctx(NoCall(), Store(os.path.join(a.out, "calls.jsonl")), a.model, k=a.k, agents=a.agents, rounds=a.rounds)
537
+ print(json.dumps(run(ctx, T.load_task(a.task, a.n), [int(x) for x in a.strategies.split(",")], a.out, progress=None)))
538
+ return 0
539
+ items = load_items(a.task, a.n, stratify=a.stratify, evidence=a.evidence)
540
+ os.makedirs(a.out, exist_ok=True)
541
+ ctx = Ctx(OllamaClient(num_ctx=a.num_ctx), Store(os.path.join(a.out, "calls.jsonl")), a.model, a.models.split(",") if a.models else None, a.k, a.agents, a.rounds, a.seed,
542
+ max_tokens=a.max_tokens, think_tokens=a.think_tokens, price_in=a.price_in, price_out=a.price_out, think_default=a.think)
543
+ if a.manifest:
544
+ write_manifest(a.manifest, a, items)
545
+ print(json.dumps(run(ctx, items, [int(x) for x in a.strategies.split(",")], a.out, workers=a.workers)))
546
+ return 0
547
+
548
+
549
+
550
+ from . import verified_debate as _vd # noqa: E402 (registers strategy 12; imported last because verified_debate imports this module lazily)
551
+ STRATS[12] = _vd.s12_verified_debate
552
+ STRATS[13] = _vd.s13_shuffled_debate
553
+
554
+ if __name__ == "__main__":
555
+ sys.exit(main())