pace-eval 0.2.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cme/__init__.py +11 -0
- cme/__main__.py +5 -0
- cme/analysis.py +223 -0
- cme/audit.py +145 -0
- cme/baseline.py +162 -0
- cme/cli.py +279 -0
- cme/client.py +165 -0
- cme/dynamics.py +212 -0
- cme/examples/crewai_integration.py +16 -0
- cme/examples/demo_pool.jsonl +360 -0
- cme/examples/demo_run.jsonl +480 -0
- cme/examples/langgraph_integration.py +27 -0
- cme/examples/make_demo.py +37 -0
- cme/examples/openai_quickstart.py +18 -0
- cme/experiment.py +555 -0
- cme/frontier.py +143 -0
- cme/hidden_profile.py +135 -0
- cme/proxy.py +106 -0
- cme/rebuttal.py +76 -0
- cme/report.py +197 -0
- cme/rollout_card.py +137 -0
- cme/tasks.py +266 -0
- cme/verified_debate.py +127 -0
- cme/verifier.py +92 -0
- mae_eval/__init__.py +11 -0
- mae_eval/__main__.py +5 -0
- mae_eval/auditor.py +112 -0
- mae_eval/classifier.py +62 -0
- mae_eval/cli.py +210 -0
- mae_eval/tracker.py +162 -0
- mae_eval/vector.py +91 -0
- pace/__init__.py +11 -0
- pace/__main__.py +5 -0
- pace/analysis.py +129 -0
- pace/audit.py +145 -0
- pace/baseline.py +162 -0
- pace/card.py +205 -0
- pace/cli.py +898 -0
- pace/client.py +165 -0
- pace/dynamics.py +372 -0
- pace/escape_benchmarks.py +162 -0
- pace/examples/crewai_integration.py +16 -0
- pace/examples/demo_pool.jsonl +360 -0
- pace/examples/demo_run.jsonl +480 -0
- pace/examples/langgraph_integration.py +27 -0
- pace/examples/make_demo.py +37 -0
- pace/examples/openai_quickstart.py +18 -0
- pace/experiment.py +555 -0
- pace/frontier.py +114 -0
- pace/hidden_profile.py +301 -0
- pace/importers.py +219 -0
- pace/inflate.py +213 -0
- pace/influence.py +231 -0
- pace/meta_engine.py +246 -0
- pace/middleware.py +230 -0
- pace/open_ended.py +114 -0
- pace/plotting.py +132 -0
- pace/pool_sizer.py +154 -0
- pace/protocol_v3.py +414 -0
- pace/proxy.py +106 -0
- pace/rebuttal.py +219 -0
- pace/report.py +197 -0
- pace/rollout_card.py +137 -0
- pace/schema.py +74 -0
- pace/simulator.py +199 -0
- pace/studio.html +1040 -0
- pace/tasks.py +266 -0
- pace/verified_debate.py +127 -0
- pace/verifier.py +92 -0
- pace/workbench.html +26 -0
- pace/workbench.py +119 -0
- pace_eval-0.2.2.dist-info/METADATA +205 -0
- pace_eval-0.2.2.dist-info/RECORD +83 -0
- pace_eval-0.2.2.dist-info/WHEEL +5 -0
- pace_eval-0.2.2.dist-info/entry_points.txt +4 -0
- pace_eval-0.2.2.dist-info/licenses/LICENSE +202 -0
- pace_eval-0.2.2.dist-info/top_level.txt +4 -0
- pace_lab/__init__.py +9 -0
- pace_lab/clients.py +568 -0
- pace_lab/concurrency_test.py +127 -0
- pace_lab/data.py +445 -0
- pace_lab/graders.py +626 -0
- pace_lab/strategies.py +345 -0
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
"""Instrument a LangGraph graph: wrap the client the nodes use and label each node's calls. Needs `pip install langgraph openai`.
|
|
2
|
+
Written from the public LangGraph API and not run in this repository's test environment (langgraph is not installed there)."""
|
|
3
|
+
from typing import TypedDict
|
|
4
|
+
|
|
5
|
+
import pace
|
|
6
|
+
from langgraph.graph import END, START, StateGraph
|
|
7
|
+
from openai import OpenAI
|
|
8
|
+
|
|
9
|
+
client = pace.wrap(OpenAI(base_url="http://localhost:11434/v1", api_key="ollama"), log_path="pace_trace.jsonl")
|
|
10
|
+
class State(TypedDict):
|
|
11
|
+
question: str
|
|
12
|
+
notes: list
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def agent(name: str, rnd: int):
|
|
16
|
+
def node(state: State) -> dict:
|
|
17
|
+
with pace.track(agent=name, round=rnd): # labels apply to the calls inside this node only
|
|
18
|
+
r = client.chat.completions.create(model="llama3.1:8b", messages=[{"role": "user", "content": state["question"] + "\nNotes: " + str(state["notes"])}])
|
|
19
|
+
return {"notes": state["notes"] + [r.choices[0].message.content]}
|
|
20
|
+
return node
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
g = StateGraph(State)
|
|
24
|
+
g.add_node("a1", agent("a1", 1)); g.add_node("a2", agent("a2", 2))
|
|
25
|
+
g.add_edge(START, "a1"); g.add_edge("a1", "a2"); g.add_edge("a2", END)
|
|
26
|
+
with pace.track(item_id="item_001", system="mas"):
|
|
27
|
+
out = g.compile().invoke({"question": "What is 17 * 24?", "notes": []})
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
"""Generate the SYNTHETIC demo data used by the quickstart notebook and tests: demo_pool.jsonl and demo_run.jsonl.
|
|
2
|
+
|
|
3
|
+
Nothing here comes from a model. Items have a hidden per-item success probability; sampled completions are correct with that probability;
|
|
4
|
+
three illustrative "systems" are drawn from fixed rules so the report has something to compare. The numbers say nothing about any real method.
|
|
5
|
+
Run: python code/cme/examples/make_demo.py (seed 7, 40 items, 8 samples per item)
|
|
6
|
+
"""
|
|
7
|
+
import json
|
|
8
|
+
import os
|
|
9
|
+
|
|
10
|
+
import numpy as np
|
|
11
|
+
|
|
12
|
+
HERE = os.path.dirname(os.path.abspath(__file__))
|
|
13
|
+
rng = np.random.default_rng(7)
|
|
14
|
+
N, K = 40, 8
|
|
15
|
+
p_item = rng.beta(2.0, 2.0, N)
|
|
16
|
+
pool, run = [], []
|
|
17
|
+
for i in range(N):
|
|
18
|
+
item = "demo%02d" % i
|
|
19
|
+
tok_p, tok_c = int(rng.integers(60, 90)), int(rng.integers(180, 320))
|
|
20
|
+
|
|
21
|
+
def rec(arm, idx, ok):
|
|
22
|
+
return {"item": item, "arm": arm, "idx": idx, "gold": "G", "answer": "G" if ok else "w%d" % int(rng.integers(0, 3)), "correct": bool(ok), "prompt_tokens": tok_p,
|
|
23
|
+
"completion_tokens": int(tok_c * rng.uniform(0.8, 1.2)), "reasoning_tokens": None, "latency": float(rng.uniform(2, 6)), "model": "synthetic"}
|
|
24
|
+
pool.append(rec("greedy", 0, rng.random() < p_item[i] * 0.95))
|
|
25
|
+
pool += [rec("sample", j, rng.random() < p_item[i]) for j in range(K)]
|
|
26
|
+
# three illustrative systems: five-call debate with a small boost, three-call voting on fresh samples, one longer single call
|
|
27
|
+
for system, calls, boost, mult in (("debate", 5, 0.02, 1.9), ("vote3", 3, 0.0, 1.0), ("long_call", 1, 0.03, 4.0)):
|
|
28
|
+
ok = rng.random() < min(0.98, p_item[i] + boost)
|
|
29
|
+
for c in range(calls):
|
|
30
|
+
run.append({"system": system, "item": item, "prompt_tokens": int(tok_p * (1 + 0.5 * (system == "debate"))), "completion_tokens": int(tok_c * mult / calls * rng.uniform(0.9, 1.1)),
|
|
31
|
+
"n_completions": 1, "start": 0.0, "end": float(rng.uniform(3, 8))})
|
|
32
|
+
run.append({"system": system, "item": item, "correct": bool(ok)})
|
|
33
|
+
with open(os.path.join(HERE, "demo_pool.jsonl"), "w", encoding="utf-8", newline="\n") as fh:
|
|
34
|
+
fh.write("".join(json.dumps(r) + "\n" for r in pool))
|
|
35
|
+
with open(os.path.join(HERE, "demo_run.jsonl"), "w", encoding="utf-8", newline="\n") as fh:
|
|
36
|
+
fh.write("".join(json.dumps(r) + "\n" for r in run))
|
|
37
|
+
print("wrote demo_pool.jsonl (%d rows) and demo_run.jsonl (%d rows), synthetic" % (len(pool), len(run)))
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
"""Log every call of an OpenAI-compatible client with pace.wrap, then audit the log. Needs `pip install openai` and a server (here Ollama)."""
|
|
2
|
+
import pace
|
|
3
|
+
from openai import OpenAI
|
|
4
|
+
|
|
5
|
+
LOG = "pace_trace.jsonl"
|
|
6
|
+
client = pace.wrap(OpenAI(base_url="http://localhost:11434/v1", api_key="ollama"), log_path=LOG) # prices omitted: cost stays null
|
|
7
|
+
ask = lambda q: client.chat.completions.create(model="llama3.1:8b", messages=[{"role": "user", "content": q}]).choices[0].message.content
|
|
8
|
+
|
|
9
|
+
with pace.track(item_id="item_001", system="sas", agent="solver", round=0): # the single-agent baseline: one call
|
|
10
|
+
answer = ask("What is 17 * 24? Answer with the number only.")
|
|
11
|
+
with pace.track(item_id="item_001", system="mas"): # the multi-agent system: three agents, two rounds
|
|
12
|
+
drafts = []
|
|
13
|
+
for rnd in (1, 2):
|
|
14
|
+
for agent in ("a1", "a2", "a3"):
|
|
15
|
+
with pace.track(agent=agent, round=rnd):
|
|
16
|
+
drafts.append(ask("Previous answers: %s\nWhat is 17 * 24? Answer with the number only." % drafts[-3:]))
|
|
17
|
+
pace.log_result(LOG, "item_001", correct=drafts[-1].strip() == "408", system="mas")
|
|
18
|
+
print("then: python -m pace audit %s --baseline-config sas.json" % LOG) # sas.json: {"n_samples": 1}
|
pace/experiment.py
ADDED
|
@@ -0,0 +1,555 @@
|
|
|
1
|
+
"""Controlled-experiment harness: strategies 1-9 on the same items with the same model (Phase 2).
|
|
2
|
+
|
|
3
|
+
Strategy numbers follow docs/PHASED_EXPERIMENT_AND_TOOL_ROADMAP.md section 2:
|
|
4
|
+
1 greedy, one call 2 pool of k=16 independent samples (self-consistency curve for k = 1..16)
|
|
5
|
+
3 one call with more thinking 4 vanilla debate (5 copies of one model, 3 rounds; round 1 = pool samples 0-4)
|
|
6
|
+
5 debate with confidence-weighted aggregation 6 debate among different models (+ 6v: vote by the same mix, no communication)
|
|
7
|
+
7 placebo debate (peer messages replaced by unrelated text of matched length) 8 mixture-of-agents aggregator (2 layers)
|
|
8
|
+
9 verifier (code tasks): agents receive unit-test feedback; the independent arm gets the same verifier (best of the pool)
|
|
9
|
+
Round-1 answers of strategies 4, 5, 7 and 8 are the first five pool samples, so debate and voting differ only by the messages; the
|
|
10
|
+
tokens of reused calls are still charged to the strategy that uses them. Every call is cached in calls.jsonl (resumable), every
|
|
11
|
+
strategy outcome is written to results.jsonl. Nothing here is a result: results exist only for runs whose files exist.
|
|
12
|
+
Limitation: Ollama's prompt_eval_count can exclude prompt tokens served from its KV cache, so prompt tokens may be undercounted on
|
|
13
|
+
repeated prefixes; completion tokens and wall time are exact for the call.
|
|
14
|
+
"""
|
|
15
|
+
from __future__ import annotations
|
|
16
|
+
|
|
17
|
+
import argparse
|
|
18
|
+
import hashlib
|
|
19
|
+
import json
|
|
20
|
+
import os
|
|
21
|
+
import random
|
|
22
|
+
import sys
|
|
23
|
+
import threading
|
|
24
|
+
import time
|
|
25
|
+
import urllib.error
|
|
26
|
+
import urllib.request
|
|
27
|
+
from collections import Counter
|
|
28
|
+
from typing import Dict, List, Optional
|
|
29
|
+
|
|
30
|
+
from . import tasks as T
|
|
31
|
+
|
|
32
|
+
THINK_MODELS = ("qwen3:", "deepseek-r1:", "qwen3.5:") # Ollama 'think' flag is passed only to these
|
|
33
|
+
SOLVE = "{q}\n\nThink step by step. Finish with two final lines: 'Answer: <answer>' and 'Confidence: <0-100>'."
|
|
34
|
+
SOLVE_CODE = "{q}\n\nAfter the code block add a final line 'Confidence: <0-100>'."
|
|
35
|
+
EXTENDED = ("{q}\n\nReason for much longer than usual: work the problem out fully, then re-check your work independently twice before answering. "
|
|
36
|
+
"Finish with two final lines: 'Answer: <answer>' and 'Confidence: <0-100>'.")
|
|
37
|
+
DEBATE = ("{q}\n\nYour previous response:\n{own}\n\nOther agents responded:\n{peers}\n\nUse their reasoning as additional information, but verify it yourself. "
|
|
38
|
+
"Give your updated solution. Finish with two final lines: 'Answer: <answer>' and 'Confidence: <0-100>'.")
|
|
39
|
+
MOA = ("{q}\n\nCandidate responses from other solvers:\n{cands}\n\nSynthesize the best final solution, correcting errors you find. "
|
|
40
|
+
"Finish with two final lines: 'Answer: <answer>' and 'Confidence: <0-100>'.")
|
|
41
|
+
FEEDBACK = "{q}\n\nYour previous code:\n```python\n{code}\n```\nIt was run against the unit tests and {status}.\nGive a corrected version in a ```python block, then a final line 'Confidence: <0-100>'."
|
|
42
|
+
TAIL = 800
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
class OllamaClient:
|
|
46
|
+
"""Ollama native /api/chat client: real token counts (prompt_eval_count, eval_count) and wall time per call."""
|
|
47
|
+
|
|
48
|
+
def __init__(self, host: str = "http://localhost:11434", num_ctx: int = 8192, timeout: float = 1800.0):
|
|
49
|
+
self.host, self.num_ctx, self.timeout = host.rstrip("/"), num_ctx, timeout
|
|
50
|
+
|
|
51
|
+
def chat(self, model: str, prompt: str, temperature: float, seed: int, max_tokens: int = 2048, think: Optional[bool] = None) -> dict:
|
|
52
|
+
body = {"model": model, "messages": [{"role": "user", "content": prompt}], "stream": False,
|
|
53
|
+
"options": {"temperature": temperature, "seed": seed, "num_predict": max_tokens, "num_ctx": self.num_ctx}}
|
|
54
|
+
if think is not None and model.startswith(THINK_MODELS):
|
|
55
|
+
body["think"] = think
|
|
56
|
+
req = urllib.request.Request(self.host + "/api/chat", data=json.dumps(body).encode(), headers={"Content-Type": "application/json"})
|
|
57
|
+
t0 = time.time()
|
|
58
|
+
with urllib.request.urlopen(req, timeout=self.timeout) as r:
|
|
59
|
+
d = json.loads(r.read())
|
|
60
|
+
m = d.get("message", {})
|
|
61
|
+
return {"text": m.get("content") or "", "thinking_chars": len(m.get("thinking") or ""), "prompt_tokens": d.get("prompt_eval_count"),
|
|
62
|
+
"completion_tokens": d.get("eval_count"), "latency": time.time() - t0, "done_reason": d.get("done_reason")}
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
class Store:
|
|
66
|
+
"""Append-only JSONL cache of calls keyed by a string; reloaded on start so interrupted runs resume."""
|
|
67
|
+
|
|
68
|
+
def __init__(self, path: str):
|
|
69
|
+
self.path, self.d = path, {}
|
|
70
|
+
self._lock = threading.Lock()
|
|
71
|
+
if os.path.exists(path):
|
|
72
|
+
for l in open(path, encoding="utf-8"):
|
|
73
|
+
if l.strip():
|
|
74
|
+
r = json.loads(l)
|
|
75
|
+
self.d[r["key"]] = r
|
|
76
|
+
|
|
77
|
+
def put(self, rec: dict) -> None:
|
|
78
|
+
with self._lock:
|
|
79
|
+
self.d[rec["key"]] = rec
|
|
80
|
+
with open(self.path, "a", encoding="utf-8") as fh:
|
|
81
|
+
fh.write(json.dumps(rec) + "\n")
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
class Ctx:
|
|
85
|
+
def __init__(self, client, store: Store, model: str, models: Optional[List[str]] = None, k: int = 16, agents: int = 5, rounds: int = 3,
|
|
86
|
+
seed: int = 0, temperature: float = 0.7, max_tokens: int = 2048, think_tokens: int = 8192,
|
|
87
|
+
price_in: Optional[float] = None, price_out: Optional[float] = None, retries: int = 3, think_default: bool = False):
|
|
88
|
+
self.client, self.store, self.model = client, store, model
|
|
89
|
+
self.models = models or [model]
|
|
90
|
+
self.k, self.agents, self.rounds, self.seed, self.temperature = k, agents, rounds, seed, temperature
|
|
91
|
+
self.max_tokens, self.think_tokens = max_tokens, think_tokens
|
|
92
|
+
self.price_in, self.price_out, self.retries = price_in, price_out, retries
|
|
93
|
+
self.think_default = think_default # value of the Ollama `think` flag for ordinary calls (True lets reasoning models report thinking separately)
|
|
94
|
+
self.used: List[dict] = []
|
|
95
|
+
self._digests: Dict[str, Optional[str]] = {}
|
|
96
|
+
|
|
97
|
+
def call(self, key: str, model: str, prompt: str, temperature: float, max_tokens: Optional[int] = None, think: Optional[bool] = None,
|
|
98
|
+
comm_chars: int = 0) -> dict:
|
|
99
|
+
rec = self.store.d.get(key)
|
|
100
|
+
if rec is None:
|
|
101
|
+
seed = int(hashlib.sha256((str(self.seed) + key).encode()).hexdigest()[:8], 16) if temperature > 0 else self.seed
|
|
102
|
+
r = None
|
|
103
|
+
for attempt in range(self.retries + 1):
|
|
104
|
+
try:
|
|
105
|
+
r = self.client.chat(model, prompt, temperature, seed, max_tokens or self.max_tokens, think)
|
|
106
|
+
break
|
|
107
|
+
except (urllib.error.URLError, TimeoutError, ConnectionError): # transient server errors only
|
|
108
|
+
if attempt == self.retries:
|
|
109
|
+
raise
|
|
110
|
+
time.sleep(5 * (attempt + 1))
|
|
111
|
+
cost = None
|
|
112
|
+
if None not in (self.price_in, self.price_out, r.get("prompt_tokens"), r.get("completion_tokens")):
|
|
113
|
+
cost = (r["prompt_tokens"] * self.price_in + r["completion_tokens"] * self.price_out) / 1e6 # local calls: prices 0.0
|
|
114
|
+
if model not in self._digests:
|
|
115
|
+
from .baseline import ollama_digest
|
|
116
|
+
self._digests[model] = ollama_digest(model, getattr(self.client, "host", "http://localhost:11434")) if hasattr(self.client, "host") else None
|
|
117
|
+
rec = {"key": key, "model": model, "model_digest": self._digests[model], "temperature": temperature, "seed": seed, "think": think,
|
|
118
|
+
"cost_usd": cost, "reasoning_tokens": None, "prompt_chars": len(prompt), "comm_chars": comm_chars, **r}
|
|
119
|
+
self.store.put(rec)
|
|
120
|
+
self.used.append(rec)
|
|
121
|
+
return rec
|
|
122
|
+
|
|
123
|
+
def begin(self):
|
|
124
|
+
self.used = []
|
|
125
|
+
|
|
126
|
+
def fork(self) -> "Ctx":
|
|
127
|
+
"""A worker context sharing the client and the call store, with its own `used` list (thread safe)."""
|
|
128
|
+
c = Ctx(self.client, self.store, self.model, self.models, self.k, self.agents, self.rounds, self.seed, self.temperature, self.max_tokens,
|
|
129
|
+
self.think_tokens, self.price_in, self.price_out, self.retries, self.think_default)
|
|
130
|
+
c._digests = self._digests
|
|
131
|
+
return c
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
def _solve_prompt(item, q=None):
|
|
135
|
+
return (SOLVE_CODE if item["kind"] == "code" else SOLVE).format(q=q or item["prompt"])
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
def ans_of(item, rec) -> str:
|
|
139
|
+
"""Normalised answer of a call (for code: pass/fail marker computed by running the tests)."""
|
|
140
|
+
if item["kind"] == "code":
|
|
141
|
+
return "pass" if T.score(item, rec["text"]) else "fail"
|
|
142
|
+
return T.normalize_answer(item, T.parse_answer(rec["text"]))
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
def vote(answers: List[str], gold_answer_fn, weights: Optional[List[float]] = None):
|
|
146
|
+
"""Plurality (or weighted) vote. Returns (leaders, credit) where credit is 1/m if the gold is among m tied leaders."""
|
|
147
|
+
c: Counter = Counter()
|
|
148
|
+
for a, w in zip(answers, weights or [1.0] * len(answers)):
|
|
149
|
+
if a != "":
|
|
150
|
+
c[a] += w
|
|
151
|
+
if not c:
|
|
152
|
+
return [], 0.0
|
|
153
|
+
top = max(c.values())
|
|
154
|
+
leaders = sorted(a for a, v in c.items() if abs(v - top) < 1e-12)
|
|
155
|
+
return leaders, (1.0 / len(leaders)) if any(gold_answer_fn(a) for a in leaders) else 0.0
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
def _is_gold(item):
|
|
159
|
+
if item["kind"] == "text":
|
|
160
|
+
gs = {T._norm_text(x) for x in item["gold"]}
|
|
161
|
+
return lambda a: a in gs
|
|
162
|
+
if item["kind"] == "num":
|
|
163
|
+
return lambda a: a == T._norm_num(item["gold"])
|
|
164
|
+
if item["kind"] == "name":
|
|
165
|
+
return lambda a: a == str(item["gold"]).lower()
|
|
166
|
+
return lambda a: a == str(item["gold"]).upper()
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
def _tail(text: str, n: int = TAIL) -> str:
|
|
170
|
+
t = (text or "").strip()
|
|
171
|
+
return t[-n:] if len(t) > n else t
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
def pool_rec(ctx: Ctx, item: dict, idx: int, model: Optional[str] = None) -> dict:
|
|
175
|
+
m = model or ctx.model
|
|
176
|
+
return ctx.call("pool|%s|%s|%d" % (m, item["id"], idx), m, _solve_prompt(item), ctx.temperature, think=ctx.think_default)
|
|
177
|
+
|
|
178
|
+
|
|
179
|
+
def round1(ctx: Ctx, item: dict, n: int) -> List[dict]:
|
|
180
|
+
"""Round-1 responses: pool samples 0..n-1, or one call per agent view when evidence is split."""
|
|
181
|
+
if item.get("views"):
|
|
182
|
+
return [ctx.call("r1v|%s|%s|%d" % (ctx.model, item["id"], i), ctx.model, _solve_prompt(item, item["views"][i % len(item["views"])]), ctx.temperature, think=ctx.think_default)
|
|
183
|
+
for i in range(n)]
|
|
184
|
+
return [pool_rec(ctx, item, i) for i in range(n)]
|
|
185
|
+
|
|
186
|
+
|
|
187
|
+
def _result(ctx, strategy, item, credit, final, extra=None):
|
|
188
|
+
toks = lambda f: sum((r.get(f) or 0) for r in ctx.used)
|
|
189
|
+
out = {"strategy": strategy, "model": ctx.model if len(ctx.models) == 1 else "+".join(ctx.models), "item": item["id"], "task": item["task"],
|
|
190
|
+
"correct": credit, "final": final, "calls": len(ctx.used), "prompt_tokens": toks("prompt_tokens"), "completion_tokens": toks("completion_tokens"),
|
|
191
|
+
"latency_sum": toks("latency"), "keys": [r["key"] for r in ctx.used]}
|
|
192
|
+
if extra:
|
|
193
|
+
out.update(extra)
|
|
194
|
+
return out
|
|
195
|
+
|
|
196
|
+
|
|
197
|
+
# ------------------------------------------------------------------------------------ strategies
|
|
198
|
+
def s1_greedy(ctx, item):
|
|
199
|
+
r = ctx.call("greedy|%s|%s" % (ctx.model, item["id"]), ctx.model, _solve_prompt(item), 0.0, think=ctx.think_default)
|
|
200
|
+
return _result(ctx, "S1", item, float(T.score(item, r["text"])), ans_of(item, r))
|
|
201
|
+
|
|
202
|
+
|
|
203
|
+
def s2_pool(ctx, item):
|
|
204
|
+
recs = [pool_rec(ctx, item, i) for i in range(ctx.k)]
|
|
205
|
+
if item["kind"] == "code":
|
|
206
|
+
passes = [ans_of(item, r) == "pass" for r in recs]
|
|
207
|
+
return _result(ctx, "S2", item, sum(passes) / len(passes), "pass@1 mean", {"pass_any": any(passes)})
|
|
208
|
+
leaders, cr = vote([ans_of(item, r) for r in recs], _is_gold(item))
|
|
209
|
+
res = _result(ctx, "S2", item, cr, "|".join(leaders))
|
|
210
|
+
if item.get("views"): # split evidence: independent agents each see one view, no communication, then vote
|
|
211
|
+
ctx.begin()
|
|
212
|
+
v = round1(ctx, item, ctx.agents)
|
|
213
|
+
lv, crv = vote([ans_of(item, r) for r in v], _is_gold(item))
|
|
214
|
+
return [res, _result(ctx, "S2v", item, crv, "|".join(lv))]
|
|
215
|
+
return res
|
|
216
|
+
|
|
217
|
+
|
|
218
|
+
def s3_think(ctx, item):
|
|
219
|
+
cap = ctx.model.startswith(THINK_MODELS)
|
|
220
|
+
prompt = _solve_prompt(item) if cap else EXTENDED.format(q=item["prompt"]) if item["kind"] != "code" else _solve_prompt(item)
|
|
221
|
+
r = ctx.call("think|%s|%s" % (ctx.model, item["id"]), ctx.model, prompt, 0.0, ctx.think_tokens if cap else 4096, think=True if cap else None)
|
|
222
|
+
return _result(ctx, "S3", item, float(T.score(item, r["text"])), ans_of(item, r), {"mode": "think-flag" if cap else "prompt-extended", "thinking_chars": r.get("thinking_chars")})
|
|
223
|
+
|
|
224
|
+
|
|
225
|
+
def _peers(ctx, texts, j, show_conf: bool, placebo_texts=None, transform=None):
|
|
226
|
+
lines = []
|
|
227
|
+
for q, t in enumerate(texts):
|
|
228
|
+
if q == j:
|
|
229
|
+
continue
|
|
230
|
+
body = _tail(t)
|
|
231
|
+
if transform is not None:
|
|
232
|
+
body = transform(body, q, j)
|
|
233
|
+
if placebo_texts is not None:
|
|
234
|
+
p = placebo_texts[q % len(placebo_texts)]
|
|
235
|
+
body = (p * (len(body) // max(1, len(p)) + 1))[: len(body)]
|
|
236
|
+
conf = T.parse_confidence(t)
|
|
237
|
+
lines.append("Agent %d%s: %s" % (q + 1, (" (confidence %.0f%%)" % (100 * conf)) if (show_conf and conf is not None) else "", body))
|
|
238
|
+
return "\n\n".join(lines)
|
|
239
|
+
|
|
240
|
+
|
|
241
|
+
def _debate(ctx, item, strategy, models, r1_recs, show_conf, weighted, placebo_pool=None, peer_transform=None):
|
|
242
|
+
cur = list(r1_recs)
|
|
243
|
+
rounds_ans = [[ans_of(item, r) for r in cur]]
|
|
244
|
+
rounds_tok = [[[r.get("prompt_tokens"), r.get("completion_tokens")] for r in cur]]
|
|
245
|
+
for rd in range(2, ctx.rounds + 1):
|
|
246
|
+
texts = [r["text"] for r in cur]
|
|
247
|
+
nxt = []
|
|
248
|
+
for j in range(len(cur)):
|
|
249
|
+
peers = _peers(ctx, texts, j, show_conf, placebo_pool, peer_transform)
|
|
250
|
+
# split evidence: an agent sees only its own view in every round (before 2026-11-12 the later rounds showed the full-evidence prompt to every agent; see docs/PREREGISTRATION_ADDENDUM_2.md)
|
|
251
|
+
q = item["views"][j % len(item["views"])] if item.get("views") else item["prompt"]
|
|
252
|
+
prompt = DEBATE.format(q=q, own=_tail(texts[j]), peers=peers)
|
|
253
|
+
nxt.append(ctx.call("%s|%s|%s|r%d|a%d" % (strategy, "+".join(models), item["id"], rd, j), models[j], prompt, ctx.temperature, think=ctx.think_default,
|
|
254
|
+
comm_chars=len(peers)))
|
|
255
|
+
cur = nxt
|
|
256
|
+
rounds_ans.append([ans_of(item, r) for r in cur])
|
|
257
|
+
rounds_tok.append([[r.get("prompt_tokens"), r.get("completion_tokens")] for r in cur])
|
|
258
|
+
answers = [ans_of(item, r) for r in cur]
|
|
259
|
+
w = [T.parse_confidence(r["text"]) or 0.5 for r in cur] if weighted else None
|
|
260
|
+
leaders, cr = vote(answers, _is_gold(item), w)
|
|
261
|
+
changed = sum(1 for a, b in zip([ans_of(item, r) for r in r1_recs], answers) if a != b)
|
|
262
|
+
return leaders, cr, {"answers_changed": changed, "round_answers": rounds_ans, "round_tokens": rounds_tok}
|
|
263
|
+
|
|
264
|
+
|
|
265
|
+
def s4_debate(ctx, item):
|
|
266
|
+
# Items with split evidence run as S4v (own keys, own label) so that results of the earlier implementation, which leaked the full evidence in rounds 2 and later, are never reused.
|
|
267
|
+
name = "S4v" if item.get("views") else "S4"
|
|
268
|
+
leaders, cr, ex = _debate(ctx, item, name, [ctx.model] * ctx.agents, round1(ctx, item, ctx.agents), False, False)
|
|
269
|
+
return _result(ctx, name, item, cr, "|".join(leaders), ex)
|
|
270
|
+
|
|
271
|
+
|
|
272
|
+
def s5_conf_debate(ctx, item):
|
|
273
|
+
leaders, cr, ex = _debate(ctx, item, "S5", [ctx.model] * ctx.agents, round1(ctx, item, ctx.agents), True, True)
|
|
274
|
+
return _result(ctx, "S5", item, cr, "|".join(leaders), ex)
|
|
275
|
+
|
|
276
|
+
|
|
277
|
+
def s6_hetero(ctx, item):
|
|
278
|
+
"""Debate among different models, and (S6v) vote by the same mix with no communication. Returns two results."""
|
|
279
|
+
ms = [ctx.models[i % len(ctx.models)] for i in range(ctx.agents)]
|
|
280
|
+
r1 = [pool_rec(ctx, item, i // len(ctx.models), ms[i]) for i in range(ctx.agents)]
|
|
281
|
+
lv, crv = vote([ans_of(item, r) for r in r1], _is_gold(item))
|
|
282
|
+
vote_res = _result(ctx, "S6v", item, crv, "|".join(lv))
|
|
283
|
+
leaders, cr, ex = _debate(ctx, item, "S6", ms, r1, False, False)
|
|
284
|
+
deb = _result(ctx, "S6", item, cr, "|".join(leaders), ex)
|
|
285
|
+
return [vote_res, deb]
|
|
286
|
+
|
|
287
|
+
|
|
288
|
+
def s7_placebo(ctx, item, others: List[dict]):
|
|
289
|
+
r1 = round1(ctx, item, ctx.agents)
|
|
290
|
+
n_used = len(ctx.used)
|
|
291
|
+
foreign = [pool_rec(ctx, others[i % len(others)], i)["text"] for i in range(ctx.agents)]
|
|
292
|
+
del ctx.used[n_used:] # the borrowed text is input material for the control, not compute spent on this item
|
|
293
|
+
leaders, cr, ex = _debate(ctx, item, "S7", [ctx.model] * ctx.agents, r1, False, False, placebo_pool=[_tail(t) for t in foreign])
|
|
294
|
+
return _result(ctx, "S7", item, cr, "|".join(leaders), ex)
|
|
295
|
+
|
|
296
|
+
|
|
297
|
+
def s8_moa(ctx, item):
|
|
298
|
+
props = round1(ctx, item, ctx.agents)
|
|
299
|
+
cands = "\n\n".join("[%d] %s" % (i + 1, _tail(r["text"])) for i, r in enumerate(props))
|
|
300
|
+
l1 = [ctx.call("S8|L1|%s|%s|%d" % (ctx.model, item["id"], i), ctx.model, MOA.format(q=item["prompt"], cands=cands), ctx.temperature, think=ctx.think_default) for i in range(ctx.agents)]
|
|
301
|
+
cands2 = "\n\n".join("[%d] %s" % (i + 1, _tail(r["text"])) for i, r in enumerate(l1))
|
|
302
|
+
fin = ctx.call("S8|L2|%s|%s" % (ctx.model, item["id"]), ctx.model, MOA.format(q=item["prompt"], cands=cands2), 0.0, think=ctx.think_default)
|
|
303
|
+
return _result(ctx, "S8", item, float(T.score(item, fin["text"])), ans_of(item, fin))
|
|
304
|
+
|
|
305
|
+
|
|
306
|
+
def s9_verifier(ctx, item):
|
|
307
|
+
"""Code only. Feedback arm: agent revises with unit-test results (up to rounds-1 revisions). Independent arm: best of the pool with the same verifier."""
|
|
308
|
+
if item["kind"] != "code":
|
|
309
|
+
raise ValueError("strategy 9 needs a code task")
|
|
310
|
+
ok = lambda rec: T.score(item, rec["text"])
|
|
311
|
+
pool = [pool_rec(ctx, item, i) for i in range(ctx.agents)]
|
|
312
|
+
ind = any(ok(r) for r in pool)
|
|
313
|
+
ind_res = _result(ctx, "S9i", item, float(ind), "verified best-of-%d" % ctx.agents)
|
|
314
|
+
ctx.begin()
|
|
315
|
+
cur, solved = pool[0], ok(pool[0])
|
|
316
|
+
for rd in range(2, ctx.rounds + 1):
|
|
317
|
+
if solved:
|
|
318
|
+
break
|
|
319
|
+
prompt = FEEDBACK.format(q=item["prompt"], code=T.extract_code(cur["text"]), status="failed" if not solved else "passed")
|
|
320
|
+
cur = ctx.call("S9|%s|%s|r%d" % (ctx.model, item["id"], rd), ctx.model, prompt, ctx.temperature, think=ctx.think_default)
|
|
321
|
+
solved = ok(cur)
|
|
322
|
+
ctx.used = [pool[0]] + [r for r in ctx.used if r is not pool[0]]
|
|
323
|
+
fb_res = _result(ctx, "S9", item, float(solved), "feedback loop")
|
|
324
|
+
return [ind_res, fb_res]
|
|
325
|
+
|
|
326
|
+
|
|
327
|
+
def s10_views_vote(ctx, item):
|
|
328
|
+
"""Independent agents on split evidence, plurality vote (the S2v arm of strategy 2 without the 16-sample pool)."""
|
|
329
|
+
if not item.get("views"):
|
|
330
|
+
raise ValueError("strategy 10 needs a split-evidence task")
|
|
331
|
+
v = round1(ctx, item, ctx.agents)
|
|
332
|
+
lv, crv = vote([ans_of(item, r) for r in v], _is_gold(item))
|
|
333
|
+
return _result(ctx, "S2v", item, crv, "|".join(lv))
|
|
334
|
+
|
|
335
|
+
|
|
336
|
+
def load_items(task, n, stratify=False, evidence="split"):
|
|
337
|
+
"""Task items; evidence='shared' removes the per-agent views of split-evidence items, so every agent sees all evidence (the redundant-evidence arm of the 2x2 design)."""
|
|
338
|
+
items = T.load_task(task, n, stratify=stratify)
|
|
339
|
+
if evidence == "shared":
|
|
340
|
+
items = [{k: v for k, v in it.items() if k != "views"} for it in items]
|
|
341
|
+
elif evidence != "split":
|
|
342
|
+
raise ValueError("evidence must be 'split' or 'shared'")
|
|
343
|
+
return items
|
|
344
|
+
|
|
345
|
+
|
|
346
|
+
STRATS = {1: s1_greedy, 2: s2_pool, 3: s3_think, 4: s4_debate, 5: s5_conf_debate, 6: s6_hetero, 7: s7_placebo, 8: s8_moa, 9: s9_verifier, 10: s10_views_vote}
|
|
347
|
+
|
|
348
|
+
|
|
349
|
+
def run(ctx: Ctx, items: List[dict], strategies: List[int], out_dir: str, progress=print, workers: int = 1) -> dict:
|
|
350
|
+
"""Run strategies over items. Strategy 2 always runs first (the pool feeds debate round 1 and the placebo text). With workers > 1,
|
|
351
|
+
items of one strategy run concurrently; item failures are logged to errors.jsonl and the item is skipped, never filled in."""
|
|
352
|
+
from concurrent.futures import ThreadPoolExecutor
|
|
353
|
+
os.makedirs(out_dir, exist_ok=True)
|
|
354
|
+
rpath, epath = os.path.join(out_dir, "results.jsonl"), os.path.join(out_dir, "errors.jsonl")
|
|
355
|
+
order = sorted(set(strategies), key=lambda x: (x != 2, strategies.index(x)))
|
|
356
|
+
if 7 in order and 2 not in order:
|
|
357
|
+
order.insert(0, 2)
|
|
358
|
+
done = set()
|
|
359
|
+
if os.path.exists(rpath):
|
|
360
|
+
for l in open(rpath, encoding="utf-8"):
|
|
361
|
+
if l.strip():
|
|
362
|
+
r = json.loads(l)
|
|
363
|
+
done.add((r["strategy"], r["item"]))
|
|
364
|
+
lock = threading.Lock()
|
|
365
|
+
counts = {"new": 0, "errors": 0, "done_items": 0}
|
|
366
|
+
t0 = time.time()
|
|
367
|
+
label = {1: "S1", 2: "S2", 3: "S3", 4: "S4", 5: "S5", 6: "S6", 7: "S7", 8: "S8", 9: "S9", 10: "S2v", 11: "S4v", 12: "S12", 13: "S13"}
|
|
368
|
+
|
|
369
|
+
def work(args):
|
|
370
|
+
s, item = args
|
|
371
|
+
lab = "S4v" if (s == 4 and item.get("views")) else label[s]
|
|
372
|
+
if (lab, item["id"]) in done:
|
|
373
|
+
return
|
|
374
|
+
c = ctx.fork()
|
|
375
|
+
c.begin()
|
|
376
|
+
try:
|
|
377
|
+
res = STRATS[s](c, item, [x for x in items if x is not item]) if s == 7 else STRATS[s](c, item)
|
|
378
|
+
except Exception as e: # record and continue: no result is invented for a failed item
|
|
379
|
+
with lock:
|
|
380
|
+
counts["errors"] += 1
|
|
381
|
+
with open(epath, "a", encoding="utf-8") as fh:
|
|
382
|
+
fh.write(json.dumps({"strategy": label[s], "item": item["id"], "error": repr(e)[:300]}) + "\n")
|
|
383
|
+
return
|
|
384
|
+
with lock:
|
|
385
|
+
for r in (res if isinstance(res, list) else [res]):
|
|
386
|
+
with open(rpath, "a", encoding="utf-8") as fh:
|
|
387
|
+
fh.write(json.dumps(r) + "\n")
|
|
388
|
+
counts["new"] += 1
|
|
389
|
+
counts["done_items"] += 1
|
|
390
|
+
if progress and counts["done_items"] % 10 == 0:
|
|
391
|
+
progress("%s: %d items finished, %.0fs, errors %d" % (label[s], counts["done_items"], time.time() - t0, counts["errors"]))
|
|
392
|
+
|
|
393
|
+
for s in order:
|
|
394
|
+
if s == 9 and any(i["kind"] != "code" for i in items):
|
|
395
|
+
raise ValueError("strategy 9 needs a code task")
|
|
396
|
+
tasks = [(s, it) for it in items]
|
|
397
|
+
if workers > 1:
|
|
398
|
+
with ThreadPoolExecutor(workers) as ex:
|
|
399
|
+
list(ex.map(work, tasks))
|
|
400
|
+
else:
|
|
401
|
+
for t in tasks:
|
|
402
|
+
work(t)
|
|
403
|
+
return {"new_results": counts["new"], "errors": counts["errors"], "seconds": time.time() - t0}
|
|
404
|
+
|
|
405
|
+
|
|
406
|
+
def summarize(out_dir: str, project_n: Optional[int] = None, price_in: Optional[float] = None, price_out: Optional[float] = None) -> List[dict]:
|
|
407
|
+
rows = [json.loads(l) for l in open(os.path.join(out_dir, "results.jsonl"), encoding="utf-8") if l.strip()]
|
|
408
|
+
by: Dict[tuple, List[dict]] = {}
|
|
409
|
+
for r in rows:
|
|
410
|
+
by.setdefault((r["strategy"], r["model"], r["task"]), []).append(r)
|
|
411
|
+
out = []
|
|
412
|
+
for (s, m, t), rs in sorted(by.items()):
|
|
413
|
+
n = len(rs)
|
|
414
|
+
row = {"strategy": s, "model": m, "task": t, "n_items": n, "accuracy": sum(r["correct"] for r in rs) / n,
|
|
415
|
+
"calls_per_item": sum(r["calls"] for r in rs) / n, "prompt_tokens_per_item": sum(r["prompt_tokens"] for r in rs) / n,
|
|
416
|
+
"completion_tokens_per_item": sum(r["completion_tokens"] for r in rs) / n, "latency_sum_per_item_s": sum(r["latency_sum"] for r in rs) / n}
|
|
417
|
+
if project_n:
|
|
418
|
+
row["projected_calls"] = row["calls_per_item"] * project_n
|
|
419
|
+
row["projected_hours_serial"] = row["latency_sum_per_item_s"] * project_n / 3600
|
|
420
|
+
if price_in is not None and price_out is not None:
|
|
421
|
+
row["projected_cost_usd"] = (row["prompt_tokens_per_item"] * price_in + row["completion_tokens_per_item"] * price_out) * project_n / 1e6
|
|
422
|
+
out.append(row)
|
|
423
|
+
return out
|
|
424
|
+
|
|
425
|
+
|
|
426
|
+
def export_pool(store: Store, model: str, items: List[dict], path: str, k: int = 16) -> int:
|
|
427
|
+
"""Write the S1/S2 calls in the `pace baseline` cache format so `pace report --pool` can read them. Non-code tasks only.
|
|
428
|
+
The gold label is the literal 'G'; a correct response gets answer 'G', a wrong one its own normalised answer plus '_wrong'."""
|
|
429
|
+
n = 0
|
|
430
|
+
with open(path, "w", encoding="utf-8") as fh:
|
|
431
|
+
for it in items:
|
|
432
|
+
if it["kind"] == "code":
|
|
433
|
+
continue
|
|
434
|
+
recs = [("greedy", 0, store.d.get("greedy|%s|%s" % (model, it["id"])))] + [("sample", i, store.d.get("pool|%s|%s|%d" % (model, it["id"], i))) for i in range(k)]
|
|
435
|
+
for arm, i, r in recs:
|
|
436
|
+
if r is None:
|
|
437
|
+
continue
|
|
438
|
+
ok = T.score(it, r["text"])
|
|
439
|
+
fh.write(json.dumps({"item": it["id"], "arm": arm, "idx": i, "gold": "G", "answer": "G" if ok else (ans_of(it, r) or "?") + "_wrong", "correct": ok,
|
|
440
|
+
"prompt_tokens": r["prompt_tokens"], "completion_tokens": r["completion_tokens"], "reasoning_tokens": None,
|
|
441
|
+
"latency": r["latency"], "model": model}) + "\n")
|
|
442
|
+
n += 1
|
|
443
|
+
return n
|
|
444
|
+
|
|
445
|
+
|
|
446
|
+
def write_manifest(path: str, a, items: List[dict]) -> None:
|
|
447
|
+
"""Append one manifest entry: what was run, with which preregistration file (SHA-256), git commit, server version, and item ids."""
|
|
448
|
+
import subprocess
|
|
449
|
+
root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
|
450
|
+
prereg = os.path.join(root, "docs", "PREREGISTRATION.md")
|
|
451
|
+
sha = hashlib.sha256(open(prereg, "rb").read()).hexdigest() if os.path.exists(prereg) else None
|
|
452
|
+
try:
|
|
453
|
+
commit = subprocess.run(["git", "rev-parse", "HEAD"], cwd=root, capture_output=True, text=True, timeout=10).stdout.strip() or None
|
|
454
|
+
dirty = bool(subprocess.run(["git", "status", "--porcelain", "--", "docs/PREREGISTRATION.md", "code"], cwd=root, capture_output=True, text=True, timeout=10).stdout.strip())
|
|
455
|
+
except Exception:
|
|
456
|
+
commit, dirty = None, None
|
|
457
|
+
try:
|
|
458
|
+
with urllib.request.urlopen("http://localhost:11434/api/version", timeout=5) as r:
|
|
459
|
+
ver = json.loads(r.read()).get("version")
|
|
460
|
+
except Exception:
|
|
461
|
+
ver = None
|
|
462
|
+
from .baseline import ollama_digest
|
|
463
|
+
entry = {"time": time.strftime("%Y-%m-%d %H:%M:%S"), "preregistration_sha256": sha, "git_commit": commit, "code_or_prereg_uncommitted_changes": dirty,
|
|
464
|
+
"ollama_version": ver, "model": a.model, "model_digest": ollama_digest(a.model), "task": a.task, "n_items": len(items),
|
|
465
|
+
"item_ids_sha256": hashlib.sha256("|".join(i["id"] for i in items).encode()).hexdigest(), "strategies": a.strategies, "k": a.k, "agents": a.agents,
|
|
466
|
+
"rounds": a.rounds, "seed": a.seed, "max_tokens": a.max_tokens, "think_tokens": a.think_tokens, "workers": a.workers, "out": a.out}
|
|
467
|
+
os.makedirs(os.path.dirname(os.path.abspath(path)), exist_ok=True)
|
|
468
|
+
with open(path, "a", encoding="utf-8") as fh:
|
|
469
|
+
fh.write(json.dumps(entry) + "\n")
|
|
470
|
+
|
|
471
|
+
|
|
472
|
+
def main(argv=None) -> int:
|
|
473
|
+
ap = argparse.ArgumentParser(prog="pace-experiment", description=__doc__.split("\n")[0])
|
|
474
|
+
sub = ap.add_subparsers(dest="cmd", required=True)
|
|
475
|
+
r = sub.add_parser("run")
|
|
476
|
+
r.add_argument("--model", required=True)
|
|
477
|
+
r.add_argument("--models", default=None, help="comma-separated model list for strategy 6")
|
|
478
|
+
r.add_argument("--task", required=True, choices=list(T.SOURCES) + ["hidden_profile"])
|
|
479
|
+
r.add_argument("--n", type=int, default=50)
|
|
480
|
+
r.add_argument("--stratify", action="store_true", help="draw items round-robin over categories/subjects")
|
|
481
|
+
r.add_argument("--num-ctx", type=int, default=8192, help="Ollama context length; must exceed prompt + max tokens")
|
|
482
|
+
r.add_argument("--strategies", default="1")
|
|
483
|
+
r.add_argument("--out", required=True)
|
|
484
|
+
r.add_argument("--k", type=int, default=16)
|
|
485
|
+
r.add_argument("--agents", type=int, default=5)
|
|
486
|
+
r.add_argument("--rounds", type=int, default=3)
|
|
487
|
+
r.add_argument("--seed", type=int, default=0)
|
|
488
|
+
r.add_argument("--max-tokens", type=int, default=2048)
|
|
489
|
+
r.add_argument("--think-tokens", type=int, default=8192)
|
|
490
|
+
r.add_argument("--workers", type=int, default=1, help="concurrent items (Ollama serves them in parallel slots)")
|
|
491
|
+
r.add_argument("--price-in", type=float, default=0.0, help="USD per million input tokens (0.0 for local models)")
|
|
492
|
+
r.add_argument("--price-out", type=float, default=0.0)
|
|
493
|
+
r.add_argument("--think", action="store_true", help="pass think=true on ordinary calls (reasoning models then report thinking separately)")
|
|
494
|
+
r.add_argument("--manifest", default=None, help="write a run manifest (preregistration hash, git commit, versions) to this path")
|
|
495
|
+
r.add_argument("--evidence", choices=["split", "shared"], default="split", help="split-evidence tasks: each agent sees its own view (split) or all evidence (shared)")
|
|
496
|
+
tr = sub.add_parser("trace", help="write the round-by-round JSONL of a debate strategy for `pace dynamics` (reads the cache, never calls a model)")
|
|
497
|
+
tr.add_argument("--model", required=True)
|
|
498
|
+
tr.add_argument("--task", required=True, choices=list(T.SOURCES) + ["hidden_profile"])
|
|
499
|
+
tr.add_argument("--n", type=int, default=50)
|
|
500
|
+
tr.add_argument("--strategy", required=True, help="S4, S4v, S5, S12 or S12u")
|
|
501
|
+
tr.add_argument("--out", required=True, help="the cell directory of the run")
|
|
502
|
+
tr.add_argument("--agents", type=int, default=5)
|
|
503
|
+
tr.add_argument("--evidence", choices=["split", "shared"], default="split")
|
|
504
|
+
tr.add_argument("--dest", required=True)
|
|
505
|
+
rs = sub.add_parser("rescore", help="recompute results.jsonl from cached calls (never calls a model)")
|
|
506
|
+
rs.add_argument("--model", required=True)
|
|
507
|
+
rs.add_argument("--task", required=True, choices=list(T.SOURCES) + ["hidden_profile"])
|
|
508
|
+
rs.add_argument("--n", type=int, default=50)
|
|
509
|
+
rs.add_argument("--strategies", default="1")
|
|
510
|
+
rs.add_argument("--out", required=True)
|
|
511
|
+
rs.add_argument("--k", type=int, default=16)
|
|
512
|
+
rs.add_argument("--agents", type=int, default=5)
|
|
513
|
+
rs.add_argument("--rounds", type=int, default=3)
|
|
514
|
+
s = sub.add_parser("summary")
|
|
515
|
+
s.add_argument("--out", required=True)
|
|
516
|
+
s.add_argument("--project-n", type=int, default=None)
|
|
517
|
+
s.add_argument("--price-in", type=float, default=None)
|
|
518
|
+
s.add_argument("--price-out", type=float, default=None)
|
|
519
|
+
a = ap.parse_args(argv)
|
|
520
|
+
if a.cmd == "trace":
|
|
521
|
+
from .verified_debate import export_trace
|
|
522
|
+
n_rows = export_trace(a.out, load_items(a.task, a.n, evidence=a.evidence), a.strategy, a.model, a.agents, a.dest)
|
|
523
|
+
print("wrote %d agent-round records to %s" % (n_rows, a.dest))
|
|
524
|
+
return 0
|
|
525
|
+
if a.cmd == "summary":
|
|
526
|
+
for row in summarize(a.out, a.project_n, a.price_in, a.price_out):
|
|
527
|
+
print(json.dumps({k: (round(v, 4) if isinstance(v, float) else v) for k, v in row.items()}))
|
|
528
|
+
return 0
|
|
529
|
+
if a.cmd == "rescore":
|
|
530
|
+
class NoCall:
|
|
531
|
+
def chat(self, *x, **y):
|
|
532
|
+
raise RuntimeError("rescore must not call a model: a call is missing from the cache")
|
|
533
|
+
rp = os.path.join(a.out, "results.jsonl")
|
|
534
|
+
if os.path.exists(rp):
|
|
535
|
+
os.replace(rp, rp + ".bak")
|
|
536
|
+
ctx = Ctx(NoCall(), Store(os.path.join(a.out, "calls.jsonl")), a.model, k=a.k, agents=a.agents, rounds=a.rounds)
|
|
537
|
+
print(json.dumps(run(ctx, T.load_task(a.task, a.n), [int(x) for x in a.strategies.split(",")], a.out, progress=None)))
|
|
538
|
+
return 0
|
|
539
|
+
items = load_items(a.task, a.n, stratify=a.stratify, evidence=a.evidence)
|
|
540
|
+
os.makedirs(a.out, exist_ok=True)
|
|
541
|
+
ctx = Ctx(OllamaClient(num_ctx=a.num_ctx), Store(os.path.join(a.out, "calls.jsonl")), a.model, a.models.split(",") if a.models else None, a.k, a.agents, a.rounds, a.seed,
|
|
542
|
+
max_tokens=a.max_tokens, think_tokens=a.think_tokens, price_in=a.price_in, price_out=a.price_out, think_default=a.think)
|
|
543
|
+
if a.manifest:
|
|
544
|
+
write_manifest(a.manifest, a, items)
|
|
545
|
+
print(json.dumps(run(ctx, items, [int(x) for x in a.strategies.split(",")], a.out, workers=a.workers)))
|
|
546
|
+
return 0
|
|
547
|
+
|
|
548
|
+
|
|
549
|
+
|
|
550
|
+
from . import verified_debate as _vd # noqa: E402 (registers strategy 12; imported last because verified_debate imports this module lazily)
|
|
551
|
+
STRATS[12] = _vd.s12_verified_debate
|
|
552
|
+
STRATS[13] = _vd.s13_shuffled_debate
|
|
553
|
+
|
|
554
|
+
if __name__ == "__main__":
|
|
555
|
+
sys.exit(main())
|