pace-eval 0.2.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cme/__init__.py +11 -0
- cme/__main__.py +5 -0
- cme/analysis.py +223 -0
- cme/audit.py +145 -0
- cme/baseline.py +162 -0
- cme/cli.py +279 -0
- cme/client.py +165 -0
- cme/dynamics.py +212 -0
- cme/examples/crewai_integration.py +16 -0
- cme/examples/demo_pool.jsonl +360 -0
- cme/examples/demo_run.jsonl +480 -0
- cme/examples/langgraph_integration.py +27 -0
- cme/examples/make_demo.py +37 -0
- cme/examples/openai_quickstart.py +18 -0
- cme/experiment.py +555 -0
- cme/frontier.py +143 -0
- cme/hidden_profile.py +135 -0
- cme/proxy.py +106 -0
- cme/rebuttal.py +76 -0
- cme/report.py +197 -0
- cme/rollout_card.py +137 -0
- cme/tasks.py +266 -0
- cme/verified_debate.py +127 -0
- cme/verifier.py +92 -0
- mae_eval/__init__.py +11 -0
- mae_eval/__main__.py +5 -0
- mae_eval/auditor.py +112 -0
- mae_eval/classifier.py +62 -0
- mae_eval/cli.py +210 -0
- mae_eval/tracker.py +162 -0
- mae_eval/vector.py +91 -0
- pace/__init__.py +11 -0
- pace/__main__.py +5 -0
- pace/analysis.py +129 -0
- pace/audit.py +145 -0
- pace/baseline.py +162 -0
- pace/card.py +205 -0
- pace/cli.py +898 -0
- pace/client.py +165 -0
- pace/dynamics.py +372 -0
- pace/escape_benchmarks.py +162 -0
- pace/examples/crewai_integration.py +16 -0
- pace/examples/demo_pool.jsonl +360 -0
- pace/examples/demo_run.jsonl +480 -0
- pace/examples/langgraph_integration.py +27 -0
- pace/examples/make_demo.py +37 -0
- pace/examples/openai_quickstart.py +18 -0
- pace/experiment.py +555 -0
- pace/frontier.py +114 -0
- pace/hidden_profile.py +301 -0
- pace/importers.py +219 -0
- pace/inflate.py +213 -0
- pace/influence.py +231 -0
- pace/meta_engine.py +246 -0
- pace/middleware.py +230 -0
- pace/open_ended.py +114 -0
- pace/plotting.py +132 -0
- pace/pool_sizer.py +154 -0
- pace/protocol_v3.py +414 -0
- pace/proxy.py +106 -0
- pace/rebuttal.py +219 -0
- pace/report.py +197 -0
- pace/rollout_card.py +137 -0
- pace/schema.py +74 -0
- pace/simulator.py +199 -0
- pace/studio.html +1040 -0
- pace/tasks.py +266 -0
- pace/verified_debate.py +127 -0
- pace/verifier.py +92 -0
- pace/workbench.html +26 -0
- pace/workbench.py +119 -0
- pace_eval-0.2.2.dist-info/METADATA +205 -0
- pace_eval-0.2.2.dist-info/RECORD +83 -0
- pace_eval-0.2.2.dist-info/WHEEL +5 -0
- pace_eval-0.2.2.dist-info/entry_points.txt +4 -0
- pace_eval-0.2.2.dist-info/licenses/LICENSE +202 -0
- pace_eval-0.2.2.dist-info/top_level.txt +4 -0
- pace_lab/__init__.py +9 -0
- pace_lab/clients.py +568 -0
- pace_lab/concurrency_test.py +127 -0
- pace_lab/data.py +445 -0
- pace_lab/graders.py +626 -0
- pace_lab/strategies.py +345 -0
pace/tasks.py
ADDED
|
@@ -0,0 +1,266 @@
|
|
|
1
|
+
"""Task loaders and scorers for the controlled experiment (Phase 2 pilots).
|
|
2
|
+
|
|
3
|
+
Data come from the public Hugging Face datasets-server JSON API (no parquet reader needed) and are cached under
|
|
4
|
+
data/experiments/<task>.jsonl with a provenance header. Subsets are drawn with a fixed seed (42) from the filtered pool so a run is
|
|
5
|
+
reproducible. Items share one schema:
|
|
6
|
+
{"id", "task", "prompt", "gold", "kind": "mc" | "num" | "text" | "code", "meta", "views": optional list of agent-specific prompts}
|
|
7
|
+
`views` is used only by the split-evidence positive control, where each agent sees part of the evidence.
|
|
8
|
+
|
|
9
|
+
Substitutions to disclose: plain MBPP (the `test_list` asserts, 500 test items) stands in for MBPP+; the positive-control items are
|
|
10
|
+
synthetic (generated here, seed 42), they are not a published benchmark.
|
|
11
|
+
"""
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
import json
|
|
15
|
+
import os
|
|
16
|
+
import random
|
|
17
|
+
import re
|
|
18
|
+
import subprocess
|
|
19
|
+
import sys
|
|
20
|
+
import tempfile
|
|
21
|
+
import time
|
|
22
|
+
import urllib.error
|
|
23
|
+
import urllib.parse
|
|
24
|
+
import urllib.request
|
|
25
|
+
from typing import Dict, List, Optional
|
|
26
|
+
|
|
27
|
+
ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
|
28
|
+
DATA = os.path.join(ROOT, "data", "experiments")
|
|
29
|
+
API = "https://datasets-server.huggingface.co/rows"
|
|
30
|
+
LETTERS = "ABCDEFGHIJ"
|
|
31
|
+
|
|
32
|
+
SOURCES = {
|
|
33
|
+
"mmlu_pro": ("TIGER-Lab/MMLU-Pro", "default", "test"),
|
|
34
|
+
"math500_l45": ("HuggingFaceH4/MATH-500", "default", "test"),
|
|
35
|
+
"musique_4hop": ("bdsaglam/musique", "answerable", "validation"),
|
|
36
|
+
"mbpp": ("google-research-datasets/mbpp", "full", "test"),
|
|
37
|
+
"gsm8k": ("openai/gsm8k", "main", "test"),
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def _get(dataset: str, config: str, split: str, offset: int, length: int) -> List[dict]:
|
|
42
|
+
q = urllib.parse.urlencode({"dataset": dataset, "config": config, "split": split, "offset": offset, "length": length})
|
|
43
|
+
for attempt in range(8):
|
|
44
|
+
try:
|
|
45
|
+
with urllib.request.urlopen(API + "?" + q, timeout=60) as r:
|
|
46
|
+
return [x["row"] for x in json.loads(r.read())["rows"]]
|
|
47
|
+
except urllib.error.HTTPError as e:
|
|
48
|
+
time.sleep(min(120, float(e.headers.get("Retry-After", 0) or 0) or 20 * (attempt + 1))) # 429 rate limit: back off
|
|
49
|
+
except Exception:
|
|
50
|
+
time.sleep(2 * (attempt + 1))
|
|
51
|
+
raise RuntimeError("datasets-server unavailable for %s offset %d" % (dataset, offset))
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def _fetch_all(dataset: str, config: str, split: str, total: int) -> List[dict]:
|
|
55
|
+
rows: List[dict] = []
|
|
56
|
+
for off in range(0, total, 100):
|
|
57
|
+
rows += _get(dataset, config, split, off, min(100, total - off))
|
|
58
|
+
return rows
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def _mc_prompt(q: str, opts: List[str]) -> str:
|
|
62
|
+
return q.strip() + "\n\nOptions:\n" + "\n".join("%s. %s" % (LETTERS[i], o) for i, o in enumerate(opts)) + "\n\nGive the letter of the correct option."
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def load_task(task: str, n: int, seed: int = 42, refresh: bool = False, stratify: bool = False) -> List[dict]:
|
|
66
|
+
"""Return n items for a task (deterministic subset). Fetches and caches the filtered pool on first use."""
|
|
67
|
+
if task == "hidden_profile":
|
|
68
|
+
return hidden_profile(n, seed)
|
|
69
|
+
os.makedirs(DATA, exist_ok=True)
|
|
70
|
+
path = os.path.join(DATA, task + ".jsonl")
|
|
71
|
+
if refresh or not os.path.exists(path):
|
|
72
|
+
ds, cfg, sp = SOURCES[task]
|
|
73
|
+
if task == "mmlu_pro":
|
|
74
|
+
starts = sorted(random.Random(seed).sample(range(0, 12000, 100), 40)) # forty seeded 100-row blocks over the whole split (about 4,000 rows); the split is ordered by category, so one block is one category and 40 blocks cover all of them
|
|
75
|
+
rows = []
|
|
76
|
+
for st in starts:
|
|
77
|
+
rows += _get(ds, cfg, sp, st, 100)
|
|
78
|
+
time.sleep(0.5) # stay under the datasets-server rate limit
|
|
79
|
+
items = [{"id": "mmlu%05d" % r["question_id"], "task": task, "prompt": _mc_prompt(r["question"], r["options"]), "gold": r["answer"],
|
|
80
|
+
"kind": "mc", "meta": {"category": r["category"], "n_options": len(r["options"])}} for r in rows]
|
|
81
|
+
elif task == "math500_l45":
|
|
82
|
+
rows = [r for r in _fetch_all(ds, cfg, sp, 500) if r["level"] in (4, 5)]
|
|
83
|
+
items = [{"id": r["unique_id"], "task": task, "prompt": r["problem"].strip(), "gold": r["answer"], "kind": "num",
|
|
84
|
+
"meta": {"level": r["level"], "subject": r["subject"]}} for r in rows]
|
|
85
|
+
elif task == "musique_4hop":
|
|
86
|
+
rows = [r for r in _fetch_all(ds, cfg, sp, 2417) if str(r["id"]).startswith("4hop")]
|
|
87
|
+
items = []
|
|
88
|
+
for r in rows:
|
|
89
|
+
ctx = "\n\n".join("[%s] %s" % (p["title"], p["paragraph_text"]) for p in r["paragraphs"])
|
|
90
|
+
items.append({"id": r["id"], "task": task, "prompt": "Paragraphs:\n" + ctx + "\n\nQuestion: " + r["question"] + "\nAnswer with a short phrase.",
|
|
91
|
+
"gold": [r["answer"]] + list(r.get("answer_aliases") or []), "kind": "text", "meta": {}})
|
|
92
|
+
elif task == "gsm8k":
|
|
93
|
+
rows = _fetch_all(ds, cfg, sp, 1319)
|
|
94
|
+
items = [{"id": "gsm%04d" % i, "task": task, "prompt": r["question"].strip(), "gold": r["answer"].split("####")[-1].strip().replace(",", ""), "kind": "num", "meta": {}}
|
|
95
|
+
for i, r in enumerate(rows)]
|
|
96
|
+
elif task == "mbpp":
|
|
97
|
+
rows = _fetch_all(ds, cfg, sp, 500)
|
|
98
|
+
items = [{"id": "mbpp%d" % r["task_id"], "task": task,
|
|
99
|
+
"prompt": r["text"].strip() + "\nYour code must pass this test:\n" + r["test_list"][0] + "\nReturn only the Python function in a ```python block.",
|
|
100
|
+
"gold": {"tests": r["test_list"], "setup": r["test_setup_code"]}, "kind": "code", "meta": {}} for r in rows]
|
|
101
|
+
else:
|
|
102
|
+
raise ValueError("unknown task " + task)
|
|
103
|
+
with open(path, "w", encoding="utf-8") as fh:
|
|
104
|
+
fh.write(json.dumps({"_provenance": {"source": SOURCES[task], "api": API, "fetched": time.strftime("%Y-%m-%d"), "pool_size": len(items), "seed": seed}}) + "\n")
|
|
105
|
+
for it in items:
|
|
106
|
+
fh.write(json.dumps(it) + "\n")
|
|
107
|
+
items = [json.loads(l) for l in open(path, encoding="utf-8") if l.strip() and not l.startswith('{"_provenance"')]
|
|
108
|
+
random.Random(seed).shuffle(items)
|
|
109
|
+
if stratify: # round-robin over categories (MMLU-Pro category, MATH subject) in the seeded order, so each stratum contributes about equally
|
|
110
|
+
key = lambda it: str(it["meta"].get("category") or it["meta"].get("subject") or "")
|
|
111
|
+
groups = {}
|
|
112
|
+
for it in items:
|
|
113
|
+
groups.setdefault(key(it), []).append(it)
|
|
114
|
+
picked, i = [], 0
|
|
115
|
+
while len(picked) < n and any(i < len(g) for g in groups.values()):
|
|
116
|
+
picked += [g[i] for _, g in sorted(groups.items()) if i < len(g)]
|
|
117
|
+
i += 1
|
|
118
|
+
return picked[:n]
|
|
119
|
+
return items[:n]
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
# ----------------------------------------------------------------------------- scoring
|
|
123
|
+
def _norm_num(s: str) -> str:
|
|
124
|
+
s = str(s).strip().strip("$").replace("\\!", "").replace("\\,", "").replace("\\left", "").replace("\\right", "").replace("dfrac", "frac").replace("tfrac", "frac")
|
|
125
|
+
s = re.sub(r"\\text\{([^}]*)\}", r"\1", s)
|
|
126
|
+
s = re.sub(r"\\boxed\{(.*)\}", r"\1", s)
|
|
127
|
+
s = s.replace("\\$", "").replace("$", "").replace(" ", "").rstrip(".").lower()
|
|
128
|
+
s = re.sub(r"^(-?\d+)/(\d+)$", r"\\frac{\1}{\2}", s)
|
|
129
|
+
s = re.sub(r"^(-?\d+)\.0+$", r"\1", s)
|
|
130
|
+
s = re.sub(r"(degrees?|units?)$", "", s)
|
|
131
|
+
s = s.replace("^{\\circ}", "").replace("^\\circ", "").replace("\\%", "").replace("%", "")
|
|
132
|
+
return s.replace(",", "") if re.fullmatch(r"-?[\d,]+(\.\d+)?", s) else s
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
def _norm_text(s: str) -> str:
|
|
136
|
+
s = re.sub(r"\b(a|an|the)\b", " ", str(s).lower())
|
|
137
|
+
return " ".join(re.sub(r"[^\w\s]", "", s).split())
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
_ANS = re.compile(r"answer\s*[:=]\s*(.+)", re.I)
|
|
141
|
+
_CONF = re.compile(r"confidence\s*[:=]\s*(\d+(?:\.\d+)?)", re.I)
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
BOXED = "\\boxed{"
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
def last_boxed(text: str) -> str:
|
|
148
|
+
"""Contents of the last boxed{...} expression (brace-balanced), or ''."""
|
|
149
|
+
text = text or ""
|
|
150
|
+
i = text.rfind(BOXED)
|
|
151
|
+
if i < 0:
|
|
152
|
+
return ""
|
|
153
|
+
start = i + len(BOXED)
|
|
154
|
+
j, depth = start, 1
|
|
155
|
+
while j < len(text) and depth:
|
|
156
|
+
depth += {"{": 1, "}": -1}.get(text[j], 0)
|
|
157
|
+
j += 1
|
|
158
|
+
return text[start: j - 1] if depth == 0 else ""
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
_IS = re.compile(r"answer is:?\s*\$?\(?([^\n$]+)", re.I)
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
def parse_answer(text: str) -> str:
|
|
165
|
+
"""Final 'Answer:' payload (first line of it); else the last boxed content; else the payload of 'the answer is ...'; else ''."""
|
|
166
|
+
m = _ANS.findall(text or "")
|
|
167
|
+
if m:
|
|
168
|
+
got = m[-1].strip().split("\n")[0].strip()
|
|
169
|
+
return last_boxed(got) or got
|
|
170
|
+
return last_boxed(text) or ((_IS.findall(text or "") or [""])[-1].strip().rstrip(".)"))
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
def parse_confidence(text: str) -> Optional[float]:
|
|
174
|
+
m = _CONF.findall(text or "")
|
|
175
|
+
return min(100.0, max(0.0, float(m[-1]))) / 100.0 if m else None
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+
def extract_code(text: str) -> str:
|
|
179
|
+
m = re.findall(r"```(?:python)?\n(.*?)```", text or "", re.S)
|
|
180
|
+
return m[-1] if m else (text or "")
|
|
181
|
+
|
|
182
|
+
|
|
183
|
+
def run_tests(code: str, tests: List[str], setup: str = "", timeout: float = 10.0) -> bool:
|
|
184
|
+
"""Execute generated code plus the asserts in a separate Python process with a timeout (no sandbox beyond that; run on a trusted dev machine)."""
|
|
185
|
+
src = (setup or "") + "\n" + code + "\n" + "\n".join(tests) + "\n"
|
|
186
|
+
with tempfile.TemporaryDirectory() as d:
|
|
187
|
+
p = os.path.join(d, "t.py")
|
|
188
|
+
open(p, "w", encoding="utf-8").write(src)
|
|
189
|
+
try:
|
|
190
|
+
return subprocess.run([sys.executable, "-I", p], cwd=d, capture_output=True, timeout=timeout).returncode == 0
|
|
191
|
+
except subprocess.TimeoutExpired:
|
|
192
|
+
return False
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
def normalize_answer(item: dict, ans: str) -> str:
|
|
196
|
+
k = item["kind"]
|
|
197
|
+
if k == "mc":
|
|
198
|
+
m = re.search(r"\b([A-J])\b", (ans or "").upper())
|
|
199
|
+
return m.group(1) if m else ""
|
|
200
|
+
if k == "num":
|
|
201
|
+
return _norm_num(ans)
|
|
202
|
+
if k == "text":
|
|
203
|
+
return _norm_text(ans)
|
|
204
|
+
if k == "name":
|
|
205
|
+
m = re.search(r"\b(avery|blake|casey|drew)\b", (ans or "").lower())
|
|
206
|
+
return m.group(1) if m else ""
|
|
207
|
+
return (ans or "").strip()
|
|
208
|
+
|
|
209
|
+
|
|
210
|
+
def score(item: dict, ans_text: str) -> bool:
|
|
211
|
+
"""Is a response correct? For code, ans_text is the full response (code is extracted and run)."""
|
|
212
|
+
if item["kind"] == "code":
|
|
213
|
+
return run_tests(extract_code(ans_text), item["gold"]["tests"], item["gold"].get("setup", ""))
|
|
214
|
+
a = normalize_answer(item, parse_answer(ans_text))
|
|
215
|
+
g = item["gold"]
|
|
216
|
+
if item["kind"] == "text":
|
|
217
|
+
return a != "" and any(a == _norm_text(x) for x in g)
|
|
218
|
+
if item["kind"] == "num":
|
|
219
|
+
return a != "" and a == _norm_num(g)
|
|
220
|
+
if item["kind"] == "name":
|
|
221
|
+
return a == str(g).lower()
|
|
222
|
+
return a == str(g).strip().upper()
|
|
223
|
+
|
|
224
|
+
|
|
225
|
+
# ----------------------------------------------------------------------------- positive control
|
|
226
|
+
def hidden_profile(n: int, seed: int = 42, n_agents: int = 5) -> List[dict]:
|
|
227
|
+
"""Synthetic split-evidence task (hidden-profile design, Stasser and Titus 1985).
|
|
228
|
+
|
|
229
|
+
Four candidates are scored 1-5 on 10 criteria with equal weights; the correct choice has the highest total. Criteria 1-4 are shared
|
|
230
|
+
(every agent sees them) and favour a decoy; criteria 5-10 are split so that each agent sees only two of the six and the pooled
|
|
231
|
+
total favours the target. A single agent shown all evidence (`prompt`) can compute the answer; one agent's `views[i]` cannot,
|
|
232
|
+
and independent voting over views is expected to favour the decoy. Generated, not published: report it as synthetic."""
|
|
233
|
+
rng = random.Random(seed)
|
|
234
|
+
names = ["Avery", "Blake", "Casey", "Drew"]
|
|
235
|
+
out = []
|
|
236
|
+
while len(out) < n:
|
|
237
|
+
target, decoy = rng.sample(range(4), 2)
|
|
238
|
+
sc = [[rng.randint(2, 4) for _ in range(10)] for _ in range(4)]
|
|
239
|
+
for j in range(4): # shared criteria: the decoy leads clearly
|
|
240
|
+
for c in range(4):
|
|
241
|
+
sc[c][j] = min(sc[c][j], 3)
|
|
242
|
+
sc[decoy][j] = 5
|
|
243
|
+
sc[target][j] = 2
|
|
244
|
+
for j in range(4, 10): # split criteria: the target leads
|
|
245
|
+
for c in range(4):
|
|
246
|
+
sc[c][j] = min(sc[c][j], 3)
|
|
247
|
+
sc[target][j] = 5
|
|
248
|
+
sc[decoy][j] = 2
|
|
249
|
+
tot = [sum(r) for r in sc]
|
|
250
|
+
if sorted(tot)[-1] == sorted(tot)[-2] or tot.index(max(tot)) != target:
|
|
251
|
+
continue
|
|
252
|
+
crit = ["criterion %d" % (j + 1) for j in range(10)]
|
|
253
|
+
facts = lambda js: "\n".join("%s on %s: %d" % (names[c], crit[j], sc[c][j]) for j in js for c in range(4))
|
|
254
|
+
head = "A committee must pick one of four candidates (Avery, Blake, Casey, Drew) using 10 equally weighted criteria, each scored 1 to 5. The best candidate has the highest total score.\n\n"
|
|
255
|
+
tail = "\n\nWhich candidate has the highest total over all 10 criteria? Give the name."
|
|
256
|
+
full = head + "All scores:\n" + facts(range(10)) + tail
|
|
257
|
+
order = list(range(4, 10))
|
|
258
|
+
rng.shuffle(order)
|
|
259
|
+
views = []
|
|
260
|
+
for a in range(n_agents):
|
|
261
|
+
mine = sorted([order[(2 * a) % 6], order[(2 * a + 1) % 6]])
|
|
262
|
+
views.append(head + "You see only part of the evidence. Scores you can see:\n" + facts(list(range(4)) + mine) +
|
|
263
|
+
"\n\nOther committee members see the other criteria. Which candidate has the highest total over all 10 criteria? Give the name.")
|
|
264
|
+
out.append({"id": "hp%04d" % len(out), "task": "hidden_profile", "prompt": full, "gold": names[target], "kind": "name", "meta": {"synthetic": True, "totals": tot},
|
|
265
|
+
"views": views})
|
|
266
|
+
return out
|
pace/verified_debate.py
ADDED
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
"""Route II harness and debate-trace export.
|
|
2
|
+
|
|
3
|
+
s12_verified_debate (strategy 12): numeric tasks only. Every response must end with 'Expression: ...', 'Answer: ...' and 'Confidence: ...'. Two debates start from the same round-1 responses:
|
|
4
|
+
S12u ungated: agents update freely (the same prompts as S12);
|
|
5
|
+
S12 gated: an agent's changed answer is accepted only if pace.verifier accepts that response (the expression evaluates to the stated answer); otherwise the agent keeps its
|
|
6
|
+
previous accepted response. The gate is deterministic and runs no model-written code.
|
|
7
|
+
The only difference between S12u and S12 is the gate, so the change in transition rates between them (right-to-wrong in particular) is attributable to the verifier. The verifier checks
|
|
8
|
+
arithmetic, not the reading of the problem (see pace.verifier), so it is partial.
|
|
9
|
+
Results carry round_answers and round_tokens per agent and round, which export_trace writes as the JSONL that `pace dynamics` reads.
|
|
10
|
+
"""
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
import json
|
|
14
|
+
import os
|
|
15
|
+
from typing import List, Optional
|
|
16
|
+
|
|
17
|
+
VSOLVE = ("{q}\n\nThink step by step. Finish with three final lines: 'Expression: <one arithmetic expression using only numbers and + - * / ( ) that computes the answer from the "
|
|
18
|
+
"quantities in the problem>', 'Answer: <answer>' and 'Confidence: <0-100>'.")
|
|
19
|
+
VDEBATE = ("{q}\n\nYour previous response:\n{own}\n\nOther agents responded:\n{peers}\n\nUse their reasoning as additional information, but verify it yourself. Give your updated solution. "
|
|
20
|
+
"Finish with three final lines: 'Expression: <one arithmetic expression using only numbers and + - * / ( )>', 'Answer: <answer>' and 'Confidence: <0-100>'.")
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def _round1(ctx, item):
|
|
24
|
+
return [ctx.call("V1|%s|%s|%d" % (ctx.model, item["id"], i), ctx.model, VSOLVE.format(q=item["prompt"]), ctx.temperature, think=ctx.think_default) for i in range(ctx.agents)]
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def _debate(ctx, item, name, r1, gated):
|
|
28
|
+
from . import experiment as E
|
|
29
|
+
from .verifier import verify
|
|
30
|
+
state = list(r1)
|
|
31
|
+
answers = [[E.ans_of(item, r) for r in state]]
|
|
32
|
+
tokens = [[[r.get("prompt_tokens"), r.get("completion_tokens")] for r in state]]
|
|
33
|
+
rejected = changed = 0
|
|
34
|
+
for rd in range(2, ctx.rounds + 1):
|
|
35
|
+
texts = [r["text"] for r in state]
|
|
36
|
+
nxt, tok = [], []
|
|
37
|
+
for j in range(len(state)):
|
|
38
|
+
peers = E._peers(ctx, texts, j, False, None)
|
|
39
|
+
new = ctx.call("%s|%s|%s|r%d|a%d" % (name, ctx.model, item["id"], rd, j), ctx.model, VDEBATE.format(q=item["prompt"], own=E._tail(texts[j]), peers=peers), ctx.temperature,
|
|
40
|
+
think=ctx.think_default, comm_chars=len(peers))
|
|
41
|
+
tok.append([new.get("prompt_tokens"), new.get("completion_tokens")])
|
|
42
|
+
old_a, new_a = E.ans_of(item, state[j]), E.ans_of(item, new)
|
|
43
|
+
if new_a != old_a:
|
|
44
|
+
if gated and not verify(new["text"])[0]:
|
|
45
|
+
rejected += 1
|
|
46
|
+
nxt.append(state[j])
|
|
47
|
+
continue
|
|
48
|
+
changed += 1
|
|
49
|
+
nxt.append(new)
|
|
50
|
+
state = nxt
|
|
51
|
+
answers.append([E.ans_of(item, r) for r in state])
|
|
52
|
+
tokens.append(tok)
|
|
53
|
+
leaders, cr = E.vote(answers[-1], E._is_gold(item))
|
|
54
|
+
return leaders, cr, {"round_answers": answers, "round_tokens": tokens, "rejected": rejected, "changed": changed}
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def s12_verified_debate(ctx, item):
|
|
58
|
+
from . import experiment as E
|
|
59
|
+
if item["kind"] != "num":
|
|
60
|
+
raise ValueError("strategy 12 needs a numeric task")
|
|
61
|
+
r1 = _round1(ctx, item)
|
|
62
|
+
base = list(ctx.used)
|
|
63
|
+
lu, cu, eu = _debate(ctx, item, "S12u", r1, False)
|
|
64
|
+
res_u = E._result(ctx, "S12u", item, cu, "|".join(lu), eu)
|
|
65
|
+
ctx.used = list(base)
|
|
66
|
+
lg, cg, eg = _debate(ctx, item, "S12", r1, True)
|
|
67
|
+
res_g = E._result(ctx, "S12", item, cg, "|".join(lg), eg)
|
|
68
|
+
return [res_u, res_g]
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def export_trace(out_dir: str, items: List[dict], strategy: str, model: str, agents: int, path: str) -> int:
|
|
72
|
+
"""Write one JSONL record per agent answer and round for every finished item of `strategy`: item, agent, round, correct, prompt_tokens, completion_tokens, strategy.
|
|
73
|
+
Strategies S12 and S12u use the accepted answers stored in their results; S4, S4v, S5 and S7 are reconstructed from the cached calls (round 1 is the pool sample or the agent's view)."""
|
|
74
|
+
from . import experiment as E
|
|
75
|
+
by_id = {it["id"]: it for it in items}
|
|
76
|
+
store = E.Store(os.path.join(out_dir, "calls.jsonl"))
|
|
77
|
+
n = 0
|
|
78
|
+
with open(path, "w", encoding="utf-8") as fh:
|
|
79
|
+
for line in open(os.path.join(out_dir, "results.jsonl"), encoding="utf-8"):
|
|
80
|
+
if not line.strip():
|
|
81
|
+
continue
|
|
82
|
+
r = json.loads(line)
|
|
83
|
+
if r["strategy"] != strategy or r["item"] not in by_id:
|
|
84
|
+
continue
|
|
85
|
+
item, gold = by_id[r["item"]], E._is_gold(by_id[r["item"]])
|
|
86
|
+
rows = []
|
|
87
|
+
if "round_answers" in r:
|
|
88
|
+
for rd, (ans, tok) in enumerate(zip(r["round_answers"], r["round_tokens"]), 1):
|
|
89
|
+
for a, (an, (pt, ct)) in enumerate(zip(ans, tok)):
|
|
90
|
+
rows.append((a, rd, bool(gold(an)), pt, ct))
|
|
91
|
+
else:
|
|
92
|
+
rounds = max(int(k.split("|")[3][1:]) for k in r["keys"] if "|r" in k) if any("|r" in k for k in r["keys"]) else 1
|
|
93
|
+
for a in range(agents):
|
|
94
|
+
for rd in range(1, rounds + 1):
|
|
95
|
+
if rd == 1:
|
|
96
|
+
key = ("r1v|%s|%s|%d" if item.get("views") else "pool|%s|%s|%d") % (model, item["id"], a)
|
|
97
|
+
else:
|
|
98
|
+
key = "%s|%s|%s|r%d|a%d" % (strategy, "+".join([model] * agents), item["id"], rd, a) # the debate key names the model of every agent
|
|
99
|
+
rec = store.d.get(key)
|
|
100
|
+
if rec is None:
|
|
101
|
+
raise KeyError("call %s is not in the cache" % key)
|
|
102
|
+
rows.append((a, rd, bool(gold(E.ans_of(item, rec))), rec.get("prompt_tokens"), rec.get("completion_tokens")))
|
|
103
|
+
for a, rd, ok, pt, ct in rows:
|
|
104
|
+
rec = {"item": r["item"], "agent": a, "round": rd, "correct": ok, "strategy": strategy}
|
|
105
|
+
if pt is not None and ct is not None:
|
|
106
|
+
rec["prompt_tokens"], rec["completion_tokens"] = pt, ct
|
|
107
|
+
fh.write(json.dumps(rec) + "\n")
|
|
108
|
+
n += 1
|
|
109
|
+
return n
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def _shuffle_sentences(body: str, q: str, j: int) -> str:
|
|
113
|
+
"""Peer message with its sentences in a deterministic random order (same words, same length, the reasoning order destroyed)."""
|
|
114
|
+
import hashlib
|
|
115
|
+
import random
|
|
116
|
+
import re
|
|
117
|
+
parts = [x for x in re.split(r"(?<=[.!?])\s+", body.strip()) if x]
|
|
118
|
+
random.Random(int(hashlib.sha256(("%s|%d" % (body[:40], j)).encode()).hexdigest()[:8], 16)).shuffle(parts)
|
|
119
|
+
return " ".join(parts)
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def s13_shuffled_debate(ctx, item):
|
|
123
|
+
"""Debate whose peer messages have their sentences shuffled: the words and the length are kept, the argument order is not. Round 1 is the first `agents` pool samples."""
|
|
124
|
+
from . import experiment as E
|
|
125
|
+
leaders, cr, ex = E._debate(ctx, item, "S13", [ctx.model] * ctx.agents, E.round1(ctx, item, ctx.agents), False, False, peer_transform=_shuffle_sentences)
|
|
126
|
+
return E._result(ctx, "S13", item, cr, "|".join(leaders), ex)
|
|
127
|
+
|
pace/verifier.py
ADDED
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
"""Deterministic arithmetic verifier for Route II (an external check in the system's tool budget, R_sys > 0).
|
|
2
|
+
|
|
3
|
+
A response passes if it contains a line `Expression: <arithmetic expression>` and a line `Answer: <number>` and the expression, evaluated by a restricted parser, equals the answer.
|
|
4
|
+
The parser accepts numbers, the operators + - * / // % ** and parentheses, and nothing else: there are no names, calls, attributes or strings, and no code is executed (ast is used
|
|
5
|
+
only to parse). It checks that the calculation is consistent with the stated answer. It does not check that the expression models the problem, so a wrong plan with correct arithmetic
|
|
6
|
+
passes: the verifier is partial, with a false-positive rate for wrong answers above zero.
|
|
7
|
+
"""
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import ast
|
|
11
|
+
import operator
|
|
12
|
+
import re
|
|
13
|
+
from fractions import Fraction
|
|
14
|
+
from typing import Optional, Tuple
|
|
15
|
+
|
|
16
|
+
_BIN = {ast.Add: operator.add, ast.Sub: operator.sub, ast.Mult: operator.mul, ast.Div: operator.truediv, ast.FloorDiv: operator.floordiv, ast.Mod: operator.mod, ast.Pow: operator.pow}
|
|
17
|
+
_UN = {ast.UAdd: operator.pos, ast.USub: operator.neg}
|
|
18
|
+
MAX_NODES, MAX_EXP, MAX_ABS = 200, 12, 10 ** 15
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def _clean(expr: str) -> str:
|
|
22
|
+
e = expr.strip().strip("`$ ").rstrip(".")
|
|
23
|
+
e = e.replace("\u00d7", "*").replace("\u00f7", "/").replace("\u2212", "-").replace("^", "**").replace("\\times", "*").replace("\\cdot", "*").replace("\\div", "/")
|
|
24
|
+
e = re.sub(r"(?<=\d),(?=\d{3}\b)", "", e) # thousands separators
|
|
25
|
+
e = re.sub(r"\$|\\left|\\right", "", e)
|
|
26
|
+
e = re.sub(r"(\d+(?:\.\d+)?)\s*%(?=\s*(?:$|[-+*/)]))", r"(\1/100)", e) # 12% -> (12/100) when % is a postfix; 7%4 stays a modulo
|
|
27
|
+
return e
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def safe_eval(expr: str) -> Fraction:
|
|
31
|
+
"""Evaluate an arithmetic expression exactly (Fraction); raises ValueError for anything outside the grammar or for oversized values."""
|
|
32
|
+
tree = ast.parse(_clean(expr), mode="eval")
|
|
33
|
+
if sum(1 for _ in ast.walk(tree)) > MAX_NODES:
|
|
34
|
+
raise ValueError("expression too long")
|
|
35
|
+
|
|
36
|
+
def ev(n):
|
|
37
|
+
if isinstance(n, ast.Expression):
|
|
38
|
+
return ev(n.body)
|
|
39
|
+
if isinstance(n, ast.Constant) and isinstance(n.value, (int, float)) and not isinstance(n.value, bool):
|
|
40
|
+
return Fraction(str(n.value))
|
|
41
|
+
if isinstance(n, ast.UnaryOp) and type(n.op) in _UN:
|
|
42
|
+
return _UN[type(n.op)](ev(n.operand))
|
|
43
|
+
if isinstance(n, ast.BinOp) and type(n.op) in _BIN:
|
|
44
|
+
a, b = ev(n.left), ev(n.right)
|
|
45
|
+
if isinstance(n.op, ast.Pow):
|
|
46
|
+
if b.denominator != 1 or abs(b) > MAX_EXP:
|
|
47
|
+
raise ValueError("exponent not allowed")
|
|
48
|
+
v = a ** int(b)
|
|
49
|
+
elif isinstance(n.op, (ast.Div, ast.FloorDiv, ast.Mod)) and b == 0:
|
|
50
|
+
raise ValueError("division by zero")
|
|
51
|
+
else:
|
|
52
|
+
v = _BIN[type(n.op)](a, b)
|
|
53
|
+
if abs(v) > MAX_ABS:
|
|
54
|
+
raise ValueError("value too large")
|
|
55
|
+
return Fraction(v)
|
|
56
|
+
raise ValueError("not an arithmetic expression")
|
|
57
|
+
return ev(tree)
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def _number(text: str) -> Optional[Fraction]:
|
|
61
|
+
t = text.strip().strip("*$ ").rstrip(".").replace(",", "")
|
|
62
|
+
t = re.sub(r"\\boxed\{(.*)\}", r"\1", t)
|
|
63
|
+
t = re.sub(r"[A-Za-z%\\ ]+$", "", t).strip() # trailing units
|
|
64
|
+
try:
|
|
65
|
+
if re.fullmatch(r"-?\d+/\d+", t):
|
|
66
|
+
return Fraction(t)
|
|
67
|
+
return Fraction(t)
|
|
68
|
+
except (ValueError, ZeroDivisionError):
|
|
69
|
+
return None
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def parse_lines(text: str) -> Tuple[Optional[str], Optional[str]]:
|
|
73
|
+
ex = re.findall(r"(?im)^\s*\**expression\**\s*:\s*(.+)$", text or "")
|
|
74
|
+
an = re.findall(r"(?im)^\s*\**answer\**\s*:\s*(.+)$", text or "")
|
|
75
|
+
return (ex[-1].strip() if ex else None, an[-1].strip() if an else None)
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def verify(text: str) -> Tuple[bool, str]:
|
|
79
|
+
"""(passed, reason). Reasons: missing line, not arithmetic, answer not a number, mismatch, ok."""
|
|
80
|
+
expr, ans = parse_lines(text)
|
|
81
|
+
if expr is None or ans is None:
|
|
82
|
+
return False, "missing expression or answer line"
|
|
83
|
+
try:
|
|
84
|
+
val = safe_eval(expr)
|
|
85
|
+
except (SyntaxError, ValueError, ZeroDivisionError, RecursionError, OverflowError):
|
|
86
|
+
return False, "not an arithmetic expression"
|
|
87
|
+
a = _number(ans)
|
|
88
|
+
if a is None:
|
|
89
|
+
return False, "answer is not a number"
|
|
90
|
+
d = len(re.sub(r"[^0-9.]", "", ans).partition(".")[2]) # an answer rounded to d decimals matches the exact value rounded to d decimals
|
|
91
|
+
ok = val == a or (d > 0 and round(val, d) == a)
|
|
92
|
+
return ok, "ok" if ok else "expression does not equal the answer"
|
pace/workbench.html
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
<!doctype html>
|
|
2
|
+
<html lang="en">
|
|
3
|
+
<head>
|
|
4
|
+
<meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
|
|
5
|
+
<title>PACE Workbench</title>
|
|
6
|
+
<style>
|
|
7
|
+
:root{--ink:#17202a;--muted:#607080;--paper:#f4f1ea;--panel:#fffdf8;--line:#d8d0c3;--accent:#bb4b35;--teal:#176b68}*{box-sizing:border-box}body{margin:0;background:linear-gradient(135deg,#f4f1ea,#e5eee9);color:var(--ink);font:15px Georgia,serif}header{padding:28px 5vw 20px;display:flex;justify-content:space-between;align-items:end}h1{margin:0;font-size:clamp(28px,5vw,54px);letter-spacing:0}header p{max-width:480px;color:var(--muted);margin:0}.tabs{display:flex;gap:8px;flex-wrap:wrap;padding:0 5vw 16px}.tabs button{border:1px solid var(--line);background:var(--panel);padding:10px 13px;border-radius:4px;color:var(--ink);cursor:pointer}.tabs button.active{background:var(--accent);color:white}.page{padding:0 5vw 48px;max-width:1300px;margin:auto}.panel{background:var(--panel);border:1px solid var(--line);padding:22px;border-radius:6px;box-shadow:0 8px 24px #17202a0b}.grid{display:grid;grid-template-columns:repeat(auto-fit,minmax(250px,1fr));gap:16px}.metric{font-size:30px;color:var(--teal);margin:8px 0}.drop{border:2px dashed var(--line);padding:28px;text-align:center;border-radius:6px;margin:12px 0}.controls{display:flex;gap:16px;flex-wrap:wrap;align-items:center}.controls label{display:grid;gap:5px}.controls input{accent-color:var(--accent)}svg{width:100%;height:230px;background:#f7f5ef;border:1px solid var(--line)}pre{white-space:pre-wrap;background:#202b31;color:#f5eee1;padding:14px;min-height:100px}button.action{background:var(--teal);color:white;border:0;border-radius:4px;padding:10px 13px;cursor:pointer}.hidden{display:none}.small{color:var(--muted);font-size:13px}
|
|
8
|
+
</style></head>
|
|
9
|
+
<body>
|
|
10
|
+
<header><div><div class="small">PACE / LOCAL ANALYSIS DESK</div><h1>Workbench</h1></div><p>Interactive parity-aware inspection for evaluation cards, empirical dynamics, and the nine-study corpus.</p></header>
|
|
11
|
+
<nav class="tabs" aria-label="Workbench tabs">
|
|
12
|
+
<button data-tab="card7" class="active">7-Field Card & Trace Auditor</button><button data-tab="itemwise">Itemwise (k_i) & Cache Pricing Lab</button><button data-tab="constructive">Placebo Overhead & Verifier Drift Lab</button><button data-tab="callgraph">Call-Graph DAG & Stopping Inspector</button><button data-tab="corpus9">9-Study Corpus & Subfield Explorer</button><button data-tab="rebuttal">Experiment Planner & Rebuttal Builder</button>
|
|
13
|
+
</nav><main class="page">
|
|
14
|
+
<section id="card7" class="tab"><div class="panel"><h2>7-field card and trace auditor</h2><div id="drop" class="drop">Drop a JSONL, CSV, or JSON run here <input id="file" type="file" accept=".jsonl,.csv,.json"></div><div class="grid"><div><div class="small">Preloaded confirmatory calls</div><div class="metric">27,500</div></div><div><div class="small">Seven-field status</div><div class="metric">PACE-Recommended</div></div></div><button class="action" onclick="exportCard('json')">Export JSON</button> <button class="action" onclick="exportCard('md')">Export Markdown</button> <button class="action" onclick="exportCard('tex')">Export LaTeX</button> <button class="action" onclick="exportCard('svg')">Export SVG</button><pre id="cardOut">baseline_rung: unreported\nresource_vector: tokens / calls / cost\nfrontier_gap: unreported</pre></div></section>
|
|
15
|
+
<section id="itemwise" class="tab hidden"><div class="panel"><h2>Itemwise frontier and cache pricing</h2><div class="controls"><label>alpha cache <input id="alphaCache" type="range" min="0" max="1" step=".05" value=".25"><output>0.25</output></label><label>alpha out <input id="alphaOut" type="range" min="0" max="2" step=".1" value="1"><output>1.0</output></label><label>pool k <input id="poolK" type="range" min="1" max="32" value="8"><output>8</output></label></div><svg id="frontierSvg" viewBox="0 0 700 230"><path d="M35 195H670M35 195V25" stroke="#607080"/><path id="frontierLine" d="M45 180 Q220 100 380 125 T650 55" fill="none" stroke="#bb4b35" stroke-width="4"/><text x="45" y="218">Expected</text><text x="520" y="218">Itemwise k_i</text></svg><p id="cacheReadout" class="small">Adjusted input pricing updates with the sliders.</p></div></section>
|
|
16
|
+
<section id="constructive" class="tab hidden"><div class="panel"><h2>Constructive objectives</h2><div class="grid"><div><div class="small">Architecture contribution</div><div class="metric">reported</div></div><div><div class="small">Overhead contribution</div><div class="metric">reported</div></div><div><div class="small">Verifier drift</div><div class="metric">8 live traces</div></div></div><svg viewBox="0 0 700 230"><path d="M60 190H650M60 190V30" stroke="#607080"/><rect x="110" y="105" width="90" height="85" fill="#176b68"/><rect x="270" y="70" width="90" height="120" fill="#bb4b35"/><path d="M470 175L570 55" stroke="#176b68" stroke-width="5"/><text x="100" y="215">arch</text><text x="255" y="215">overhead</text><text x="470" y="215">threshold phase</text></svg></div></section>
|
|
17
|
+
<section id="callgraph" class="tab hidden"><div class="panel"><h2>Call graph and stopping inspector</h2><div class="metric">Pi_agent / Pi_single</div><p class="small">Classifies cross-thread reads from imported traces and summarizes early stopping.</p><svg viewBox="0 0 700 230"><path d="M90 110L270 60M90 110L270 165M270 60L480 110M270 165L480 110M480 110L620 110" stroke="#176b68" stroke-width="4" fill="none"/><circle cx="90" cy="110" r="24" fill="#bb4b35"/><circle cx="270" cy="60" r="24" fill="#f0c36b"/><circle cx="270" cy="165" r="24" fill="#f0c36b"/><circle cx="480" cy="110" r="24" fill="#176b68"/><circle cx="620" cy="110" r="24" fill="#bb4b35"/></svg></div></section>
|
|
18
|
+
<section id="corpus9" class="tab hidden"><div class="panel"><h2>Nine-study corpus explorer</h2><div class="controls"><label>Study <select id="study"><option>All</option><option>Jwalapuram</option><option>Zhang</option><option>Ke</option></select></label><label>Category <select><option>All</option><option>A</option><option>B</option><option>C</option></select></label><label>phi <input id="phi" type="range" min="0" max=".9" step=".05" value=".5"><output>0.50</output></label></div><div class="grid"><div><div class="small">Contrasts</div><div class="metric">832</div></div><div><div class="small">Noise band</div><div class="metric">phi=<span id="phiValue">0.50</span></div></div><div><div class="small">Subfield</div><div class="metric">5 routes</div></div></div></div></section>
|
|
19
|
+
<section id="rebuttal" class="tab hidden"><div class="panel"><h2>Experiment planner and rebuttal builder</h2><div class="controls"><label>k_tok <input id="ktok" type="number" value="8"></label><label>Power <input id="power" type="number" value="0.8" step=".05"></label><button class="action" onclick="plan()">Calculate</button></div><pre id="planOut">Set k_tok and power, then calculate a compact setup.</pre></div></section>
|
|
20
|
+
</main>
|
|
21
|
+
<script>
|
|
22
|
+
const tabs=[...document.querySelectorAll('[data-tab]')];tabs.forEach(b=>b.onclick=()=>{tabs.forEach(x=>x.classList.toggle('active',x===b));document.querySelectorAll('.tab').forEach(x=>x.classList.toggle('hidden',x.id!==b.dataset.tab))});
|
|
23
|
+
function exportCard(kind){let text=document.getElementById('cardOut').textContent;if(kind==='svg')text='<svg xmlns="http://www.w3.org/2000/svg" width="640" height="180"><text x="20" y="40">PACE evaluation card v7</text><text x="20" y="80">PACE-Recommended / empirical trace</text></svg>';let blob=new Blob([text],{type:'text/plain'}),a=document.createElement('a');a.href=URL.createObjectURL(blob);a.download='pace-card-v7.'+(kind==='tex'?'tex':kind==='md'?'md':kind);a.click()}
|
|
24
|
+
function bindRange(id,cb){let e=document.getElementById(id);e.oninput=()=>{e.nextElementSibling.value=e.value;cb(e.value)}}bindRange('alphaCache',v=>document.getElementById('cacheReadout').textContent='Cache alpha '+v+'; output alpha '+document.getElementById('alphaOut').value+'; pool k '+document.getElementById('poolK').value);bindRange('alphaOut',v=>document.getElementById('cacheReadout').textContent='Cache alpha '+document.getElementById('alphaCache').value+'; output alpha '+v+'; pool k '+document.getElementById('poolK').value);bindRange('poolK',v=>document.getElementById('cacheReadout').textContent='Cache alpha '+document.getElementById('alphaCache').value+'; output alpha '+document.getElementById('alphaOut').value+'; pool k '+v);bindRange('phi',v=>document.getElementById('phiValue').textContent=Number(v).toFixed(2));
|
|
25
|
+
document.getElementById('file').onchange=e=>{let f=e.target.files[0];if(f)document.getElementById('cardOut').textContent='Loaded '+f.name+'\nDrop parsed locally for card-v7 export.'};document.getElementById('drop').ondragover=e=>e.preventDefault();document.getElementById('drop').ondrop=e=>{e.preventDefault();let f=e.dataTransfer.files[0];if(f)document.getElementById('cardOut').textContent='Loaded '+f.name+'\nDrop parsed locally for card-v7 export.'};function plan(){let k=Number(document.getElementById('ktok').value),p=Number(document.getElementById('power').value);document.getElementById('planOut').textContent='k_tok='+k+'\npower='+p+'\nSetup: report baseline rung, resource vector, frontier gap, and stopping ceiling.'}
|
|
26
|
+
</script></body></html>
|
pace/workbench.py
ADDED
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
"""Local stdlib HTTP workbench and card-v7 file helpers."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import csv
|
|
6
|
+
import json
|
|
7
|
+
import threading
|
|
8
|
+
import webbrowser
|
|
9
|
+
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
|
10
|
+
from pathlib import Path
|
|
11
|
+
from urllib.parse import urlparse
|
|
12
|
+
|
|
13
|
+
from .protocol_v3 import build_evaluation_card_v7, card_v7_to_latex, card_v7_to_markdown
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def load_rows(path):
|
|
17
|
+
path = Path(path)
|
|
18
|
+
if path.suffix.lower() == ".jsonl":
|
|
19
|
+
return [
|
|
20
|
+
json.loads(line)
|
|
21
|
+
for line in path.read_text(encoding="utf-8").splitlines()
|
|
22
|
+
if line.strip()
|
|
23
|
+
]
|
|
24
|
+
if path.suffix.lower() == ".csv":
|
|
25
|
+
with path.open(newline="", encoding="utf-8") as handle:
|
|
26
|
+
return list(csv.DictReader(handle))
|
|
27
|
+
data = json.loads(path.read_text(encoding="utf-8"))
|
|
28
|
+
return (
|
|
29
|
+
data
|
|
30
|
+
if isinstance(data, list)
|
|
31
|
+
else data.get("rows", data.get("records", [data]))
|
|
32
|
+
)
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def card_from_file(path, baseline=None):
|
|
36
|
+
return build_evaluation_card_v7(
|
|
37
|
+
load_rows(path), load_rows(baseline) if baseline else None
|
|
38
|
+
)
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def export_html(path):
|
|
42
|
+
source = Path(__file__).with_name("workbench.html")
|
|
43
|
+
Path(path).write_text(source.read_text(encoding="utf-8"), encoding="utf-8")
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
class _Handler(BaseHTTPRequestHandler):
|
|
47
|
+
server_version = "PACEWorkbench/1.0"
|
|
48
|
+
|
|
49
|
+
def _json(self, payload, status=200):
|
|
50
|
+
data = json.dumps(payload, default=str).encode("utf-8")
|
|
51
|
+
self.send_response(status)
|
|
52
|
+
self.send_header("Content-Type", "application/json")
|
|
53
|
+
self.send_header("Content-Length", str(len(data)))
|
|
54
|
+
self.end_headers()
|
|
55
|
+
self.wfile.write(data)
|
|
56
|
+
|
|
57
|
+
def do_GET(self):
|
|
58
|
+
route = urlparse(self.path).path
|
|
59
|
+
if route == "/api/health":
|
|
60
|
+
return self._json({"ok": True, "service": "pace-workbench"})
|
|
61
|
+
if route == "/api/summary":
|
|
62
|
+
root = Path(__file__).resolve().parents[2]
|
|
63
|
+
summary = root / "supplement" / "live_empirical" / "empirical_dynamics.json"
|
|
64
|
+
return self._json(
|
|
65
|
+
json.loads(summary.read_text(encoding="utf-8"))
|
|
66
|
+
if summary.exists()
|
|
67
|
+
else {"status": "unreported"}
|
|
68
|
+
)
|
|
69
|
+
if route == "/":
|
|
70
|
+
data = Path(__file__).with_name("workbench.html").read_bytes()
|
|
71
|
+
self.send_response(200)
|
|
72
|
+
self.send_header("Content-Type", "text/html; charset=utf-8")
|
|
73
|
+
self.send_header("Content-Length", str(len(data)))
|
|
74
|
+
self.end_headers()
|
|
75
|
+
self.wfile.write(data)
|
|
76
|
+
return
|
|
77
|
+
self._json({"error": "not found"}, 404)
|
|
78
|
+
|
|
79
|
+
def do_POST(self):
|
|
80
|
+
route = urlparse(self.path).path
|
|
81
|
+
length = int(self.headers.get("Content-Length", "0"))
|
|
82
|
+
try:
|
|
83
|
+
body = json.loads(self.rfile.read(length) or b"{}")
|
|
84
|
+
if route == "/api/evaluate-card-v7":
|
|
85
|
+
return self._json(
|
|
86
|
+
build_evaluation_card_v7(
|
|
87
|
+
body.get("rows", []), body.get("baseline_rows")
|
|
88
|
+
)
|
|
89
|
+
)
|
|
90
|
+
if route == "/api/constructive":
|
|
91
|
+
from .protocol_v3 import evaluate_constructive_objectives
|
|
92
|
+
|
|
93
|
+
return self._json(
|
|
94
|
+
evaluate_constructive_objectives(
|
|
95
|
+
body.get("comm_rows", []),
|
|
96
|
+
body.get("placebo_rows", []),
|
|
97
|
+
body.get("full_base_rows", []),
|
|
98
|
+
body.get("ver_rows"),
|
|
99
|
+
)
|
|
100
|
+
)
|
|
101
|
+
except (ValueError, KeyError, TypeError) as exc:
|
|
102
|
+
return self._json({"error": str(exc)}, 400)
|
|
103
|
+
self._json({"error": "not found"}, 404)
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def serve(host="127.0.0.1", port=8765, no_browser=False):
|
|
107
|
+
server = ThreadingHTTPServer((host, port), _Handler)
|
|
108
|
+
if not no_browser:
|
|
109
|
+
threading.Timer(0.25, lambda: webbrowser.open(f"http://{host}:{port}/")).start()
|
|
110
|
+
return server
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
__all__ = [
|
|
114
|
+
"card_from_file",
|
|
115
|
+
"card_v7_to_latex",
|
|
116
|
+
"card_v7_to_markdown",
|
|
117
|
+
"export_html",
|
|
118
|
+
"serve",
|
|
119
|
+
]
|