pace-eval 0.2.2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (83) hide show
  1. cme/__init__.py +11 -0
  2. cme/__main__.py +5 -0
  3. cme/analysis.py +223 -0
  4. cme/audit.py +145 -0
  5. cme/baseline.py +162 -0
  6. cme/cli.py +279 -0
  7. cme/client.py +165 -0
  8. cme/dynamics.py +212 -0
  9. cme/examples/crewai_integration.py +16 -0
  10. cme/examples/demo_pool.jsonl +360 -0
  11. cme/examples/demo_run.jsonl +480 -0
  12. cme/examples/langgraph_integration.py +27 -0
  13. cme/examples/make_demo.py +37 -0
  14. cme/examples/openai_quickstart.py +18 -0
  15. cme/experiment.py +555 -0
  16. cme/frontier.py +143 -0
  17. cme/hidden_profile.py +135 -0
  18. cme/proxy.py +106 -0
  19. cme/rebuttal.py +76 -0
  20. cme/report.py +197 -0
  21. cme/rollout_card.py +137 -0
  22. cme/tasks.py +266 -0
  23. cme/verified_debate.py +127 -0
  24. cme/verifier.py +92 -0
  25. mae_eval/__init__.py +11 -0
  26. mae_eval/__main__.py +5 -0
  27. mae_eval/auditor.py +112 -0
  28. mae_eval/classifier.py +62 -0
  29. mae_eval/cli.py +210 -0
  30. mae_eval/tracker.py +162 -0
  31. mae_eval/vector.py +91 -0
  32. pace/__init__.py +11 -0
  33. pace/__main__.py +5 -0
  34. pace/analysis.py +129 -0
  35. pace/audit.py +145 -0
  36. pace/baseline.py +162 -0
  37. pace/card.py +205 -0
  38. pace/cli.py +898 -0
  39. pace/client.py +165 -0
  40. pace/dynamics.py +372 -0
  41. pace/escape_benchmarks.py +162 -0
  42. pace/examples/crewai_integration.py +16 -0
  43. pace/examples/demo_pool.jsonl +360 -0
  44. pace/examples/demo_run.jsonl +480 -0
  45. pace/examples/langgraph_integration.py +27 -0
  46. pace/examples/make_demo.py +37 -0
  47. pace/examples/openai_quickstart.py +18 -0
  48. pace/experiment.py +555 -0
  49. pace/frontier.py +114 -0
  50. pace/hidden_profile.py +301 -0
  51. pace/importers.py +219 -0
  52. pace/inflate.py +213 -0
  53. pace/influence.py +231 -0
  54. pace/meta_engine.py +246 -0
  55. pace/middleware.py +230 -0
  56. pace/open_ended.py +114 -0
  57. pace/plotting.py +132 -0
  58. pace/pool_sizer.py +154 -0
  59. pace/protocol_v3.py +414 -0
  60. pace/proxy.py +106 -0
  61. pace/rebuttal.py +219 -0
  62. pace/report.py +197 -0
  63. pace/rollout_card.py +137 -0
  64. pace/schema.py +74 -0
  65. pace/simulator.py +199 -0
  66. pace/studio.html +1040 -0
  67. pace/tasks.py +266 -0
  68. pace/verified_debate.py +127 -0
  69. pace/verifier.py +92 -0
  70. pace/workbench.html +26 -0
  71. pace/workbench.py +119 -0
  72. pace_eval-0.2.2.dist-info/METADATA +205 -0
  73. pace_eval-0.2.2.dist-info/RECORD +83 -0
  74. pace_eval-0.2.2.dist-info/WHEEL +5 -0
  75. pace_eval-0.2.2.dist-info/entry_points.txt +4 -0
  76. pace_eval-0.2.2.dist-info/licenses/LICENSE +202 -0
  77. pace_eval-0.2.2.dist-info/top_level.txt +4 -0
  78. pace_lab/__init__.py +9 -0
  79. pace_lab/clients.py +568 -0
  80. pace_lab/concurrency_test.py +127 -0
  81. pace_lab/data.py +445 -0
  82. pace_lab/graders.py +626 -0
  83. pace_lab/strategies.py +345 -0
@@ -0,0 +1,127 @@
1
+ """Local HTTP harness for testing pace_lab client concurrency and retry accounting."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import json
6
+ import threading
7
+ import time
8
+ from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
9
+ from typing import Any
10
+
11
+ from .clients import LLMClient, LLMRequest
12
+
13
+
14
+ class MockOpenAIHandler(BaseHTTPRequestHandler):
15
+ latency_per_token = 0.0
16
+ response_tokens = 8
17
+ failures_remaining = 0
18
+ failure_status = 429
19
+
20
+ def do_POST(self) -> None:
21
+ length = int(self.headers.get("Content-Length", "0"))
22
+ self.rfile.read(length)
23
+ if type(self).failures_remaining:
24
+ type(self).failures_remaining -= 1
25
+ self.send_response(type(self).failure_status)
26
+ self.end_headers()
27
+ return
28
+ time.sleep(type(self).latency_per_token * type(self).response_tokens)
29
+ payload = {
30
+ "choices": [{"message": {"content": "Answer: A"}, "finish_reason": "stop"}],
31
+ "usage": {
32
+ "prompt_tokens": 5,
33
+ "completion_tokens": type(self).response_tokens,
34
+ },
35
+ }
36
+ body = json.dumps(payload).encode("utf-8")
37
+ self.send_response(200)
38
+ self.send_header("Content-Type", "application/json")
39
+ self.send_header("Content-Length", str(len(body)))
40
+ self.end_headers()
41
+ self.wfile.write(body)
42
+
43
+ def log_message(self, format: str, *args: Any) -> None:
44
+ return
45
+
46
+
47
+ class LocalMockServer:
48
+ """Context manager for an ephemeral threaded OpenAI-compatible endpoint."""
49
+
50
+ def __init__(
51
+ self,
52
+ latency_per_token: float = 0.0,
53
+ response_tokens: int = 8,
54
+ failures: int = 0,
55
+ failure_status: int = 429,
56
+ ):
57
+ self.settings = (latency_per_token, response_tokens, failures, failure_status)
58
+ self.server: ThreadingHTTPServer | None = None
59
+ self.thread: threading.Thread | None = None
60
+
61
+ def __enter__(self) -> LocalMockServer:
62
+ latency, tokens, failures, status = self.settings
63
+ MockOpenAIHandler.latency_per_token, MockOpenAIHandler.response_tokens = (
64
+ latency,
65
+ tokens,
66
+ )
67
+ MockOpenAIHandler.failures_remaining, MockOpenAIHandler.failure_status = (
68
+ failures,
69
+ status,
70
+ )
71
+ self.server = ThreadingHTTPServer(("127.0.0.1", 0), MockOpenAIHandler)
72
+ self.thread = threading.Thread(target=self.server.serve_forever, daemon=True)
73
+ self.thread.start()
74
+ return self
75
+
76
+ @property
77
+ def url(self) -> str:
78
+ if self.server is None:
79
+ raise RuntimeError("server is not running")
80
+ return "http://127.0.0.1:%d/v1" % self.server.server_port
81
+
82
+ def __exit__(self, exc_type, exc_value, traceback) -> bool:
83
+ if self.server is not None:
84
+ self.server.shutdown()
85
+ self.server.server_close()
86
+ if self.thread is not None:
87
+ self.thread.join(timeout=2)
88
+ return False
89
+
90
+
91
+ def run_concurrency_test(
92
+ workers: int = 1, requests: int = 8, latency_per_token: float = 0.001
93
+ ) -> dict:
94
+ """Run local threaded calls and return throughput/timing accounting."""
95
+ if workers < 1 or requests < 1:
96
+ raise ValueError("workers and requests must be positive")
97
+ with LocalMockServer(latency_per_token=latency_per_token) as server:
98
+ client = LLMClient(
99
+ ".", transport=None, groq_url=server.url, max_retries=0, timeout=5
100
+ )
101
+ # The real transport is intentionally used only against this local endpoint.
102
+ import concurrent.futures
103
+
104
+ start = time.perf_counter()
105
+
106
+ def call(index: int):
107
+ return client.call(
108
+ LLMRequest(
109
+ "groq", "mock", "Answer A", max_tokens=16, item_id=str(index)
110
+ )
111
+ )
112
+
113
+ with concurrent.futures.ThreadPoolExecutor(max_workers=workers) as pool:
114
+ results = list(pool.map(call, range(requests)))
115
+ elapsed = time.perf_counter() - start
116
+ return {
117
+ "workers": workers,
118
+ "requests": requests,
119
+ "elapsed_s": elapsed,
120
+ "ok": sum(result.ok for result in results),
121
+ "prompt_tokens": sum(
122
+ result.record.get("prompt_tokens") or 0 for result in results
123
+ ),
124
+ "completion_tokens": sum(
125
+ result.record.get("completion_tokens") or 0 for result in results
126
+ ),
127
+ }
pace_lab/data.py ADDED
@@ -0,0 +1,445 @@
1
+ """pace_lab.data: benchmark loaders that save the item files and a manifest (ids + SHA-256 hashes) under data/pace_lab/.
2
+
3
+ Datasets (item schema as in cme.tasks: {id, task, prompt, gold, kind, meta[, views]}):
4
+ mmlu_pro TIGER-Lab/MMLU-Pro test split, 1,000 items stratified by category (proportional allocation, largest remainder), seed 2026
5
+ math500_l45 HuggingFaceH4/MATH-500 test split, every item of level 4 or 5 (no sampling)
6
+ hp_numeric / hp_nl hidden-profile, 500 synthetic items, seed 42, evidence as a numeric table / as prose
7
+ hp_numeric_shared / hp_nl_shared the shared-evidence twin of each: same scores and gold, but every agent sees ALL evidence (no hidden profile)
8
+
9
+ Source of real data: the public Hugging Face datasets-server JSON API (same endpoint as cme.tasks). The full raw pool is cached under
10
+ data/pace_lab/raw/ so a rebuild is offline and byte-identical. Nothing is fabricated: if the download fails, `build_*` raises and no item file is written.
11
+ Hidden-profile items are SYNTHETIC (meta.synthetic is true) and come from the existing generator (code/cme/hidden_profile.py, loaded read-only by
12
+ file path; code/pace/hidden_profile.py if the package was renamed). The natural-language rendering and the twin are made here from the generator's scores.
13
+
14
+ Manifest (data/pace_lab/manifest.json): per dataset the source, selection rule, seed, n, file name, file sha256, the ordered id list and an
15
+ item_sha256 map (sha256 of the canonical JSON of each item). `verify_manifest` re-hashes everything.
16
+
17
+ Command line: python -m pace_lab.data build {mmlu_pro,math500_l45,hidden_profile,all} [--refresh] [--from-experiments-cache] | verify
18
+ """
19
+ from __future__ import annotations
20
+
21
+ import argparse
22
+ import hashlib
23
+ import importlib.util
24
+ import json
25
+ import math
26
+ import os
27
+ import random
28
+ import re
29
+ import sys
30
+ import time
31
+ import urllib.error
32
+ import urllib.parse
33
+ import urllib.request
34
+ from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple
35
+
36
+ CODE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
37
+ ROOT = os.path.dirname(CODE_DIR)
38
+ DATA_DIR = os.path.join(ROOT, "data", "pace_lab")
39
+ HF_ROWS_API = "https://datasets-server.huggingface.co/rows"
40
+ LETTERS = "ABCDEFGHIJ"
41
+ SOURCES = {
42
+ "mmlu_pro": ("TIGER-Lab/MMLU-Pro", "default", "test"),
43
+ "math500_l45": ("HuggingFaceH4/MATH-500", "default", "test"),
44
+ }
45
+ RowFetcher = Callable[[str, str, str, int, int], Tuple[List[dict], int]] # (dataset, config, split, offset, length) -> (rows, num_rows_total)
46
+
47
+
48
+ # ----------------------------------------------------------------------------- hashing and files
49
+ def canonical_json(obj: Any) -> str:
50
+ return json.dumps(obj, sort_keys=True, ensure_ascii=False, separators=(",", ":"))
51
+
52
+
53
+ def sha256_text(s: str) -> str:
54
+ return hashlib.sha256(s.encode("utf-8")).hexdigest()
55
+
56
+
57
+ def item_hash(item: dict) -> str:
58
+ return sha256_text(canonical_json(item))
59
+
60
+
61
+ def file_sha256(path: str) -> str:
62
+ h = hashlib.sha256()
63
+ with open(path, "rb") as fh:
64
+ for chunk in iter(lambda: fh.read(1 << 20), b""):
65
+ h.update(chunk)
66
+ return h.hexdigest()
67
+
68
+
69
+ def write_jsonl(path: str, rows: Sequence[dict], header: Optional[dict] = None) -> None:
70
+ os.makedirs(os.path.dirname(path), exist_ok=True)
71
+ with open(path, "w", encoding="utf-8", newline="\n") as fh:
72
+ if header is not None:
73
+ fh.write(json.dumps({"_provenance": header}, ensure_ascii=False, sort_keys=True) + "\n")
74
+ for r in rows:
75
+ fh.write(canonical_json(r) + "\n")
76
+
77
+
78
+ def read_jsonl(path: str) -> List[dict]:
79
+ out = []
80
+ with open(path, encoding="utf-8") as fh:
81
+ for line in fh:
82
+ if line.strip():
83
+ rec = json.loads(line)
84
+ if "_provenance" not in rec or len(rec) > 1:
85
+ out.append(rec)
86
+ return out
87
+
88
+
89
+ def read_provenance(path: str) -> Optional[dict]:
90
+ with open(path, encoding="utf-8") as fh:
91
+ first = fh.readline()
92
+ rec = json.loads(first) if first.strip() else {}
93
+ return rec.get("_provenance") if len(rec) == 1 else None
94
+
95
+
96
+ # ----------------------------------------------------------------------------- manifest
97
+ def manifest_path(data_dir: Optional[str] = None) -> str:
98
+ return os.path.join(data_dir or DATA_DIR, "manifest.json")
99
+
100
+
101
+ def load_manifest(data_dir: Optional[str] = None) -> dict:
102
+ p = manifest_path(data_dir)
103
+ if os.path.isfile(p):
104
+ with open(p, encoding="utf-8") as fh:
105
+ return json.load(fh)
106
+ return {"schema": 1, "datasets": {}}
107
+
108
+
109
+ def register(name: str, items: List[dict], info: dict, data_dir: Optional[str] = None) -> dict:
110
+ """Write data/pace_lab/<name>.jsonl and add its entry (ids, per-item sha256, file sha256, provenance) to manifest.json."""
111
+ d = data_dir or DATA_DIR
112
+ os.makedirs(d, exist_ok=True)
113
+ fname = name + ".jsonl"
114
+ path = os.path.join(d, fname)
115
+ write_jsonl(path, items)
116
+ entry = dict(info)
117
+ entry.update({"file": fname, "n": len(items), "file_sha256": file_sha256(path), "ids": [it["id"] for it in items],
118
+ "item_sha256": {it["id"]: item_hash(it) for it in items}})
119
+ if len(set(entry["ids"])) != len(entry["ids"]):
120
+ raise ValueError("duplicate item ids in " + name)
121
+ m = load_manifest(d)
122
+ m["datasets"][name] = entry
123
+ m["updated"] = time.strftime("%Y-%m-%dT%H:%M:%S%z")
124
+ with open(manifest_path(d), "w", encoding="utf-8", newline="\n") as fh:
125
+ json.dump(m, fh, indent=1, sort_keys=True, ensure_ascii=False)
126
+ fh.write("\n")
127
+ return entry
128
+
129
+
130
+ def verify_manifest(data_dir: Optional[str] = None) -> List[str]:
131
+ """Re-hash every file and item listed in the manifest. Returns a list of problems (empty = all good)."""
132
+ d = data_dir or DATA_DIR
133
+ problems: List[str] = []
134
+ for name, e in load_manifest(d).get("datasets", {}).items():
135
+ path = os.path.join(d, e["file"])
136
+ if not os.path.isfile(path):
137
+ problems.append("%s: file missing" % name)
138
+ continue
139
+ if file_sha256(path) != e["file_sha256"]:
140
+ problems.append("%s: file hash differs" % name)
141
+ items = read_jsonl(path)
142
+ if [it["id"] for it in items] != e["ids"]:
143
+ problems.append("%s: id list differs" % name)
144
+ for it in items:
145
+ if e["item_sha256"].get(it["id"]) != item_hash(it):
146
+ problems.append("%s: item %s hash differs" % (name, it["id"]))
147
+ return problems
148
+
149
+
150
+ def load_dataset(name: str, data_dir: Optional[str] = None, verify: bool = True) -> List[dict]:
151
+ d = data_dir or DATA_DIR
152
+ e = load_manifest(d)["datasets"][name]
153
+ items = read_jsonl(os.path.join(d, e["file"]))
154
+ if verify and [item_hash(i) for i in items] != [e["item_sha256"][i["id"]] for i in items]:
155
+ raise ValueError("%s: items do not match the manifest hashes" % name)
156
+ return items
157
+
158
+
159
+ # ----------------------------------------------------------------------------- Hugging Face rows API
160
+ def hf_fetcher(dataset: str, config: str, split: str, offset: int, length: int) -> Tuple[List[dict], int]:
161
+ """One page from the datasets-server (max length 100), with retries; returns (rows, num_rows_total)."""
162
+ q = urllib.parse.urlencode({"dataset": dataset, "config": config, "split": split, "offset": offset, "length": length})
163
+ last = "unknown"
164
+ for attempt in range(8):
165
+ try:
166
+ with urllib.request.urlopen(HF_ROWS_API + "?" + q, timeout=60) as r:
167
+ data = json.loads(r.read())
168
+ return [x["row"] for x in data["rows"]], int(data["num_rows_total"])
169
+ except urllib.error.HTTPError as e:
170
+ last = "HTTP %d" % e.code
171
+ time.sleep(min(120, float(e.headers.get("Retry-After", 0) or 0) or 15 * (attempt + 1)))
172
+ except Exception as e: # noqa: BLE001
173
+ last = type(e).__name__
174
+ time.sleep(2 * (attempt + 1))
175
+ raise RuntimeError("datasets-server unavailable for %s offset %d (%s)" % (dataset, offset, last))
176
+
177
+
178
+ def fetch_split(dataset: str, config: str, split: str, fetcher: RowFetcher = hf_fetcher, page: int = 100, pause: float = 0.5,
179
+ sleep: Callable[[float], None] = time.sleep) -> List[dict]:
180
+ rows: List[dict] = []
181
+ offset, total = 0, None
182
+ while total is None or offset < total:
183
+ got, total = fetcher(dataset, config, split, offset, page)
184
+ if not got:
185
+ break
186
+ rows += got
187
+ offset += len(got)
188
+ if pause:
189
+ sleep(pause)
190
+ if total is not None and len(rows) != total:
191
+ raise RuntimeError("fetched %d of %d rows from %s" % (len(rows), total, dataset))
192
+ return rows
193
+
194
+
195
+ # ----------------------------------------------------------------------------- item builders
196
+ def mc_prompt(question: str, options: Sequence[str]) -> str:
197
+ return question.strip() + "\n\nOptions:\n" + "\n".join("%s. %s" % (LETTERS[i], o) for i, o in enumerate(options)) + "\n\nGive the letter of the correct option."
198
+
199
+
200
+ def mmlu_pro_items(rows: Sequence[dict]) -> List[dict]:
201
+ out = []
202
+ for r in rows:
203
+ opts = list(r["options"])
204
+ if not 2 <= len(opts) <= 10 or r["answer"] not in LETTERS[:len(opts)]:
205
+ raise ValueError("unexpected MMLU-Pro row %r" % r.get("question_id"))
206
+ out.append({"id": "mmlu%05d" % r["question_id"], "task": "mmlu_pro", "prompt": mc_prompt(r["question"], opts), "gold": r["answer"], "kind": "mc",
207
+ "meta": {"category": r["category"], "n_options": len(opts), "question_id": r["question_id"], "src": r.get("src")}})
208
+ return out
209
+
210
+
211
+ def math500_items(rows: Sequence[dict], levels=(4, 5)) -> List[dict]:
212
+ return [{"id": r["unique_id"], "task": "math500_l45", "prompt": r["problem"].strip(), "gold": r["answer"], "kind": "num",
213
+ "meta": {"level": r["level"], "subject": r["subject"]}} for r in rows if r["level"] in levels]
214
+
215
+
216
+ def allocate(counts: Dict[str, int], n: int) -> Dict[str, int]:
217
+ """Proportional allocation of n draws over strata (largest remainder; ties by stratum name)."""
218
+ total = sum(counts.values())
219
+ if n > total:
220
+ raise ValueError("cannot draw %d items from a pool of %d" % (n, total))
221
+ quota = {k: n * c / total for k, c in counts.items()}
222
+ base = {k: int(math.floor(q)) for k, q in quota.items()}
223
+ left = n - sum(base.values())
224
+ for k in sorted(counts, key=lambda k: (-(quota[k] - base[k]), k))[:left]:
225
+ base[k] += 1
226
+ return base
227
+
228
+
229
+ def stratified_sample(items: Sequence[dict], n: int, seed: int, key: Callable[[dict], str]) -> List[dict]:
230
+ """n items stratified by key: proportional allocation, a seeded random.Random(seed).sample inside each stratum (strata in name order,
231
+ pool in id order), returned in id order."""
232
+ groups: Dict[str, List[dict]] = {}
233
+ for it in sorted(items, key=lambda i: i["id"]):
234
+ groups.setdefault(key(it), []).append(it)
235
+ alloc = allocate({k: len(v) for k, v in groups.items()}, n)
236
+ rng = random.Random(seed)
237
+ picked: List[dict] = []
238
+ for k in sorted(groups):
239
+ picked += rng.sample(groups[k], alloc[k])
240
+ return sorted(picked, key=lambda i: i["id"])
241
+
242
+
243
+ # ----------------------------------------------------------------------------- real-data builders
244
+ def _pool(name: str, fetcher: RowFetcher, data_dir: str, refresh: bool, pause: float, sleep) -> Tuple[List[dict], dict]:
245
+ ds, cfg, sp = SOURCES[name]
246
+ raw = os.path.join(data_dir, "raw", name + "_pool.jsonl")
247
+ if os.path.isfile(raw) and not refresh:
248
+ prov = read_provenance(raw) or {}
249
+ return read_jsonl(raw), dict(prov, pool_file="raw/%s_pool.jsonl" % name, pool_sha256=file_sha256(raw))
250
+ rows = fetch_split(ds, cfg, sp, fetcher, pause=pause, sleep=sleep)
251
+ prov = {"source": [ds, cfg, sp], "api": HF_ROWS_API, "fetched": time.strftime("%Y-%m-%d"), "pool_rows": len(rows)}
252
+ write_jsonl(raw, rows, prov)
253
+ return rows, dict(prov, pool_file="raw/%s_pool.jsonl" % name, pool_sha256=file_sha256(raw))
254
+
255
+
256
+ def build_mmlu_pro(n: int = 1000, seed: int = 2026, fetcher: RowFetcher = hf_fetcher, data_dir: Optional[str] = None, refresh: bool = False,
257
+ pause: float = 0.5, sleep=time.sleep) -> dict:
258
+ d = data_dir or DATA_DIR
259
+ rows, prov = _pool("mmlu_pro", fetcher, d, refresh, pause, sleep)
260
+ pool = mmlu_pro_items(rows)
261
+ items = stratified_sample(pool, n, seed, key=lambda it: it["meta"]["category"])
262
+ cats = {}
263
+ for it in items:
264
+ cats[it["meta"]["category"]] = cats.get(it["meta"]["category"], 0) + 1
265
+ info = {"source": prov["source"], "selection": "stratified by category, proportional allocation (largest remainder), random.Random(seed).sample inside each stratum",
266
+ "seed": seed, "pool_size": len(pool), "pool_file": prov["pool_file"], "pool_sha256": prov["pool_sha256"], "fetched": prov.get("fetched"),
267
+ "per_category": dict(sorted(cats.items())), "synthetic": False}
268
+ return register("mmlu_pro", items, info, d)
269
+
270
+
271
+ def build_math500_l45(fetcher: RowFetcher = hf_fetcher, data_dir: Optional[str] = None, refresh: bool = False, pause: float = 0.5, sleep=time.sleep) -> dict:
272
+ d = data_dir or DATA_DIR
273
+ rows, prov = _pool("math500_l45", fetcher, d, refresh, pause, sleep)
274
+ items = sorted(math500_items(rows), key=lambda i: i["id"])
275
+ info = {"source": prov["source"], "selection": "all MATH-500 items with level 4 or 5, id order (no sampling)", "seed": None, "pool_size": len(rows),
276
+ "pool_file": prov["pool_file"], "pool_sha256": prov["pool_sha256"], "fetched": prov.get("fetched"), "synthetic": False,
277
+ "per_level": {str(l): sum(1 for i in items if i["meta"]["level"] == l) for l in (4, 5)}}
278
+ return register("math500_l45", items, info, d)
279
+
280
+
281
+ def items_from_experiments_cache(path: str) -> List[dict]:
282
+ """Read an item file in the cme.tasks cache format (data/experiments/<task>.jsonl). That MMLU-Pro cache is a 4,000-row seeded sample, not the
283
+ whole split, so a build from it is NOT the full-pool stratified draw; use only when the download is impossible and say so."""
284
+ return read_jsonl(path)
285
+
286
+
287
+ # ----------------------------------------------------------------------------- hidden profile
288
+ def load_generator():
289
+ """Load code/cme/hidden_profile.py (read-only, by file path; falls back to code/pace/ after a rename)."""
290
+ for pkg in ("cme", "pace"):
291
+ p = os.path.join(CODE_DIR, pkg, "hidden_profile.py")
292
+ if os.path.isfile(p):
293
+ spec = importlib.util.spec_from_file_location("pace_lab._hp_generator", p)
294
+ mod = importlib.util.module_from_spec(spec)
295
+ spec.loader.exec_module(mod)
296
+ mod.__file_path__ = p
297
+ return mod
298
+ raise FileNotFoundError("hidden_profile.py not found under code/cme or code/pace")
299
+
300
+
301
+ NAMES = ("Avery", "Blake", "Casey", "Drew")
302
+ _NL_SENT = re.compile(r"On criterion (\d+), Avery scored (\d+), Blake scored (\d+), Casey scored (\d+) and Drew scored (\d+)\.")
303
+ _ALL_HEAD = "\n\nAll scores:\n"
304
+
305
+
306
+ def parse_numeric(text: str) -> Dict[int, Dict[str, int]]:
307
+ got: Dict[int, Dict[str, int]] = {}
308
+ for line in text.splitlines():
309
+ parts = line.split(",")
310
+ if len(parts) == 5 and parts[0].isdigit():
311
+ got[int(parts[0])] = dict(zip(NAMES, map(int, parts[1:])))
312
+ return got
313
+
314
+
315
+ def parse_nl(text: str) -> Dict[int, Dict[str, int]]:
316
+ return {int(m[0]): dict(zip(NAMES, map(int, m[1:]))) for m in _NL_SENT.findall(text)}
317
+
318
+
319
+ def render_nl(scores: Dict[int, Dict[str, int]]) -> str:
320
+ return " ".join("On criterion %d, Avery scored %d, Blake scored %d, Casey scored %d and Drew scored %d." % ((j,) + tuple(scores[j][n] for n in NAMES))
321
+ for j in sorted(scores))
322
+
323
+
324
+ def _to_nl(item: dict) -> dict:
325
+ """Prose version of a numeric hidden-profile item: same scores, same gold, same intro and question sentences."""
326
+ prompt = item["prompt"]
327
+ i = prompt.index(_ALL_HEAD)
328
+ head, rest = prompt[:i], prompt[i + len(_ALL_HEAD):]
329
+ q = rest.rindex("\n\nWhich candidate")
330
+ new_prompt = head + _ALL_HEAD + render_nl(parse_numeric(rest[:q])) + rest[q:]
331
+ views = []
332
+ for v in item["views"]:
333
+ marker = "You see only part of the evidence. Scores you can see:\n"
334
+ a = v.index(marker)
335
+ b = v.index("\n\nOther committee members")
336
+ views.append(v[:a + len(marker)] + render_nl(parse_numeric(v[a + len(marker):b])) + v[b:])
337
+ return dict(item, prompt=new_prompt, views=views)
338
+
339
+
340
+ def _twin(item: dict) -> dict:
341
+ """Shared-evidence twin: identical scores, gold and `prompt`, but every agent's view contains ALL the evidence."""
342
+ shared = item["prompt"].replace("All scores:", "All committee members see the same evidence. Scores you can see:", 1)
343
+ return dict(item, views=[shared for _ in item["views"]])
344
+
345
+
346
+ def _hp_label(item: dict, prefix: str, fmt: str, evidence: str, suffix: str) -> dict:
347
+ base = item["id"]
348
+ meta = dict(item["meta"], base_id=base, format=fmt, evidence=evidence, synthetic=True)
349
+ if evidence == "shared":
350
+ meta["twin_of"] = prefix + base[2:]
351
+ return dict(item, id=prefix + base[2:] + suffix, meta=meta)
352
+
353
+
354
+ def validate_hidden_profile(items: Sequence[dict], parser: Callable[[str], Dict[int, Dict[str, int]]]) -> Dict[str, int]:
355
+ """Check from the prompt text: gold strictly leads the pooled evidence; split items: decoy strictly leads every view, every split criterion is seen
356
+ by some agent; shared twins: every view contains all criteria and gold strictly leads it."""
357
+ bad = dict(items=len(items), target_not_leading_pooled=0, decoy_not_leading_a_view=0, uncovered_split_criteria=0, twin_view_incomplete=0)
358
+
359
+ def leader(sc):
360
+ tot = {n: sum(sc[j][n] for j in sc) for n in NAMES}
361
+ best = max(tot.values())
362
+ top = [n for n in NAMES if tot[n] == best]
363
+ return top[0] if len(top) == 1 else None
364
+ for it in items:
365
+ k = it["meta"]["knobs"]
366
+ full = parser(it["prompt"])
367
+ if len(full) != k["criteria"] or leader(full) != it["gold"]:
368
+ bad["target_not_leading_pooled"] += 1
369
+ shared_n = k["criteria"] - k["split"]
370
+ seen, view_bad = set(), False
371
+ for v in it["views"]:
372
+ part = parser(v)
373
+ if it["meta"].get("evidence") == "shared":
374
+ if len(part) != k["criteria"] or leader(part) != it["gold"]:
375
+ bad["twin_view_incomplete"] += 1
376
+ continue
377
+ seen |= {j for j in part if j > shared_n}
378
+ if leader(part) != it["meta"]["decoy"]:
379
+ view_bad = True
380
+ if it["meta"].get("evidence") != "shared":
381
+ bad["decoy_not_leading_a_view"] += view_bad
382
+ bad["uncovered_split_criteria"] += len(set(range(shared_n + 1, k["criteria"] + 1)) - seen) > 0
383
+ return bad
384
+
385
+
386
+ def build_hidden_profile(n: int = 500, seed: int = 42, data_dir: Optional[str] = None) -> Dict[str, dict]:
387
+ d = data_dir or DATA_DIR
388
+ hp = load_generator()
389
+ num = hp.generate(n, seed=seed, fmt="numeric")
390
+ txt = hp.generate(n, seed=seed, fmt="text")
391
+ if [x["gold"] for x in num] != [x["gold"] for x in txt]:
392
+ raise RuntimeError("generator formats disagree on gold answers")
393
+ gen_path = hp.__file_path__
394
+ sets = {
395
+ "hp_numeric": [_hp_label(x, "hpnum", "numeric", "split", "") for x in num],
396
+ "hp_numeric_shared": [_hp_label(_twin(x), "hpnum", "numeric", "shared", "_shared") for x in num],
397
+ }
398
+ nl = [_to_nl(x) for x in num]
399
+ for a, b in zip(num, nl): # the prose must carry exactly the same scores
400
+ if parse_nl(b["prompt"]) != parse_numeric(a["prompt"][a["prompt"].index(_ALL_HEAD):]) or \
401
+ [parse_nl(v) for v in b["views"]] != [parse_numeric(v[v.index("Scores you can see:"):]) for v in a["views"]]:
402
+ raise RuntimeError("NL rendering of %s does not round-trip" % a["id"])
403
+ sets["hp_nl"] = [_hp_label(x, "hpnl", "nl", "split", "") for x in nl]
404
+ sets["hp_nl_shared"] = [_hp_label(_twin(x), "hpnl", "nl", "shared", "_shared") for x in nl]
405
+ parsers = {"hp_numeric": parse_numeric, "hp_numeric_shared": parse_numeric, "hp_nl": parse_nl, "hp_nl_shared": parse_nl}
406
+ out = {}
407
+ for name, items in sets.items():
408
+ counts = validate_hidden_profile(items, parsers[name])
409
+ if any(v for k2, v in counts.items() if k2 != "items"):
410
+ raise RuntimeError("%s failed validation: %s" % (name, counts))
411
+ info = {"source": "synthetic: code/%s/hidden_profile.py generate()" % os.path.basename(os.path.dirname(gen_path)), "generator_sha256": file_sha256(gen_path),
412
+ "selection": "first n items of the seeded generator", "seed": seed, "synthetic": True, "validation": counts,
413
+ "knobs": items[0]["meta"]["knobs"], "format": items[0]["meta"]["format"], "evidence": items[0]["meta"]["evidence"]}
414
+ out[name] = register(name, items, info, d)
415
+ return out
416
+
417
+
418
+ # ----------------------------------------------------------------------------- command line
419
+ def main(argv: Optional[Sequence[str]] = None) -> int:
420
+ ap = argparse.ArgumentParser(prog="python -m pace_lab.data", description=__doc__.split("\n")[0])
421
+ sub = ap.add_subparsers(dest="cmd", required=True)
422
+ b = sub.add_parser("build")
423
+ b.add_argument("what", choices=["mmlu_pro", "math500_l45", "hidden_profile", "all"])
424
+ b.add_argument("--refresh", action="store_true", help="re-download the raw pool")
425
+ b.add_argument("--data-dir", default=None)
426
+ sub.add_parser("verify")
427
+ args = ap.parse_args(argv)
428
+ if args.cmd == "verify":
429
+ problems = verify_manifest()
430
+ print("\n".join(problems) if problems else "manifest OK: %s" % ", ".join(sorted(load_manifest()["datasets"])))
431
+ return 1 if problems else 0
432
+ todo = ["mmlu_pro", "math500_l45", "hidden_profile"] if args.what == "all" else [args.what]
433
+ for w in todo:
434
+ if w == "mmlu_pro":
435
+ e = build_mmlu_pro(data_dir=args.data_dir, refresh=args.refresh)
436
+ elif w == "math500_l45":
437
+ e = build_math500_l45(data_dir=args.data_dir, refresh=args.refresh)
438
+ else:
439
+ e = build_hidden_profile(data_dir=args.data_dir)
440
+ print(w, "ok", json.dumps({k: v for k, v in (e.items() if "n" in e else [(kk, vv["n"]) for kk, vv in e.items()]) if k in ("n",) or w == "hidden_profile"}))
441
+ return 0
442
+
443
+
444
+ if __name__ == "__main__":
445
+ sys.exit(main())