pace-eval 0.2.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cme/__init__.py +11 -0
- cme/__main__.py +5 -0
- cme/analysis.py +223 -0
- cme/audit.py +145 -0
- cme/baseline.py +162 -0
- cme/cli.py +279 -0
- cme/client.py +165 -0
- cme/dynamics.py +212 -0
- cme/examples/crewai_integration.py +16 -0
- cme/examples/demo_pool.jsonl +360 -0
- cme/examples/demo_run.jsonl +480 -0
- cme/examples/langgraph_integration.py +27 -0
- cme/examples/make_demo.py +37 -0
- cme/examples/openai_quickstart.py +18 -0
- cme/experiment.py +555 -0
- cme/frontier.py +143 -0
- cme/hidden_profile.py +135 -0
- cme/proxy.py +106 -0
- cme/rebuttal.py +76 -0
- cme/report.py +197 -0
- cme/rollout_card.py +137 -0
- cme/tasks.py +266 -0
- cme/verified_debate.py +127 -0
- cme/verifier.py +92 -0
- mae_eval/__init__.py +11 -0
- mae_eval/__main__.py +5 -0
- mae_eval/auditor.py +112 -0
- mae_eval/classifier.py +62 -0
- mae_eval/cli.py +210 -0
- mae_eval/tracker.py +162 -0
- mae_eval/vector.py +91 -0
- pace/__init__.py +11 -0
- pace/__main__.py +5 -0
- pace/analysis.py +129 -0
- pace/audit.py +145 -0
- pace/baseline.py +162 -0
- pace/card.py +205 -0
- pace/cli.py +898 -0
- pace/client.py +165 -0
- pace/dynamics.py +372 -0
- pace/escape_benchmarks.py +162 -0
- pace/examples/crewai_integration.py +16 -0
- pace/examples/demo_pool.jsonl +360 -0
- pace/examples/demo_run.jsonl +480 -0
- pace/examples/langgraph_integration.py +27 -0
- pace/examples/make_demo.py +37 -0
- pace/examples/openai_quickstart.py +18 -0
- pace/experiment.py +555 -0
- pace/frontier.py +114 -0
- pace/hidden_profile.py +301 -0
- pace/importers.py +219 -0
- pace/inflate.py +213 -0
- pace/influence.py +231 -0
- pace/meta_engine.py +246 -0
- pace/middleware.py +230 -0
- pace/open_ended.py +114 -0
- pace/plotting.py +132 -0
- pace/pool_sizer.py +154 -0
- pace/protocol_v3.py +414 -0
- pace/proxy.py +106 -0
- pace/rebuttal.py +219 -0
- pace/report.py +197 -0
- pace/rollout_card.py +137 -0
- pace/schema.py +74 -0
- pace/simulator.py +199 -0
- pace/studio.html +1040 -0
- pace/tasks.py +266 -0
- pace/verified_debate.py +127 -0
- pace/verifier.py +92 -0
- pace/workbench.html +26 -0
- pace/workbench.py +119 -0
- pace_eval-0.2.2.dist-info/METADATA +205 -0
- pace_eval-0.2.2.dist-info/RECORD +83 -0
- pace_eval-0.2.2.dist-info/WHEEL +5 -0
- pace_eval-0.2.2.dist-info/entry_points.txt +4 -0
- pace_eval-0.2.2.dist-info/licenses/LICENSE +202 -0
- pace_eval-0.2.2.dist-info/top_level.txt +4 -0
- pace_lab/__init__.py +9 -0
- pace_lab/clients.py +568 -0
- pace_lab/concurrency_test.py +127 -0
- pace_lab/data.py +445 -0
- pace_lab/graders.py +626 -0
- pace_lab/strategies.py +345 -0
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
"""Local HTTP harness for testing pace_lab client concurrency and retry accounting."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
import threading
|
|
7
|
+
import time
|
|
8
|
+
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
|
9
|
+
from typing import Any
|
|
10
|
+
|
|
11
|
+
from .clients import LLMClient, LLMRequest
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class MockOpenAIHandler(BaseHTTPRequestHandler):
|
|
15
|
+
latency_per_token = 0.0
|
|
16
|
+
response_tokens = 8
|
|
17
|
+
failures_remaining = 0
|
|
18
|
+
failure_status = 429
|
|
19
|
+
|
|
20
|
+
def do_POST(self) -> None:
|
|
21
|
+
length = int(self.headers.get("Content-Length", "0"))
|
|
22
|
+
self.rfile.read(length)
|
|
23
|
+
if type(self).failures_remaining:
|
|
24
|
+
type(self).failures_remaining -= 1
|
|
25
|
+
self.send_response(type(self).failure_status)
|
|
26
|
+
self.end_headers()
|
|
27
|
+
return
|
|
28
|
+
time.sleep(type(self).latency_per_token * type(self).response_tokens)
|
|
29
|
+
payload = {
|
|
30
|
+
"choices": [{"message": {"content": "Answer: A"}, "finish_reason": "stop"}],
|
|
31
|
+
"usage": {
|
|
32
|
+
"prompt_tokens": 5,
|
|
33
|
+
"completion_tokens": type(self).response_tokens,
|
|
34
|
+
},
|
|
35
|
+
}
|
|
36
|
+
body = json.dumps(payload).encode("utf-8")
|
|
37
|
+
self.send_response(200)
|
|
38
|
+
self.send_header("Content-Type", "application/json")
|
|
39
|
+
self.send_header("Content-Length", str(len(body)))
|
|
40
|
+
self.end_headers()
|
|
41
|
+
self.wfile.write(body)
|
|
42
|
+
|
|
43
|
+
def log_message(self, format: str, *args: Any) -> None:
|
|
44
|
+
return
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
class LocalMockServer:
|
|
48
|
+
"""Context manager for an ephemeral threaded OpenAI-compatible endpoint."""
|
|
49
|
+
|
|
50
|
+
def __init__(
|
|
51
|
+
self,
|
|
52
|
+
latency_per_token: float = 0.0,
|
|
53
|
+
response_tokens: int = 8,
|
|
54
|
+
failures: int = 0,
|
|
55
|
+
failure_status: int = 429,
|
|
56
|
+
):
|
|
57
|
+
self.settings = (latency_per_token, response_tokens, failures, failure_status)
|
|
58
|
+
self.server: ThreadingHTTPServer | None = None
|
|
59
|
+
self.thread: threading.Thread | None = None
|
|
60
|
+
|
|
61
|
+
def __enter__(self) -> LocalMockServer:
|
|
62
|
+
latency, tokens, failures, status = self.settings
|
|
63
|
+
MockOpenAIHandler.latency_per_token, MockOpenAIHandler.response_tokens = (
|
|
64
|
+
latency,
|
|
65
|
+
tokens,
|
|
66
|
+
)
|
|
67
|
+
MockOpenAIHandler.failures_remaining, MockOpenAIHandler.failure_status = (
|
|
68
|
+
failures,
|
|
69
|
+
status,
|
|
70
|
+
)
|
|
71
|
+
self.server = ThreadingHTTPServer(("127.0.0.1", 0), MockOpenAIHandler)
|
|
72
|
+
self.thread = threading.Thread(target=self.server.serve_forever, daemon=True)
|
|
73
|
+
self.thread.start()
|
|
74
|
+
return self
|
|
75
|
+
|
|
76
|
+
@property
|
|
77
|
+
def url(self) -> str:
|
|
78
|
+
if self.server is None:
|
|
79
|
+
raise RuntimeError("server is not running")
|
|
80
|
+
return "http://127.0.0.1:%d/v1" % self.server.server_port
|
|
81
|
+
|
|
82
|
+
def __exit__(self, exc_type, exc_value, traceback) -> bool:
|
|
83
|
+
if self.server is not None:
|
|
84
|
+
self.server.shutdown()
|
|
85
|
+
self.server.server_close()
|
|
86
|
+
if self.thread is not None:
|
|
87
|
+
self.thread.join(timeout=2)
|
|
88
|
+
return False
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def run_concurrency_test(
|
|
92
|
+
workers: int = 1, requests: int = 8, latency_per_token: float = 0.001
|
|
93
|
+
) -> dict:
|
|
94
|
+
"""Run local threaded calls and return throughput/timing accounting."""
|
|
95
|
+
if workers < 1 or requests < 1:
|
|
96
|
+
raise ValueError("workers and requests must be positive")
|
|
97
|
+
with LocalMockServer(latency_per_token=latency_per_token) as server:
|
|
98
|
+
client = LLMClient(
|
|
99
|
+
".", transport=None, groq_url=server.url, max_retries=0, timeout=5
|
|
100
|
+
)
|
|
101
|
+
# The real transport is intentionally used only against this local endpoint.
|
|
102
|
+
import concurrent.futures
|
|
103
|
+
|
|
104
|
+
start = time.perf_counter()
|
|
105
|
+
|
|
106
|
+
def call(index: int):
|
|
107
|
+
return client.call(
|
|
108
|
+
LLMRequest(
|
|
109
|
+
"groq", "mock", "Answer A", max_tokens=16, item_id=str(index)
|
|
110
|
+
)
|
|
111
|
+
)
|
|
112
|
+
|
|
113
|
+
with concurrent.futures.ThreadPoolExecutor(max_workers=workers) as pool:
|
|
114
|
+
results = list(pool.map(call, range(requests)))
|
|
115
|
+
elapsed = time.perf_counter() - start
|
|
116
|
+
return {
|
|
117
|
+
"workers": workers,
|
|
118
|
+
"requests": requests,
|
|
119
|
+
"elapsed_s": elapsed,
|
|
120
|
+
"ok": sum(result.ok for result in results),
|
|
121
|
+
"prompt_tokens": sum(
|
|
122
|
+
result.record.get("prompt_tokens") or 0 for result in results
|
|
123
|
+
),
|
|
124
|
+
"completion_tokens": sum(
|
|
125
|
+
result.record.get("completion_tokens") or 0 for result in results
|
|
126
|
+
),
|
|
127
|
+
}
|
pace_lab/data.py
ADDED
|
@@ -0,0 +1,445 @@
|
|
|
1
|
+
"""pace_lab.data: benchmark loaders that save the item files and a manifest (ids + SHA-256 hashes) under data/pace_lab/.
|
|
2
|
+
|
|
3
|
+
Datasets (item schema as in cme.tasks: {id, task, prompt, gold, kind, meta[, views]}):
|
|
4
|
+
mmlu_pro TIGER-Lab/MMLU-Pro test split, 1,000 items stratified by category (proportional allocation, largest remainder), seed 2026
|
|
5
|
+
math500_l45 HuggingFaceH4/MATH-500 test split, every item of level 4 or 5 (no sampling)
|
|
6
|
+
hp_numeric / hp_nl hidden-profile, 500 synthetic items, seed 42, evidence as a numeric table / as prose
|
|
7
|
+
hp_numeric_shared / hp_nl_shared the shared-evidence twin of each: same scores and gold, but every agent sees ALL evidence (no hidden profile)
|
|
8
|
+
|
|
9
|
+
Source of real data: the public Hugging Face datasets-server JSON API (same endpoint as cme.tasks). The full raw pool is cached under
|
|
10
|
+
data/pace_lab/raw/ so a rebuild is offline and byte-identical. Nothing is fabricated: if the download fails, `build_*` raises and no item file is written.
|
|
11
|
+
Hidden-profile items are SYNTHETIC (meta.synthetic is true) and come from the existing generator (code/cme/hidden_profile.py, loaded read-only by
|
|
12
|
+
file path; code/pace/hidden_profile.py if the package was renamed). The natural-language rendering and the twin are made here from the generator's scores.
|
|
13
|
+
|
|
14
|
+
Manifest (data/pace_lab/manifest.json): per dataset the source, selection rule, seed, n, file name, file sha256, the ordered id list and an
|
|
15
|
+
item_sha256 map (sha256 of the canonical JSON of each item). `verify_manifest` re-hashes everything.
|
|
16
|
+
|
|
17
|
+
Command line: python -m pace_lab.data build {mmlu_pro,math500_l45,hidden_profile,all} [--refresh] [--from-experiments-cache] | verify
|
|
18
|
+
"""
|
|
19
|
+
from __future__ import annotations
|
|
20
|
+
|
|
21
|
+
import argparse
|
|
22
|
+
import hashlib
|
|
23
|
+
import importlib.util
|
|
24
|
+
import json
|
|
25
|
+
import math
|
|
26
|
+
import os
|
|
27
|
+
import random
|
|
28
|
+
import re
|
|
29
|
+
import sys
|
|
30
|
+
import time
|
|
31
|
+
import urllib.error
|
|
32
|
+
import urllib.parse
|
|
33
|
+
import urllib.request
|
|
34
|
+
from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple
|
|
35
|
+
|
|
36
|
+
CODE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
|
37
|
+
ROOT = os.path.dirname(CODE_DIR)
|
|
38
|
+
DATA_DIR = os.path.join(ROOT, "data", "pace_lab")
|
|
39
|
+
HF_ROWS_API = "https://datasets-server.huggingface.co/rows"
|
|
40
|
+
LETTERS = "ABCDEFGHIJ"
|
|
41
|
+
SOURCES = {
|
|
42
|
+
"mmlu_pro": ("TIGER-Lab/MMLU-Pro", "default", "test"),
|
|
43
|
+
"math500_l45": ("HuggingFaceH4/MATH-500", "default", "test"),
|
|
44
|
+
}
|
|
45
|
+
RowFetcher = Callable[[str, str, str, int, int], Tuple[List[dict], int]] # (dataset, config, split, offset, length) -> (rows, num_rows_total)
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
# ----------------------------------------------------------------------------- hashing and files
|
|
49
|
+
def canonical_json(obj: Any) -> str:
|
|
50
|
+
return json.dumps(obj, sort_keys=True, ensure_ascii=False, separators=(",", ":"))
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def sha256_text(s: str) -> str:
|
|
54
|
+
return hashlib.sha256(s.encode("utf-8")).hexdigest()
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def item_hash(item: dict) -> str:
|
|
58
|
+
return sha256_text(canonical_json(item))
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def file_sha256(path: str) -> str:
|
|
62
|
+
h = hashlib.sha256()
|
|
63
|
+
with open(path, "rb") as fh:
|
|
64
|
+
for chunk in iter(lambda: fh.read(1 << 20), b""):
|
|
65
|
+
h.update(chunk)
|
|
66
|
+
return h.hexdigest()
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def write_jsonl(path: str, rows: Sequence[dict], header: Optional[dict] = None) -> None:
|
|
70
|
+
os.makedirs(os.path.dirname(path), exist_ok=True)
|
|
71
|
+
with open(path, "w", encoding="utf-8", newline="\n") as fh:
|
|
72
|
+
if header is not None:
|
|
73
|
+
fh.write(json.dumps({"_provenance": header}, ensure_ascii=False, sort_keys=True) + "\n")
|
|
74
|
+
for r in rows:
|
|
75
|
+
fh.write(canonical_json(r) + "\n")
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def read_jsonl(path: str) -> List[dict]:
|
|
79
|
+
out = []
|
|
80
|
+
with open(path, encoding="utf-8") as fh:
|
|
81
|
+
for line in fh:
|
|
82
|
+
if line.strip():
|
|
83
|
+
rec = json.loads(line)
|
|
84
|
+
if "_provenance" not in rec or len(rec) > 1:
|
|
85
|
+
out.append(rec)
|
|
86
|
+
return out
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def read_provenance(path: str) -> Optional[dict]:
|
|
90
|
+
with open(path, encoding="utf-8") as fh:
|
|
91
|
+
first = fh.readline()
|
|
92
|
+
rec = json.loads(first) if first.strip() else {}
|
|
93
|
+
return rec.get("_provenance") if len(rec) == 1 else None
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
# ----------------------------------------------------------------------------- manifest
|
|
97
|
+
def manifest_path(data_dir: Optional[str] = None) -> str:
|
|
98
|
+
return os.path.join(data_dir or DATA_DIR, "manifest.json")
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def load_manifest(data_dir: Optional[str] = None) -> dict:
|
|
102
|
+
p = manifest_path(data_dir)
|
|
103
|
+
if os.path.isfile(p):
|
|
104
|
+
with open(p, encoding="utf-8") as fh:
|
|
105
|
+
return json.load(fh)
|
|
106
|
+
return {"schema": 1, "datasets": {}}
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def register(name: str, items: List[dict], info: dict, data_dir: Optional[str] = None) -> dict:
|
|
110
|
+
"""Write data/pace_lab/<name>.jsonl and add its entry (ids, per-item sha256, file sha256, provenance) to manifest.json."""
|
|
111
|
+
d = data_dir or DATA_DIR
|
|
112
|
+
os.makedirs(d, exist_ok=True)
|
|
113
|
+
fname = name + ".jsonl"
|
|
114
|
+
path = os.path.join(d, fname)
|
|
115
|
+
write_jsonl(path, items)
|
|
116
|
+
entry = dict(info)
|
|
117
|
+
entry.update({"file": fname, "n": len(items), "file_sha256": file_sha256(path), "ids": [it["id"] for it in items],
|
|
118
|
+
"item_sha256": {it["id"]: item_hash(it) for it in items}})
|
|
119
|
+
if len(set(entry["ids"])) != len(entry["ids"]):
|
|
120
|
+
raise ValueError("duplicate item ids in " + name)
|
|
121
|
+
m = load_manifest(d)
|
|
122
|
+
m["datasets"][name] = entry
|
|
123
|
+
m["updated"] = time.strftime("%Y-%m-%dT%H:%M:%S%z")
|
|
124
|
+
with open(manifest_path(d), "w", encoding="utf-8", newline="\n") as fh:
|
|
125
|
+
json.dump(m, fh, indent=1, sort_keys=True, ensure_ascii=False)
|
|
126
|
+
fh.write("\n")
|
|
127
|
+
return entry
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
def verify_manifest(data_dir: Optional[str] = None) -> List[str]:
|
|
131
|
+
"""Re-hash every file and item listed in the manifest. Returns a list of problems (empty = all good)."""
|
|
132
|
+
d = data_dir or DATA_DIR
|
|
133
|
+
problems: List[str] = []
|
|
134
|
+
for name, e in load_manifest(d).get("datasets", {}).items():
|
|
135
|
+
path = os.path.join(d, e["file"])
|
|
136
|
+
if not os.path.isfile(path):
|
|
137
|
+
problems.append("%s: file missing" % name)
|
|
138
|
+
continue
|
|
139
|
+
if file_sha256(path) != e["file_sha256"]:
|
|
140
|
+
problems.append("%s: file hash differs" % name)
|
|
141
|
+
items = read_jsonl(path)
|
|
142
|
+
if [it["id"] for it in items] != e["ids"]:
|
|
143
|
+
problems.append("%s: id list differs" % name)
|
|
144
|
+
for it in items:
|
|
145
|
+
if e["item_sha256"].get(it["id"]) != item_hash(it):
|
|
146
|
+
problems.append("%s: item %s hash differs" % (name, it["id"]))
|
|
147
|
+
return problems
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
def load_dataset(name: str, data_dir: Optional[str] = None, verify: bool = True) -> List[dict]:
|
|
151
|
+
d = data_dir or DATA_DIR
|
|
152
|
+
e = load_manifest(d)["datasets"][name]
|
|
153
|
+
items = read_jsonl(os.path.join(d, e["file"]))
|
|
154
|
+
if verify and [item_hash(i) for i in items] != [e["item_sha256"][i["id"]] for i in items]:
|
|
155
|
+
raise ValueError("%s: items do not match the manifest hashes" % name)
|
|
156
|
+
return items
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
# ----------------------------------------------------------------------------- Hugging Face rows API
|
|
160
|
+
def hf_fetcher(dataset: str, config: str, split: str, offset: int, length: int) -> Tuple[List[dict], int]:
|
|
161
|
+
"""One page from the datasets-server (max length 100), with retries; returns (rows, num_rows_total)."""
|
|
162
|
+
q = urllib.parse.urlencode({"dataset": dataset, "config": config, "split": split, "offset": offset, "length": length})
|
|
163
|
+
last = "unknown"
|
|
164
|
+
for attempt in range(8):
|
|
165
|
+
try:
|
|
166
|
+
with urllib.request.urlopen(HF_ROWS_API + "?" + q, timeout=60) as r:
|
|
167
|
+
data = json.loads(r.read())
|
|
168
|
+
return [x["row"] for x in data["rows"]], int(data["num_rows_total"])
|
|
169
|
+
except urllib.error.HTTPError as e:
|
|
170
|
+
last = "HTTP %d" % e.code
|
|
171
|
+
time.sleep(min(120, float(e.headers.get("Retry-After", 0) or 0) or 15 * (attempt + 1)))
|
|
172
|
+
except Exception as e: # noqa: BLE001
|
|
173
|
+
last = type(e).__name__
|
|
174
|
+
time.sleep(2 * (attempt + 1))
|
|
175
|
+
raise RuntimeError("datasets-server unavailable for %s offset %d (%s)" % (dataset, offset, last))
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+
def fetch_split(dataset: str, config: str, split: str, fetcher: RowFetcher = hf_fetcher, page: int = 100, pause: float = 0.5,
|
|
179
|
+
sleep: Callable[[float], None] = time.sleep) -> List[dict]:
|
|
180
|
+
rows: List[dict] = []
|
|
181
|
+
offset, total = 0, None
|
|
182
|
+
while total is None or offset < total:
|
|
183
|
+
got, total = fetcher(dataset, config, split, offset, page)
|
|
184
|
+
if not got:
|
|
185
|
+
break
|
|
186
|
+
rows += got
|
|
187
|
+
offset += len(got)
|
|
188
|
+
if pause:
|
|
189
|
+
sleep(pause)
|
|
190
|
+
if total is not None and len(rows) != total:
|
|
191
|
+
raise RuntimeError("fetched %d of %d rows from %s" % (len(rows), total, dataset))
|
|
192
|
+
return rows
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
# ----------------------------------------------------------------------------- item builders
|
|
196
|
+
def mc_prompt(question: str, options: Sequence[str]) -> str:
|
|
197
|
+
return question.strip() + "\n\nOptions:\n" + "\n".join("%s. %s" % (LETTERS[i], o) for i, o in enumerate(options)) + "\n\nGive the letter of the correct option."
|
|
198
|
+
|
|
199
|
+
|
|
200
|
+
def mmlu_pro_items(rows: Sequence[dict]) -> List[dict]:
|
|
201
|
+
out = []
|
|
202
|
+
for r in rows:
|
|
203
|
+
opts = list(r["options"])
|
|
204
|
+
if not 2 <= len(opts) <= 10 or r["answer"] not in LETTERS[:len(opts)]:
|
|
205
|
+
raise ValueError("unexpected MMLU-Pro row %r" % r.get("question_id"))
|
|
206
|
+
out.append({"id": "mmlu%05d" % r["question_id"], "task": "mmlu_pro", "prompt": mc_prompt(r["question"], opts), "gold": r["answer"], "kind": "mc",
|
|
207
|
+
"meta": {"category": r["category"], "n_options": len(opts), "question_id": r["question_id"], "src": r.get("src")}})
|
|
208
|
+
return out
|
|
209
|
+
|
|
210
|
+
|
|
211
|
+
def math500_items(rows: Sequence[dict], levels=(4, 5)) -> List[dict]:
|
|
212
|
+
return [{"id": r["unique_id"], "task": "math500_l45", "prompt": r["problem"].strip(), "gold": r["answer"], "kind": "num",
|
|
213
|
+
"meta": {"level": r["level"], "subject": r["subject"]}} for r in rows if r["level"] in levels]
|
|
214
|
+
|
|
215
|
+
|
|
216
|
+
def allocate(counts: Dict[str, int], n: int) -> Dict[str, int]:
|
|
217
|
+
"""Proportional allocation of n draws over strata (largest remainder; ties by stratum name)."""
|
|
218
|
+
total = sum(counts.values())
|
|
219
|
+
if n > total:
|
|
220
|
+
raise ValueError("cannot draw %d items from a pool of %d" % (n, total))
|
|
221
|
+
quota = {k: n * c / total for k, c in counts.items()}
|
|
222
|
+
base = {k: int(math.floor(q)) for k, q in quota.items()}
|
|
223
|
+
left = n - sum(base.values())
|
|
224
|
+
for k in sorted(counts, key=lambda k: (-(quota[k] - base[k]), k))[:left]:
|
|
225
|
+
base[k] += 1
|
|
226
|
+
return base
|
|
227
|
+
|
|
228
|
+
|
|
229
|
+
def stratified_sample(items: Sequence[dict], n: int, seed: int, key: Callable[[dict], str]) -> List[dict]:
|
|
230
|
+
"""n items stratified by key: proportional allocation, a seeded random.Random(seed).sample inside each stratum (strata in name order,
|
|
231
|
+
pool in id order), returned in id order."""
|
|
232
|
+
groups: Dict[str, List[dict]] = {}
|
|
233
|
+
for it in sorted(items, key=lambda i: i["id"]):
|
|
234
|
+
groups.setdefault(key(it), []).append(it)
|
|
235
|
+
alloc = allocate({k: len(v) for k, v in groups.items()}, n)
|
|
236
|
+
rng = random.Random(seed)
|
|
237
|
+
picked: List[dict] = []
|
|
238
|
+
for k in sorted(groups):
|
|
239
|
+
picked += rng.sample(groups[k], alloc[k])
|
|
240
|
+
return sorted(picked, key=lambda i: i["id"])
|
|
241
|
+
|
|
242
|
+
|
|
243
|
+
# ----------------------------------------------------------------------------- real-data builders
|
|
244
|
+
def _pool(name: str, fetcher: RowFetcher, data_dir: str, refresh: bool, pause: float, sleep) -> Tuple[List[dict], dict]:
|
|
245
|
+
ds, cfg, sp = SOURCES[name]
|
|
246
|
+
raw = os.path.join(data_dir, "raw", name + "_pool.jsonl")
|
|
247
|
+
if os.path.isfile(raw) and not refresh:
|
|
248
|
+
prov = read_provenance(raw) or {}
|
|
249
|
+
return read_jsonl(raw), dict(prov, pool_file="raw/%s_pool.jsonl" % name, pool_sha256=file_sha256(raw))
|
|
250
|
+
rows = fetch_split(ds, cfg, sp, fetcher, pause=pause, sleep=sleep)
|
|
251
|
+
prov = {"source": [ds, cfg, sp], "api": HF_ROWS_API, "fetched": time.strftime("%Y-%m-%d"), "pool_rows": len(rows)}
|
|
252
|
+
write_jsonl(raw, rows, prov)
|
|
253
|
+
return rows, dict(prov, pool_file="raw/%s_pool.jsonl" % name, pool_sha256=file_sha256(raw))
|
|
254
|
+
|
|
255
|
+
|
|
256
|
+
def build_mmlu_pro(n: int = 1000, seed: int = 2026, fetcher: RowFetcher = hf_fetcher, data_dir: Optional[str] = None, refresh: bool = False,
|
|
257
|
+
pause: float = 0.5, sleep=time.sleep) -> dict:
|
|
258
|
+
d = data_dir or DATA_DIR
|
|
259
|
+
rows, prov = _pool("mmlu_pro", fetcher, d, refresh, pause, sleep)
|
|
260
|
+
pool = mmlu_pro_items(rows)
|
|
261
|
+
items = stratified_sample(pool, n, seed, key=lambda it: it["meta"]["category"])
|
|
262
|
+
cats = {}
|
|
263
|
+
for it in items:
|
|
264
|
+
cats[it["meta"]["category"]] = cats.get(it["meta"]["category"], 0) + 1
|
|
265
|
+
info = {"source": prov["source"], "selection": "stratified by category, proportional allocation (largest remainder), random.Random(seed).sample inside each stratum",
|
|
266
|
+
"seed": seed, "pool_size": len(pool), "pool_file": prov["pool_file"], "pool_sha256": prov["pool_sha256"], "fetched": prov.get("fetched"),
|
|
267
|
+
"per_category": dict(sorted(cats.items())), "synthetic": False}
|
|
268
|
+
return register("mmlu_pro", items, info, d)
|
|
269
|
+
|
|
270
|
+
|
|
271
|
+
def build_math500_l45(fetcher: RowFetcher = hf_fetcher, data_dir: Optional[str] = None, refresh: bool = False, pause: float = 0.5, sleep=time.sleep) -> dict:
|
|
272
|
+
d = data_dir or DATA_DIR
|
|
273
|
+
rows, prov = _pool("math500_l45", fetcher, d, refresh, pause, sleep)
|
|
274
|
+
items = sorted(math500_items(rows), key=lambda i: i["id"])
|
|
275
|
+
info = {"source": prov["source"], "selection": "all MATH-500 items with level 4 or 5, id order (no sampling)", "seed": None, "pool_size": len(rows),
|
|
276
|
+
"pool_file": prov["pool_file"], "pool_sha256": prov["pool_sha256"], "fetched": prov.get("fetched"), "synthetic": False,
|
|
277
|
+
"per_level": {str(l): sum(1 for i in items if i["meta"]["level"] == l) for l in (4, 5)}}
|
|
278
|
+
return register("math500_l45", items, info, d)
|
|
279
|
+
|
|
280
|
+
|
|
281
|
+
def items_from_experiments_cache(path: str) -> List[dict]:
|
|
282
|
+
"""Read an item file in the cme.tasks cache format (data/experiments/<task>.jsonl). That MMLU-Pro cache is a 4,000-row seeded sample, not the
|
|
283
|
+
whole split, so a build from it is NOT the full-pool stratified draw; use only when the download is impossible and say so."""
|
|
284
|
+
return read_jsonl(path)
|
|
285
|
+
|
|
286
|
+
|
|
287
|
+
# ----------------------------------------------------------------------------- hidden profile
|
|
288
|
+
def load_generator():
|
|
289
|
+
"""Load code/cme/hidden_profile.py (read-only, by file path; falls back to code/pace/ after a rename)."""
|
|
290
|
+
for pkg in ("cme", "pace"):
|
|
291
|
+
p = os.path.join(CODE_DIR, pkg, "hidden_profile.py")
|
|
292
|
+
if os.path.isfile(p):
|
|
293
|
+
spec = importlib.util.spec_from_file_location("pace_lab._hp_generator", p)
|
|
294
|
+
mod = importlib.util.module_from_spec(spec)
|
|
295
|
+
spec.loader.exec_module(mod)
|
|
296
|
+
mod.__file_path__ = p
|
|
297
|
+
return mod
|
|
298
|
+
raise FileNotFoundError("hidden_profile.py not found under code/cme or code/pace")
|
|
299
|
+
|
|
300
|
+
|
|
301
|
+
NAMES = ("Avery", "Blake", "Casey", "Drew")
|
|
302
|
+
_NL_SENT = re.compile(r"On criterion (\d+), Avery scored (\d+), Blake scored (\d+), Casey scored (\d+) and Drew scored (\d+)\.")
|
|
303
|
+
_ALL_HEAD = "\n\nAll scores:\n"
|
|
304
|
+
|
|
305
|
+
|
|
306
|
+
def parse_numeric(text: str) -> Dict[int, Dict[str, int]]:
|
|
307
|
+
got: Dict[int, Dict[str, int]] = {}
|
|
308
|
+
for line in text.splitlines():
|
|
309
|
+
parts = line.split(",")
|
|
310
|
+
if len(parts) == 5 and parts[0].isdigit():
|
|
311
|
+
got[int(parts[0])] = dict(zip(NAMES, map(int, parts[1:])))
|
|
312
|
+
return got
|
|
313
|
+
|
|
314
|
+
|
|
315
|
+
def parse_nl(text: str) -> Dict[int, Dict[str, int]]:
|
|
316
|
+
return {int(m[0]): dict(zip(NAMES, map(int, m[1:]))) for m in _NL_SENT.findall(text)}
|
|
317
|
+
|
|
318
|
+
|
|
319
|
+
def render_nl(scores: Dict[int, Dict[str, int]]) -> str:
|
|
320
|
+
return " ".join("On criterion %d, Avery scored %d, Blake scored %d, Casey scored %d and Drew scored %d." % ((j,) + tuple(scores[j][n] for n in NAMES))
|
|
321
|
+
for j in sorted(scores))
|
|
322
|
+
|
|
323
|
+
|
|
324
|
+
def _to_nl(item: dict) -> dict:
|
|
325
|
+
"""Prose version of a numeric hidden-profile item: same scores, same gold, same intro and question sentences."""
|
|
326
|
+
prompt = item["prompt"]
|
|
327
|
+
i = prompt.index(_ALL_HEAD)
|
|
328
|
+
head, rest = prompt[:i], prompt[i + len(_ALL_HEAD):]
|
|
329
|
+
q = rest.rindex("\n\nWhich candidate")
|
|
330
|
+
new_prompt = head + _ALL_HEAD + render_nl(parse_numeric(rest[:q])) + rest[q:]
|
|
331
|
+
views = []
|
|
332
|
+
for v in item["views"]:
|
|
333
|
+
marker = "You see only part of the evidence. Scores you can see:\n"
|
|
334
|
+
a = v.index(marker)
|
|
335
|
+
b = v.index("\n\nOther committee members")
|
|
336
|
+
views.append(v[:a + len(marker)] + render_nl(parse_numeric(v[a + len(marker):b])) + v[b:])
|
|
337
|
+
return dict(item, prompt=new_prompt, views=views)
|
|
338
|
+
|
|
339
|
+
|
|
340
|
+
def _twin(item: dict) -> dict:
|
|
341
|
+
"""Shared-evidence twin: identical scores, gold and `prompt`, but every agent's view contains ALL the evidence."""
|
|
342
|
+
shared = item["prompt"].replace("All scores:", "All committee members see the same evidence. Scores you can see:", 1)
|
|
343
|
+
return dict(item, views=[shared for _ in item["views"]])
|
|
344
|
+
|
|
345
|
+
|
|
346
|
+
def _hp_label(item: dict, prefix: str, fmt: str, evidence: str, suffix: str) -> dict:
|
|
347
|
+
base = item["id"]
|
|
348
|
+
meta = dict(item["meta"], base_id=base, format=fmt, evidence=evidence, synthetic=True)
|
|
349
|
+
if evidence == "shared":
|
|
350
|
+
meta["twin_of"] = prefix + base[2:]
|
|
351
|
+
return dict(item, id=prefix + base[2:] + suffix, meta=meta)
|
|
352
|
+
|
|
353
|
+
|
|
354
|
+
def validate_hidden_profile(items: Sequence[dict], parser: Callable[[str], Dict[int, Dict[str, int]]]) -> Dict[str, int]:
|
|
355
|
+
"""Check from the prompt text: gold strictly leads the pooled evidence; split items: decoy strictly leads every view, every split criterion is seen
|
|
356
|
+
by some agent; shared twins: every view contains all criteria and gold strictly leads it."""
|
|
357
|
+
bad = dict(items=len(items), target_not_leading_pooled=0, decoy_not_leading_a_view=0, uncovered_split_criteria=0, twin_view_incomplete=0)
|
|
358
|
+
|
|
359
|
+
def leader(sc):
|
|
360
|
+
tot = {n: sum(sc[j][n] for j in sc) for n in NAMES}
|
|
361
|
+
best = max(tot.values())
|
|
362
|
+
top = [n for n in NAMES if tot[n] == best]
|
|
363
|
+
return top[0] if len(top) == 1 else None
|
|
364
|
+
for it in items:
|
|
365
|
+
k = it["meta"]["knobs"]
|
|
366
|
+
full = parser(it["prompt"])
|
|
367
|
+
if len(full) != k["criteria"] or leader(full) != it["gold"]:
|
|
368
|
+
bad["target_not_leading_pooled"] += 1
|
|
369
|
+
shared_n = k["criteria"] - k["split"]
|
|
370
|
+
seen, view_bad = set(), False
|
|
371
|
+
for v in it["views"]:
|
|
372
|
+
part = parser(v)
|
|
373
|
+
if it["meta"].get("evidence") == "shared":
|
|
374
|
+
if len(part) != k["criteria"] or leader(part) != it["gold"]:
|
|
375
|
+
bad["twin_view_incomplete"] += 1
|
|
376
|
+
continue
|
|
377
|
+
seen |= {j for j in part if j > shared_n}
|
|
378
|
+
if leader(part) != it["meta"]["decoy"]:
|
|
379
|
+
view_bad = True
|
|
380
|
+
if it["meta"].get("evidence") != "shared":
|
|
381
|
+
bad["decoy_not_leading_a_view"] += view_bad
|
|
382
|
+
bad["uncovered_split_criteria"] += len(set(range(shared_n + 1, k["criteria"] + 1)) - seen) > 0
|
|
383
|
+
return bad
|
|
384
|
+
|
|
385
|
+
|
|
386
|
+
def build_hidden_profile(n: int = 500, seed: int = 42, data_dir: Optional[str] = None) -> Dict[str, dict]:
|
|
387
|
+
d = data_dir or DATA_DIR
|
|
388
|
+
hp = load_generator()
|
|
389
|
+
num = hp.generate(n, seed=seed, fmt="numeric")
|
|
390
|
+
txt = hp.generate(n, seed=seed, fmt="text")
|
|
391
|
+
if [x["gold"] for x in num] != [x["gold"] for x in txt]:
|
|
392
|
+
raise RuntimeError("generator formats disagree on gold answers")
|
|
393
|
+
gen_path = hp.__file_path__
|
|
394
|
+
sets = {
|
|
395
|
+
"hp_numeric": [_hp_label(x, "hpnum", "numeric", "split", "") for x in num],
|
|
396
|
+
"hp_numeric_shared": [_hp_label(_twin(x), "hpnum", "numeric", "shared", "_shared") for x in num],
|
|
397
|
+
}
|
|
398
|
+
nl = [_to_nl(x) for x in num]
|
|
399
|
+
for a, b in zip(num, nl): # the prose must carry exactly the same scores
|
|
400
|
+
if parse_nl(b["prompt"]) != parse_numeric(a["prompt"][a["prompt"].index(_ALL_HEAD):]) or \
|
|
401
|
+
[parse_nl(v) for v in b["views"]] != [parse_numeric(v[v.index("Scores you can see:"):]) for v in a["views"]]:
|
|
402
|
+
raise RuntimeError("NL rendering of %s does not round-trip" % a["id"])
|
|
403
|
+
sets["hp_nl"] = [_hp_label(x, "hpnl", "nl", "split", "") for x in nl]
|
|
404
|
+
sets["hp_nl_shared"] = [_hp_label(_twin(x), "hpnl", "nl", "shared", "_shared") for x in nl]
|
|
405
|
+
parsers = {"hp_numeric": parse_numeric, "hp_numeric_shared": parse_numeric, "hp_nl": parse_nl, "hp_nl_shared": parse_nl}
|
|
406
|
+
out = {}
|
|
407
|
+
for name, items in sets.items():
|
|
408
|
+
counts = validate_hidden_profile(items, parsers[name])
|
|
409
|
+
if any(v for k2, v in counts.items() if k2 != "items"):
|
|
410
|
+
raise RuntimeError("%s failed validation: %s" % (name, counts))
|
|
411
|
+
info = {"source": "synthetic: code/%s/hidden_profile.py generate()" % os.path.basename(os.path.dirname(gen_path)), "generator_sha256": file_sha256(gen_path),
|
|
412
|
+
"selection": "first n items of the seeded generator", "seed": seed, "synthetic": True, "validation": counts,
|
|
413
|
+
"knobs": items[0]["meta"]["knobs"], "format": items[0]["meta"]["format"], "evidence": items[0]["meta"]["evidence"]}
|
|
414
|
+
out[name] = register(name, items, info, d)
|
|
415
|
+
return out
|
|
416
|
+
|
|
417
|
+
|
|
418
|
+
# ----------------------------------------------------------------------------- command line
|
|
419
|
+
def main(argv: Optional[Sequence[str]] = None) -> int:
|
|
420
|
+
ap = argparse.ArgumentParser(prog="python -m pace_lab.data", description=__doc__.split("\n")[0])
|
|
421
|
+
sub = ap.add_subparsers(dest="cmd", required=True)
|
|
422
|
+
b = sub.add_parser("build")
|
|
423
|
+
b.add_argument("what", choices=["mmlu_pro", "math500_l45", "hidden_profile", "all"])
|
|
424
|
+
b.add_argument("--refresh", action="store_true", help="re-download the raw pool")
|
|
425
|
+
b.add_argument("--data-dir", default=None)
|
|
426
|
+
sub.add_parser("verify")
|
|
427
|
+
args = ap.parse_args(argv)
|
|
428
|
+
if args.cmd == "verify":
|
|
429
|
+
problems = verify_manifest()
|
|
430
|
+
print("\n".join(problems) if problems else "manifest OK: %s" % ", ".join(sorted(load_manifest()["datasets"])))
|
|
431
|
+
return 1 if problems else 0
|
|
432
|
+
todo = ["mmlu_pro", "math500_l45", "hidden_profile"] if args.what == "all" else [args.what]
|
|
433
|
+
for w in todo:
|
|
434
|
+
if w == "mmlu_pro":
|
|
435
|
+
e = build_mmlu_pro(data_dir=args.data_dir, refresh=args.refresh)
|
|
436
|
+
elif w == "math500_l45":
|
|
437
|
+
e = build_math500_l45(data_dir=args.data_dir, refresh=args.refresh)
|
|
438
|
+
else:
|
|
439
|
+
e = build_hidden_profile(data_dir=args.data_dir)
|
|
440
|
+
print(w, "ok", json.dumps({k: v for k, v in (e.items() if "n" in e else [(kk, vv["n"]) for kk, vv in e.items()]) if k in ("n",) or w == "hidden_profile"}))
|
|
441
|
+
return 0
|
|
442
|
+
|
|
443
|
+
|
|
444
|
+
if __name__ == "__main__":
|
|
445
|
+
sys.exit(main())
|