pace-eval 0.2.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cme/__init__.py +11 -0
- cme/__main__.py +5 -0
- cme/analysis.py +223 -0
- cme/audit.py +145 -0
- cme/baseline.py +162 -0
- cme/cli.py +279 -0
- cme/client.py +165 -0
- cme/dynamics.py +212 -0
- cme/examples/crewai_integration.py +16 -0
- cme/examples/demo_pool.jsonl +360 -0
- cme/examples/demo_run.jsonl +480 -0
- cme/examples/langgraph_integration.py +27 -0
- cme/examples/make_demo.py +37 -0
- cme/examples/openai_quickstart.py +18 -0
- cme/experiment.py +555 -0
- cme/frontier.py +143 -0
- cme/hidden_profile.py +135 -0
- cme/proxy.py +106 -0
- cme/rebuttal.py +76 -0
- cme/report.py +197 -0
- cme/rollout_card.py +137 -0
- cme/tasks.py +266 -0
- cme/verified_debate.py +127 -0
- cme/verifier.py +92 -0
- mae_eval/__init__.py +11 -0
- mae_eval/__main__.py +5 -0
- mae_eval/auditor.py +112 -0
- mae_eval/classifier.py +62 -0
- mae_eval/cli.py +210 -0
- mae_eval/tracker.py +162 -0
- mae_eval/vector.py +91 -0
- pace/__init__.py +11 -0
- pace/__main__.py +5 -0
- pace/analysis.py +129 -0
- pace/audit.py +145 -0
- pace/baseline.py +162 -0
- pace/card.py +205 -0
- pace/cli.py +898 -0
- pace/client.py +165 -0
- pace/dynamics.py +372 -0
- pace/escape_benchmarks.py +162 -0
- pace/examples/crewai_integration.py +16 -0
- pace/examples/demo_pool.jsonl +360 -0
- pace/examples/demo_run.jsonl +480 -0
- pace/examples/langgraph_integration.py +27 -0
- pace/examples/make_demo.py +37 -0
- pace/examples/openai_quickstart.py +18 -0
- pace/experiment.py +555 -0
- pace/frontier.py +114 -0
- pace/hidden_profile.py +301 -0
- pace/importers.py +219 -0
- pace/inflate.py +213 -0
- pace/influence.py +231 -0
- pace/meta_engine.py +246 -0
- pace/middleware.py +230 -0
- pace/open_ended.py +114 -0
- pace/plotting.py +132 -0
- pace/pool_sizer.py +154 -0
- pace/protocol_v3.py +414 -0
- pace/proxy.py +106 -0
- pace/rebuttal.py +219 -0
- pace/report.py +197 -0
- pace/rollout_card.py +137 -0
- pace/schema.py +74 -0
- pace/simulator.py +199 -0
- pace/studio.html +1040 -0
- pace/tasks.py +266 -0
- pace/verified_debate.py +127 -0
- pace/verifier.py +92 -0
- pace/workbench.html +26 -0
- pace/workbench.py +119 -0
- pace_eval-0.2.2.dist-info/METADATA +205 -0
- pace_eval-0.2.2.dist-info/RECORD +83 -0
- pace_eval-0.2.2.dist-info/WHEEL +5 -0
- pace_eval-0.2.2.dist-info/entry_points.txt +4 -0
- pace_eval-0.2.2.dist-info/licenses/LICENSE +202 -0
- pace_eval-0.2.2.dist-info/top_level.txt +4 -0
- pace_lab/__init__.py +9 -0
- pace_lab/clients.py +568 -0
- pace_lab/concurrency_test.py +127 -0
- pace_lab/data.py +445 -0
- pace_lab/graders.py +626 -0
- pace_lab/strategies.py +345 -0
pace/middleware.py
ADDED
|
@@ -0,0 +1,230 @@
|
|
|
1
|
+
"""Runtime accounting middleware, DAG latency, and scaffold decomposition."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import functools
|
|
6
|
+
import json
|
|
7
|
+
from collections.abc import Callable
|
|
8
|
+
from typing import Any
|
|
9
|
+
|
|
10
|
+
from .card import build_card
|
|
11
|
+
|
|
12
|
+
MODEL_PRICING_USD_PER_1M = {
|
|
13
|
+
name: prices
|
|
14
|
+
for name, prices in {
|
|
15
|
+
"gpt-4o": (5.0, 1.25, 15.0),
|
|
16
|
+
"gpt-4.1": (2.0, 0.5, 8.0),
|
|
17
|
+
"gpt-5": (1.25, 0.125, 10.0),
|
|
18
|
+
"o1": (15.0, 7.5, 60.0),
|
|
19
|
+
"o3": (2.0, 0.5, 8.0),
|
|
20
|
+
"claude-3-5-sonnet": (3.0, 0.3, 15.0),
|
|
21
|
+
"claude-3-7-sonnet": (3.0, 0.3, 15.0),
|
|
22
|
+
"claude-4": (3.0, 0.3, 15.0),
|
|
23
|
+
"gemini-1.5-pro": (1.25, 0.315, 5.0),
|
|
24
|
+
"gemini-2.5-pro": (1.25, 0.125, 10.0),
|
|
25
|
+
"deepseek-r1": (0.55, 0.14, 2.19),
|
|
26
|
+
"deepseek-v3": (0.27, 0.07, 1.1),
|
|
27
|
+
"qwen2.5": (0.4, 0.1, 1.6),
|
|
28
|
+
"llama-3.1-70b": (0.88, 0.0, 0.88),
|
|
29
|
+
"llama-3.3-70b": (0.88, 0.0, 0.88),
|
|
30
|
+
"gpt-4-turbo": (10.0, 0.0, 30.0),
|
|
31
|
+
"claude-3-opus": (15.0, 1.5, 75.0),
|
|
32
|
+
"claude-3-haiku": (0.25, 0.03, 1.25),
|
|
33
|
+
"gemini-1.5-flash": (0.075, 0.01875, 0.3),
|
|
34
|
+
"mistral-large": (2.0, 0.0, 6.0),
|
|
35
|
+
"command-r-plus": (2.5, 0.0, 10.0),
|
|
36
|
+
}.items()
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def estimate_call_cost(
|
|
41
|
+
model: str,
|
|
42
|
+
prompt_tokens: float,
|
|
43
|
+
completion_tokens: float,
|
|
44
|
+
cached_prompt_tokens: float = 0.0,
|
|
45
|
+
reasoning_tokens: float = 0.0,
|
|
46
|
+
pricing: dict | None = None,
|
|
47
|
+
) -> dict:
|
|
48
|
+
"""Estimate USD cost while reporting uncached, cached, reasoning, and completion components."""
|
|
49
|
+
table = pricing or MODEL_PRICING_USD_PER_1M
|
|
50
|
+
if (
|
|
51
|
+
model not in table
|
|
52
|
+
or min(prompt_tokens, completion_tokens, cached_prompt_tokens, reasoning_tokens)
|
|
53
|
+
< 0
|
|
54
|
+
or cached_prompt_tokens > prompt_tokens
|
|
55
|
+
):
|
|
56
|
+
raise ValueError("unknown model or invalid token counts")
|
|
57
|
+
input_price, cached_price, output_price = table[model]
|
|
58
|
+
uncached = prompt_tokens - cached_prompt_tokens
|
|
59
|
+
parts = {
|
|
60
|
+
"uncached_prompt": uncached * input_price / 1e6,
|
|
61
|
+
"cached_prompt": cached_prompt_tokens * cached_price / 1e6,
|
|
62
|
+
"reasoning": reasoning_tokens * output_price / 1e6,
|
|
63
|
+
"completion": completion_tokens * output_price / 1e6,
|
|
64
|
+
}
|
|
65
|
+
return {
|
|
66
|
+
"model": model,
|
|
67
|
+
"input_tokens": prompt_tokens,
|
|
68
|
+
"cached_prompt_tokens": cached_prompt_tokens,
|
|
69
|
+
"reasoning_tokens": reasoning_tokens,
|
|
70
|
+
"completion_tokens": completion_tokens,
|
|
71
|
+
"cost_usd": sum(parts.values()),
|
|
72
|
+
"components": parts,
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def compute_dag_latency(calls: list[dict]) -> dict:
|
|
77
|
+
if not calls:
|
|
78
|
+
raise ValueError("calls cannot be empty")
|
|
79
|
+
durations = {
|
|
80
|
+
str(call["call_id"]): float(
|
|
81
|
+
call.get(
|
|
82
|
+
"duration_ms",
|
|
83
|
+
float(call.get("end_ms", 0)) - float(call.get("start_ms", 0)),
|
|
84
|
+
)
|
|
85
|
+
)
|
|
86
|
+
for call in calls
|
|
87
|
+
}
|
|
88
|
+
if any(value < 0 for value in durations.values()):
|
|
89
|
+
raise ValueError("durations cannot be negative")
|
|
90
|
+
total = sum(durations.values())
|
|
91
|
+
parents = {
|
|
92
|
+
str(call["call_id"]): [str(dep) for dep in call.get("depends_on", [])]
|
|
93
|
+
for call in calls
|
|
94
|
+
}
|
|
95
|
+
memo = {}
|
|
96
|
+
|
|
97
|
+
def longest(node):
|
|
98
|
+
if node in memo:
|
|
99
|
+
return memo[node]
|
|
100
|
+
memo[node] = durations[node] + max(
|
|
101
|
+
(longest(parent) for parent in parents[node]), default=0.0
|
|
102
|
+
)
|
|
103
|
+
return memo[node]
|
|
104
|
+
|
|
105
|
+
critical = (
|
|
106
|
+
max((longest(node) for node in durations), default=0.0)
|
|
107
|
+
if any(parents.values())
|
|
108
|
+
else sum(
|
|
109
|
+
max(
|
|
110
|
+
durations[str(call["call_id"])]
|
|
111
|
+
for call in calls
|
|
112
|
+
if call.get("round", 0) == round_no
|
|
113
|
+
)
|
|
114
|
+
for round_no in sorted({call.get("round", 0) for call in calls})
|
|
115
|
+
)
|
|
116
|
+
)
|
|
117
|
+
workers = max(
|
|
118
|
+
sum(1 for call in calls if call.get("round", 0) == round_no)
|
|
119
|
+
for round_no in {call.get("round", 0) for call in calls}
|
|
120
|
+
)
|
|
121
|
+
return {
|
|
122
|
+
"total_sequential_ms": total,
|
|
123
|
+
"critical_path_ms": critical,
|
|
124
|
+
"concurrency_speedup": total / max(critical, 1e-9),
|
|
125
|
+
"max_parallel_workers": workers,
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
def _word_count(value: Any) -> int:
|
|
130
|
+
if isinstance(value, str):
|
|
131
|
+
return len(value.split())
|
|
132
|
+
if isinstance(value, list):
|
|
133
|
+
return sum(
|
|
134
|
+
_word_count(item.get("content", "") if isinstance(item, dict) else item)
|
|
135
|
+
for item in value
|
|
136
|
+
)
|
|
137
|
+
return 0
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
def decompose_scaffold_tokens(calls: list[dict]) -> dict:
|
|
141
|
+
if not calls:
|
|
142
|
+
raise ValueError("calls cannot be empty")
|
|
143
|
+
total = sum(
|
|
144
|
+
_word_count(call.get("prompt_text", call.get("messages", "")))
|
|
145
|
+
+ float(call.get("completion_tokens", 0))
|
|
146
|
+
for call in calls
|
|
147
|
+
)
|
|
148
|
+
c0 = _word_count(calls[0].get("prompt_text", calls[0].get("messages", "")))
|
|
149
|
+
echoed = sum(
|
|
150
|
+
max(0, _word_count(call.get("prompt_text", call.get("messages", ""))) - c0)
|
|
151
|
+
for call in calls[1:]
|
|
152
|
+
)
|
|
153
|
+
reasoning = sum(
|
|
154
|
+
float(call.get("completion_tokens", call.get("reasoning_tokens", 0)))
|
|
155
|
+
for call in calls
|
|
156
|
+
)
|
|
157
|
+
return {
|
|
158
|
+
"c0": c0,
|
|
159
|
+
"echoed_peer_tokens": echoed,
|
|
160
|
+
"reasoning_tokens": reasoning,
|
|
161
|
+
"total_tokens": total,
|
|
162
|
+
"scaffold_dilution_ratio": 1 - reasoning / total if total else 0.0,
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
class PaceSession:
|
|
167
|
+
"""Collect decorated call records and flush validated JSONL on exit."""
|
|
168
|
+
|
|
169
|
+
current: PaceSession | None = None
|
|
170
|
+
|
|
171
|
+
def __init__(
|
|
172
|
+
self,
|
|
173
|
+
path: str,
|
|
174
|
+
item: str = "item_0",
|
|
175
|
+
agent: str = "agent_0",
|
|
176
|
+
round_num: int = 1,
|
|
177
|
+
):
|
|
178
|
+
self.path, self.item, self.agent, self.round_num, self.records = (
|
|
179
|
+
path,
|
|
180
|
+
item,
|
|
181
|
+
agent,
|
|
182
|
+
round_num,
|
|
183
|
+
[],
|
|
184
|
+
)
|
|
185
|
+
|
|
186
|
+
def __enter__(self):
|
|
187
|
+
self._previous, PaceSession.current = PaceSession.current, self
|
|
188
|
+
return self
|
|
189
|
+
|
|
190
|
+
def __exit__(self, exc_type, exc_value, traceback):
|
|
191
|
+
if exc_type is None:
|
|
192
|
+
with open(self.path, "w", encoding="utf-8") as handle:
|
|
193
|
+
handle.writelines(json.dumps(row) + "\n" for row in self.records)
|
|
194
|
+
PaceSession.current = self._previous
|
|
195
|
+
return False
|
|
196
|
+
|
|
197
|
+
def record(
|
|
198
|
+
self,
|
|
199
|
+
correct: bool,
|
|
200
|
+
answer: str | None = None,
|
|
201
|
+
prompt_tokens: float = 0,
|
|
202
|
+
completion_tokens: float = 0,
|
|
203
|
+
) -> None:
|
|
204
|
+
self.records.append(
|
|
205
|
+
{
|
|
206
|
+
"item": self.item,
|
|
207
|
+
"agent": self.agent,
|
|
208
|
+
"round": self.round_num,
|
|
209
|
+
"correct": bool(correct),
|
|
210
|
+
"answer": answer,
|
|
211
|
+
"prompt_tokens": prompt_tokens,
|
|
212
|
+
"completion_tokens": completion_tokens,
|
|
213
|
+
}
|
|
214
|
+
)
|
|
215
|
+
|
|
216
|
+
def card(self) -> dict:
|
|
217
|
+
return build_card(self.records)
|
|
218
|
+
|
|
219
|
+
|
|
220
|
+
def pace_track(function: Callable) -> Callable:
|
|
221
|
+
"""Decorate a function and record its boolean result in the active session."""
|
|
222
|
+
|
|
223
|
+
@functools.wraps(function)
|
|
224
|
+
def wrapper(*args, **kwargs):
|
|
225
|
+
result = function(*args, **kwargs)
|
|
226
|
+
if PaceSession.current is not None:
|
|
227
|
+
PaceSession.current.record(bool(result), answer=str(result))
|
|
228
|
+
return result
|
|
229
|
+
|
|
230
|
+
return wrapper
|
pace/open_ended.py
ADDED
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
"""Noncommunicating open-ended sample aggregation utilities."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import difflib
|
|
6
|
+
import math
|
|
7
|
+
import re
|
|
8
|
+
from collections import Counter
|
|
9
|
+
|
|
10
|
+
import numpy as np
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def _tokens(text: str) -> list[str]:
|
|
14
|
+
return re.findall(r"[A-Za-z0-9_]+", text.lower())
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def _features(text: str, metric: str, n: int) -> Counter:
|
|
18
|
+
source = text if metric == "ngram_cosine" else " ".join(_tokens(text))
|
|
19
|
+
return Counter(source[i : i + n] for i in range(max(0, len(source) - n + 1)))
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def similarity_matrix(
|
|
23
|
+
candidates: list[str], metric: str = "ngram_cosine", n: int = 3
|
|
24
|
+
) -> np.ndarray:
|
|
25
|
+
if (
|
|
26
|
+
metric
|
|
27
|
+
not in {"ngram_cosine", "token_jaccard", "sequence_ratio", "code_structural"}
|
|
28
|
+
or n < 1
|
|
29
|
+
):
|
|
30
|
+
raise ValueError("unsupported metric or n")
|
|
31
|
+
size = len(candidates)
|
|
32
|
+
matrix = np.eye(size)
|
|
33
|
+
for i in range(size):
|
|
34
|
+
for j in range(i):
|
|
35
|
+
if metric == "sequence_ratio":
|
|
36
|
+
score = difflib.SequenceMatcher(
|
|
37
|
+
None, candidates[i], candidates[j]
|
|
38
|
+
).ratio()
|
|
39
|
+
elif metric == "token_jaccard":
|
|
40
|
+
left, right = set(_tokens(candidates[i])), set(_tokens(candidates[j]))
|
|
41
|
+
score = len(left & right) / len(left | right) if left | right else 1.0
|
|
42
|
+
else:
|
|
43
|
+
left, right = (
|
|
44
|
+
_features(candidates[i], metric, n),
|
|
45
|
+
_features(candidates[j], metric, n),
|
|
46
|
+
)
|
|
47
|
+
if metric == "code_structural":
|
|
48
|
+
left = Counter(_tokens(candidates[i]) + candidates[i].splitlines())
|
|
49
|
+
right = Counter(_tokens(candidates[j]) + candidates[j].splitlines())
|
|
50
|
+
dot = sum(left[k] * right[k] for k in set(left) | set(right))
|
|
51
|
+
denom = math.sqrt(
|
|
52
|
+
sum(v * v for v in left.values())
|
|
53
|
+
* sum(v * v for v in right.values())
|
|
54
|
+
)
|
|
55
|
+
score = dot / denom if denom else 0.0
|
|
56
|
+
matrix[i, j] = matrix[j, i] = score
|
|
57
|
+
return matrix
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def mbr_centroid_select(
|
|
61
|
+
candidates: list[str],
|
|
62
|
+
metric: str = "ngram_cosine",
|
|
63
|
+
token_costs: list[float] | None = None,
|
|
64
|
+
) -> dict:
|
|
65
|
+
if not candidates:
|
|
66
|
+
raise ValueError("candidates cannot be empty")
|
|
67
|
+
if token_costs is not None and len(token_costs) != len(candidates):
|
|
68
|
+
raise ValueError("token_costs must match candidates")
|
|
69
|
+
matrix = similarity_matrix(candidates, metric)
|
|
70
|
+
scores = (matrix.sum(axis=1) - 1) / max(1, len(candidates) - 1)
|
|
71
|
+
index = int(np.argmax(scores))
|
|
72
|
+
off_diag = (matrix.sum() - len(candidates)) / max(
|
|
73
|
+
1, len(candidates) * (len(candidates) - 1)
|
|
74
|
+
)
|
|
75
|
+
return {
|
|
76
|
+
"index": index,
|
|
77
|
+
"selected": candidates[index],
|
|
78
|
+
"consensus_score": float(scores[index]),
|
|
79
|
+
"diversity_dispersion": float(1 - off_diag),
|
|
80
|
+
"total_token_cost": float(sum(token_costs)) if token_costs else None,
|
|
81
|
+
"similarity": matrix.tolist(),
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def unbiased_pass_at_k_frontier(
|
|
86
|
+
n_total_samples: int,
|
|
87
|
+
c_correct_samples: int,
|
|
88
|
+
k_values: list[int],
|
|
89
|
+
mean_tokens_per_sample: float,
|
|
90
|
+
) -> list[dict]:
|
|
91
|
+
if (
|
|
92
|
+
not 0 <= c_correct_samples <= n_total_samples
|
|
93
|
+
or n_total_samples < 1
|
|
94
|
+
or mean_tokens_per_sample < 0
|
|
95
|
+
):
|
|
96
|
+
raise ValueError("invalid sample counts or token cost")
|
|
97
|
+
out = []
|
|
98
|
+
for k in k_values:
|
|
99
|
+
if not 1 <= k <= n_total_samples:
|
|
100
|
+
raise ValueError("k must be in [1, n_total_samples]")
|
|
101
|
+
bad = (
|
|
102
|
+
math.comb(n_total_samples - c_correct_samples, k)
|
|
103
|
+
if n_total_samples - c_correct_samples >= k
|
|
104
|
+
else 0
|
|
105
|
+
)
|
|
106
|
+
out.append(
|
|
107
|
+
{
|
|
108
|
+
"k": k,
|
|
109
|
+
"pass_at_k": 1 - bad / math.comb(n_total_samples, k),
|
|
110
|
+
"token_budget": k * mean_tokens_per_sample,
|
|
111
|
+
"verifier_calls": k,
|
|
112
|
+
}
|
|
113
|
+
)
|
|
114
|
+
return out
|
pace/plotting.py
ADDED
|
@@ -0,0 +1,132 @@
|
|
|
1
|
+
"""Pure SVG publication figures with no plotting dependency."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import html
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def _svg(body: str, title: str, width: int = 800, height: int = 440) -> str:
|
|
9
|
+
return (
|
|
10
|
+
'<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 %d %d" role="img"><title>%s</title><style>text{font-family:Georgia,serif;fill:#17212b}.axis{stroke:#17212b}.grid{stroke:#d8d0c3}.bar{fill:#bb4d2e}.line{fill:none;stroke:#1d6b68;stroke-width:3}.point{fill:#bb4d2e}</style>%s</svg>'
|
|
11
|
+
% (width, height, html.escape(title), body)
|
|
12
|
+
)
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def svg_waterfall_decomposition(
|
|
16
|
+
p_greedy: float,
|
|
17
|
+
p_sampled: float,
|
|
18
|
+
p_mas: float,
|
|
19
|
+
title: str = "PACE Accounting Decomposition",
|
|
20
|
+
) -> str:
|
|
21
|
+
values = [p_greedy, p_sampled - p_greedy, p_sampled, p_mas - p_sampled]
|
|
22
|
+
labels = [
|
|
23
|
+
"Category A",
|
|
24
|
+
"Compute Scaling Effect",
|
|
25
|
+
"Category B",
|
|
26
|
+
"Coordination Residual",
|
|
27
|
+
]
|
|
28
|
+
body = '<line class="axis" x1="55" y1="380" x2="760" y2="380"/>'
|
|
29
|
+
for i, (value, label) in enumerate(zip(values, labels)):
|
|
30
|
+
x = 75 + i * 170
|
|
31
|
+
y = 380 - abs(value) * 280
|
|
32
|
+
h = abs(value) * 280
|
|
33
|
+
body += (
|
|
34
|
+
'<rect class="bar" x="%d" y="%.1f" width="100" height="%.1f"/><text x="%d" y="405" font-size="12">%s</text><text x="%d" y="%.1f" font-size="12">%.3f</text>'
|
|
35
|
+
% (x, y, h, x, html.escape(label), x, y - 6, value)
|
|
36
|
+
)
|
|
37
|
+
return _svg(body, title)
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def svg_phase_diagram(
|
|
41
|
+
rho_grid: list[float] | None = None,
|
|
42
|
+
k_team: int = 5,
|
|
43
|
+
empirical_point: tuple[float, float] | None = None,
|
|
44
|
+
) -> str:
|
|
45
|
+
rho_grid = rho_grid or [0, 0.25, 0.5, 0.75]
|
|
46
|
+
body = '<line class="axis" x1="55" y1="190" x2="370" y2="190"/><line class="axis" x1="55" y1="30" x2="55" y2="190"/><line class="axis" x1="430" y1="190" x2="760" y2="190"/><line class="axis" x1="430" y1="30" x2="430" y2="190"/><text x="65" y="22">A: K_eff vs K</text><text x="440" y="22">B: beneficial phase boundary</text>'
|
|
47
|
+
for index, rho in enumerate(rho_grid):
|
|
48
|
+
points = []
|
|
49
|
+
for k in range(1, k_team + 1):
|
|
50
|
+
points.append(
|
|
51
|
+
"%d,%.1f" % (55 + k * 55, 190 - (k / (1 + (k - 1) * rho)) * 25)
|
|
52
|
+
)
|
|
53
|
+
body += (
|
|
54
|
+
'<polyline class="line" points="%s"/><text x="%d" y="210" font-size="11">rho=%.2f</text>'
|
|
55
|
+
% (" ".join(points), 60 + index * 75, rho)
|
|
56
|
+
)
|
|
57
|
+
boundary = []
|
|
58
|
+
for i in range(1, 10):
|
|
59
|
+
kappa = i / 2
|
|
60
|
+
boundary.append("%.1f,%.1f" % (430 + i * 30, 190 - (kappa / (1 + kappa)) * 150))
|
|
61
|
+
body += '<polyline class="line" points="%s"/>' % " ".join(boundary)
|
|
62
|
+
if empirical_point:
|
|
63
|
+
body += '<circle class="point" cx="%.1f" cy="%.1f" r="6"/>' % (
|
|
64
|
+
430 + empirical_point[0] * 30,
|
|
65
|
+
190 - empirical_point[1] * 150,
|
|
66
|
+
)
|
|
67
|
+
return _svg(body, "PACE Phase Diagram")
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def svg_frontier_curve(
|
|
71
|
+
frontier_points: list[dict], mas_points: list[dict] | None = None
|
|
72
|
+
) -> str:
|
|
73
|
+
body = '<line class="axis" x1="55" y1="380" x2="760" y2="380"/><line class="axis" x1="55" y1="30" x2="55" y2="380"/>'
|
|
74
|
+
points = [
|
|
75
|
+
"%.1f,%.1f"
|
|
76
|
+
% (
|
|
77
|
+
55
|
|
78
|
+
+ float(row["tokens"])
|
|
79
|
+
/ max(1, max(float(item["tokens"]) for item in frontier_points))
|
|
80
|
+
* 680,
|
|
81
|
+
380 - float(row["accuracy"]) * 330,
|
|
82
|
+
)
|
|
83
|
+
for row in frontier_points
|
|
84
|
+
]
|
|
85
|
+
body += '<polyline class="line" points="%s"/>' % " ".join(points)
|
|
86
|
+
for row in mas_points or []:
|
|
87
|
+
body += (
|
|
88
|
+
'<circle class="point" cx="%.1f" cy="%.1f" r="5"/><text x="%.1f" y="%.1f" font-size="12">%s</text>'
|
|
89
|
+
% (
|
|
90
|
+
55
|
|
91
|
+
+ float(row["tokens"])
|
|
92
|
+
/ max(1, max(float(item["tokens"]) for item in frontier_points))
|
|
93
|
+
* 680,
|
|
94
|
+
380 - float(row["accuracy"]) * 330,
|
|
95
|
+
60
|
|
96
|
+
+ float(row["tokens"])
|
|
97
|
+
/ max(1, max(float(item["tokens"]) for item in frontier_points))
|
|
98
|
+
* 680,
|
|
99
|
+
375 - float(row["accuracy"]) * 330,
|
|
100
|
+
html.escape(str(row.get("label", "MAS"))),
|
|
101
|
+
)
|
|
102
|
+
)
|
|
103
|
+
return _svg(body, "PACE Frontier Curve")
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def svg_influence_curves(
|
|
107
|
+
by_correct_peers: dict, fit_result: dict, k_agents: int
|
|
108
|
+
) -> str:
|
|
109
|
+
body = '<line class="axis" x1="55" y1="380" x2="760" y2="380"/><line class="axis" x1="55" y1="30" x2="55" y2="380"/>'
|
|
110
|
+
for key, cell in by_correct_peers.items():
|
|
111
|
+
own, peers = (
|
|
112
|
+
key
|
|
113
|
+
if isinstance(key, tuple)
|
|
114
|
+
else (
|
|
115
|
+
int(str(key).split("=")[1].split(",")[0]),
|
|
116
|
+
int(str(key).split("=")[-1]),
|
|
117
|
+
)
|
|
118
|
+
)
|
|
119
|
+
x = peers / max(1, k_agents - 1)
|
|
120
|
+
y = cell.get("p_switch", cell.get("p_correct_next", 0))
|
|
121
|
+
body += '<circle class="point" cx="%.1f" cy="%.1f" r="4"/>' % (
|
|
122
|
+
55 + x * 680,
|
|
123
|
+
380 - y * 330,
|
|
124
|
+
)
|
|
125
|
+
for name, model in fit_result.get("models", {}).items():
|
|
126
|
+
power = 0 if name == "constant" else 1 if name == "linear_martingale" else 2
|
|
127
|
+
points = [
|
|
128
|
+
"%.1f,%.1f" % (55 + x * 680, 380 - min(1, model["a"] * x**power) * 330)
|
|
129
|
+
for x in [i / 20 for i in range(21)]
|
|
130
|
+
]
|
|
131
|
+
body += '<polyline class="line" points="%s"/>' % " ".join(points)
|
|
132
|
+
return _svg(body, "PACE Peer Influence Curves")
|
pace/pool_sizer.py
ADDED
|
@@ -0,0 +1,154 @@
|
|
|
1
|
+
"""Constructive token budgets and split-sample frontiers."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import math
|
|
6
|
+
|
|
7
|
+
import numpy as np
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def compute_k_tok(
|
|
11
|
+
k_agents: int,
|
|
12
|
+
t_rounds: int,
|
|
13
|
+
c0_prompt_tokens: float,
|
|
14
|
+
l_message_tokens: float,
|
|
15
|
+
topology: str = "last_round_peers",
|
|
16
|
+
) -> dict:
|
|
17
|
+
if k_agents < 1 or t_rounds < 1:
|
|
18
|
+
raise ValueError("agents and rounds must be positive")
|
|
19
|
+
if topology not in {"last_round_peers", "last_round_all", "full_history_echo"}:
|
|
20
|
+
raise ValueError("unknown topology: %s" % topology)
|
|
21
|
+
denominator = c0_prompt_tokens + l_message_tokens
|
|
22
|
+
ratio = l_message_tokens / denominator if denominator > 0 else 0.0
|
|
23
|
+
base = k_agents * t_rounds
|
|
24
|
+
if topology == "last_round_peers":
|
|
25
|
+
coefficient = (t_rounds - 1) * k_agents * (k_agents - 1)
|
|
26
|
+
k_tok = base + coefficient * ratio
|
|
27
|
+
elif topology == "last_round_all":
|
|
28
|
+
coefficient = (t_rounds - 1) * k_agents**2
|
|
29
|
+
k_tok = base + coefficient * ratio
|
|
30
|
+
else:
|
|
31
|
+
coefficient = 0.5 * k_agents**2 * t_rounds * (t_rounds - 1)
|
|
32
|
+
k_tok = base + coefficient * ratio
|
|
33
|
+
|
|
34
|
+
def b_eff(budget: float) -> float:
|
|
35
|
+
return float(budget) / k_tok if k_tok else float("inf")
|
|
36
|
+
|
|
37
|
+
recommended = max(8, 1)
|
|
38
|
+
while recommended < math.ceil(k_tok):
|
|
39
|
+
recommended *= 2
|
|
40
|
+
return {
|
|
41
|
+
"k_tok": float(k_tok),
|
|
42
|
+
"r": float(ratio),
|
|
43
|
+
"topology": topology,
|
|
44
|
+
"bounds": [float(base), float(base + coefficient)],
|
|
45
|
+
"k_min": float(base),
|
|
46
|
+
"k_max": float(base + coefficient),
|
|
47
|
+
"b_eff": b_eff,
|
|
48
|
+
"recommended_pool_size": recommended,
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def isotonic_pava_frontier(
|
|
53
|
+
budgets: list[float], accuracies: list[float], weights: list[float] | None = None
|
|
54
|
+
) -> list[dict]:
|
|
55
|
+
if len(budgets) != len(accuracies) or not budgets:
|
|
56
|
+
raise ValueError("budgets and accuracies must have equal nonzero length")
|
|
57
|
+
weights = weights or [1.0] * len(budgets)
|
|
58
|
+
if len(weights) != len(budgets):
|
|
59
|
+
raise ValueError("weights must match budgets")
|
|
60
|
+
data = sorted(
|
|
61
|
+
zip(map(float, budgets), map(float, accuracies), map(float, weights)),
|
|
62
|
+
key=lambda row: row[0],
|
|
63
|
+
)
|
|
64
|
+
blocks = []
|
|
65
|
+
for budget, accuracy, weight in data:
|
|
66
|
+
blocks.append([budget, accuracy, weight, 1])
|
|
67
|
+
while len(blocks) > 1 and blocks[-2][1] > blocks[-1][1]:
|
|
68
|
+
right = blocks.pop()
|
|
69
|
+
left = blocks.pop()
|
|
70
|
+
weight_sum = left[2] + right[2]
|
|
71
|
+
blocks.append(
|
|
72
|
+
[
|
|
73
|
+
left[0],
|
|
74
|
+
(left[1] * left[2] + right[1] * right[2]) / weight_sum,
|
|
75
|
+
weight_sum,
|
|
76
|
+
left[3] + right[3],
|
|
77
|
+
]
|
|
78
|
+
)
|
|
79
|
+
fitted = []
|
|
80
|
+
cursor = 0
|
|
81
|
+
for budget, accuracy, weight, count in blocks:
|
|
82
|
+
fitted.extend(
|
|
83
|
+
{
|
|
84
|
+
"budget": data[cursor + i][0],
|
|
85
|
+
"accuracy": data[cursor + i][1],
|
|
86
|
+
"fitted_accuracy": accuracy,
|
|
87
|
+
"weight": data[cursor + i][2],
|
|
88
|
+
}
|
|
89
|
+
for i in range(count)
|
|
90
|
+
)
|
|
91
|
+
cursor += count
|
|
92
|
+
return fitted
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def _fold_gap(scores, tokens, mas_scores, mas_tokens, train, test):
|
|
96
|
+
mas_budget = float(np.mean(mas_tokens[train]))
|
|
97
|
+
candidates = [
|
|
98
|
+
(k, float(np.mean(scores[train, k])))
|
|
99
|
+
for k in range(scores.shape[1])
|
|
100
|
+
if float(np.mean(tokens[train, k])) <= mas_budget
|
|
101
|
+
]
|
|
102
|
+
if not candidates:
|
|
103
|
+
k = 0
|
|
104
|
+
else:
|
|
105
|
+
k = max(candidates, key=lambda pair: pair[1])[0]
|
|
106
|
+
return (
|
|
107
|
+
float(np.mean(mas_scores[test]) - np.mean(scores[test, k])),
|
|
108
|
+
int(k + 1),
|
|
109
|
+
mas_budget,
|
|
110
|
+
)
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def split_sample_frontier_gap(
|
|
114
|
+
item_pool_scores: np.ndarray,
|
|
115
|
+
item_pool_tokens: np.ndarray,
|
|
116
|
+
mas_item_scores: np.ndarray,
|
|
117
|
+
mas_item_tokens: np.ndarray,
|
|
118
|
+
seed: int = 0,
|
|
119
|
+
) -> dict:
|
|
120
|
+
scores = np.asarray(item_pool_scores, dtype=float)
|
|
121
|
+
tokens = np.asarray(item_pool_tokens, dtype=float)
|
|
122
|
+
if scores.ndim != 2:
|
|
123
|
+
raise ValueError("item_pool_scores must be items by candidate k")
|
|
124
|
+
if tokens.ndim == 1:
|
|
125
|
+
tokens = np.broadcast_to(tokens, scores.shape)
|
|
126
|
+
mas_scores, mas_tokens = (
|
|
127
|
+
np.asarray(mas_item_scores, dtype=float).reshape(-1),
|
|
128
|
+
np.asarray(mas_item_tokens, dtype=float).reshape(-1),
|
|
129
|
+
)
|
|
130
|
+
n = scores.shape[0]
|
|
131
|
+
if (
|
|
132
|
+
tokens.shape != scores.shape
|
|
133
|
+
or len(mas_scores) != n
|
|
134
|
+
or len(mas_tokens) != n
|
|
135
|
+
or n < 2
|
|
136
|
+
):
|
|
137
|
+
raise ValueError("all arrays must align and contain at least two items")
|
|
138
|
+
rng = np.random.default_rng(seed)
|
|
139
|
+
order = rng.permutation(n)
|
|
140
|
+
split = n // 2
|
|
141
|
+
a, b = order[:split], order[split:]
|
|
142
|
+
gap_a, k_a, budget_a = _fold_gap(scores, tokens, mas_scores, mas_tokens, a, b)
|
|
143
|
+
gap_b, k_b, budget_b = _fold_gap(scores, tokens, mas_scores, mas_tokens, b, a)
|
|
144
|
+
return {
|
|
145
|
+
"gap": float((gap_a + gap_b) / 2),
|
|
146
|
+
"fold_a_gap": gap_a,
|
|
147
|
+
"fold_b_gap": gap_b,
|
|
148
|
+
"selected_k_fold_a": k_a,
|
|
149
|
+
"selected_k_fold_b": k_b,
|
|
150
|
+
"budget_fold_a": budget_a,
|
|
151
|
+
"budget_fold_b": budget_b,
|
|
152
|
+
"n_items": n,
|
|
153
|
+
"seed": seed,
|
|
154
|
+
}
|