pace-eval 0.2.2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (83) hide show
  1. cme/__init__.py +11 -0
  2. cme/__main__.py +5 -0
  3. cme/analysis.py +223 -0
  4. cme/audit.py +145 -0
  5. cme/baseline.py +162 -0
  6. cme/cli.py +279 -0
  7. cme/client.py +165 -0
  8. cme/dynamics.py +212 -0
  9. cme/examples/crewai_integration.py +16 -0
  10. cme/examples/demo_pool.jsonl +360 -0
  11. cme/examples/demo_run.jsonl +480 -0
  12. cme/examples/langgraph_integration.py +27 -0
  13. cme/examples/make_demo.py +37 -0
  14. cme/examples/openai_quickstart.py +18 -0
  15. cme/experiment.py +555 -0
  16. cme/frontier.py +143 -0
  17. cme/hidden_profile.py +135 -0
  18. cme/proxy.py +106 -0
  19. cme/rebuttal.py +76 -0
  20. cme/report.py +197 -0
  21. cme/rollout_card.py +137 -0
  22. cme/tasks.py +266 -0
  23. cme/verified_debate.py +127 -0
  24. cme/verifier.py +92 -0
  25. mae_eval/__init__.py +11 -0
  26. mae_eval/__main__.py +5 -0
  27. mae_eval/auditor.py +112 -0
  28. mae_eval/classifier.py +62 -0
  29. mae_eval/cli.py +210 -0
  30. mae_eval/tracker.py +162 -0
  31. mae_eval/vector.py +91 -0
  32. pace/__init__.py +11 -0
  33. pace/__main__.py +5 -0
  34. pace/analysis.py +129 -0
  35. pace/audit.py +145 -0
  36. pace/baseline.py +162 -0
  37. pace/card.py +205 -0
  38. pace/cli.py +898 -0
  39. pace/client.py +165 -0
  40. pace/dynamics.py +372 -0
  41. pace/escape_benchmarks.py +162 -0
  42. pace/examples/crewai_integration.py +16 -0
  43. pace/examples/demo_pool.jsonl +360 -0
  44. pace/examples/demo_run.jsonl +480 -0
  45. pace/examples/langgraph_integration.py +27 -0
  46. pace/examples/make_demo.py +37 -0
  47. pace/examples/openai_quickstart.py +18 -0
  48. pace/experiment.py +555 -0
  49. pace/frontier.py +114 -0
  50. pace/hidden_profile.py +301 -0
  51. pace/importers.py +219 -0
  52. pace/inflate.py +213 -0
  53. pace/influence.py +231 -0
  54. pace/meta_engine.py +246 -0
  55. pace/middleware.py +230 -0
  56. pace/open_ended.py +114 -0
  57. pace/plotting.py +132 -0
  58. pace/pool_sizer.py +154 -0
  59. pace/protocol_v3.py +414 -0
  60. pace/proxy.py +106 -0
  61. pace/rebuttal.py +219 -0
  62. pace/report.py +197 -0
  63. pace/rollout_card.py +137 -0
  64. pace/schema.py +74 -0
  65. pace/simulator.py +199 -0
  66. pace/studio.html +1040 -0
  67. pace/tasks.py +266 -0
  68. pace/verified_debate.py +127 -0
  69. pace/verifier.py +92 -0
  70. pace/workbench.html +26 -0
  71. pace/workbench.py +119 -0
  72. pace_eval-0.2.2.dist-info/METADATA +205 -0
  73. pace_eval-0.2.2.dist-info/RECORD +83 -0
  74. pace_eval-0.2.2.dist-info/WHEEL +5 -0
  75. pace_eval-0.2.2.dist-info/entry_points.txt +4 -0
  76. pace_eval-0.2.2.dist-info/licenses/LICENSE +202 -0
  77. pace_eval-0.2.2.dist-info/top_level.txt +4 -0
  78. pace_lab/__init__.py +9 -0
  79. pace_lab/clients.py +568 -0
  80. pace_lab/concurrency_test.py +127 -0
  81. pace_lab/data.py +445 -0
  82. pace_lab/graders.py +626 -0
  83. pace_lab/strategies.py +345 -0
pace/middleware.py ADDED
@@ -0,0 +1,230 @@
1
+ """Runtime accounting middleware, DAG latency, and scaffold decomposition."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import functools
6
+ import json
7
+ from collections.abc import Callable
8
+ from typing import Any
9
+
10
+ from .card import build_card
11
+
12
+ MODEL_PRICING_USD_PER_1M = {
13
+ name: prices
14
+ for name, prices in {
15
+ "gpt-4o": (5.0, 1.25, 15.0),
16
+ "gpt-4.1": (2.0, 0.5, 8.0),
17
+ "gpt-5": (1.25, 0.125, 10.0),
18
+ "o1": (15.0, 7.5, 60.0),
19
+ "o3": (2.0, 0.5, 8.0),
20
+ "claude-3-5-sonnet": (3.0, 0.3, 15.0),
21
+ "claude-3-7-sonnet": (3.0, 0.3, 15.0),
22
+ "claude-4": (3.0, 0.3, 15.0),
23
+ "gemini-1.5-pro": (1.25, 0.315, 5.0),
24
+ "gemini-2.5-pro": (1.25, 0.125, 10.0),
25
+ "deepseek-r1": (0.55, 0.14, 2.19),
26
+ "deepseek-v3": (0.27, 0.07, 1.1),
27
+ "qwen2.5": (0.4, 0.1, 1.6),
28
+ "llama-3.1-70b": (0.88, 0.0, 0.88),
29
+ "llama-3.3-70b": (0.88, 0.0, 0.88),
30
+ "gpt-4-turbo": (10.0, 0.0, 30.0),
31
+ "claude-3-opus": (15.0, 1.5, 75.0),
32
+ "claude-3-haiku": (0.25, 0.03, 1.25),
33
+ "gemini-1.5-flash": (0.075, 0.01875, 0.3),
34
+ "mistral-large": (2.0, 0.0, 6.0),
35
+ "command-r-plus": (2.5, 0.0, 10.0),
36
+ }.items()
37
+ }
38
+
39
+
40
+ def estimate_call_cost(
41
+ model: str,
42
+ prompt_tokens: float,
43
+ completion_tokens: float,
44
+ cached_prompt_tokens: float = 0.0,
45
+ reasoning_tokens: float = 0.0,
46
+ pricing: dict | None = None,
47
+ ) -> dict:
48
+ """Estimate USD cost while reporting uncached, cached, reasoning, and completion components."""
49
+ table = pricing or MODEL_PRICING_USD_PER_1M
50
+ if (
51
+ model not in table
52
+ or min(prompt_tokens, completion_tokens, cached_prompt_tokens, reasoning_tokens)
53
+ < 0
54
+ or cached_prompt_tokens > prompt_tokens
55
+ ):
56
+ raise ValueError("unknown model or invalid token counts")
57
+ input_price, cached_price, output_price = table[model]
58
+ uncached = prompt_tokens - cached_prompt_tokens
59
+ parts = {
60
+ "uncached_prompt": uncached * input_price / 1e6,
61
+ "cached_prompt": cached_prompt_tokens * cached_price / 1e6,
62
+ "reasoning": reasoning_tokens * output_price / 1e6,
63
+ "completion": completion_tokens * output_price / 1e6,
64
+ }
65
+ return {
66
+ "model": model,
67
+ "input_tokens": prompt_tokens,
68
+ "cached_prompt_tokens": cached_prompt_tokens,
69
+ "reasoning_tokens": reasoning_tokens,
70
+ "completion_tokens": completion_tokens,
71
+ "cost_usd": sum(parts.values()),
72
+ "components": parts,
73
+ }
74
+
75
+
76
+ def compute_dag_latency(calls: list[dict]) -> dict:
77
+ if not calls:
78
+ raise ValueError("calls cannot be empty")
79
+ durations = {
80
+ str(call["call_id"]): float(
81
+ call.get(
82
+ "duration_ms",
83
+ float(call.get("end_ms", 0)) - float(call.get("start_ms", 0)),
84
+ )
85
+ )
86
+ for call in calls
87
+ }
88
+ if any(value < 0 for value in durations.values()):
89
+ raise ValueError("durations cannot be negative")
90
+ total = sum(durations.values())
91
+ parents = {
92
+ str(call["call_id"]): [str(dep) for dep in call.get("depends_on", [])]
93
+ for call in calls
94
+ }
95
+ memo = {}
96
+
97
+ def longest(node):
98
+ if node in memo:
99
+ return memo[node]
100
+ memo[node] = durations[node] + max(
101
+ (longest(parent) for parent in parents[node]), default=0.0
102
+ )
103
+ return memo[node]
104
+
105
+ critical = (
106
+ max((longest(node) for node in durations), default=0.0)
107
+ if any(parents.values())
108
+ else sum(
109
+ max(
110
+ durations[str(call["call_id"])]
111
+ for call in calls
112
+ if call.get("round", 0) == round_no
113
+ )
114
+ for round_no in sorted({call.get("round", 0) for call in calls})
115
+ )
116
+ )
117
+ workers = max(
118
+ sum(1 for call in calls if call.get("round", 0) == round_no)
119
+ for round_no in {call.get("round", 0) for call in calls}
120
+ )
121
+ return {
122
+ "total_sequential_ms": total,
123
+ "critical_path_ms": critical,
124
+ "concurrency_speedup": total / max(critical, 1e-9),
125
+ "max_parallel_workers": workers,
126
+ }
127
+
128
+
129
+ def _word_count(value: Any) -> int:
130
+ if isinstance(value, str):
131
+ return len(value.split())
132
+ if isinstance(value, list):
133
+ return sum(
134
+ _word_count(item.get("content", "") if isinstance(item, dict) else item)
135
+ for item in value
136
+ )
137
+ return 0
138
+
139
+
140
+ def decompose_scaffold_tokens(calls: list[dict]) -> dict:
141
+ if not calls:
142
+ raise ValueError("calls cannot be empty")
143
+ total = sum(
144
+ _word_count(call.get("prompt_text", call.get("messages", "")))
145
+ + float(call.get("completion_tokens", 0))
146
+ for call in calls
147
+ )
148
+ c0 = _word_count(calls[0].get("prompt_text", calls[0].get("messages", "")))
149
+ echoed = sum(
150
+ max(0, _word_count(call.get("prompt_text", call.get("messages", ""))) - c0)
151
+ for call in calls[1:]
152
+ )
153
+ reasoning = sum(
154
+ float(call.get("completion_tokens", call.get("reasoning_tokens", 0)))
155
+ for call in calls
156
+ )
157
+ return {
158
+ "c0": c0,
159
+ "echoed_peer_tokens": echoed,
160
+ "reasoning_tokens": reasoning,
161
+ "total_tokens": total,
162
+ "scaffold_dilution_ratio": 1 - reasoning / total if total else 0.0,
163
+ }
164
+
165
+
166
+ class PaceSession:
167
+ """Collect decorated call records and flush validated JSONL on exit."""
168
+
169
+ current: PaceSession | None = None
170
+
171
+ def __init__(
172
+ self,
173
+ path: str,
174
+ item: str = "item_0",
175
+ agent: str = "agent_0",
176
+ round_num: int = 1,
177
+ ):
178
+ self.path, self.item, self.agent, self.round_num, self.records = (
179
+ path,
180
+ item,
181
+ agent,
182
+ round_num,
183
+ [],
184
+ )
185
+
186
+ def __enter__(self):
187
+ self._previous, PaceSession.current = PaceSession.current, self
188
+ return self
189
+
190
+ def __exit__(self, exc_type, exc_value, traceback):
191
+ if exc_type is None:
192
+ with open(self.path, "w", encoding="utf-8") as handle:
193
+ handle.writelines(json.dumps(row) + "\n" for row in self.records)
194
+ PaceSession.current = self._previous
195
+ return False
196
+
197
+ def record(
198
+ self,
199
+ correct: bool,
200
+ answer: str | None = None,
201
+ prompt_tokens: float = 0,
202
+ completion_tokens: float = 0,
203
+ ) -> None:
204
+ self.records.append(
205
+ {
206
+ "item": self.item,
207
+ "agent": self.agent,
208
+ "round": self.round_num,
209
+ "correct": bool(correct),
210
+ "answer": answer,
211
+ "prompt_tokens": prompt_tokens,
212
+ "completion_tokens": completion_tokens,
213
+ }
214
+ )
215
+
216
+ def card(self) -> dict:
217
+ return build_card(self.records)
218
+
219
+
220
+ def pace_track(function: Callable) -> Callable:
221
+ """Decorate a function and record its boolean result in the active session."""
222
+
223
+ @functools.wraps(function)
224
+ def wrapper(*args, **kwargs):
225
+ result = function(*args, **kwargs)
226
+ if PaceSession.current is not None:
227
+ PaceSession.current.record(bool(result), answer=str(result))
228
+ return result
229
+
230
+ return wrapper
pace/open_ended.py ADDED
@@ -0,0 +1,114 @@
1
+ """Noncommunicating open-ended sample aggregation utilities."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import difflib
6
+ import math
7
+ import re
8
+ from collections import Counter
9
+
10
+ import numpy as np
11
+
12
+
13
+ def _tokens(text: str) -> list[str]:
14
+ return re.findall(r"[A-Za-z0-9_]+", text.lower())
15
+
16
+
17
+ def _features(text: str, metric: str, n: int) -> Counter:
18
+ source = text if metric == "ngram_cosine" else " ".join(_tokens(text))
19
+ return Counter(source[i : i + n] for i in range(max(0, len(source) - n + 1)))
20
+
21
+
22
+ def similarity_matrix(
23
+ candidates: list[str], metric: str = "ngram_cosine", n: int = 3
24
+ ) -> np.ndarray:
25
+ if (
26
+ metric
27
+ not in {"ngram_cosine", "token_jaccard", "sequence_ratio", "code_structural"}
28
+ or n < 1
29
+ ):
30
+ raise ValueError("unsupported metric or n")
31
+ size = len(candidates)
32
+ matrix = np.eye(size)
33
+ for i in range(size):
34
+ for j in range(i):
35
+ if metric == "sequence_ratio":
36
+ score = difflib.SequenceMatcher(
37
+ None, candidates[i], candidates[j]
38
+ ).ratio()
39
+ elif metric == "token_jaccard":
40
+ left, right = set(_tokens(candidates[i])), set(_tokens(candidates[j]))
41
+ score = len(left & right) / len(left | right) if left | right else 1.0
42
+ else:
43
+ left, right = (
44
+ _features(candidates[i], metric, n),
45
+ _features(candidates[j], metric, n),
46
+ )
47
+ if metric == "code_structural":
48
+ left = Counter(_tokens(candidates[i]) + candidates[i].splitlines())
49
+ right = Counter(_tokens(candidates[j]) + candidates[j].splitlines())
50
+ dot = sum(left[k] * right[k] for k in set(left) | set(right))
51
+ denom = math.sqrt(
52
+ sum(v * v for v in left.values())
53
+ * sum(v * v for v in right.values())
54
+ )
55
+ score = dot / denom if denom else 0.0
56
+ matrix[i, j] = matrix[j, i] = score
57
+ return matrix
58
+
59
+
60
+ def mbr_centroid_select(
61
+ candidates: list[str],
62
+ metric: str = "ngram_cosine",
63
+ token_costs: list[float] | None = None,
64
+ ) -> dict:
65
+ if not candidates:
66
+ raise ValueError("candidates cannot be empty")
67
+ if token_costs is not None and len(token_costs) != len(candidates):
68
+ raise ValueError("token_costs must match candidates")
69
+ matrix = similarity_matrix(candidates, metric)
70
+ scores = (matrix.sum(axis=1) - 1) / max(1, len(candidates) - 1)
71
+ index = int(np.argmax(scores))
72
+ off_diag = (matrix.sum() - len(candidates)) / max(
73
+ 1, len(candidates) * (len(candidates) - 1)
74
+ )
75
+ return {
76
+ "index": index,
77
+ "selected": candidates[index],
78
+ "consensus_score": float(scores[index]),
79
+ "diversity_dispersion": float(1 - off_diag),
80
+ "total_token_cost": float(sum(token_costs)) if token_costs else None,
81
+ "similarity": matrix.tolist(),
82
+ }
83
+
84
+
85
+ def unbiased_pass_at_k_frontier(
86
+ n_total_samples: int,
87
+ c_correct_samples: int,
88
+ k_values: list[int],
89
+ mean_tokens_per_sample: float,
90
+ ) -> list[dict]:
91
+ if (
92
+ not 0 <= c_correct_samples <= n_total_samples
93
+ or n_total_samples < 1
94
+ or mean_tokens_per_sample < 0
95
+ ):
96
+ raise ValueError("invalid sample counts or token cost")
97
+ out = []
98
+ for k in k_values:
99
+ if not 1 <= k <= n_total_samples:
100
+ raise ValueError("k must be in [1, n_total_samples]")
101
+ bad = (
102
+ math.comb(n_total_samples - c_correct_samples, k)
103
+ if n_total_samples - c_correct_samples >= k
104
+ else 0
105
+ )
106
+ out.append(
107
+ {
108
+ "k": k,
109
+ "pass_at_k": 1 - bad / math.comb(n_total_samples, k),
110
+ "token_budget": k * mean_tokens_per_sample,
111
+ "verifier_calls": k,
112
+ }
113
+ )
114
+ return out
pace/plotting.py ADDED
@@ -0,0 +1,132 @@
1
+ """Pure SVG publication figures with no plotting dependency."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import html
6
+
7
+
8
+ def _svg(body: str, title: str, width: int = 800, height: int = 440) -> str:
9
+ return (
10
+ '<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 %d %d" role="img"><title>%s</title><style>text{font-family:Georgia,serif;fill:#17212b}.axis{stroke:#17212b}.grid{stroke:#d8d0c3}.bar{fill:#bb4d2e}.line{fill:none;stroke:#1d6b68;stroke-width:3}.point{fill:#bb4d2e}</style>%s</svg>'
11
+ % (width, height, html.escape(title), body)
12
+ )
13
+
14
+
15
+ def svg_waterfall_decomposition(
16
+ p_greedy: float,
17
+ p_sampled: float,
18
+ p_mas: float,
19
+ title: str = "PACE Accounting Decomposition",
20
+ ) -> str:
21
+ values = [p_greedy, p_sampled - p_greedy, p_sampled, p_mas - p_sampled]
22
+ labels = [
23
+ "Category A",
24
+ "Compute Scaling Effect",
25
+ "Category B",
26
+ "Coordination Residual",
27
+ ]
28
+ body = '<line class="axis" x1="55" y1="380" x2="760" y2="380"/>'
29
+ for i, (value, label) in enumerate(zip(values, labels)):
30
+ x = 75 + i * 170
31
+ y = 380 - abs(value) * 280
32
+ h = abs(value) * 280
33
+ body += (
34
+ '<rect class="bar" x="%d" y="%.1f" width="100" height="%.1f"/><text x="%d" y="405" font-size="12">%s</text><text x="%d" y="%.1f" font-size="12">%.3f</text>'
35
+ % (x, y, h, x, html.escape(label), x, y - 6, value)
36
+ )
37
+ return _svg(body, title)
38
+
39
+
40
+ def svg_phase_diagram(
41
+ rho_grid: list[float] | None = None,
42
+ k_team: int = 5,
43
+ empirical_point: tuple[float, float] | None = None,
44
+ ) -> str:
45
+ rho_grid = rho_grid or [0, 0.25, 0.5, 0.75]
46
+ body = '<line class="axis" x1="55" y1="190" x2="370" y2="190"/><line class="axis" x1="55" y1="30" x2="55" y2="190"/><line class="axis" x1="430" y1="190" x2="760" y2="190"/><line class="axis" x1="430" y1="30" x2="430" y2="190"/><text x="65" y="22">A: K_eff vs K</text><text x="440" y="22">B: beneficial phase boundary</text>'
47
+ for index, rho in enumerate(rho_grid):
48
+ points = []
49
+ for k in range(1, k_team + 1):
50
+ points.append(
51
+ "%d,%.1f" % (55 + k * 55, 190 - (k / (1 + (k - 1) * rho)) * 25)
52
+ )
53
+ body += (
54
+ '<polyline class="line" points="%s"/><text x="%d" y="210" font-size="11">rho=%.2f</text>'
55
+ % (" ".join(points), 60 + index * 75, rho)
56
+ )
57
+ boundary = []
58
+ for i in range(1, 10):
59
+ kappa = i / 2
60
+ boundary.append("%.1f,%.1f" % (430 + i * 30, 190 - (kappa / (1 + kappa)) * 150))
61
+ body += '<polyline class="line" points="%s"/>' % " ".join(boundary)
62
+ if empirical_point:
63
+ body += '<circle class="point" cx="%.1f" cy="%.1f" r="6"/>' % (
64
+ 430 + empirical_point[0] * 30,
65
+ 190 - empirical_point[1] * 150,
66
+ )
67
+ return _svg(body, "PACE Phase Diagram")
68
+
69
+
70
+ def svg_frontier_curve(
71
+ frontier_points: list[dict], mas_points: list[dict] | None = None
72
+ ) -> str:
73
+ body = '<line class="axis" x1="55" y1="380" x2="760" y2="380"/><line class="axis" x1="55" y1="30" x2="55" y2="380"/>'
74
+ points = [
75
+ "%.1f,%.1f"
76
+ % (
77
+ 55
78
+ + float(row["tokens"])
79
+ / max(1, max(float(item["tokens"]) for item in frontier_points))
80
+ * 680,
81
+ 380 - float(row["accuracy"]) * 330,
82
+ )
83
+ for row in frontier_points
84
+ ]
85
+ body += '<polyline class="line" points="%s"/>' % " ".join(points)
86
+ for row in mas_points or []:
87
+ body += (
88
+ '<circle class="point" cx="%.1f" cy="%.1f" r="5"/><text x="%.1f" y="%.1f" font-size="12">%s</text>'
89
+ % (
90
+ 55
91
+ + float(row["tokens"])
92
+ / max(1, max(float(item["tokens"]) for item in frontier_points))
93
+ * 680,
94
+ 380 - float(row["accuracy"]) * 330,
95
+ 60
96
+ + float(row["tokens"])
97
+ / max(1, max(float(item["tokens"]) for item in frontier_points))
98
+ * 680,
99
+ 375 - float(row["accuracy"]) * 330,
100
+ html.escape(str(row.get("label", "MAS"))),
101
+ )
102
+ )
103
+ return _svg(body, "PACE Frontier Curve")
104
+
105
+
106
+ def svg_influence_curves(
107
+ by_correct_peers: dict, fit_result: dict, k_agents: int
108
+ ) -> str:
109
+ body = '<line class="axis" x1="55" y1="380" x2="760" y2="380"/><line class="axis" x1="55" y1="30" x2="55" y2="380"/>'
110
+ for key, cell in by_correct_peers.items():
111
+ own, peers = (
112
+ key
113
+ if isinstance(key, tuple)
114
+ else (
115
+ int(str(key).split("=")[1].split(",")[0]),
116
+ int(str(key).split("=")[-1]),
117
+ )
118
+ )
119
+ x = peers / max(1, k_agents - 1)
120
+ y = cell.get("p_switch", cell.get("p_correct_next", 0))
121
+ body += '<circle class="point" cx="%.1f" cy="%.1f" r="4"/>' % (
122
+ 55 + x * 680,
123
+ 380 - y * 330,
124
+ )
125
+ for name, model in fit_result.get("models", {}).items():
126
+ power = 0 if name == "constant" else 1 if name == "linear_martingale" else 2
127
+ points = [
128
+ "%.1f,%.1f" % (55 + x * 680, 380 - min(1, model["a"] * x**power) * 330)
129
+ for x in [i / 20 for i in range(21)]
130
+ ]
131
+ body += '<polyline class="line" points="%s"/>' % " ".join(points)
132
+ return _svg(body, "PACE Peer Influence Curves")
pace/pool_sizer.py ADDED
@@ -0,0 +1,154 @@
1
+ """Constructive token budgets and split-sample frontiers."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import math
6
+
7
+ import numpy as np
8
+
9
+
10
+ def compute_k_tok(
11
+ k_agents: int,
12
+ t_rounds: int,
13
+ c0_prompt_tokens: float,
14
+ l_message_tokens: float,
15
+ topology: str = "last_round_peers",
16
+ ) -> dict:
17
+ if k_agents < 1 or t_rounds < 1:
18
+ raise ValueError("agents and rounds must be positive")
19
+ if topology not in {"last_round_peers", "last_round_all", "full_history_echo"}:
20
+ raise ValueError("unknown topology: %s" % topology)
21
+ denominator = c0_prompt_tokens + l_message_tokens
22
+ ratio = l_message_tokens / denominator if denominator > 0 else 0.0
23
+ base = k_agents * t_rounds
24
+ if topology == "last_round_peers":
25
+ coefficient = (t_rounds - 1) * k_agents * (k_agents - 1)
26
+ k_tok = base + coefficient * ratio
27
+ elif topology == "last_round_all":
28
+ coefficient = (t_rounds - 1) * k_agents**2
29
+ k_tok = base + coefficient * ratio
30
+ else:
31
+ coefficient = 0.5 * k_agents**2 * t_rounds * (t_rounds - 1)
32
+ k_tok = base + coefficient * ratio
33
+
34
+ def b_eff(budget: float) -> float:
35
+ return float(budget) / k_tok if k_tok else float("inf")
36
+
37
+ recommended = max(8, 1)
38
+ while recommended < math.ceil(k_tok):
39
+ recommended *= 2
40
+ return {
41
+ "k_tok": float(k_tok),
42
+ "r": float(ratio),
43
+ "topology": topology,
44
+ "bounds": [float(base), float(base + coefficient)],
45
+ "k_min": float(base),
46
+ "k_max": float(base + coefficient),
47
+ "b_eff": b_eff,
48
+ "recommended_pool_size": recommended,
49
+ }
50
+
51
+
52
+ def isotonic_pava_frontier(
53
+ budgets: list[float], accuracies: list[float], weights: list[float] | None = None
54
+ ) -> list[dict]:
55
+ if len(budgets) != len(accuracies) or not budgets:
56
+ raise ValueError("budgets and accuracies must have equal nonzero length")
57
+ weights = weights or [1.0] * len(budgets)
58
+ if len(weights) != len(budgets):
59
+ raise ValueError("weights must match budgets")
60
+ data = sorted(
61
+ zip(map(float, budgets), map(float, accuracies), map(float, weights)),
62
+ key=lambda row: row[0],
63
+ )
64
+ blocks = []
65
+ for budget, accuracy, weight in data:
66
+ blocks.append([budget, accuracy, weight, 1])
67
+ while len(blocks) > 1 and blocks[-2][1] > blocks[-1][1]:
68
+ right = blocks.pop()
69
+ left = blocks.pop()
70
+ weight_sum = left[2] + right[2]
71
+ blocks.append(
72
+ [
73
+ left[0],
74
+ (left[1] * left[2] + right[1] * right[2]) / weight_sum,
75
+ weight_sum,
76
+ left[3] + right[3],
77
+ ]
78
+ )
79
+ fitted = []
80
+ cursor = 0
81
+ for budget, accuracy, weight, count in blocks:
82
+ fitted.extend(
83
+ {
84
+ "budget": data[cursor + i][0],
85
+ "accuracy": data[cursor + i][1],
86
+ "fitted_accuracy": accuracy,
87
+ "weight": data[cursor + i][2],
88
+ }
89
+ for i in range(count)
90
+ )
91
+ cursor += count
92
+ return fitted
93
+
94
+
95
+ def _fold_gap(scores, tokens, mas_scores, mas_tokens, train, test):
96
+ mas_budget = float(np.mean(mas_tokens[train]))
97
+ candidates = [
98
+ (k, float(np.mean(scores[train, k])))
99
+ for k in range(scores.shape[1])
100
+ if float(np.mean(tokens[train, k])) <= mas_budget
101
+ ]
102
+ if not candidates:
103
+ k = 0
104
+ else:
105
+ k = max(candidates, key=lambda pair: pair[1])[0]
106
+ return (
107
+ float(np.mean(mas_scores[test]) - np.mean(scores[test, k])),
108
+ int(k + 1),
109
+ mas_budget,
110
+ )
111
+
112
+
113
+ def split_sample_frontier_gap(
114
+ item_pool_scores: np.ndarray,
115
+ item_pool_tokens: np.ndarray,
116
+ mas_item_scores: np.ndarray,
117
+ mas_item_tokens: np.ndarray,
118
+ seed: int = 0,
119
+ ) -> dict:
120
+ scores = np.asarray(item_pool_scores, dtype=float)
121
+ tokens = np.asarray(item_pool_tokens, dtype=float)
122
+ if scores.ndim != 2:
123
+ raise ValueError("item_pool_scores must be items by candidate k")
124
+ if tokens.ndim == 1:
125
+ tokens = np.broadcast_to(tokens, scores.shape)
126
+ mas_scores, mas_tokens = (
127
+ np.asarray(mas_item_scores, dtype=float).reshape(-1),
128
+ np.asarray(mas_item_tokens, dtype=float).reshape(-1),
129
+ )
130
+ n = scores.shape[0]
131
+ if (
132
+ tokens.shape != scores.shape
133
+ or len(mas_scores) != n
134
+ or len(mas_tokens) != n
135
+ or n < 2
136
+ ):
137
+ raise ValueError("all arrays must align and contain at least two items")
138
+ rng = np.random.default_rng(seed)
139
+ order = rng.permutation(n)
140
+ split = n // 2
141
+ a, b = order[:split], order[split:]
142
+ gap_a, k_a, budget_a = _fold_gap(scores, tokens, mas_scores, mas_tokens, a, b)
143
+ gap_b, k_b, budget_b = _fold_gap(scores, tokens, mas_scores, mas_tokens, b, a)
144
+ return {
145
+ "gap": float((gap_a + gap_b) / 2),
146
+ "fold_a_gap": gap_a,
147
+ "fold_b_gap": gap_b,
148
+ "selected_k_fold_a": k_a,
149
+ "selected_k_fold_b": k_b,
150
+ "budget_fold_a": budget_a,
151
+ "budget_fold_b": budget_b,
152
+ "n_items": n,
153
+ "seed": seed,
154
+ }