fusionkit 0.3.1__tar.gz → 0.4.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
Metadata-Version: 2.3
|
|
2
2
|
Name: fusionkit
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.4.0
|
|
4
4
|
Summary: Command line interface for fusionkit: local response-level model fusion.
|
|
5
|
-
Requires-Dist: fusionkit-core==0.
|
|
6
|
-
Requires-Dist: fusionkit-evals==0.
|
|
7
|
-
Requires-Dist: fusionkit-mlx==0.
|
|
8
|
-
Requires-Dist: fusionkit-server==0.
|
|
5
|
+
Requires-Dist: fusionkit-core==0.4.0
|
|
6
|
+
Requires-Dist: fusionkit-evals==0.4.0
|
|
7
|
+
Requires-Dist: fusionkit-mlx==0.4.0
|
|
8
|
+
Requires-Dist: fusionkit-server==0.4.0
|
|
9
9
|
Requires-Dist: typer>=0.20.0
|
|
10
10
|
Requires-Python: >=3.11
|
|
@@ -1,13 +1,13 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "fusionkit"
|
|
3
|
-
version = "0.
|
|
3
|
+
version = "0.4.0"
|
|
4
4
|
description = "Command line interface for fusionkit: local response-level model fusion."
|
|
5
5
|
requires-python = ">=3.11"
|
|
6
6
|
dependencies = [
|
|
7
|
-
"fusionkit-core==0.
|
|
8
|
-
"fusionkit-evals==0.
|
|
9
|
-
"fusionkit-mlx==0.
|
|
10
|
-
"fusionkit-server==0.
|
|
7
|
+
"fusionkit-core==0.4.0",
|
|
8
|
+
"fusionkit-evals==0.4.0",
|
|
9
|
+
"fusionkit-mlx==0.4.0",
|
|
10
|
+
"fusionkit-server==0.4.0",
|
|
11
11
|
"typer>=0.20.0",
|
|
12
12
|
]
|
|
13
13
|
|
|
@@ -0,0 +1,616 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import asyncio
|
|
4
|
+
import json
|
|
5
|
+
import os
|
|
6
|
+
import shlex
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
from typing import Annotated, cast
|
|
9
|
+
|
|
10
|
+
import typer
|
|
11
|
+
import uvicorn
|
|
12
|
+
from fusionkit_core.clients import build_clients
|
|
13
|
+
from fusionkit_core.config import FusionMode, load_config
|
|
14
|
+
from fusionkit_core.fusion import FusionEngine
|
|
15
|
+
from fusionkit_core.prompts import SYSTEM_PROMPT_DEFAULTS
|
|
16
|
+
from fusionkit_evals.bench_history import BenchRunRecord, append_run, drift_vs_previous
|
|
17
|
+
from fusionkit_evals.benchmark import BenchmarkRunner, load_jsonl_samples, write_jsonl_results
|
|
18
|
+
from fusionkit_evals.benchmark_panel import get_benchmark_panel
|
|
19
|
+
from fusionkit_evals.candidate_bank import (
|
|
20
|
+
PreparedTask,
|
|
21
|
+
bank_signature,
|
|
22
|
+
build_candidate_bank,
|
|
23
|
+
load_bank,
|
|
24
|
+
save_bank,
|
|
25
|
+
)
|
|
26
|
+
from fusionkit_evals.fusion_bench import (
|
|
27
|
+
CommandHandoffKitExecutor,
|
|
28
|
+
FusionBenchReport,
|
|
29
|
+
FusionBenchRunner,
|
|
30
|
+
build_fusion_bench_report,
|
|
31
|
+
load_benchmark_tasks,
|
|
32
|
+
load_fusion_bench_jsonl,
|
|
33
|
+
write_fusion_bench_jsonl,
|
|
34
|
+
)
|
|
35
|
+
from fusionkit_evals.fusion_reports import (
|
|
36
|
+
write_fusion_bench_html_report,
|
|
37
|
+
write_fusion_bench_markdown_report,
|
|
38
|
+
write_fusion_bench_report_jsonl,
|
|
39
|
+
)
|
|
40
|
+
from fusionkit_evals.livecodebench_data import (
|
|
41
|
+
LCB_PROMPT_SUFFIX,
|
|
42
|
+
load_manifest,
|
|
43
|
+
load_problems,
|
|
44
|
+
prepare_tasks,
|
|
45
|
+
)
|
|
46
|
+
from fusionkit_evals.pareto import load_points, write_pareto_report
|
|
47
|
+
from fusionkit_evals.prompt_tuning import (
|
|
48
|
+
LLMProposer,
|
|
49
|
+
TunableRole,
|
|
50
|
+
TunerRuntime,
|
|
51
|
+
TuningResult,
|
|
52
|
+
optimize,
|
|
53
|
+
select_decision_tasks,
|
|
54
|
+
split_dev_val,
|
|
55
|
+
)
|
|
56
|
+
from fusionkit_evals.public_bench import (
|
|
57
|
+
PUBLIC_BENCHMARK_INFO,
|
|
58
|
+
PUBLIC_BENCHMARK_SUITES,
|
|
59
|
+
CommandExternalBenchmarkExecutor,
|
|
60
|
+
ExternalBenchmarkRequest,
|
|
61
|
+
PublicBenchmarkSuite,
|
|
62
|
+
baselines_for,
|
|
63
|
+
run_public_benchmark,
|
|
64
|
+
write_external_runs_jsonl,
|
|
65
|
+
)
|
|
66
|
+
from fusionkit_evals.public_bench_report import (
|
|
67
|
+
build_benchmark_comparison,
|
|
68
|
+
write_benchmark_comparison_markdown,
|
|
69
|
+
)
|
|
70
|
+
from fusionkit_evals.sandbox import SandboxConfig, build_sandbox
|
|
71
|
+
from fusionkit_evals.tiny import (
|
|
72
|
+
load_tiny_tasks,
|
|
73
|
+
run_tiny_benchmark,
|
|
74
|
+
write_tiny_benchmark_report,
|
|
75
|
+
write_tiny_jsonl,
|
|
76
|
+
)
|
|
77
|
+
from fusionkit_server.app import create_app
|
|
78
|
+
from fusionkit_server.openai_endpoint import build_endpoint, serve_single_endpoint
|
|
79
|
+
|
|
80
|
+
app = typer.Typer(help="Local model fusion toolkit.")
|
|
81
|
+
|
|
82
|
+
prompts_app = typer.Typer(help="Inspect and export the built-in fusion prompts.")
|
|
83
|
+
app.add_typer(prompts_app, name="prompts")
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
@prompts_app.command("dump")
|
|
87
|
+
def prompts_dump(
|
|
88
|
+
dir: Annotated[
|
|
89
|
+
Path | None,
|
|
90
|
+
typer.Option("--dir", help="write each default prompt to <dir>/<id>.md instead of stdout"),
|
|
91
|
+
] = None,
|
|
92
|
+
) -> None:
|
|
93
|
+
"""Emit the built-in system prompts so a consumer can scaffold editable overrides.
|
|
94
|
+
|
|
95
|
+
With no options this prints a JSON object mapping each prompt id (e.g.
|
|
96
|
+
``judge``, ``trajectory-step``) to its default text. With ``--dir`` it writes
|
|
97
|
+
one ``<id>.md`` file per prompt. This keeps the CLI's scaffolded
|
|
98
|
+
``.fusionkit/prompts`` defaults in lockstep with this package's source.
|
|
99
|
+
"""
|
|
100
|
+
if dir is not None:
|
|
101
|
+
dir.mkdir(parents=True, exist_ok=True)
|
|
102
|
+
for prompt_id, text in SYSTEM_PROMPT_DEFAULTS.items():
|
|
103
|
+
(dir / f"{prompt_id}.md").write_text(text + "\n")
|
|
104
|
+
typer.echo(json.dumps({"dir": str(dir), "count": len(SYSTEM_PROMPT_DEFAULTS)}))
|
|
105
|
+
return
|
|
106
|
+
typer.echo(json.dumps(SYSTEM_PROMPT_DEFAULTS, indent=2))
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
@app.command()
|
|
110
|
+
def serve(
|
|
111
|
+
config: Annotated[Path, typer.Option("--config", "-c")],
|
|
112
|
+
host: str = "127.0.0.1",
|
|
113
|
+
port: int = 8080,
|
|
114
|
+
) -> None:
|
|
115
|
+
fusion_config = load_config(config)
|
|
116
|
+
api = create_app(fusion_config)
|
|
117
|
+
uvicorn.run(api, host=host, port=port)
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
@app.command("serve-endpoint")
|
|
121
|
+
def serve_endpoint(
|
|
122
|
+
id: Annotated[str, typer.Option("--id", help="endpoint id exposed via /v1/models")],
|
|
123
|
+
model: Annotated[str, typer.Option("--model", help="provider model name (e.g. gpt-5.5)")],
|
|
124
|
+
port: Annotated[int, typer.Option("--port")],
|
|
125
|
+
provider: Annotated[str, typer.Option("--provider")] = "openai",
|
|
126
|
+
base_url: Annotated[
|
|
127
|
+
str | None, typer.Option("--base-url", help="override the provider base URL")
|
|
128
|
+
] = None,
|
|
129
|
+
api_key_env: Annotated[
|
|
130
|
+
str | None, typer.Option("--api-key-env", help="env var holding the API key")
|
|
131
|
+
] = None,
|
|
132
|
+
timeout_s: Annotated[float, typer.Option("--timeout-s")] = 120.0,
|
|
133
|
+
host: Annotated[str, typer.Option("--host")] = "127.0.0.1",
|
|
134
|
+
) -> None:
|
|
135
|
+
"""Front a single provider model as an OpenAI Chat Completions endpoint."""
|
|
136
|
+
endpoint = build_endpoint(
|
|
137
|
+
id=id,
|
|
138
|
+
model=model,
|
|
139
|
+
provider=provider,
|
|
140
|
+
base_url=base_url,
|
|
141
|
+
api_key_env=api_key_env,
|
|
142
|
+
timeout_s=timeout_s,
|
|
143
|
+
)
|
|
144
|
+
serve_single_endpoint(endpoint, host=host, port=port)
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
@app.command("eval")
|
|
148
|
+
def run_eval(
|
|
149
|
+
config: Annotated[Path, typer.Option("--config", "-c")],
|
|
150
|
+
samples: Annotated[Path, typer.Option("--samples", "-s")],
|
|
151
|
+
output: Annotated[Path, typer.Option("--output", "-o")],
|
|
152
|
+
mode: FusionMode = "single",
|
|
153
|
+
config_id: str = "local",
|
|
154
|
+
) -> None:
|
|
155
|
+
fusion_config = load_config(config)
|
|
156
|
+
clients = build_clients(fusion_config)
|
|
157
|
+
engine = FusionEngine(config=fusion_config, clients=clients)
|
|
158
|
+
runner = BenchmarkRunner(engine)
|
|
159
|
+
results = asyncio.run(runner.run_samples(load_jsonl_samples(samples), config_id, mode))
|
|
160
|
+
write_jsonl_results(output, results)
|
|
161
|
+
typer.echo(json.dumps({"results": len(results), "output": str(output)}))
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
@app.command()
|
|
165
|
+
def pareto(
|
|
166
|
+
points: Annotated[Path, typer.Option("--points", "-p")],
|
|
167
|
+
output: Annotated[Path, typer.Option("--output", "-o")],
|
|
168
|
+
) -> None:
|
|
169
|
+
write_pareto_report(output, load_points(points))
|
|
170
|
+
typer.echo(json.dumps({"output": str(output)}))
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
@app.command("tiny-bench")
|
|
174
|
+
def tiny_bench(
|
|
175
|
+
config: Annotated[Path, typer.Option("--config", "-c")],
|
|
176
|
+
output: Annotated[Path, typer.Option("--output", "-o")],
|
|
177
|
+
report: Annotated[Path | None, typer.Option("--report", "-r")] = None,
|
|
178
|
+
mode: FusionMode = "panel",
|
|
179
|
+
config_id: str = "local",
|
|
180
|
+
) -> None:
|
|
181
|
+
fusion_config = load_config(config)
|
|
182
|
+
clients = build_clients(fusion_config)
|
|
183
|
+
engine = FusionEngine(config=fusion_config, clients=clients)
|
|
184
|
+
results = asyncio.run(
|
|
185
|
+
run_tiny_benchmark(
|
|
186
|
+
engine,
|
|
187
|
+
config_id=config_id,
|
|
188
|
+
mode=mode,
|
|
189
|
+
tasks=load_tiny_tasks(),
|
|
190
|
+
model_versions={endpoint.id: endpoint.model for endpoint in fusion_config.endpoints},
|
|
191
|
+
)
|
|
192
|
+
)
|
|
193
|
+
write_tiny_jsonl(output, results)
|
|
194
|
+
response = {"results": len(results), "output": str(output)}
|
|
195
|
+
if report is not None:
|
|
196
|
+
write_tiny_benchmark_report(report, results)
|
|
197
|
+
response["report"] = str(report)
|
|
198
|
+
typer.echo(json.dumps(response))
|
|
199
|
+
|
|
200
|
+
|
|
201
|
+
@app.command("fusion-bench")
|
|
202
|
+
def fusion_bench(
|
|
203
|
+
config: Annotated[Path, typer.Option("--config", "-c")],
|
|
204
|
+
output: Annotated[Path, typer.Option("--output", "-o")],
|
|
205
|
+
manifest: Annotated[Path | None, typer.Option("--manifest", "-m")] = None,
|
|
206
|
+
run_root: Annotated[Path, typer.Option("--run-root")] = Path(".fusionkit/fusion-bench"),
|
|
207
|
+
report_jsonl: Annotated[Path | None, typer.Option("--report-jsonl")] = None,
|
|
208
|
+
report_markdown: Annotated[
|
|
209
|
+
Path | None,
|
|
210
|
+
typer.Option("--report", "-r", "--report-markdown"),
|
|
211
|
+
] = None,
|
|
212
|
+
report_html: Annotated[Path | None, typer.Option("--report-html")] = None,
|
|
213
|
+
handoff_command: Annotated[
|
|
214
|
+
str | None,
|
|
215
|
+
typer.Option(
|
|
216
|
+
"--handoff-command",
|
|
217
|
+
help=(
|
|
218
|
+
"Optional HandoffKit-compatible command. It receives task JSON on stdin "
|
|
219
|
+
"and emits model-fusion contract records on stdout."
|
|
220
|
+
),
|
|
221
|
+
),
|
|
222
|
+
] = None,
|
|
223
|
+
handoff_timeout_s: Annotated[
|
|
224
|
+
float,
|
|
225
|
+
typer.Option("--handoff-timeout-s", min=1.0),
|
|
226
|
+
] = 300.0,
|
|
227
|
+
mode: FusionMode = "panel",
|
|
228
|
+
config_id: str = "local",
|
|
229
|
+
) -> None:
|
|
230
|
+
fusion_config = load_config(config)
|
|
231
|
+
clients = build_clients(fusion_config)
|
|
232
|
+
engine = FusionEngine(config=fusion_config, clients=clients)
|
|
233
|
+
runner = FusionBenchRunner(
|
|
234
|
+
engine,
|
|
235
|
+
run_root=run_root,
|
|
236
|
+
config_id=config_id,
|
|
237
|
+
mode=mode,
|
|
238
|
+
model_versions={endpoint.id: endpoint.model for endpoint in fusion_config.endpoints},
|
|
239
|
+
handoff_executor=(
|
|
240
|
+
CommandHandoffKitExecutor(
|
|
241
|
+
shlex.split(handoff_command),
|
|
242
|
+
timeout_s=handoff_timeout_s,
|
|
243
|
+
)
|
|
244
|
+
if handoff_command is not None
|
|
245
|
+
else None
|
|
246
|
+
),
|
|
247
|
+
)
|
|
248
|
+
tasks = load_benchmark_tasks(manifest) if manifest else load_benchmark_tasks()
|
|
249
|
+
rows = asyncio.run(runner.run_tasks(tasks))
|
|
250
|
+
write_fusion_bench_jsonl(output, rows)
|
|
251
|
+
response: dict[str, int | str] = {"rows": len(rows), "output": str(output)}
|
|
252
|
+
response.update(
|
|
253
|
+
_write_fusion_bench_reports(
|
|
254
|
+
build_fusion_bench_report(rows),
|
|
255
|
+
report_jsonl=report_jsonl,
|
|
256
|
+
report_markdown=report_markdown,
|
|
257
|
+
report_html=report_html,
|
|
258
|
+
)
|
|
259
|
+
)
|
|
260
|
+
typer.echo(json.dumps(response))
|
|
261
|
+
|
|
262
|
+
|
|
263
|
+
@app.command("fusion-bench-report")
|
|
264
|
+
def fusion_bench_report(
|
|
265
|
+
input_path: Annotated[Path, typer.Option("--input", "-i")],
|
|
266
|
+
report_jsonl: Annotated[Path | None, typer.Option("--jsonl")] = None,
|
|
267
|
+
report_markdown: Annotated[Path | None, typer.Option("--markdown", "-m")] = None,
|
|
268
|
+
report_html: Annotated[Path | None, typer.Option("--html")] = None,
|
|
269
|
+
) -> None:
|
|
270
|
+
rows = load_fusion_bench_jsonl(input_path)
|
|
271
|
+
report = build_fusion_bench_report(rows)
|
|
272
|
+
response: dict[str, int | str] = {
|
|
273
|
+
"rows": len(rows),
|
|
274
|
+
"tasks": report.aggregate.total_tasks,
|
|
275
|
+
"skipped": report.aggregate.skipped_tasks,
|
|
276
|
+
"failed": report.aggregate.failed_tasks,
|
|
277
|
+
}
|
|
278
|
+
response.update(
|
|
279
|
+
_write_fusion_bench_reports(
|
|
280
|
+
report,
|
|
281
|
+
report_jsonl=report_jsonl,
|
|
282
|
+
report_markdown=report_markdown,
|
|
283
|
+
report_html=report_html,
|
|
284
|
+
)
|
|
285
|
+
)
|
|
286
|
+
typer.echo(json.dumps(response))
|
|
287
|
+
|
|
288
|
+
|
|
289
|
+
@app.command("public-bench")
|
|
290
|
+
def public_bench(
|
|
291
|
+
suite: Annotated[str, typer.Option("--suite", help="public benchmark suite to run")],
|
|
292
|
+
panel: Annotated[
|
|
293
|
+
str,
|
|
294
|
+
typer.Option("--panel", help="benchmark panel id (e.g. decorrelated-peers)"),
|
|
295
|
+
] = "decorrelated-peers",
|
|
296
|
+
gateway_base_url: Annotated[
|
|
297
|
+
str, typer.Option("--gateway-base-url")
|
|
298
|
+
] = "http://127.0.0.1:8080",
|
|
299
|
+
gateway_model: Annotated[
|
|
300
|
+
str | None,
|
|
301
|
+
typer.Option("--gateway-model", help="gateway model alias (defaults per suite)"),
|
|
302
|
+
] = None,
|
|
303
|
+
runner_command: Annotated[
|
|
304
|
+
str | None,
|
|
305
|
+
typer.Option(
|
|
306
|
+
"--runner-command",
|
|
307
|
+
help=(
|
|
308
|
+
"External runner adapter. Receives the request JSON on stdin and emits a "
|
|
309
|
+
"normalized run envelope on stdout. Omit to produce an 'unavailable' report."
|
|
310
|
+
),
|
|
311
|
+
),
|
|
312
|
+
] = None,
|
|
313
|
+
subset: Annotated[
|
|
314
|
+
int | None,
|
|
315
|
+
typer.Option("--subset", min=1, help="run only the first N tasks (subset-first)"),
|
|
316
|
+
] = None,
|
|
317
|
+
runner_timeout_s: Annotated[float, typer.Option("--runner-timeout-s", min=1.0)] = 1800.0,
|
|
318
|
+
output: Annotated[Path | None, typer.Option("--output", "-o")] = None,
|
|
319
|
+
report_markdown: Annotated[Path | None, typer.Option("--report", "-r")] = None,
|
|
320
|
+
ledger: Annotated[
|
|
321
|
+
Path | None,
|
|
322
|
+
typer.Option("--ledger", help="append the run to a history ledger and report drift"),
|
|
323
|
+
] = None,
|
|
324
|
+
) -> None:
|
|
325
|
+
"""Run a public coding benchmark against the gateway and compare to leaderboards."""
|
|
326
|
+
|
|
327
|
+
resolved_suite = _resolve_public_suite(suite)
|
|
328
|
+
info = PUBLIC_BENCHMARK_INFO[resolved_suite]
|
|
329
|
+
benchmark_panel = get_benchmark_panel(panel)
|
|
330
|
+
request = ExternalBenchmarkRequest(
|
|
331
|
+
suite=resolved_suite,
|
|
332
|
+
mount_mode=info.mount_mode,
|
|
333
|
+
gateway_base_url=gateway_base_url,
|
|
334
|
+
gateway_model=gateway_model or info.default_gateway_model,
|
|
335
|
+
panel_id=benchmark_panel.panel_id,
|
|
336
|
+
subset=subset,
|
|
337
|
+
)
|
|
338
|
+
executor = (
|
|
339
|
+
CommandExternalBenchmarkExecutor(
|
|
340
|
+
shlex.split(runner_command),
|
|
341
|
+
timeout_s=runner_timeout_s,
|
|
342
|
+
)
|
|
343
|
+
if runner_command is not None
|
|
344
|
+
else None
|
|
345
|
+
)
|
|
346
|
+
run = asyncio.run(run_public_benchmark(request, executor))
|
|
347
|
+
comparison = build_benchmark_comparison(run, benchmark_panel)
|
|
348
|
+
response: dict[str, int | str | float | None] = {
|
|
349
|
+
"suite": run.suite,
|
|
350
|
+
"panel": run.panel_id,
|
|
351
|
+
"availability": run.availability,
|
|
352
|
+
"fusion_score": comparison.fusion_score,
|
|
353
|
+
"fusion_ci": _fmt_ci(comparison.fusion_ci_low, comparison.fusion_ci_high),
|
|
354
|
+
"measured_oracle": comparison.measured_oracle,
|
|
355
|
+
"measured_regret": comparison.measured_regret,
|
|
356
|
+
"oracle_headroom": comparison.oracle_headroom,
|
|
357
|
+
"lopsided_panel": int(comparison.lopsided),
|
|
358
|
+
"infra_error_tasks": run.infra_error_tasks,
|
|
359
|
+
"excluded_tasks": run.excluded_tasks,
|
|
360
|
+
}
|
|
361
|
+
if output is not None:
|
|
362
|
+
write_external_runs_jsonl(output, [run])
|
|
363
|
+
response["output"] = str(output)
|
|
364
|
+
if report_markdown is not None:
|
|
365
|
+
write_benchmark_comparison_markdown(report_markdown, comparison)
|
|
366
|
+
response["report"] = str(report_markdown)
|
|
367
|
+
if ledger is not None:
|
|
368
|
+
record = BenchRunRecord(
|
|
369
|
+
suite=run.suite,
|
|
370
|
+
panel_id=run.panel_id,
|
|
371
|
+
resolved_tasks=run.resolved_tasks,
|
|
372
|
+
score=comparison.fusion_score,
|
|
373
|
+
ci_low=comparison.fusion_ci_low,
|
|
374
|
+
ci_high=comparison.fusion_ci_high,
|
|
375
|
+
cache_signature=_optional_str(run.raw_metadata.get("cache_signature")),
|
|
376
|
+
repo_sha=_optional_str(run.provenance.get("repo_sha")),
|
|
377
|
+
)
|
|
378
|
+
drift = drift_vs_previous(ledger, record)
|
|
379
|
+
append_run(ledger, record)
|
|
380
|
+
response["ledger"] = str(ledger)
|
|
381
|
+
if drift is not None:
|
|
382
|
+
response["drift_delta"] = drift.delta
|
|
383
|
+
response["regressed"] = int(drift.regressed)
|
|
384
|
+
typer.echo(json.dumps(response))
|
|
385
|
+
|
|
386
|
+
|
|
387
|
+
def _fmt_ci(low: float | None, high: float | None) -> str | None:
|
|
388
|
+
if low is None or high is None:
|
|
389
|
+
return None
|
|
390
|
+
return f"[{low:.4f}, {high:.4f}]"
|
|
391
|
+
|
|
392
|
+
|
|
393
|
+
def _optional_str(value: object) -> str | None:
|
|
394
|
+
return value if isinstance(value, str) and value else None
|
|
395
|
+
|
|
396
|
+
|
|
397
|
+
@app.command("public-bench-baselines")
|
|
398
|
+
def public_bench_baselines(
|
|
399
|
+
suite: Annotated[
|
|
400
|
+
str | None,
|
|
401
|
+
typer.Option("--suite", help="limit to one suite; omit for all suites"),
|
|
402
|
+
] = None,
|
|
403
|
+
) -> None:
|
|
404
|
+
"""Print the published leaderboard baselines used for comparison."""
|
|
405
|
+
|
|
406
|
+
suites = (PUBLIC_BENCHMARK_SUITES if suite is None else (_resolve_public_suite(suite),))
|
|
407
|
+
payload = {
|
|
408
|
+
target: [baseline.model_dump(mode="json") for baseline in baselines_for(target)]
|
|
409
|
+
for target in suites
|
|
410
|
+
}
|
|
411
|
+
typer.echo(json.dumps(payload, indent=2))
|
|
412
|
+
|
|
413
|
+
|
|
414
|
+
_ROLE_PROMPT_FILE = {
|
|
415
|
+
"judge_system": "judge.md",
|
|
416
|
+
"synthesizer_system": "synthesizer.md",
|
|
417
|
+
"verifier_system": "verifier.md",
|
|
418
|
+
}
|
|
419
|
+
|
|
420
|
+
|
|
421
|
+
@app.command("tune-prompts")
|
|
422
|
+
def tune_prompts(
|
|
423
|
+
config: Annotated[Path, typer.Option("--config", "-c", help="panel FusionConfig YAML")],
|
|
424
|
+
bank: Annotated[
|
|
425
|
+
Path, typer.Option("--bank", help="candidate bank path (built if missing)")
|
|
426
|
+
] = Path(".fusionkit/tuning/bank.json"),
|
|
427
|
+
role: Annotated[str, typer.Option("--role", help="prompt role to tune")] = "synthesizer_system",
|
|
428
|
+
subset: Annotated[int, typer.Option("--subset", min=1)] = 40,
|
|
429
|
+
bank_max_tests: Annotated[
|
|
430
|
+
int, typer.Option("--bank-max-tests", help="cap tests/task when building the bank (0=all)")
|
|
431
|
+
] = 0,
|
|
432
|
+
optimizer_model: Annotated[
|
|
433
|
+
str | None, typer.Option("--optimizer-model", help="endpoint id (default: judge)")
|
|
434
|
+
] = None,
|
|
435
|
+
max_iterations: Annotated[int, typer.Option("--max-iterations", min=1)] = 8,
|
|
436
|
+
patience: Annotated[int, typer.Option("--patience", min=1)] = 3,
|
|
437
|
+
val_fraction: Annotated[float, typer.Option("--val-fraction", min=0.1, max=0.9)] = 0.4,
|
|
438
|
+
seed: Annotated[int, typer.Option("--seed")] = 0,
|
|
439
|
+
verify: Annotated[bool, typer.Option("--verify/--no-verify")] = False,
|
|
440
|
+
test_timeout_s: Annotated[float, typer.Option("--test-timeout-s", min=1.0)] = 8.0,
|
|
441
|
+
concurrency: Annotated[int, typer.Option("--concurrency", min=1)] = 4,
|
|
442
|
+
prompts_out: Annotated[Path, typer.Option("--prompts-out")] = Path(".fusionkit/prompts"),
|
|
443
|
+
cache_dir: Annotated[Path, typer.Option("--cache-dir")] = Path(".fusionkit/tuning/cache"),
|
|
444
|
+
report: Annotated[Path | None, typer.Option("--report", "-r")] = None,
|
|
445
|
+
ledger: Annotated[Path | None, typer.Option("--ledger")] = None,
|
|
446
|
+
) -> None:
|
|
447
|
+
"""Automated LLM-driven tuning of judge/synth/verifier prompts over a frozen bank."""
|
|
448
|
+
|
|
449
|
+
if role not in _ROLE_PROMPT_FILE:
|
|
450
|
+
raise typer.BadParameter(f"role must be one of {sorted(_ROLE_PROMPT_FILE)}")
|
|
451
|
+
resolved_role = cast(TunableRole, role)
|
|
452
|
+
fusion_config = load_config(config)
|
|
453
|
+
clients = build_clients(fusion_config)
|
|
454
|
+
engine = FusionEngine(config=fusion_config, clients=clients)
|
|
455
|
+
sandbox = build_sandbox(SandboxConfig(backend=os.environ.get("BENCH_SANDBOX", "local")))
|
|
456
|
+
|
|
457
|
+
if bank.exists():
|
|
458
|
+
candidate_bank = load_bank(bank)
|
|
459
|
+
else:
|
|
460
|
+
problems = load_problems(
|
|
461
|
+
subset,
|
|
462
|
+
version=os.environ.get("LCB_VERSION", "release_v6"),
|
|
463
|
+
min_date=os.environ.get("LCB_MIN_DATE", "2025-01-01"),
|
|
464
|
+
manifest=load_manifest(os.environ.get("LCB_MANIFEST")),
|
|
465
|
+
)
|
|
466
|
+
prepared = [
|
|
467
|
+
PreparedTask(**task) for task in prepare_tasks(problems, max_tests=bank_max_tests)
|
|
468
|
+
]
|
|
469
|
+
signature = bank_signature(engine, prompt_suffix=LCB_PROMPT_SUFFIX)
|
|
470
|
+
candidate_bank = asyncio.run(
|
|
471
|
+
build_candidate_bank(
|
|
472
|
+
engine,
|
|
473
|
+
sandbox,
|
|
474
|
+
prepared,
|
|
475
|
+
signature=signature,
|
|
476
|
+
test_timeout_s=test_timeout_s,
|
|
477
|
+
concurrency=concurrency,
|
|
478
|
+
)
|
|
479
|
+
)
|
|
480
|
+
save_bank(bank, candidate_bank)
|
|
481
|
+
|
|
482
|
+
decision = select_decision_tasks(candidate_bank)
|
|
483
|
+
if len(decision) < 2:
|
|
484
|
+
typer.echo(
|
|
485
|
+
json.dumps(
|
|
486
|
+
{
|
|
487
|
+
"error": "not enough decision tasks to tune",
|
|
488
|
+
"decision_tasks": len(decision),
|
|
489
|
+
"total_tasks": len(candidate_bank.tasks),
|
|
490
|
+
}
|
|
491
|
+
)
|
|
492
|
+
)
|
|
493
|
+
raise typer.Exit(code=1)
|
|
494
|
+
split = split_dev_val(decision, val_fraction=val_fraction, seed=seed)
|
|
495
|
+
by_id = {task.task_id: task for task in candidate_bank.tasks}
|
|
496
|
+
dev_tasks = [by_id[task_id] for task_id in split.dev]
|
|
497
|
+
val_tasks = [by_id[task_id] for task_id in split.val]
|
|
498
|
+
|
|
499
|
+
runtime = TunerRuntime(
|
|
500
|
+
clients=clients,
|
|
501
|
+
judge_id=fusion_config.resolved_judge_model,
|
|
502
|
+
synth_id=fusion_config.resolved_synthesizer_model,
|
|
503
|
+
bank_signature=candidate_bank.signature,
|
|
504
|
+
sandbox=sandbox,
|
|
505
|
+
cache_dir=cache_dir,
|
|
506
|
+
judge_sampling=fusion_config.sampling.model_copy(update={"temperature": 0.0}),
|
|
507
|
+
synth_sampling=fusion_config.sampling,
|
|
508
|
+
test_timeout_s=test_timeout_s,
|
|
509
|
+
verify=verify,
|
|
510
|
+
concurrency=concurrency,
|
|
511
|
+
)
|
|
512
|
+
proposer = LLMProposer(
|
|
513
|
+
clients[optimizer_model or fusion_config.resolved_judge_model],
|
|
514
|
+
fusion_config.sampling,
|
|
515
|
+
)
|
|
516
|
+
result = asyncio.run(
|
|
517
|
+
optimize(
|
|
518
|
+
runtime,
|
|
519
|
+
dev_tasks=dev_tasks,
|
|
520
|
+
val_tasks=val_tasks,
|
|
521
|
+
proposer=proposer,
|
|
522
|
+
role=resolved_role,
|
|
523
|
+
max_iterations=max_iterations,
|
|
524
|
+
patience=patience,
|
|
525
|
+
)
|
|
526
|
+
)
|
|
527
|
+
|
|
528
|
+
tuned_prompt = result.best_variant.role_text(resolved_role)
|
|
529
|
+
response: dict[str, object] = {
|
|
530
|
+
"role": result.role,
|
|
531
|
+
"decision_tasks": len(decision),
|
|
532
|
+
"dev": len(dev_tasks),
|
|
533
|
+
"val": len(val_tasks),
|
|
534
|
+
"baseline_dev": result.baseline_dev.score,
|
|
535
|
+
"best_dev": result.best_dev.score,
|
|
536
|
+
"baseline_val": result.baseline_val.score,
|
|
537
|
+
"best_val": result.best_val.score,
|
|
538
|
+
"val_ci": _fmt_ci(result.best_val.ci_low, result.best_val.ci_high),
|
|
539
|
+
"accepted_trials": sum(1 for trial in result.trials if trial.accepted),
|
|
540
|
+
"trials": len(result.trials),
|
|
541
|
+
"improved": result.best_val.score > result.baseline_val.score,
|
|
542
|
+
}
|
|
543
|
+
if tuned_prompt is not None and result.best_val.score > result.baseline_val.score:
|
|
544
|
+
prompts_out.mkdir(parents=True, exist_ok=True)
|
|
545
|
+
out_file = prompts_out / _ROLE_PROMPT_FILE[role]
|
|
546
|
+
out_file.write_text(tuned_prompt + "\n", encoding="utf-8")
|
|
547
|
+
response["prompts_out"] = str(out_file)
|
|
548
|
+
if report is not None:
|
|
549
|
+
report.parent.mkdir(parents=True, exist_ok=True)
|
|
550
|
+
report.write_text(_format_tuning_report(result), encoding="utf-8")
|
|
551
|
+
response["report"] = str(report)
|
|
552
|
+
if ledger is not None:
|
|
553
|
+
append_run(
|
|
554
|
+
ledger,
|
|
555
|
+
BenchRunRecord(
|
|
556
|
+
suite="prompt-tuning",
|
|
557
|
+
panel_id=result.role,
|
|
558
|
+
resolved_tasks=len(val_tasks),
|
|
559
|
+
score=result.best_val.score,
|
|
560
|
+
ci_low=result.best_val.ci_low,
|
|
561
|
+
ci_high=result.best_val.ci_high,
|
|
562
|
+
),
|
|
563
|
+
)
|
|
564
|
+
response["ledger"] = str(ledger)
|
|
565
|
+
typer.echo(json.dumps(response))
|
|
566
|
+
|
|
567
|
+
|
|
568
|
+
def _format_tuning_report(result: TuningResult) -> str:
|
|
569
|
+
lines = [
|
|
570
|
+
"# Prompt Tuning Report",
|
|
571
|
+
"",
|
|
572
|
+
f"- Role: {result.role}",
|
|
573
|
+
f"- Baseline dev: {result.baseline_dev.score:.4f} -> best dev: {result.best_dev.score:.4f}",
|
|
574
|
+
f"- Baseline val: {result.baseline_val.score:.4f} -> best val: {result.best_val.score:.4f} "
|
|
575
|
+
f"(95% CI [{result.best_val.ci_low:.4f}, {result.best_val.ci_high:.4f}])",
|
|
576
|
+
f"- Promoted: {result.best_val.score > result.baseline_val.score}",
|
|
577
|
+
"",
|
|
578
|
+
"## Trials (dev)",
|
|
579
|
+
"",
|
|
580
|
+
"| Iter | dev score | wins | losses | accepted |",
|
|
581
|
+
"| ---: | ---: | ---: | ---: | :--: |",
|
|
582
|
+
]
|
|
583
|
+
for trial in result.trials:
|
|
584
|
+
lines.append(
|
|
585
|
+
f"| {trial.iteration} | {trial.dev_score:.4f} | {trial.wins} | {trial.losses} | "
|
|
586
|
+
f"{'yes' if trial.accepted else 'no'} |"
|
|
587
|
+
)
|
|
588
|
+
lines.append("")
|
|
589
|
+
return "\n".join(lines)
|
|
590
|
+
|
|
591
|
+
|
|
592
|
+
def _resolve_public_suite(suite: str) -> PublicBenchmarkSuite:
|
|
593
|
+
if suite not in PUBLIC_BENCHMARK_SUITES:
|
|
594
|
+
known = ", ".join(PUBLIC_BENCHMARK_SUITES)
|
|
595
|
+
raise typer.BadParameter(f"unknown suite {suite!r}; choose one of: {known}")
|
|
596
|
+
return cast(PublicBenchmarkSuite, suite)
|
|
597
|
+
|
|
598
|
+
|
|
599
|
+
def _write_fusion_bench_reports(
|
|
600
|
+
report: FusionBenchReport,
|
|
601
|
+
*,
|
|
602
|
+
report_jsonl: Path | None,
|
|
603
|
+
report_markdown: Path | None,
|
|
604
|
+
report_html: Path | None,
|
|
605
|
+
) -> dict[str, str]:
|
|
606
|
+
outputs = {}
|
|
607
|
+
if report_jsonl is not None:
|
|
608
|
+
write_fusion_bench_report_jsonl(report_jsonl, report)
|
|
609
|
+
outputs["report_jsonl"] = str(report_jsonl)
|
|
610
|
+
if report_markdown is not None:
|
|
611
|
+
write_fusion_bench_markdown_report(report_markdown, report)
|
|
612
|
+
outputs["report_markdown"] = str(report_markdown)
|
|
613
|
+
if report_html is not None:
|
|
614
|
+
write_fusion_bench_html_report(report_html, report)
|
|
615
|
+
outputs["report_html"] = str(report_html)
|
|
616
|
+
return outputs
|
|
@@ -1,266 +0,0 @@
|
|
|
1
|
-
from __future__ import annotations
|
|
2
|
-
|
|
3
|
-
import asyncio
|
|
4
|
-
import json
|
|
5
|
-
import shlex
|
|
6
|
-
from pathlib import Path
|
|
7
|
-
from typing import Annotated
|
|
8
|
-
|
|
9
|
-
import typer
|
|
10
|
-
import uvicorn
|
|
11
|
-
from fusionkit_core.clients import build_clients
|
|
12
|
-
from fusionkit_core.config import FusionMode, load_config
|
|
13
|
-
from fusionkit_core.fusion import FusionEngine
|
|
14
|
-
from fusionkit_core.prompts import SYSTEM_PROMPT_DEFAULTS
|
|
15
|
-
from fusionkit_evals.benchmark import BenchmarkRunner, load_jsonl_samples, write_jsonl_results
|
|
16
|
-
from fusionkit_evals.fusion_bench import (
|
|
17
|
-
CommandHandoffKitExecutor,
|
|
18
|
-
FusionBenchReport,
|
|
19
|
-
FusionBenchRunner,
|
|
20
|
-
build_fusion_bench_report,
|
|
21
|
-
load_benchmark_tasks,
|
|
22
|
-
load_fusion_bench_jsonl,
|
|
23
|
-
write_fusion_bench_jsonl,
|
|
24
|
-
)
|
|
25
|
-
from fusionkit_evals.fusion_reports import (
|
|
26
|
-
write_fusion_bench_html_report,
|
|
27
|
-
write_fusion_bench_markdown_report,
|
|
28
|
-
write_fusion_bench_report_jsonl,
|
|
29
|
-
)
|
|
30
|
-
from fusionkit_evals.pareto import load_points, write_pareto_report
|
|
31
|
-
from fusionkit_evals.tiny import (
|
|
32
|
-
load_tiny_tasks,
|
|
33
|
-
run_tiny_benchmark,
|
|
34
|
-
write_tiny_benchmark_report,
|
|
35
|
-
write_tiny_jsonl,
|
|
36
|
-
)
|
|
37
|
-
from fusionkit_server.app import create_app
|
|
38
|
-
from fusionkit_server.openai_endpoint import build_endpoint, serve_single_endpoint
|
|
39
|
-
|
|
40
|
-
app = typer.Typer(help="Local model fusion toolkit.")
|
|
41
|
-
|
|
42
|
-
prompts_app = typer.Typer(help="Inspect and export the built-in fusion prompts.")
|
|
43
|
-
app.add_typer(prompts_app, name="prompts")
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
@prompts_app.command("dump")
|
|
47
|
-
def prompts_dump(
|
|
48
|
-
dir: Annotated[
|
|
49
|
-
Path | None,
|
|
50
|
-
typer.Option("--dir", help="write each default prompt to <dir>/<id>.md instead of stdout"),
|
|
51
|
-
] = None,
|
|
52
|
-
) -> None:
|
|
53
|
-
"""Emit the built-in system prompts so a consumer can scaffold editable overrides.
|
|
54
|
-
|
|
55
|
-
With no options this prints a JSON object mapping each prompt id (e.g.
|
|
56
|
-
``judge``, ``trajectory-step``) to its default text. With ``--dir`` it writes
|
|
57
|
-
one ``<id>.md`` file per prompt. This keeps the CLI's scaffolded
|
|
58
|
-
``.fusionkit/prompts`` defaults in lockstep with this package's source.
|
|
59
|
-
"""
|
|
60
|
-
if dir is not None:
|
|
61
|
-
dir.mkdir(parents=True, exist_ok=True)
|
|
62
|
-
for prompt_id, text in SYSTEM_PROMPT_DEFAULTS.items():
|
|
63
|
-
(dir / f"{prompt_id}.md").write_text(text + "\n")
|
|
64
|
-
typer.echo(json.dumps({"dir": str(dir), "count": len(SYSTEM_PROMPT_DEFAULTS)}))
|
|
65
|
-
return
|
|
66
|
-
typer.echo(json.dumps(SYSTEM_PROMPT_DEFAULTS, indent=2))
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
@app.command()
|
|
70
|
-
def serve(
|
|
71
|
-
config: Annotated[Path, typer.Option("--config", "-c")],
|
|
72
|
-
host: str = "127.0.0.1",
|
|
73
|
-
port: int = 8080,
|
|
74
|
-
) -> None:
|
|
75
|
-
fusion_config = load_config(config)
|
|
76
|
-
api = create_app(fusion_config)
|
|
77
|
-
uvicorn.run(api, host=host, port=port)
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
@app.command("serve-endpoint")
|
|
81
|
-
def serve_endpoint(
|
|
82
|
-
id: Annotated[str, typer.Option("--id", help="endpoint id exposed via /v1/models")],
|
|
83
|
-
model: Annotated[str, typer.Option("--model", help="provider model name (e.g. gpt-5.5)")],
|
|
84
|
-
port: Annotated[int, typer.Option("--port")],
|
|
85
|
-
provider: Annotated[str, typer.Option("--provider")] = "openai",
|
|
86
|
-
base_url: Annotated[
|
|
87
|
-
str | None, typer.Option("--base-url", help="override the provider base URL")
|
|
88
|
-
] = None,
|
|
89
|
-
api_key_env: Annotated[
|
|
90
|
-
str | None, typer.Option("--api-key-env", help="env var holding the API key")
|
|
91
|
-
] = None,
|
|
92
|
-
timeout_s: Annotated[float, typer.Option("--timeout-s")] = 120.0,
|
|
93
|
-
host: Annotated[str, typer.Option("--host")] = "127.0.0.1",
|
|
94
|
-
) -> None:
|
|
95
|
-
"""Front a single provider model as an OpenAI Chat Completions endpoint."""
|
|
96
|
-
endpoint = build_endpoint(
|
|
97
|
-
id=id,
|
|
98
|
-
model=model,
|
|
99
|
-
provider=provider,
|
|
100
|
-
base_url=base_url,
|
|
101
|
-
api_key_env=api_key_env,
|
|
102
|
-
timeout_s=timeout_s,
|
|
103
|
-
)
|
|
104
|
-
serve_single_endpoint(endpoint, host=host, port=port)
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
@app.command("eval")
|
|
108
|
-
def run_eval(
|
|
109
|
-
config: Annotated[Path, typer.Option("--config", "-c")],
|
|
110
|
-
samples: Annotated[Path, typer.Option("--samples", "-s")],
|
|
111
|
-
output: Annotated[Path, typer.Option("--output", "-o")],
|
|
112
|
-
mode: FusionMode = "single",
|
|
113
|
-
config_id: str = "local",
|
|
114
|
-
) -> None:
|
|
115
|
-
fusion_config = load_config(config)
|
|
116
|
-
clients = build_clients(fusion_config)
|
|
117
|
-
engine = FusionEngine(config=fusion_config, clients=clients)
|
|
118
|
-
runner = BenchmarkRunner(engine)
|
|
119
|
-
results = asyncio.run(runner.run_samples(load_jsonl_samples(samples), config_id, mode))
|
|
120
|
-
write_jsonl_results(output, results)
|
|
121
|
-
typer.echo(json.dumps({"results": len(results), "output": str(output)}))
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
@app.command()
|
|
125
|
-
def pareto(
|
|
126
|
-
points: Annotated[Path, typer.Option("--points", "-p")],
|
|
127
|
-
output: Annotated[Path, typer.Option("--output", "-o")],
|
|
128
|
-
) -> None:
|
|
129
|
-
write_pareto_report(output, load_points(points))
|
|
130
|
-
typer.echo(json.dumps({"output": str(output)}))
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
@app.command("tiny-bench")
|
|
134
|
-
def tiny_bench(
|
|
135
|
-
config: Annotated[Path, typer.Option("--config", "-c")],
|
|
136
|
-
output: Annotated[Path, typer.Option("--output", "-o")],
|
|
137
|
-
report: Annotated[Path | None, typer.Option("--report", "-r")] = None,
|
|
138
|
-
mode: FusionMode = "panel",
|
|
139
|
-
config_id: str = "local",
|
|
140
|
-
) -> None:
|
|
141
|
-
fusion_config = load_config(config)
|
|
142
|
-
clients = build_clients(fusion_config)
|
|
143
|
-
engine = FusionEngine(config=fusion_config, clients=clients)
|
|
144
|
-
results = asyncio.run(
|
|
145
|
-
run_tiny_benchmark(
|
|
146
|
-
engine,
|
|
147
|
-
config_id=config_id,
|
|
148
|
-
mode=mode,
|
|
149
|
-
tasks=load_tiny_tasks(),
|
|
150
|
-
model_versions={endpoint.id: endpoint.model for endpoint in fusion_config.endpoints},
|
|
151
|
-
)
|
|
152
|
-
)
|
|
153
|
-
write_tiny_jsonl(output, results)
|
|
154
|
-
response = {"results": len(results), "output": str(output)}
|
|
155
|
-
if report is not None:
|
|
156
|
-
write_tiny_benchmark_report(report, results)
|
|
157
|
-
response["report"] = str(report)
|
|
158
|
-
typer.echo(json.dumps(response))
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
@app.command("fusion-bench")
|
|
162
|
-
def fusion_bench(
|
|
163
|
-
config: Annotated[Path, typer.Option("--config", "-c")],
|
|
164
|
-
output: Annotated[Path, typer.Option("--output", "-o")],
|
|
165
|
-
manifest: Annotated[Path | None, typer.Option("--manifest", "-m")] = None,
|
|
166
|
-
run_root: Annotated[Path, typer.Option("--run-root")] = Path(".fusionkit/fusion-bench"),
|
|
167
|
-
report_jsonl: Annotated[Path | None, typer.Option("--report-jsonl")] = None,
|
|
168
|
-
report_markdown: Annotated[
|
|
169
|
-
Path | None,
|
|
170
|
-
typer.Option("--report", "-r", "--report-markdown"),
|
|
171
|
-
] = None,
|
|
172
|
-
report_html: Annotated[Path | None, typer.Option("--report-html")] = None,
|
|
173
|
-
handoff_command: Annotated[
|
|
174
|
-
str | None,
|
|
175
|
-
typer.Option(
|
|
176
|
-
"--handoff-command",
|
|
177
|
-
help=(
|
|
178
|
-
"Optional HandoffKit-compatible command. It receives task JSON on stdin "
|
|
179
|
-
"and emits model-fusion contract records on stdout."
|
|
180
|
-
),
|
|
181
|
-
),
|
|
182
|
-
] = None,
|
|
183
|
-
handoff_timeout_s: Annotated[
|
|
184
|
-
float,
|
|
185
|
-
typer.Option("--handoff-timeout-s", min=1.0),
|
|
186
|
-
] = 300.0,
|
|
187
|
-
mode: FusionMode = "panel",
|
|
188
|
-
config_id: str = "local",
|
|
189
|
-
) -> None:
|
|
190
|
-
fusion_config = load_config(config)
|
|
191
|
-
clients = build_clients(fusion_config)
|
|
192
|
-
engine = FusionEngine(config=fusion_config, clients=clients)
|
|
193
|
-
runner = FusionBenchRunner(
|
|
194
|
-
engine,
|
|
195
|
-
run_root=run_root,
|
|
196
|
-
config_id=config_id,
|
|
197
|
-
mode=mode,
|
|
198
|
-
model_versions={endpoint.id: endpoint.model for endpoint in fusion_config.endpoints},
|
|
199
|
-
handoff_executor=(
|
|
200
|
-
CommandHandoffKitExecutor(
|
|
201
|
-
shlex.split(handoff_command),
|
|
202
|
-
timeout_s=handoff_timeout_s,
|
|
203
|
-
)
|
|
204
|
-
if handoff_command is not None
|
|
205
|
-
else None
|
|
206
|
-
),
|
|
207
|
-
)
|
|
208
|
-
tasks = load_benchmark_tasks(manifest) if manifest else load_benchmark_tasks()
|
|
209
|
-
rows = asyncio.run(runner.run_tasks(tasks))
|
|
210
|
-
write_fusion_bench_jsonl(output, rows)
|
|
211
|
-
response: dict[str, int | str] = {"rows": len(rows), "output": str(output)}
|
|
212
|
-
response.update(
|
|
213
|
-
_write_fusion_bench_reports(
|
|
214
|
-
build_fusion_bench_report(rows),
|
|
215
|
-
report_jsonl=report_jsonl,
|
|
216
|
-
report_markdown=report_markdown,
|
|
217
|
-
report_html=report_html,
|
|
218
|
-
)
|
|
219
|
-
)
|
|
220
|
-
typer.echo(json.dumps(response))
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
@app.command("fusion-bench-report")
|
|
224
|
-
def fusion_bench_report(
|
|
225
|
-
input_path: Annotated[Path, typer.Option("--input", "-i")],
|
|
226
|
-
report_jsonl: Annotated[Path | None, typer.Option("--jsonl")] = None,
|
|
227
|
-
report_markdown: Annotated[Path | None, typer.Option("--markdown", "-m")] = None,
|
|
228
|
-
report_html: Annotated[Path | None, typer.Option("--html")] = None,
|
|
229
|
-
) -> None:
|
|
230
|
-
rows = load_fusion_bench_jsonl(input_path)
|
|
231
|
-
report = build_fusion_bench_report(rows)
|
|
232
|
-
response: dict[str, int | str] = {
|
|
233
|
-
"rows": len(rows),
|
|
234
|
-
"tasks": report.aggregate.total_tasks,
|
|
235
|
-
"skipped": report.aggregate.skipped_tasks,
|
|
236
|
-
"failed": report.aggregate.failed_tasks,
|
|
237
|
-
}
|
|
238
|
-
response.update(
|
|
239
|
-
_write_fusion_bench_reports(
|
|
240
|
-
report,
|
|
241
|
-
report_jsonl=report_jsonl,
|
|
242
|
-
report_markdown=report_markdown,
|
|
243
|
-
report_html=report_html,
|
|
244
|
-
)
|
|
245
|
-
)
|
|
246
|
-
typer.echo(json.dumps(response))
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
def _write_fusion_bench_reports(
|
|
250
|
-
report: FusionBenchReport,
|
|
251
|
-
*,
|
|
252
|
-
report_jsonl: Path | None,
|
|
253
|
-
report_markdown: Path | None,
|
|
254
|
-
report_html: Path | None,
|
|
255
|
-
) -> dict[str, str]:
|
|
256
|
-
outputs = {}
|
|
257
|
-
if report_jsonl is not None:
|
|
258
|
-
write_fusion_bench_report_jsonl(report_jsonl, report)
|
|
259
|
-
outputs["report_jsonl"] = str(report_jsonl)
|
|
260
|
-
if report_markdown is not None:
|
|
261
|
-
write_fusion_bench_markdown_report(report_markdown, report)
|
|
262
|
-
outputs["report_markdown"] = str(report_markdown)
|
|
263
|
-
if report_html is not None:
|
|
264
|
-
write_fusion_bench_html_report(report_html, report)
|
|
265
|
-
outputs["report_html"] = str(report_html)
|
|
266
|
-
return outputs
|
|
File without changes
|