slmeval-cli 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- slmeval/__init__.py +71 -0
- slmeval/__main__.py +14 -0
- slmeval/adapters/__init__.py +24 -0
- slmeval/adapters/instrument.py +135 -0
- slmeval/adapters/langgraph.py +237 -0
- slmeval/cli/__init__.py +15 -0
- slmeval/cli/commands/__init__.py +35 -0
- slmeval/cli/commands/baseline_cmd.py +201 -0
- slmeval/cli/commands/cache_cmd.py +249 -0
- slmeval/cli/commands/dataset_cmd.py +52 -0
- slmeval/cli/commands/doctor_cmd.py +68 -0
- slmeval/cli/commands/gate_cmd.py +602 -0
- slmeval/cli/commands/init_cmd.py +144 -0
- slmeval/cli/commands/judge_cmd.py +284 -0
- slmeval/cli/commands/record_cmd.py +154 -0
- slmeval/cli/commands/report_cmd.py +98 -0
- slmeval/cli/commands/run_cmd.py +247 -0
- slmeval/cli/commands/show_cmd.py +482 -0
- slmeval/cli/commands/stubs.py +57 -0
- slmeval/cli/commands/validate_cmd.py +296 -0
- slmeval/cli/loading.py +149 -0
- slmeval/cli/main.py +91 -0
- slmeval/cli/render.py +251 -0
- slmeval/cli/wrap.py +31 -0
- slmeval/config/__init__.py +47 -0
- slmeval/config/loader.py +363 -0
- slmeval/config/locate.py +97 -0
- slmeval/config/models.py +430 -0
- slmeval/config/templates.py +246 -0
- slmeval/core/__init__.py +78 -0
- slmeval/core/adapter.py +253 -0
- slmeval/core/context.py +418 -0
- slmeval/core/outcome.py +260 -0
- slmeval/core/router.py +319 -0
- slmeval/core/runner.py +1209 -0
- slmeval/core/trace.py +188 -0
- slmeval/core/verdict.py +127 -0
- slmeval/dataset/__init__.py +8 -0
- slmeval/dataset/loader.py +130 -0
- slmeval/dataset/models.py +108 -0
- slmeval/doctor/__init__.py +7 -0
- slmeval/doctor/checks.py +380 -0
- slmeval/errors.py +76 -0
- slmeval/eval/__init__.py +64 -0
- slmeval/eval/assertions.py +356 -0
- slmeval/eval/calibration.py +364 -0
- slmeval/eval/cost.py +291 -0
- slmeval/eval/pipeline.py +273 -0
- slmeval/eval/stats.py +376 -0
- slmeval/eval/tier1.py +510 -0
- slmeval/eval/tier2.py +582 -0
- slmeval/eval/tier3.py +415 -0
- slmeval/exit_codes.py +28 -0
- slmeval/gate.py +722 -0
- slmeval/providers/__init__.py +69 -0
- slmeval/providers/base.py +426 -0
- slmeval/providers/fake.py +760 -0
- slmeval/providers/jsonx.py +239 -0
- slmeval/providers/ladder.py +339 -0
- slmeval/providers/openai_compat.py +392 -0
- slmeval/providers/registry.py +185 -0
- slmeval/providers/retry.py +81 -0
- slmeval/providers/schema.py +136 -0
- slmeval/providers/tokens.py +150 -0
- slmeval/py.typed +0 -0
- slmeval/report/__init__.py +115 -0
- slmeval/report/attribution.py +197 -0
- slmeval/report/baseline.py +353 -0
- slmeval/report/drilldown.py +1395 -0
- slmeval/report/export.py +450 -0
- slmeval/report/model.py +562 -0
- slmeval/report/progress.py +257 -0
- slmeval/report/scorecard.py +367 -0
- slmeval/sandbox/__init__.py +145 -0
- slmeval/sandbox/diagnosis.py +690 -0
- slmeval/sandbox/envelope.py +486 -0
- slmeval/sandbox/faults.py +794 -0
- slmeval/sandbox/keys.py +458 -0
- slmeval/sandbox/normalizers.py +333 -0
- slmeval/sandbox/store.py +525 -0
- slmeval/sandbox/tool.py +685 -0
- slmeval/storage/__init__.py +73 -0
- slmeval/storage/blobs.py +308 -0
- slmeval/storage/codec.py +93 -0
- slmeval/storage/db.py +1334 -0
- slmeval/storage/ingest.py +346 -0
- slmeval/storage/migrations.py +358 -0
- slmeval/storage/models.py +554 -0
- slmeval/storage/provenance.py +201 -0
- slmeval_cli-0.1.0.dist-info/METADATA +484 -0
- slmeval_cli-0.1.0.dist-info/RECORD +94 -0
- slmeval_cli-0.1.0.dist-info/WHEEL +4 -0
- slmeval_cli-0.1.0.dist-info/entry_points.txt +2 -0
- slmeval_cli-0.1.0.dist-info/licenses/LICENSE +201 -0
|
@@ -0,0 +1,247 @@
|
|
|
1
|
+
"""``slmeval run`` - execute the matrix and print the scorecard.
|
|
2
|
+
|
|
3
|
+
This is the command the whole tool exists for, so the flag surface is wide. The
|
|
4
|
+
defaults are the important part: ``replay`` tool mode, throughput scheduling,
|
|
5
|
+
no spend ceiling, and resume available by run id. A first run needs none of
|
|
6
|
+
them.
|
|
7
|
+
|
|
8
|
+
Two behaviours worth calling out because they are easy to get wrong:
|
|
9
|
+
|
|
10
|
+
**Rollups are written after the matrix, not during it.** ``pass^k`` and the
|
|
11
|
+
Wilson interval are properties of a case's whole set of repeats, and computing
|
|
12
|
+
them incrementally would mean storing a number that was briefly wrong - which
|
|
13
|
+
somebody would eventually read.
|
|
14
|
+
|
|
15
|
+
**A ``Ctrl-C`` is not an error.** It prints what was finished, names the run id
|
|
16
|
+
to resume, and exits. Treating an interrupt as a crash would throw away the
|
|
17
|
+
most valuable thing the user has at that moment: the id of the half-finished
|
|
18
|
+
run.
|
|
19
|
+
"""
|
|
20
|
+
|
|
21
|
+
from __future__ import annotations
|
|
22
|
+
|
|
23
|
+
import asyncio
|
|
24
|
+
from pathlib import Path
|
|
25
|
+
from typing import Annotated
|
|
26
|
+
|
|
27
|
+
import typer
|
|
28
|
+
from rich.console import Console
|
|
29
|
+
from rich.text import Text
|
|
30
|
+
|
|
31
|
+
from slmeval import exit_codes
|
|
32
|
+
from slmeval.cli.loading import Project, open_project
|
|
33
|
+
from slmeval.cli.render import Symbols, console, symbols
|
|
34
|
+
from slmeval.cli.wrap import handle_errors
|
|
35
|
+
from slmeval.core.runner import MatrixRunner, RunOptions, RunSummary, encode_plan
|
|
36
|
+
from slmeval.eval.pipeline import case_rollups, evaluator_for
|
|
37
|
+
from slmeval.report.export import export
|
|
38
|
+
from slmeval.report.model import build_scorecard
|
|
39
|
+
from slmeval.report.progress import ProgressObserver
|
|
40
|
+
from slmeval.report.scorecard import render_scorecard, render_summary_line
|
|
41
|
+
from slmeval.storage.models import Outcome, RunStatus
|
|
42
|
+
|
|
43
|
+
_TOOL_MODES = ("replay", "auto", "record", "live")
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
@handle_errors
|
|
47
|
+
def run(
|
|
48
|
+
config: Annotated[
|
|
49
|
+
Path | None,
|
|
50
|
+
typer.Option("--config", "-c", help="Path to eval.toml. Defaults to the nearest one."),
|
|
51
|
+
] = None,
|
|
52
|
+
candidate: Annotated[
|
|
53
|
+
list[str] | None,
|
|
54
|
+
typer.Option(
|
|
55
|
+
"--candidate",
|
|
56
|
+
help="Only these candidates. The baseline is always included - every "
|
|
57
|
+
"delta is measured against it.",
|
|
58
|
+
),
|
|
59
|
+
] = None,
|
|
60
|
+
case: Annotated[
|
|
61
|
+
list[str] | None,
|
|
62
|
+
typer.Option("--case", help="Only these case ids."),
|
|
63
|
+
] = None,
|
|
64
|
+
runs: Annotated[
|
|
65
|
+
int | None,
|
|
66
|
+
typer.Option("--runs", min=1, max=50, help="Repeats per case. Overrides runs_per_case."),
|
|
67
|
+
] = None,
|
|
68
|
+
tool_mode: Annotated[
|
|
69
|
+
str | None,
|
|
70
|
+
typer.Option(
|
|
71
|
+
"--tool-mode",
|
|
72
|
+
help="replay | auto | record | live. replay needs no network and no keys.",
|
|
73
|
+
),
|
|
74
|
+
] = None,
|
|
75
|
+
fault_rate: Annotated[
|
|
76
|
+
float | None,
|
|
77
|
+
typer.Option("--fault-rate", min=0.0, max=1.0, help="Chance any tool call is faulted."),
|
|
78
|
+
] = None,
|
|
79
|
+
concurrency: Annotated[
|
|
80
|
+
int | None,
|
|
81
|
+
typer.Option("--concurrency", min=1, max=256, help="Executions in flight, at most."),
|
|
82
|
+
] = None,
|
|
83
|
+
serial_latency: Annotated[
|
|
84
|
+
bool,
|
|
85
|
+
typer.Option(
|
|
86
|
+
"--serial-latency",
|
|
87
|
+
help="One request at a time per endpoint, after a warm-up, so latency "
|
|
88
|
+
"is reportable. Much slower, and the only mode whose p95 means anything.",
|
|
89
|
+
),
|
|
90
|
+
] = False,
|
|
91
|
+
max_spend: Annotated[
|
|
92
|
+
float | None,
|
|
93
|
+
typer.Option("--max-spend", min=0.0, help="Abort once estimated spend passes this (USD)."),
|
|
94
|
+
] = None,
|
|
95
|
+
resume: Annotated[
|
|
96
|
+
str | None,
|
|
97
|
+
typer.Option("--resume", help="Finish an interrupted run. Takes a run id or a prefix."),
|
|
98
|
+
] = None,
|
|
99
|
+
fail_fast: Annotated[
|
|
100
|
+
bool,
|
|
101
|
+
typer.Option("--fail-fast", help="Stop once a candidate fails three executions in a row."),
|
|
102
|
+
] = False,
|
|
103
|
+
gpu_hourly: Annotated[
|
|
104
|
+
float | None,
|
|
105
|
+
typer.Option(
|
|
106
|
+
"--gpu-hourly",
|
|
107
|
+
min=0.0,
|
|
108
|
+
help="Your $/hour for GPU time, used to model local cost. Reported as "
|
|
109
|
+
"modelled, never as measured.",
|
|
110
|
+
),
|
|
111
|
+
] = None,
|
|
112
|
+
json_out: Annotated[
|
|
113
|
+
Path | None,
|
|
114
|
+
typer.Option("--json", help="Also write the versioned JSON report here."),
|
|
115
|
+
] = None,
|
|
116
|
+
nodes: Annotated[
|
|
117
|
+
bool, typer.Option("--nodes/--no-nodes", help="Show the per-node attribution table.")
|
|
118
|
+
] = True,
|
|
119
|
+
ascii_mode: Annotated[
|
|
120
|
+
bool, typer.Option("--ascii", help="ASCII only, for legacy Windows consoles.")
|
|
121
|
+
] = False,
|
|
122
|
+
dry_run: Annotated[
|
|
123
|
+
bool, typer.Option("--dry-run", help="Print the plan as JSON and execute nothing.")
|
|
124
|
+
] = False,
|
|
125
|
+
quiet: Annotated[
|
|
126
|
+
bool, typer.Option("--quiet", "-q", help="No progress bars; one line per failure.")
|
|
127
|
+
] = False,
|
|
128
|
+
) -> None:
|
|
129
|
+
"""Run the candidate matrix and print the scorecard."""
|
|
130
|
+
if tool_mode is not None and tool_mode not in _TOOL_MODES:
|
|
131
|
+
raise typer.BadParameter(f"tool mode must be one of: {', '.join(_TOOL_MODES)}")
|
|
132
|
+
|
|
133
|
+
out = console()
|
|
134
|
+
sym = symbols(ascii_mode or None)
|
|
135
|
+
|
|
136
|
+
with open_project(config, gpu_hourly_usd=gpu_hourly) as project:
|
|
137
|
+
options = RunOptions.from_config(
|
|
138
|
+
project.config,
|
|
139
|
+
repeats=runs,
|
|
140
|
+
tool_mode=tool_mode,
|
|
141
|
+
fault_rate=fault_rate,
|
|
142
|
+
concurrency=concurrency,
|
|
143
|
+
serial_latency=serial_latency or None,
|
|
144
|
+
max_spend_usd=max_spend,
|
|
145
|
+
fail_fast=fail_fast or None,
|
|
146
|
+
case_ids=tuple(case) if case else None,
|
|
147
|
+
candidate_ids=tuple(candidate) if candidate else None,
|
|
148
|
+
gpu_hourly_usd=gpu_hourly,
|
|
149
|
+
)
|
|
150
|
+
runner = MatrixRunner(
|
|
151
|
+
config=project.config,
|
|
152
|
+
dataset=project.dataset,
|
|
153
|
+
target=project.target,
|
|
154
|
+
store=project.store,
|
|
155
|
+
options=options,
|
|
156
|
+
cassettes=project.cassettes,
|
|
157
|
+
evaluator=evaluator_for(project.config),
|
|
158
|
+
prices=project.prices,
|
|
159
|
+
root=project.root,
|
|
160
|
+
)
|
|
161
|
+
|
|
162
|
+
if dry_run:
|
|
163
|
+
# print, not rich-print: this is JSON on its way into a pipe, and
|
|
164
|
+
# Rich would wrap it at the terminal width into something no parser
|
|
165
|
+
# can read.
|
|
166
|
+
print(encode_plan(runner.plan(resume=resume)))
|
|
167
|
+
return
|
|
168
|
+
|
|
169
|
+
observer = ProgressObserver(console=out, ascii_mode=ascii_mode or None, quiet=quiet)
|
|
170
|
+
runner.observer = observer
|
|
171
|
+
try:
|
|
172
|
+
summary = asyncio.run(runner.run(resume=resume))
|
|
173
|
+
except KeyboardInterrupt:
|
|
174
|
+
# Everything finished is already in SQLite - that is the point of
|
|
175
|
+
# persisting per unit - so the only thing at risk here is the user
|
|
176
|
+
# knowing how to pick it back up.
|
|
177
|
+
observer.close()
|
|
178
|
+
_interrupted(out, sym, runner)
|
|
179
|
+
raise typer.Exit(exit_codes.USAGE) from None
|
|
180
|
+
|
|
181
|
+
_save_rollups(project, summary)
|
|
182
|
+
card = build_scorecard(
|
|
183
|
+
project.store, summary.run_id, prices_as_of=project.prices.as_of
|
|
184
|
+
)
|
|
185
|
+
out.print(render_scorecard(card, ascii_mode=ascii_mode or None, show_nodes=nodes))
|
|
186
|
+
line = render_summary_line(card, ascii_mode=ascii_mode or None)
|
|
187
|
+
if str(line).strip():
|
|
188
|
+
out.print(line)
|
|
189
|
+
out.print()
|
|
190
|
+
|
|
191
|
+
if json_out is not None:
|
|
192
|
+
json_out.write_text(export(card, "json"), encoding="utf-8")
|
|
193
|
+
out.print(Text(f" {sym.ok} wrote {json_out}", style="dim"))
|
|
194
|
+
out.print()
|
|
195
|
+
|
|
196
|
+
raise typer.Exit(_exit_code(summary))
|
|
197
|
+
|
|
198
|
+
|
|
199
|
+
def _save_rollups(project: Project, summary: RunSummary) -> None:
|
|
200
|
+
"""Compute and store the consistency rollups for the whole run.
|
|
201
|
+
|
|
202
|
+
After the matrix, and from the *stored* executions rather than from the
|
|
203
|
+
results in memory: a resumed run's earlier half exists only in the
|
|
204
|
+
database, and rolling up what this process happened to execute would report
|
|
205
|
+
``n=1`` for every case the first half already covered.
|
|
206
|
+
"""
|
|
207
|
+
executions = project.store.executions(summary.run_id)
|
|
208
|
+
project.store.save_rollups(case_rollups(summary.run_id, executions))
|
|
209
|
+
|
|
210
|
+
|
|
211
|
+
def _interrupted(out: Console, sym: Symbols, runner: MatrixRunner) -> None:
|
|
212
|
+
out.print()
|
|
213
|
+
out.print(
|
|
214
|
+
Text(
|
|
215
|
+
f" {sym.warn} interrupted - everything finished is already stored",
|
|
216
|
+
style="yellow",
|
|
217
|
+
)
|
|
218
|
+
)
|
|
219
|
+
if runner.last_run_id:
|
|
220
|
+
out.print(
|
|
221
|
+
Text(
|
|
222
|
+
f" resume with: slmeval run --resume {runner.last_run_id}",
|
|
223
|
+
style="cyan",
|
|
224
|
+
)
|
|
225
|
+
)
|
|
226
|
+
out.print()
|
|
227
|
+
|
|
228
|
+
|
|
229
|
+
def _exit_code(summary: RunSummary) -> int:
|
|
230
|
+
"""What the shell sees.
|
|
231
|
+
|
|
232
|
+
An unreachable provider exits 3, not 1. That distinction is the whole
|
|
233
|
+
reason the codes are a documented contract: a CI job that reports "Ollama
|
|
234
|
+
was not running" as a model regression teaches people to ignore the gate,
|
|
235
|
+
and once they do, the gate is worth nothing.
|
|
236
|
+
"""
|
|
237
|
+
if summary.results and all(
|
|
238
|
+
r.outcome is Outcome.PROVIDER_ERROR for r in summary.results
|
|
239
|
+
):
|
|
240
|
+
return exit_codes.ENVIRONMENT
|
|
241
|
+
if summary.status is RunStatus.CANCELLED:
|
|
242
|
+
return exit_codes.GATE_FAILED
|
|
243
|
+
return exit_codes.OK
|
|
244
|
+
|
|
245
|
+
|
|
246
|
+
def register(app: typer.Typer) -> None:
|
|
247
|
+
app.command(name="run", help="Run the candidate matrix and print the scorecard.")(run)
|