slmeval-cli 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- slmeval/__init__.py +71 -0
- slmeval/__main__.py +14 -0
- slmeval/adapters/__init__.py +24 -0
- slmeval/adapters/instrument.py +135 -0
- slmeval/adapters/langgraph.py +237 -0
- slmeval/cli/__init__.py +15 -0
- slmeval/cli/commands/__init__.py +35 -0
- slmeval/cli/commands/baseline_cmd.py +201 -0
- slmeval/cli/commands/cache_cmd.py +249 -0
- slmeval/cli/commands/dataset_cmd.py +52 -0
- slmeval/cli/commands/doctor_cmd.py +68 -0
- slmeval/cli/commands/gate_cmd.py +602 -0
- slmeval/cli/commands/init_cmd.py +144 -0
- slmeval/cli/commands/judge_cmd.py +284 -0
- slmeval/cli/commands/record_cmd.py +154 -0
- slmeval/cli/commands/report_cmd.py +98 -0
- slmeval/cli/commands/run_cmd.py +247 -0
- slmeval/cli/commands/show_cmd.py +482 -0
- slmeval/cli/commands/stubs.py +57 -0
- slmeval/cli/commands/validate_cmd.py +296 -0
- slmeval/cli/loading.py +149 -0
- slmeval/cli/main.py +91 -0
- slmeval/cli/render.py +251 -0
- slmeval/cli/wrap.py +31 -0
- slmeval/config/__init__.py +47 -0
- slmeval/config/loader.py +363 -0
- slmeval/config/locate.py +97 -0
- slmeval/config/models.py +430 -0
- slmeval/config/templates.py +246 -0
- slmeval/core/__init__.py +78 -0
- slmeval/core/adapter.py +253 -0
- slmeval/core/context.py +418 -0
- slmeval/core/outcome.py +260 -0
- slmeval/core/router.py +319 -0
- slmeval/core/runner.py +1209 -0
- slmeval/core/trace.py +188 -0
- slmeval/core/verdict.py +127 -0
- slmeval/dataset/__init__.py +8 -0
- slmeval/dataset/loader.py +130 -0
- slmeval/dataset/models.py +108 -0
- slmeval/doctor/__init__.py +7 -0
- slmeval/doctor/checks.py +380 -0
- slmeval/errors.py +76 -0
- slmeval/eval/__init__.py +64 -0
- slmeval/eval/assertions.py +356 -0
- slmeval/eval/calibration.py +364 -0
- slmeval/eval/cost.py +291 -0
- slmeval/eval/pipeline.py +273 -0
- slmeval/eval/stats.py +376 -0
- slmeval/eval/tier1.py +510 -0
- slmeval/eval/tier2.py +582 -0
- slmeval/eval/tier3.py +415 -0
- slmeval/exit_codes.py +28 -0
- slmeval/gate.py +722 -0
- slmeval/providers/__init__.py +69 -0
- slmeval/providers/base.py +426 -0
- slmeval/providers/fake.py +760 -0
- slmeval/providers/jsonx.py +239 -0
- slmeval/providers/ladder.py +339 -0
- slmeval/providers/openai_compat.py +392 -0
- slmeval/providers/registry.py +185 -0
- slmeval/providers/retry.py +81 -0
- slmeval/providers/schema.py +136 -0
- slmeval/providers/tokens.py +150 -0
- slmeval/py.typed +0 -0
- slmeval/report/__init__.py +115 -0
- slmeval/report/attribution.py +197 -0
- slmeval/report/baseline.py +353 -0
- slmeval/report/drilldown.py +1395 -0
- slmeval/report/export.py +450 -0
- slmeval/report/model.py +562 -0
- slmeval/report/progress.py +257 -0
- slmeval/report/scorecard.py +367 -0
- slmeval/sandbox/__init__.py +145 -0
- slmeval/sandbox/diagnosis.py +690 -0
- slmeval/sandbox/envelope.py +486 -0
- slmeval/sandbox/faults.py +794 -0
- slmeval/sandbox/keys.py +458 -0
- slmeval/sandbox/normalizers.py +333 -0
- slmeval/sandbox/store.py +525 -0
- slmeval/sandbox/tool.py +685 -0
- slmeval/storage/__init__.py +73 -0
- slmeval/storage/blobs.py +308 -0
- slmeval/storage/codec.py +93 -0
- slmeval/storage/db.py +1334 -0
- slmeval/storage/ingest.py +346 -0
- slmeval/storage/migrations.py +358 -0
- slmeval/storage/models.py +554 -0
- slmeval/storage/provenance.py +201 -0
- slmeval_cli-0.1.0.dist-info/METADATA +484 -0
- slmeval_cli-0.1.0.dist-info/RECORD +94 -0
- slmeval_cli-0.1.0.dist-info/WHEEL +4 -0
- slmeval_cli-0.1.0.dist-info/entry_points.txt +2 -0
- slmeval_cli-0.1.0.dist-info/licenses/LICENSE +201 -0
|
@@ -0,0 +1,482 @@
|
|
|
1
|
+
"""``slmeval show`` - the drill-down, from the scorecard to the exact step.
|
|
2
|
+
|
|
3
|
+
The scorecard is the headline; this is the audit trail under it. Four views,
|
|
4
|
+
each one a question somebody asks while staring at a red cell:
|
|
5
|
+
|
|
6
|
+
``show run <id>``
|
|
7
|
+
Which executions exist and how did each end? Worst first.
|
|
8
|
+
``show exec <id>``
|
|
9
|
+
What did this one actually do - trajectory, model calls, tool calls,
|
|
10
|
+
metrics, final state?
|
|
11
|
+
``show case <id> --candidate <id>``
|
|
12
|
+
Where did it part company with the baseline? Side by side, divergence
|
|
13
|
+
marked, cassette-miss diagnosis inline.
|
|
14
|
+
``show uncovered``
|
|
15
|
+
Which tool calls had no recording, so where does ``record --fill-gaps``
|
|
16
|
+
need to go?
|
|
17
|
+
|
|
18
|
+
Like ``report``, none of this loads the user's agent and none of it re-executes
|
|
19
|
+
anything. A checkout whose agent has since been refactored still has valid
|
|
20
|
+
results from last Tuesday, and a view that could not open them would be the
|
|
21
|
+
tool losing data it already holds.
|
|
22
|
+
"""
|
|
23
|
+
|
|
24
|
+
from __future__ import annotations
|
|
25
|
+
|
|
26
|
+
import json
|
|
27
|
+
from collections.abc import Sequence
|
|
28
|
+
from pathlib import Path
|
|
29
|
+
from typing import Annotated, Any
|
|
30
|
+
|
|
31
|
+
import typer
|
|
32
|
+
|
|
33
|
+
from slmeval.cli.loading import LATEST, Project, open_project, resolve_run
|
|
34
|
+
from slmeval.cli.render import console
|
|
35
|
+
from slmeval.cli.wrap import handle_errors
|
|
36
|
+
from slmeval.errors import ConfigError
|
|
37
|
+
from slmeval.report.drilldown import (
|
|
38
|
+
Diff,
|
|
39
|
+
build_diff,
|
|
40
|
+
diff_payload,
|
|
41
|
+
execution_payload,
|
|
42
|
+
gaps,
|
|
43
|
+
gaps_payload,
|
|
44
|
+
known_tool_names,
|
|
45
|
+
rebuild_diagnosis,
|
|
46
|
+
render_diff,
|
|
47
|
+
render_execution,
|
|
48
|
+
render_gaps,
|
|
49
|
+
render_run,
|
|
50
|
+
tool_steps,
|
|
51
|
+
)
|
|
52
|
+
from slmeval.sandbox.diagnosis import MissDiagnosis
|
|
53
|
+
from slmeval.storage.db import RunStore
|
|
54
|
+
from slmeval.storage.models import ExecutionDetail, ExecutionRecord, Outcome
|
|
55
|
+
|
|
56
|
+
MISS_OUTCOMES = frozenset({Outcome.CASSETTE_MISS, Outcome.UNCOVERED})
|
|
57
|
+
|
|
58
|
+
show_app = typer.Typer(
|
|
59
|
+
no_args_is_help=True,
|
|
60
|
+
help="Drill into a run, an execution, or one case against the baseline.",
|
|
61
|
+
)
|
|
62
|
+
|
|
63
|
+
_ConfigOpt = Annotated[
|
|
64
|
+
Path | None,
|
|
65
|
+
typer.Option("--config", "-c", help="Path to eval.toml. Defaults to the nearest one."),
|
|
66
|
+
]
|
|
67
|
+
_AsciiOpt = Annotated[
|
|
68
|
+
bool, typer.Option("--ascii", help="ASCII only, for legacy Windows consoles.")
|
|
69
|
+
]
|
|
70
|
+
_JsonOpt = Annotated[
|
|
71
|
+
bool, typer.Option("--json", help="Emit the view as JSON instead of a table.")
|
|
72
|
+
]
|
|
73
|
+
_RunOpt = Annotated[
|
|
74
|
+
str,
|
|
75
|
+
typer.Option("--run-id", help="A run id or prefix, or 'latest'."),
|
|
76
|
+
]
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
# --------------------------------------------------------------------------- #
|
|
80
|
+
# show run
|
|
81
|
+
# --------------------------------------------------------------------------- #
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
@show_app.command("run")
|
|
85
|
+
@handle_errors
|
|
86
|
+
def show_run(
|
|
87
|
+
run_id: Annotated[
|
|
88
|
+
str, typer.Argument(help="A run id or prefix, or 'latest'.")
|
|
89
|
+
] = LATEST,
|
|
90
|
+
config: _ConfigOpt = None,
|
|
91
|
+
candidate: Annotated[
|
|
92
|
+
str | None, typer.Option("--candidate", help="Only this candidate's cells.")
|
|
93
|
+
] = None,
|
|
94
|
+
case: Annotated[str | None, typer.Option("--case", help="Only this case.")] = None,
|
|
95
|
+
failed: Annotated[
|
|
96
|
+
bool, typer.Option("--failed", help="Only the executions that did not pass.")
|
|
97
|
+
] = False,
|
|
98
|
+
limit: Annotated[
|
|
99
|
+
int, typer.Option("--limit", min=0, help="Rows to show. 0 for all.")
|
|
100
|
+
] = 40,
|
|
101
|
+
as_json: _JsonOpt = False,
|
|
102
|
+
ascii_mode: _AsciiOpt = False,
|
|
103
|
+
) -> None:
|
|
104
|
+
"""List a run's executions, worst first."""
|
|
105
|
+
with open_project(config, load_agent=False) as project:
|
|
106
|
+
resolved = resolve_run(project.store, run_id)
|
|
107
|
+
run = project.store.require_run(resolved)
|
|
108
|
+
candidates = project.store.candidates(resolved)
|
|
109
|
+
executions = _filtered(
|
|
110
|
+
project.store.executions(resolved, candidate_id=candidate, case_id=case),
|
|
111
|
+
failed=failed,
|
|
112
|
+
)
|
|
113
|
+
if as_json:
|
|
114
|
+
_print_json(
|
|
115
|
+
{
|
|
116
|
+
"run_id": run.run_id,
|
|
117
|
+
"name": run.name,
|
|
118
|
+
"status": run.status.value,
|
|
119
|
+
"mode": run.mode,
|
|
120
|
+
"seed": run.seed,
|
|
121
|
+
"runs_per_case": run.runs_per_case,
|
|
122
|
+
"config_hash": run.config_hash,
|
|
123
|
+
"dataset_hash": run.dataset_hash,
|
|
124
|
+
"candidates": [
|
|
125
|
+
{
|
|
126
|
+
"candidate_id": c.candidate_id,
|
|
127
|
+
"role": c.role,
|
|
128
|
+
"provider": c.provider,
|
|
129
|
+
"model": c.model,
|
|
130
|
+
}
|
|
131
|
+
for c in candidates
|
|
132
|
+
],
|
|
133
|
+
"executions": [
|
|
134
|
+
{
|
|
135
|
+
"exec_id": e.exec_id,
|
|
136
|
+
"candidate_id": e.candidate_id,
|
|
137
|
+
"case_id": e.case_id,
|
|
138
|
+
"repeat_idx": e.repeat_idx,
|
|
139
|
+
"status": e.status.value,
|
|
140
|
+
"outcome": e.outcome_class.value,
|
|
141
|
+
"steps": e.steps,
|
|
142
|
+
"duration_ms": e.duration_ms,
|
|
143
|
+
"error_class": e.error_class,
|
|
144
|
+
"error_message": e.error_message,
|
|
145
|
+
}
|
|
146
|
+
for e in executions
|
|
147
|
+
],
|
|
148
|
+
}
|
|
149
|
+
)
|
|
150
|
+
return
|
|
151
|
+
out = console()
|
|
152
|
+
out.print(
|
|
153
|
+
render_run(
|
|
154
|
+
run,
|
|
155
|
+
candidates,
|
|
156
|
+
executions,
|
|
157
|
+
ascii_mode=ascii_mode or None,
|
|
158
|
+
limit=limit,
|
|
159
|
+
)
|
|
160
|
+
)
|
|
161
|
+
out.print()
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
# --------------------------------------------------------------------------- #
|
|
165
|
+
# show exec
|
|
166
|
+
# --------------------------------------------------------------------------- #
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
@show_app.command("exec")
|
|
170
|
+
@handle_errors
|
|
171
|
+
def show_exec(
|
|
172
|
+
exec_id: Annotated[
|
|
173
|
+
str,
|
|
174
|
+
typer.Argument(
|
|
175
|
+
help="An execution id, or its readable tail: gemma3-4b.refund_207.r0"
|
|
176
|
+
),
|
|
177
|
+
],
|
|
178
|
+
config: _ConfigOpt = None,
|
|
179
|
+
run_id: _RunOpt = LATEST,
|
|
180
|
+
as_json: _JsonOpt = False,
|
|
181
|
+
ascii_mode: _AsciiOpt = False,
|
|
182
|
+
) -> None:
|
|
183
|
+
"""Show one execution's timeline, metrics and final state.
|
|
184
|
+
|
|
185
|
+
The id is resolved inside one run - the latest unless ``--run-id`` says
|
|
186
|
+
otherwise - because the readable tail is identical in every run of a
|
|
187
|
+
project: ``record`` alone leaves a second execution under every one of
|
|
188
|
+
them. A full execution id skips the question.
|
|
189
|
+
"""
|
|
190
|
+
with open_project(config, load_agent=False) as project:
|
|
191
|
+
resolved = _resolve_exec(project.store, exec_id, run_id)
|
|
192
|
+
detail = project.store.reconstruct(resolved)
|
|
193
|
+
if as_json:
|
|
194
|
+
_print_json(execution_payload(detail))
|
|
195
|
+
return
|
|
196
|
+
out = console()
|
|
197
|
+
out.print(
|
|
198
|
+
render_execution(
|
|
199
|
+
detail,
|
|
200
|
+
ascii_mode=ascii_mode or None,
|
|
201
|
+
diagnoses=_diagnoses(detail, project),
|
|
202
|
+
)
|
|
203
|
+
)
|
|
204
|
+
|
|
205
|
+
|
|
206
|
+
# --------------------------------------------------------------------------- #
|
|
207
|
+
# show case
|
|
208
|
+
# --------------------------------------------------------------------------- #
|
|
209
|
+
|
|
210
|
+
|
|
211
|
+
@show_app.command("case")
|
|
212
|
+
@handle_errors
|
|
213
|
+
def show_case(
|
|
214
|
+
case_id: Annotated[str, typer.Argument(help="A case id from the dataset.")],
|
|
215
|
+
config: _ConfigOpt = None,
|
|
216
|
+
run_id: _RunOpt = LATEST,
|
|
217
|
+
candidate: Annotated[
|
|
218
|
+
list[str] | None,
|
|
219
|
+
typer.Option(
|
|
220
|
+
"--candidate",
|
|
221
|
+
help="Diff only these candidates. Defaults to every non-baseline one.",
|
|
222
|
+
),
|
|
223
|
+
] = None,
|
|
224
|
+
repeat: Annotated[
|
|
225
|
+
int | None,
|
|
226
|
+
typer.Option(
|
|
227
|
+
"--repeat",
|
|
228
|
+
min=0,
|
|
229
|
+
help="Which repeat to show. Defaults to the worst one, which is the "
|
|
230
|
+
"reason you are here.",
|
|
231
|
+
),
|
|
232
|
+
] = None,
|
|
233
|
+
as_json: _JsonOpt = False,
|
|
234
|
+
ascii_mode: _AsciiOpt = False,
|
|
235
|
+
) -> None:
|
|
236
|
+
"""Diff one case against the baseline, side by side."""
|
|
237
|
+
with open_project(config, load_agent=False) as project:
|
|
238
|
+
resolved = resolve_run(project.store, run_id)
|
|
239
|
+
store = project.store
|
|
240
|
+
executions = store.executions(resolved, case_id=case_id)
|
|
241
|
+
if not executions:
|
|
242
|
+
raise ConfigError(
|
|
243
|
+
f"run {resolved} has no execution for case {case_id!r}",
|
|
244
|
+
hint="`slmeval show run` lists the cases a run covered",
|
|
245
|
+
)
|
|
246
|
+
|
|
247
|
+
baseline_id = _baseline_id(store, resolved)
|
|
248
|
+
wanted = _wanted_candidates(executions, baseline_id, candidate)
|
|
249
|
+
diffs = [
|
|
250
|
+
_diff_for(
|
|
251
|
+
project,
|
|
252
|
+
case_id=case_id,
|
|
253
|
+
executions=executions,
|
|
254
|
+
baseline_id=baseline_id,
|
|
255
|
+
candidate_id=candidate_id,
|
|
256
|
+
repeat=repeat,
|
|
257
|
+
)
|
|
258
|
+
for candidate_id in wanted
|
|
259
|
+
]
|
|
260
|
+
|
|
261
|
+
if as_json:
|
|
262
|
+
_print_json(
|
|
263
|
+
{
|
|
264
|
+
"run_id": resolved,
|
|
265
|
+
"case_id": case_id,
|
|
266
|
+
"diffs": [diff_payload(diff) for diff in diffs],
|
|
267
|
+
}
|
|
268
|
+
)
|
|
269
|
+
return
|
|
270
|
+
|
|
271
|
+
out = console()
|
|
272
|
+
for diff in diffs:
|
|
273
|
+
out.print(render_diff(diff, ascii_mode=ascii_mode or None))
|
|
274
|
+
out.print()
|
|
275
|
+
|
|
276
|
+
|
|
277
|
+
# --------------------------------------------------------------------------- #
|
|
278
|
+
# show uncovered
|
|
279
|
+
# --------------------------------------------------------------------------- #
|
|
280
|
+
|
|
281
|
+
|
|
282
|
+
@show_app.command("uncovered")
|
|
283
|
+
@handle_errors
|
|
284
|
+
def show_uncovered(
|
|
285
|
+
run_id: Annotated[
|
|
286
|
+
str, typer.Argument(help="A run id or prefix, or 'latest'.")
|
|
287
|
+
] = LATEST,
|
|
288
|
+
config: _ConfigOpt = None,
|
|
289
|
+
candidate: Annotated[
|
|
290
|
+
str | None, typer.Option("--candidate", help="Only this candidate's misses.")
|
|
291
|
+
] = None,
|
|
292
|
+
all_misses: Annotated[
|
|
293
|
+
bool,
|
|
294
|
+
typer.Option(
|
|
295
|
+
"--all",
|
|
296
|
+
help="Include hallucinated calls too, not only the coverage gaps.",
|
|
297
|
+
),
|
|
298
|
+
] = False,
|
|
299
|
+
as_json: _JsonOpt = False,
|
|
300
|
+
ascii_mode: _AsciiOpt = False,
|
|
301
|
+
) -> None:
|
|
302
|
+
"""List the tool calls no cassette covered - the input to `record --fill-gaps`.
|
|
303
|
+
|
|
304
|
+
Only the uncovered verdict by default. The other three verdicts are model
|
|
305
|
+
failures and belong in the failure report, not in a list of recordings to
|
|
306
|
+
go and make; mixing them would have a user re-recording a hallucinated
|
|
307
|
+
order id as if it were ground truth.
|
|
308
|
+
"""
|
|
309
|
+
with open_project(config, load_agent=False) as project:
|
|
310
|
+
resolved = resolve_run(project.store, run_id)
|
|
311
|
+
executions = [
|
|
312
|
+
e
|
|
313
|
+
for e in project.store.executions(resolved, candidate_id=candidate)
|
|
314
|
+
if e.outcome_class in MISS_OUTCOMES
|
|
315
|
+
]
|
|
316
|
+
details = [
|
|
317
|
+
(e.candidate_id, project.store.reconstruct(e.exec_id)) for e in executions
|
|
318
|
+
]
|
|
319
|
+
found = gaps(details, uncovered_only=not all_misses)
|
|
320
|
+
if as_json:
|
|
321
|
+
_print_json({"run_id": resolved, **gaps_payload(found)})
|
|
322
|
+
return
|
|
323
|
+
out = console()
|
|
324
|
+
out.print(render_gaps(found, ascii_mode=ascii_mode or None, run_id=resolved))
|
|
325
|
+
|
|
326
|
+
|
|
327
|
+
# --------------------------------------------------------------------------- #
|
|
328
|
+
# Helpers
|
|
329
|
+
# --------------------------------------------------------------------------- #
|
|
330
|
+
|
|
331
|
+
|
|
332
|
+
def _resolve_exec(store: RunStore, exec_id: str, run_id: str) -> str:
|
|
333
|
+
"""One execution id, resolved inside a run before the whole project.
|
|
334
|
+
|
|
335
|
+
Inside a run first, because ``candidate.case.rN`` is not unique across a
|
|
336
|
+
project and the id a user just read off ``show run`` belongs to the run
|
|
337
|
+
they were looking at. Falling back to a project-wide search afterwards is
|
|
338
|
+
still worth doing - an id pasted from an older run's table should not need
|
|
339
|
+
its run spelled out - and that search refuses ambiguity rather than
|
|
340
|
+
guessing.
|
|
341
|
+
"""
|
|
342
|
+
if store.get_execution(exec_id) is not None:
|
|
343
|
+
return exec_id
|
|
344
|
+
scoped = f"{resolve_run(store, run_id)}."
|
|
345
|
+
within = f"{scoped}{exec_id.removeprefix(scoped)}"
|
|
346
|
+
if store.get_execution(within) is not None:
|
|
347
|
+
return within
|
|
348
|
+
return store.resolve_exec_id(exec_id)
|
|
349
|
+
|
|
350
|
+
|
|
351
|
+
def _filtered(
|
|
352
|
+
executions: Sequence[ExecutionRecord], *, failed: bool
|
|
353
|
+
) -> list[ExecutionRecord]:
|
|
354
|
+
if not failed:
|
|
355
|
+
return list(executions)
|
|
356
|
+
return [e for e in executions if e.outcome_class is not Outcome.PASSED]
|
|
357
|
+
|
|
358
|
+
|
|
359
|
+
def _baseline_id(store: RunStore, run_id: str) -> str:
|
|
360
|
+
rows = store.candidates(run_id)
|
|
361
|
+
found = next((c for c in rows if c.role == "baseline"), None)
|
|
362
|
+
return found.candidate_id if found else ""
|
|
363
|
+
|
|
364
|
+
|
|
365
|
+
def _wanted_candidates(
|
|
366
|
+
executions: Sequence[ExecutionRecord], baseline_id: str, asked: list[str] | None
|
|
367
|
+
) -> list[str]:
|
|
368
|
+
present = list(dict.fromkeys(e.candidate_id for e in executions))
|
|
369
|
+
if asked:
|
|
370
|
+
unknown = [c for c in asked if c not in present]
|
|
371
|
+
if unknown:
|
|
372
|
+
raise ConfigError(
|
|
373
|
+
f"no executions for candidate(s) {', '.join(unknown)} in this run",
|
|
374
|
+
hint=f"this run has: {', '.join(present)}",
|
|
375
|
+
)
|
|
376
|
+
return list(asked)
|
|
377
|
+
others = [c for c in present if c != baseline_id]
|
|
378
|
+
# With no candidates at all - a baseline-only run, which `record` produces -
|
|
379
|
+
# diffing the baseline against itself is the only honest thing left, and it
|
|
380
|
+
# still shows the timeline.
|
|
381
|
+
return others or present
|
|
382
|
+
|
|
383
|
+
|
|
384
|
+
def _diff_for(
|
|
385
|
+
project: Project,
|
|
386
|
+
*,
|
|
387
|
+
case_id: str,
|
|
388
|
+
executions: Sequence[ExecutionRecord],
|
|
389
|
+
baseline_id: str,
|
|
390
|
+
candidate_id: str,
|
|
391
|
+
repeat: int | None,
|
|
392
|
+
) -> Diff:
|
|
393
|
+
mine = [e for e in executions if e.candidate_id == candidate_id]
|
|
394
|
+
chosen = _pick_repeat(mine, repeat)
|
|
395
|
+
if chosen is None:
|
|
396
|
+
raise ConfigError(
|
|
397
|
+
f"candidate {candidate_id!r} has no repeat {repeat} for case {case_id!r}",
|
|
398
|
+
hint=f"stored repeats: {', '.join(str(e.repeat_idx) for e in mine)}",
|
|
399
|
+
)
|
|
400
|
+
reference = _pick_baseline(
|
|
401
|
+
[e for e in executions if e.candidate_id == baseline_id], chosen.repeat_idx
|
|
402
|
+
)
|
|
403
|
+
return build_diff(
|
|
404
|
+
case_id=case_id,
|
|
405
|
+
candidate=project.store.reconstruct(chosen.exec_id),
|
|
406
|
+
baseline=(
|
|
407
|
+
project.store.reconstruct(reference.exec_id)
|
|
408
|
+
if reference is not None and reference.exec_id != chosen.exec_id
|
|
409
|
+
else None
|
|
410
|
+
),
|
|
411
|
+
candidate_id=candidate_id,
|
|
412
|
+
baseline_id=baseline_id or "baseline",
|
|
413
|
+
cassettes=project.cassettes,
|
|
414
|
+
)
|
|
415
|
+
|
|
416
|
+
|
|
417
|
+
def _pick_repeat(
|
|
418
|
+
mine: Sequence[ExecutionRecord], repeat: int | None
|
|
419
|
+
) -> ExecutionRecord | None:
|
|
420
|
+
"""The repeat to show: the one the user asked for, or the worst one.
|
|
421
|
+
|
|
422
|
+
The worst one by default, because a user opening the drill-down for a case
|
|
423
|
+
that failed two runs in three does not want to be shown the pass. "Worst"
|
|
424
|
+
means failed before passed, and an earlier repeat before a later one so the
|
|
425
|
+
choice is reproducible.
|
|
426
|
+
"""
|
|
427
|
+
if repeat is not None:
|
|
428
|
+
return next((e for e in mine if e.repeat_idx == repeat), None)
|
|
429
|
+
if not mine:
|
|
430
|
+
return None
|
|
431
|
+
return sorted(
|
|
432
|
+
mine,
|
|
433
|
+
key=lambda e: (e.outcome_class is Outcome.PASSED, e.repeat_idx),
|
|
434
|
+
)[0]
|
|
435
|
+
|
|
436
|
+
|
|
437
|
+
def _pick_baseline(
|
|
438
|
+
rows: Sequence[ExecutionRecord], repeat_idx: int
|
|
439
|
+
) -> ExecutionRecord | None:
|
|
440
|
+
"""The baseline execution to diff against: the matching repeat if it passed.
|
|
441
|
+
|
|
442
|
+
A failing baseline repeat is a poor reference - the left column would be
|
|
443
|
+
just as wrong as the right one, and the user would be comparing two
|
|
444
|
+
failures and seeing no divergence. So a passing repeat wins over the
|
|
445
|
+
matching index, and the matching index wins over anything else.
|
|
446
|
+
"""
|
|
447
|
+
if not rows:
|
|
448
|
+
return None
|
|
449
|
+
passing = [e for e in rows if e.outcome_class is Outcome.PASSED]
|
|
450
|
+
pool = passing or list(rows)
|
|
451
|
+
exact = next((e for e in pool if e.repeat_idx == repeat_idx), None)
|
|
452
|
+
return exact or pool[0]
|
|
453
|
+
|
|
454
|
+
|
|
455
|
+
def _diagnoses(detail: ExecutionDetail, project: Project) -> list[MissDiagnosis]:
|
|
456
|
+
"""Rebuild every cassette-miss block in one execution."""
|
|
457
|
+
known = known_tool_names([detail], project.cassettes)
|
|
458
|
+
found: list[MissDiagnosis] = []
|
|
459
|
+
for step in tool_steps(detail):
|
|
460
|
+
finding = rebuild_diagnosis(
|
|
461
|
+
step,
|
|
462
|
+
case_id=detail.execution.case_id,
|
|
463
|
+
candidate_id=detail.execution.candidate_id,
|
|
464
|
+
cassettes=project.cassettes,
|
|
465
|
+
known_tools=known,
|
|
466
|
+
)
|
|
467
|
+
if finding is not None:
|
|
468
|
+
found.append(finding)
|
|
469
|
+
return found
|
|
470
|
+
|
|
471
|
+
|
|
472
|
+
def _print_json(payload: dict[str, Any]) -> None:
|
|
473
|
+
"""``print``, not rich-print: this is data on its way into a pipe, and Rich
|
|
474
|
+
would wrap and style it into something no parser can read."""
|
|
475
|
+
print(json.dumps(payload, indent=2, default=str))
|
|
476
|
+
|
|
477
|
+
|
|
478
|
+
def register(app: typer.Typer) -> None:
|
|
479
|
+
app.add_typer(show_app, name="show")
|
|
480
|
+
|
|
481
|
+
|
|
482
|
+
__all__ = ["register", "show_app"]
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
"""Commands that are planned but not built yet.
|
|
2
|
+
|
|
3
|
+
They are hidden from `--help`, so nobody is offered a command that does not work.
|
|
4
|
+
They still accept whatever arguments you throw at them, so a scripted invocation
|
|
5
|
+
gets a useful message instead of a Click usage error.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
from collections.abc import Callable
|
|
11
|
+
|
|
12
|
+
import typer
|
|
13
|
+
from rich.text import Text
|
|
14
|
+
|
|
15
|
+
from slmeval import exit_codes
|
|
16
|
+
from slmeval.cli.render import console, symbols
|
|
17
|
+
|
|
18
|
+
PLANNED: list[tuple[str, str]] = [
|
|
19
|
+
("compare", "Put two runs side by side."),
|
|
20
|
+
("cassette", "Inspect, verify and prune recorded tool responses."),
|
|
21
|
+
]
|
|
22
|
+
|
|
23
|
+
_CONTEXT = {"ignore_unknown_options": True, "allow_extra_args": True}
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def _make(name: str, summary: str) -> Callable[..., None]:
|
|
27
|
+
def command(ctx: typer.Context) -> None:
|
|
28
|
+
out = console(stderr=True)
|
|
29
|
+
sym = symbols()
|
|
30
|
+
out.print()
|
|
31
|
+
out.print(Text(f" {sym.skip} `slmeval {name}` is not implemented yet.", style="yellow"))
|
|
32
|
+
out.print(Text(f" {summary}", style="dim"))
|
|
33
|
+
out.print(Text(" It is not part of this release.", style="dim"))
|
|
34
|
+
out.print()
|
|
35
|
+
out.print(
|
|
36
|
+
Text(
|
|
37
|
+
" Available now: init, doctor, validate, dataset validate, "
|
|
38
|
+
"record, run, report, show, gate, baseline save, cache stats",
|
|
39
|
+
style="dim",
|
|
40
|
+
)
|
|
41
|
+
)
|
|
42
|
+
out.print()
|
|
43
|
+
raise typer.Exit(exit_codes.USAGE)
|
|
44
|
+
|
|
45
|
+
command.__name__ = name
|
|
46
|
+
command.__doc__ = summary
|
|
47
|
+
return command
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def register(app: typer.Typer) -> None:
|
|
51
|
+
for name, summary in PLANNED:
|
|
52
|
+
app.command(
|
|
53
|
+
name=name,
|
|
54
|
+
help=summary,
|
|
55
|
+
hidden=True,
|
|
56
|
+
context_settings=_CONTEXT,
|
|
57
|
+
)(_make(name, summary))
|