slmeval-cli 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- slmeval/__init__.py +71 -0
- slmeval/__main__.py +14 -0
- slmeval/adapters/__init__.py +24 -0
- slmeval/adapters/instrument.py +135 -0
- slmeval/adapters/langgraph.py +237 -0
- slmeval/cli/__init__.py +15 -0
- slmeval/cli/commands/__init__.py +35 -0
- slmeval/cli/commands/baseline_cmd.py +201 -0
- slmeval/cli/commands/cache_cmd.py +249 -0
- slmeval/cli/commands/dataset_cmd.py +52 -0
- slmeval/cli/commands/doctor_cmd.py +68 -0
- slmeval/cli/commands/gate_cmd.py +602 -0
- slmeval/cli/commands/init_cmd.py +144 -0
- slmeval/cli/commands/judge_cmd.py +284 -0
- slmeval/cli/commands/record_cmd.py +154 -0
- slmeval/cli/commands/report_cmd.py +98 -0
- slmeval/cli/commands/run_cmd.py +247 -0
- slmeval/cli/commands/show_cmd.py +482 -0
- slmeval/cli/commands/stubs.py +57 -0
- slmeval/cli/commands/validate_cmd.py +296 -0
- slmeval/cli/loading.py +149 -0
- slmeval/cli/main.py +91 -0
- slmeval/cli/render.py +251 -0
- slmeval/cli/wrap.py +31 -0
- slmeval/config/__init__.py +47 -0
- slmeval/config/loader.py +363 -0
- slmeval/config/locate.py +97 -0
- slmeval/config/models.py +430 -0
- slmeval/config/templates.py +246 -0
- slmeval/core/__init__.py +78 -0
- slmeval/core/adapter.py +253 -0
- slmeval/core/context.py +418 -0
- slmeval/core/outcome.py +260 -0
- slmeval/core/router.py +319 -0
- slmeval/core/runner.py +1209 -0
- slmeval/core/trace.py +188 -0
- slmeval/core/verdict.py +127 -0
- slmeval/dataset/__init__.py +8 -0
- slmeval/dataset/loader.py +130 -0
- slmeval/dataset/models.py +108 -0
- slmeval/doctor/__init__.py +7 -0
- slmeval/doctor/checks.py +380 -0
- slmeval/errors.py +76 -0
- slmeval/eval/__init__.py +64 -0
- slmeval/eval/assertions.py +356 -0
- slmeval/eval/calibration.py +364 -0
- slmeval/eval/cost.py +291 -0
- slmeval/eval/pipeline.py +273 -0
- slmeval/eval/stats.py +376 -0
- slmeval/eval/tier1.py +510 -0
- slmeval/eval/tier2.py +582 -0
- slmeval/eval/tier3.py +415 -0
- slmeval/exit_codes.py +28 -0
- slmeval/gate.py +722 -0
- slmeval/providers/__init__.py +69 -0
- slmeval/providers/base.py +426 -0
- slmeval/providers/fake.py +760 -0
- slmeval/providers/jsonx.py +239 -0
- slmeval/providers/ladder.py +339 -0
- slmeval/providers/openai_compat.py +392 -0
- slmeval/providers/registry.py +185 -0
- slmeval/providers/retry.py +81 -0
- slmeval/providers/schema.py +136 -0
- slmeval/providers/tokens.py +150 -0
- slmeval/py.typed +0 -0
- slmeval/report/__init__.py +115 -0
- slmeval/report/attribution.py +197 -0
- slmeval/report/baseline.py +353 -0
- slmeval/report/drilldown.py +1395 -0
- slmeval/report/export.py +450 -0
- slmeval/report/model.py +562 -0
- slmeval/report/progress.py +257 -0
- slmeval/report/scorecard.py +367 -0
- slmeval/sandbox/__init__.py +145 -0
- slmeval/sandbox/diagnosis.py +690 -0
- slmeval/sandbox/envelope.py +486 -0
- slmeval/sandbox/faults.py +794 -0
- slmeval/sandbox/keys.py +458 -0
- slmeval/sandbox/normalizers.py +333 -0
- slmeval/sandbox/store.py +525 -0
- slmeval/sandbox/tool.py +685 -0
- slmeval/storage/__init__.py +73 -0
- slmeval/storage/blobs.py +308 -0
- slmeval/storage/codec.py +93 -0
- slmeval/storage/db.py +1334 -0
- slmeval/storage/ingest.py +346 -0
- slmeval/storage/migrations.py +358 -0
- slmeval/storage/models.py +554 -0
- slmeval/storage/provenance.py +201 -0
- slmeval_cli-0.1.0.dist-info/METADATA +484 -0
- slmeval_cli-0.1.0.dist-info/RECORD +94 -0
- slmeval_cli-0.1.0.dist-info/WHEEL +4 -0
- slmeval_cli-0.1.0.dist-info/entry_points.txt +2 -0
- slmeval_cli-0.1.0.dist-info/licenses/LICENSE +201 -0
|
@@ -0,0 +1,554 @@
|
|
|
1
|
+
"""Row types for the trace store.
|
|
2
|
+
|
|
3
|
+
Traces are the asset. Reports, diffs, regressions and the eventual hosted
|
|
4
|
+
product are all queries over these rows, so the shape here outlives any one
|
|
5
|
+
command and is worth more care than the code that writes it.
|
|
6
|
+
|
|
7
|
+
These are frozen dataclasses rather than Pydantic models for the same reason
|
|
8
|
+
``core.trace`` is: a 90-execution matrix over a 10-step agent writes tens of
|
|
9
|
+
thousands of them, and validating every field assignment on the way to SQLite
|
|
10
|
+
would be pure overhead. Validation belongs at the boundaries - config coming
|
|
11
|
+
in, report payloads going out.
|
|
12
|
+
|
|
13
|
+
Every enum stored in a column is a :class:`~enum.StrEnum`, so the database
|
|
14
|
+
holds a readable word rather than an integer whose meaning lives only in a
|
|
15
|
+
Python file. ``sqlite3 runs.sqlite "SELECT outcome_class, COUNT(*) ..."`` has
|
|
16
|
+
to be useful to a user debugging their own benchmark; that is most of why one
|
|
17
|
+
SQLite file was the right choice.
|
|
18
|
+
"""
|
|
19
|
+
|
|
20
|
+
from __future__ import annotations
|
|
21
|
+
|
|
22
|
+
from dataclasses import dataclass, field
|
|
23
|
+
from enum import StrEnum
|
|
24
|
+
from typing import Any, Final
|
|
25
|
+
|
|
26
|
+
INLINE_LIMIT: Final = 1024
|
|
27
|
+
"""Payloads at or below this many bytes stay in the row.
|
|
28
|
+
|
|
29
|
+
Content addressing pays for itself on tool outputs - the same recorded payload
|
|
30
|
+
is replayed by every candidate, so 1,500 executions share one file - but a
|
|
31
|
+
20-byte payload in its own file costs a filesystem cluster plus an inode, and a
|
|
32
|
+
benchmark makes tens of thousands of them. Small payloads inline, large ones
|
|
33
|
+
shared; readers cannot tell the difference.
|
|
34
|
+
"""
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
class RunStatus(StrEnum):
|
|
38
|
+
"""Lifecycle of a whole run."""
|
|
39
|
+
|
|
40
|
+
RUNNING = "running"
|
|
41
|
+
"""Written before the first execution. A run left in this state is a crash
|
|
42
|
+
or a Ctrl-C, which is exactly what ``--resume`` looks for."""
|
|
43
|
+
|
|
44
|
+
COMPLETED = "completed"
|
|
45
|
+
|
|
46
|
+
FAILED = "failed"
|
|
47
|
+
"""The harness gave up - not a quality verdict. A run whose candidates all
|
|
48
|
+
scored zero is ``completed``."""
|
|
49
|
+
|
|
50
|
+
CANCELLED = "cancelled"
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
class ExecStatus(StrEnum):
|
|
54
|
+
"""What mechanically happened to one execution.
|
|
55
|
+
|
|
56
|
+
Deliberately separate from :class:`Outcome`: this says whether the agent
|
|
57
|
+
returned, and that says whether what it returned was any good. Collapsing
|
|
58
|
+
the two is how a harness ends up reporting its own timeout as a model
|
|
59
|
+
failure.
|
|
60
|
+
"""
|
|
61
|
+
|
|
62
|
+
OK = "ok"
|
|
63
|
+
"""The target returned a result. It may still have failed every assertion."""
|
|
64
|
+
|
|
65
|
+
ERROR = "error"
|
|
66
|
+
"""The target raised. ``error_class`` says what."""
|
|
67
|
+
|
|
68
|
+
TIMEOUT = "timeout"
|
|
69
|
+
|
|
70
|
+
SKIPPED = "skipped"
|
|
71
|
+
"""Planned but never run - a unit left over when a run was interrupted.
|
|
72
|
+
Recorded rather than absent, so the shape of the matrix survives a crash."""
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
class Outcome(StrEnum):
|
|
76
|
+
"""The classified verdict for one execution.
|
|
77
|
+
|
|
78
|
+
Filled by the evaluators in step 11, not by the runner. Day 8 only owns the
|
|
79
|
+
column and the vocabulary - but the vocabulary has to be settled now,
|
|
80
|
+
because the scorecard warnings are counts over these values and a rename
|
|
81
|
+
later is a migration.
|
|
82
|
+
"""
|
|
83
|
+
|
|
84
|
+
UNCLASSIFIED = "unclassified"
|
|
85
|
+
"""Stored but not yet judged. Never aggregated into a pass rate; a metric
|
|
86
|
+
that silently counts unjudged executions as failures is worse than a gap."""
|
|
87
|
+
|
|
88
|
+
PASSED = "passed"
|
|
89
|
+
|
|
90
|
+
FAILED_ASSERTION = "failed_assertion"
|
|
91
|
+
"""Ran cleanly, produced the wrong answer or state. The ordinary failure."""
|
|
92
|
+
|
|
93
|
+
SCHEMA_FAILURE = "schema_failure"
|
|
94
|
+
"""Could not produce output matching the requested schema. The headline SLM
|
|
95
|
+
failure mode, and the reason it is not folded into ``failed_assertion``."""
|
|
96
|
+
|
|
97
|
+
FORBIDDEN_TOOL = "forbidden_tool"
|
|
98
|
+
"""Called a tool the case forbids. A hard fail that must never be averaged
|
|
99
|
+
away - the scorecard prints it above the table."""
|
|
100
|
+
|
|
101
|
+
CASSETTE_MISS = "cassette_miss"
|
|
102
|
+
"""Asked for a tool call we never recorded, and the miss was diagnosed as
|
|
103
|
+
the model's doing: a hallucinated tool or parameter."""
|
|
104
|
+
|
|
105
|
+
UNCOVERED = "uncovered"
|
|
106
|
+
"""Also a miss, but diagnosed as a valid path we simply never recorded.
|
|
107
|
+
Excluded from pass rates rather than counted against the candidate: marking
|
|
108
|
+
a model down for exploring a route the baseline never took would be our own
|
|
109
|
+
gap, reported as its failure."""
|
|
110
|
+
|
|
111
|
+
TOOL_ERROR = "tool_error"
|
|
112
|
+
"""A tool raised - usually an injected fault the agent never recovered
|
|
113
|
+
from, which is the numerator of the recovery rate."""
|
|
114
|
+
|
|
115
|
+
PROVIDER_ERROR = "provider_error"
|
|
116
|
+
"""The model server never produced a response: down, unreachable, key
|
|
117
|
+
rejected, model not pulled. Separate from every quality outcome, because
|
|
118
|
+
"Ollama was not running" must never be reported as a regression."""
|
|
119
|
+
|
|
120
|
+
STEP_BUDGET = "step_budget"
|
|
121
|
+
"""Hit ``max_steps``. Usually a loop."""
|
|
122
|
+
|
|
123
|
+
TIMEOUT = "timeout"
|
|
124
|
+
|
|
125
|
+
CRASHED = "crashed"
|
|
126
|
+
"""The target raised something unrelated to the model - a bug in the agent
|
|
127
|
+
under test. The agent's own code is the user's to fix, so this counts
|
|
128
|
+
against the run but is reported as the agent failing, not the model."""
|
|
129
|
+
|
|
130
|
+
HARNESS_ERROR = "harness_error"
|
|
131
|
+
"""*We* broke. Counted separately and never attributed to the model or the
|
|
132
|
+
agent, because conflating our own bugs with model failures is exactly how a
|
|
133
|
+
benchmark loses the right to be believed. A scorecard prints these above
|
|
134
|
+
the table as a caveat on the whole run and leaves them out of every rate."""
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
TERMINAL_STATUSES: Final = frozenset({ExecStatus.OK, ExecStatus.ERROR, ExecStatus.TIMEOUT})
|
|
138
|
+
"""Statuses meaning "this unit does not need running again".
|
|
139
|
+
|
|
140
|
+
``--resume`` re-runs anything outside this set, which is why ``SKIPPED`` is not
|
|
141
|
+
in it: a unit the previous run never reached is work, not a result.
|
|
142
|
+
"""
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
NOT_THE_MODELS_FAULT: Final = frozenset({Outcome.HARNESS_ERROR, Outcome.UNCOVERED})
|
|
146
|
+
"""Outcomes excluded from a candidate's pass rate entirely.
|
|
147
|
+
|
|
148
|
+
Not scored as failures and not scored as passes - dropped from the denominator.
|
|
149
|
+
``harness_error`` is our bug and ``uncovered`` is our missing recording; either
|
|
150
|
+
one counted against a candidate would be a regression the tool invented. Both
|
|
151
|
+
are surfaced as warnings above the table instead, where they belong.
|
|
152
|
+
"""
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
@dataclass(frozen=True, slots=True)
|
|
156
|
+
class Payload:
|
|
157
|
+
"""A JSON payload living either in the row or in the blob store.
|
|
158
|
+
|
|
159
|
+
Exactly one side is set, or neither for "nothing recorded". Readers go
|
|
160
|
+
through ``RunStore.load`` and never need to know which, so the inline/blob
|
|
161
|
+
threshold can change between versions without touching a query.
|
|
162
|
+
"""
|
|
163
|
+
|
|
164
|
+
inline: str | None = None
|
|
165
|
+
digest: str | None = None
|
|
166
|
+
|
|
167
|
+
@property
|
|
168
|
+
def present(self) -> bool:
|
|
169
|
+
return self.inline is not None or self.digest is not None
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
EMPTY_PAYLOAD: Final = Payload()
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
@dataclass(frozen=True, slots=True)
|
|
176
|
+
class RunRecord:
|
|
177
|
+
"""One invocation of ``slmeval run``, with its provenance.
|
|
178
|
+
|
|
179
|
+
The provenance fields are not bookkeeping. Without ``config_hash`` and
|
|
180
|
+
``dataset_hash``, comparing two runs is guesswork dressed up as a table;
|
|
181
|
+
with them, ``slmeval compare`` can *refuse* to compare runs built on
|
|
182
|
+
different datasets unless forced. That refusal is a feature.
|
|
183
|
+
"""
|
|
184
|
+
|
|
185
|
+
run_id: str
|
|
186
|
+
name: str
|
|
187
|
+
started_at: str
|
|
188
|
+
config_json: str
|
|
189
|
+
config_hash: str
|
|
190
|
+
dataset_hash: str
|
|
191
|
+
slmeval_version: str
|
|
192
|
+
|
|
193
|
+
mode: str
|
|
194
|
+
"""``replay``, ``record`` or ``live`` - the tool mode the matrix ran under.
|
|
195
|
+
A live run and a replay run of the same config do not have comparable
|
|
196
|
+
latency numbers, and this is how a report knows not to put them in one
|
|
197
|
+
column."""
|
|
198
|
+
|
|
199
|
+
seed: int
|
|
200
|
+
|
|
201
|
+
scheduling: str = "throughput"
|
|
202
|
+
"""``throughput`` or ``latency``. Throughput runs high concurrency and its
|
|
203
|
+
latency numbers are not reportable: requests queue inside the provider, and
|
|
204
|
+
a local model measured that way reads several times slower than it is.
|
|
205
|
+
Stored so the report refuses to print a p95 column rather than printing a
|
|
206
|
+
misleading one."""
|
|
207
|
+
|
|
208
|
+
status: RunStatus = RunStatus.RUNNING
|
|
209
|
+
finished_at: str | None = None
|
|
210
|
+
dataset_path: str = ""
|
|
211
|
+
git_sha: str = ""
|
|
212
|
+
host_info: str = "{}"
|
|
213
|
+
fault_rate: float = 0.0
|
|
214
|
+
runs_per_case: int = 1
|
|
215
|
+
error_message: str | None = None
|
|
216
|
+
|
|
217
|
+
|
|
218
|
+
@dataclass(frozen=True, slots=True)
|
|
219
|
+
class CandidateRecord:
|
|
220
|
+
"""One model configuration under test in a run.
|
|
221
|
+
|
|
222
|
+
``model_cfg_json`` is the whole resolved config, not a pointer at the file:
|
|
223
|
+
``eval.toml`` will have been edited by the time anyone reads this row, and a
|
|
224
|
+
report that re-read the file would quietly describe a run that never
|
|
225
|
+
happened.
|
|
226
|
+
"""
|
|
227
|
+
|
|
228
|
+
run_id: str
|
|
229
|
+
candidate_id: str
|
|
230
|
+
model_cfg_json: str
|
|
231
|
+
|
|
232
|
+
role: str = "candidate"
|
|
233
|
+
"""``baseline`` or ``candidate``. The scorecard computes its delta column
|
|
234
|
+
against the baseline row, so which one that is has to survive the run."""
|
|
235
|
+
|
|
236
|
+
ordinal: int = 0
|
|
237
|
+
"""Declaration order, so table row order is stable across reports rather
|
|
238
|
+
than whatever the query planner happens to return today."""
|
|
239
|
+
|
|
240
|
+
provider: str = ""
|
|
241
|
+
model: str = ""
|
|
242
|
+
|
|
243
|
+
|
|
244
|
+
@dataclass(frozen=True, slots=True)
|
|
245
|
+
class ExecutionRecord:
|
|
246
|
+
"""One (candidate, case, repeat) cell of the matrix."""
|
|
247
|
+
|
|
248
|
+
exec_id: str
|
|
249
|
+
run_id: str
|
|
250
|
+
candidate_id: str
|
|
251
|
+
case_id: str
|
|
252
|
+
repeat_idx: int
|
|
253
|
+
status: ExecStatus
|
|
254
|
+
outcome_class: Outcome = Outcome.UNCLASSIFIED
|
|
255
|
+
|
|
256
|
+
attempt: int = 0
|
|
257
|
+
"""Incremented when crash isolation re-runs a unit. The retry replaces the
|
|
258
|
+
row, but the count stays: "passed on the second attempt" is a finding about
|
|
259
|
+
the candidate, not an implementation detail of the runner."""
|
|
260
|
+
|
|
261
|
+
started_at: str = ""
|
|
262
|
+
duration_ms: float = 0.0
|
|
263
|
+
steps: int = 0
|
|
264
|
+
total_prompt_tokens: int | None = None
|
|
265
|
+
total_completion_tokens: int | None = None
|
|
266
|
+
|
|
267
|
+
tokens_estimated: bool = False
|
|
268
|
+
"""True when any call here had its usage estimated rather than reported.
|
|
269
|
+
Any aggregate containing one must be marked ``~``, so the flag rides on the
|
|
270
|
+
execution and not only on the individual call."""
|
|
271
|
+
|
|
272
|
+
cost_usd: float | None = None
|
|
273
|
+
fault_vector_json: str = "[]"
|
|
274
|
+
|
|
275
|
+
fault_digest: str = ""
|
|
276
|
+
"""Digest of the faults this execution actually met. Two candidates with
|
|
277
|
+
different digests were not tested under the same conditions, and
|
|
278
|
+
``--require-fault-parity`` drops the pair rather than print both numbers as
|
|
279
|
+
if they were."""
|
|
280
|
+
|
|
281
|
+
trajectory_json: str = "[]"
|
|
282
|
+
"""Ordered node visits. Stored even though ``node_traces`` implies most of
|
|
283
|
+
it, because a node can be entered without making a model call and the
|
|
284
|
+
trajectory-distance metric counts the visit, not the call."""
|
|
285
|
+
|
|
286
|
+
warnings_json: str = "[]"
|
|
287
|
+
output: Payload = EMPTY_PAYLOAD
|
|
288
|
+
state: Payload = EMPTY_PAYLOAD
|
|
289
|
+
error_class: str | None = None
|
|
290
|
+
error_message: str | None = None
|
|
291
|
+
|
|
292
|
+
|
|
293
|
+
@dataclass(frozen=True, slots=True)
|
|
294
|
+
class NodeTraceRow:
|
|
295
|
+
"""One model call, as stored. Mirrors
|
|
296
|
+
:class:`~slmeval.core.trace.LLMCallTrace`."""
|
|
297
|
+
|
|
298
|
+
exec_id: str
|
|
299
|
+
seq: int
|
|
300
|
+
node_name: str = ""
|
|
301
|
+
node_path: str = "[]"
|
|
302
|
+
|
|
303
|
+
model_id: str = ""
|
|
304
|
+
"""The candidate id, duplicated from the execution so per-node attribution
|
|
305
|
+
is one indexed query instead of a join across the whole matrix."""
|
|
306
|
+
|
|
307
|
+
provider: str = ""
|
|
308
|
+
model: str = ""
|
|
309
|
+
latency_ms: float = 0.0
|
|
310
|
+
ttft_ms: float | None = None
|
|
311
|
+
retries: int = 0
|
|
312
|
+
retry_ms: float = 0.0
|
|
313
|
+
prompt_tokens: int | None = None
|
|
314
|
+
completion_tokens: int | None = None
|
|
315
|
+
tokens_estimated: bool = False
|
|
316
|
+
schema_name: str | None = None
|
|
317
|
+
schema_state: str = "not_requested"
|
|
318
|
+
structured_requested: str = "auto"
|
|
319
|
+
|
|
320
|
+
structured_mode: str | None = None
|
|
321
|
+
"""Which ladder rung actually worked. The most valuable field in this
|
|
322
|
+
table - "needed json_mode on 100% of calls" is a line in the scorecard."""
|
|
323
|
+
|
|
324
|
+
rungs_tried: str = "[]"
|
|
325
|
+
repair_attempts: int = 0
|
|
326
|
+
json_stage: str | None = None
|
|
327
|
+
error_class: str | None = None
|
|
328
|
+
error_message: str | None = None
|
|
329
|
+
|
|
330
|
+
raw: Payload = EMPTY_PAYLOAD
|
|
331
|
+
"""The provider's raw response, kept for failing calls only. The failure
|
|
332
|
+
report needs it; keeping it for every call would balloon the store."""
|
|
333
|
+
|
|
334
|
+
|
|
335
|
+
@dataclass(frozen=True, slots=True)
|
|
336
|
+
class ToolTraceRow:
|
|
337
|
+
"""One tool invocation, as stored. Mirrors
|
|
338
|
+
:class:`~slmeval.core.trace.ToolCallTrace`."""
|
|
339
|
+
|
|
340
|
+
exec_id: str
|
|
341
|
+
seq: int
|
|
342
|
+
tool_name: str
|
|
343
|
+
node_name: str = ""
|
|
344
|
+
node_path: str = "[]"
|
|
345
|
+
args: Payload = EMPTY_PAYLOAD
|
|
346
|
+
cassette_key: str = ""
|
|
347
|
+
outcome: str = "live"
|
|
348
|
+
match_tier: str | None = None
|
|
349
|
+
cache_hit: bool = False
|
|
350
|
+
fault: str | None = None
|
|
351
|
+
fault_injected: bool = False
|
|
352
|
+
miss_verdict: str | None = None
|
|
353
|
+
tool_version: str = ""
|
|
354
|
+
latency_ms: float = 0.0
|
|
355
|
+
output: Payload = EMPTY_PAYLOAD
|
|
356
|
+
error_message: str | None = None
|
|
357
|
+
|
|
358
|
+
|
|
359
|
+
@dataclass(frozen=True, slots=True)
|
|
360
|
+
class EvaluationRow:
|
|
361
|
+
"""One metric for one execution.
|
|
362
|
+
|
|
363
|
+
Long and narrow rather than a column per metric, because week 3 keeps
|
|
364
|
+
adding evaluators and a schema where every new metric is a migration is a
|
|
365
|
+
schema that stops gaining metrics.
|
|
366
|
+
"""
|
|
367
|
+
|
|
368
|
+
exec_id: str
|
|
369
|
+
tier: int
|
|
370
|
+
metric: str
|
|
371
|
+
value_num: float | None = None
|
|
372
|
+
value_text: str | None = None
|
|
373
|
+
detail_json: str | None = None
|
|
374
|
+
|
|
375
|
+
|
|
376
|
+
@dataclass(frozen=True, slots=True)
|
|
377
|
+
class CaseRollupRow:
|
|
378
|
+
"""Consistency math for one (candidate, case) across its repeats.
|
|
379
|
+
|
|
380
|
+
Derived from ``executions``, and stored anyway: ``pass^k`` and the Wilson
|
|
381
|
+
interval are the headline numbers, a report must not recompute them and
|
|
382
|
+
risk disagreeing with the run that printed them, and a later change to the
|
|
383
|
+
estimator must not silently restate last month's results.
|
|
384
|
+
"""
|
|
385
|
+
|
|
386
|
+
run_id: str
|
|
387
|
+
candidate_id: str
|
|
388
|
+
case_id: str
|
|
389
|
+
|
|
390
|
+
n: int
|
|
391
|
+
"""Repeats that produced a judgeable result. Always printed - a pass rate
|
|
392
|
+
without its n is a number pretending to be evidence."""
|
|
393
|
+
|
|
394
|
+
c: int
|
|
395
|
+
"""How many of them passed."""
|
|
396
|
+
|
|
397
|
+
pass_k: float | None = None
|
|
398
|
+
pass_hat_k: float | None = None
|
|
399
|
+
wilson_lo: float | None = None
|
|
400
|
+
wilson_hi: float | None = None
|
|
401
|
+
|
|
402
|
+
|
|
403
|
+
@dataclass(frozen=True, slots=True)
|
|
404
|
+
class MetricSample:
|
|
405
|
+
"""One stored metric, with the cell it belongs to.
|
|
406
|
+
|
|
407
|
+
The join the report needs and the drill-down does not: a scorecard
|
|
408
|
+
aggregates thousands of metrics across a run, and doing it by reconstructing
|
|
409
|
+
each execution in turn would make rendering a table slower than producing
|
|
410
|
+
one.
|
|
411
|
+
"""
|
|
412
|
+
|
|
413
|
+
candidate_id: str
|
|
414
|
+
case_id: str
|
|
415
|
+
repeat_idx: int
|
|
416
|
+
exec_id: str
|
|
417
|
+
tier: int
|
|
418
|
+
metric: str
|
|
419
|
+
value_num: float | None = None
|
|
420
|
+
value_text: str | None = None
|
|
421
|
+
detail_json: str | None = None
|
|
422
|
+
outcome: Outcome = Outcome.UNCLASSIFIED
|
|
423
|
+
|
|
424
|
+
|
|
425
|
+
@dataclass(frozen=True, slots=True)
|
|
426
|
+
class NodeSummaryRow:
|
|
427
|
+
"""Model calls at one node, for one candidate, aggregated.
|
|
428
|
+
|
|
429
|
+
Per-node attribution is the table that turns "this candidate is 13 points
|
|
430
|
+
worse" into "this candidate breaks in ``policy_check``", which is the
|
|
431
|
+
difference between a benchmark and a recommendation.
|
|
432
|
+
"""
|
|
433
|
+
|
|
434
|
+
candidate_id: str
|
|
435
|
+
node_name: str
|
|
436
|
+
provider: str = ""
|
|
437
|
+
model: str = ""
|
|
438
|
+
calls: int = 0
|
|
439
|
+
errors: int = 0
|
|
440
|
+
schema_valid: int = 0
|
|
441
|
+
schema_invalid: int = 0
|
|
442
|
+
repair_attempts: int = 0
|
|
443
|
+
latency_ms: float = 0.0
|
|
444
|
+
"""Total across calls. The mean is the report's to compute, so a caller
|
|
445
|
+
that wants a different statistic is not stuck with ours."""
|
|
446
|
+
prompt_tokens: int = 0
|
|
447
|
+
completion_tokens: int = 0
|
|
448
|
+
|
|
449
|
+
@property
|
|
450
|
+
def mean_latency_ms(self) -> float | None:
|
|
451
|
+
return self.latency_ms / self.calls if self.calls else None
|
|
452
|
+
|
|
453
|
+
@property
|
|
454
|
+
def schema_valid_rate(self) -> float | None:
|
|
455
|
+
judged = self.schema_valid + self.schema_invalid
|
|
456
|
+
return self.schema_valid / judged if judged else None
|
|
457
|
+
|
|
458
|
+
|
|
459
|
+
@dataclass(frozen=True, slots=True)
|
|
460
|
+
class BlameRow:
|
|
461
|
+
"""Where a candidate's failing executions went wrong, counted by node."""
|
|
462
|
+
|
|
463
|
+
candidate_id: str
|
|
464
|
+
node_name: str
|
|
465
|
+
kind: str
|
|
466
|
+
"""``model`` or ``tool`` - which half of the step failed."""
|
|
467
|
+
error_class: str = ""
|
|
468
|
+
count: int = 0
|
|
469
|
+
|
|
470
|
+
|
|
471
|
+
@dataclass(frozen=True, slots=True)
|
|
472
|
+
class ExecutionDetail:
|
|
473
|
+
"""One execution and everything recorded under it, payloads resolved.
|
|
474
|
+
|
|
475
|
+
What ``slmeval show exec <id>`` renders, and the proof of step 12's
|
|
476
|
+
acceptance test: built from rows alone, so dropping every in-memory object
|
|
477
|
+
changes nothing about what gets printed.
|
|
478
|
+
"""
|
|
479
|
+
|
|
480
|
+
execution: ExecutionRecord
|
|
481
|
+
node_traces: tuple[NodeTraceRow, ...] = ()
|
|
482
|
+
tool_traces: tuple[ToolTraceRow, ...] = ()
|
|
483
|
+
evaluations: tuple[EvaluationRow, ...] = ()
|
|
484
|
+
output: Any = None
|
|
485
|
+
state: Any = None
|
|
486
|
+
|
|
487
|
+
tool_outputs: dict[int, Any] = field(default_factory=dict)
|
|
488
|
+
"""Resolved tool return values, keyed by ``seq``. Loaded eagerly, because a
|
|
489
|
+
timeline that re-opened the blob store per row would make the drill-down
|
|
490
|
+
the slowest command in the tool."""
|
|
491
|
+
|
|
492
|
+
tool_arguments: dict[int, Any] = field(default_factory=dict)
|
|
493
|
+
|
|
494
|
+
@property
|
|
495
|
+
def trajectory(self) -> tuple[tuple[str, ...], ...]:
|
|
496
|
+
from slmeval.storage.codec import decode_paths
|
|
497
|
+
|
|
498
|
+
return decode_paths(self.execution.trajectory_json)
|
|
499
|
+
|
|
500
|
+
|
|
501
|
+
@dataclass(frozen=True, slots=True)
|
|
502
|
+
class RunDetail:
|
|
503
|
+
"""A whole run, reconstructed. ``slmeval report`` reads exactly this."""
|
|
504
|
+
|
|
505
|
+
run: RunRecord
|
|
506
|
+
candidates: tuple[CandidateRecord, ...] = ()
|
|
507
|
+
executions: tuple[ExecutionRecord, ...] = ()
|
|
508
|
+
rollups: tuple[CaseRollupRow, ...] = ()
|
|
509
|
+
|
|
510
|
+
@property
|
|
511
|
+
def baseline(self) -> CandidateRecord | None:
|
|
512
|
+
return next((c for c in self.candidates if c.role == "baseline"), None)
|
|
513
|
+
|
|
514
|
+
|
|
515
|
+
@dataclass(frozen=True, slots=True)
|
|
516
|
+
class StoreStats:
|
|
517
|
+
"""What ``slmeval cache stats`` prints."""
|
|
518
|
+
|
|
519
|
+
runs: int
|
|
520
|
+
running: int
|
|
521
|
+
executions: int
|
|
522
|
+
node_traces: int
|
|
523
|
+
tool_traces: int
|
|
524
|
+
evaluations: int
|
|
525
|
+
db_bytes: int
|
|
526
|
+
blobs: int
|
|
527
|
+
blob_bytes: int
|
|
528
|
+
oldest_run: str | None = None
|
|
529
|
+
newest_run: str | None = None
|
|
530
|
+
|
|
531
|
+
|
|
532
|
+
__all__ = [
|
|
533
|
+
"EMPTY_PAYLOAD",
|
|
534
|
+
"INLINE_LIMIT",
|
|
535
|
+
"NOT_THE_MODELS_FAULT",
|
|
536
|
+
"TERMINAL_STATUSES",
|
|
537
|
+
"BlameRow",
|
|
538
|
+
"CandidateRecord",
|
|
539
|
+
"CaseRollupRow",
|
|
540
|
+
"EvaluationRow",
|
|
541
|
+
"ExecStatus",
|
|
542
|
+
"ExecutionDetail",
|
|
543
|
+
"ExecutionRecord",
|
|
544
|
+
"MetricSample",
|
|
545
|
+
"NodeSummaryRow",
|
|
546
|
+
"NodeTraceRow",
|
|
547
|
+
"Outcome",
|
|
548
|
+
"Payload",
|
|
549
|
+
"RunDetail",
|
|
550
|
+
"RunRecord",
|
|
551
|
+
"RunStatus",
|
|
552
|
+
"StoreStats",
|
|
553
|
+
"ToolTraceRow",
|
|
554
|
+
]
|