slmeval-cli 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (94) hide show
  1. slmeval/__init__.py +71 -0
  2. slmeval/__main__.py +14 -0
  3. slmeval/adapters/__init__.py +24 -0
  4. slmeval/adapters/instrument.py +135 -0
  5. slmeval/adapters/langgraph.py +237 -0
  6. slmeval/cli/__init__.py +15 -0
  7. slmeval/cli/commands/__init__.py +35 -0
  8. slmeval/cli/commands/baseline_cmd.py +201 -0
  9. slmeval/cli/commands/cache_cmd.py +249 -0
  10. slmeval/cli/commands/dataset_cmd.py +52 -0
  11. slmeval/cli/commands/doctor_cmd.py +68 -0
  12. slmeval/cli/commands/gate_cmd.py +602 -0
  13. slmeval/cli/commands/init_cmd.py +144 -0
  14. slmeval/cli/commands/judge_cmd.py +284 -0
  15. slmeval/cli/commands/record_cmd.py +154 -0
  16. slmeval/cli/commands/report_cmd.py +98 -0
  17. slmeval/cli/commands/run_cmd.py +247 -0
  18. slmeval/cli/commands/show_cmd.py +482 -0
  19. slmeval/cli/commands/stubs.py +57 -0
  20. slmeval/cli/commands/validate_cmd.py +296 -0
  21. slmeval/cli/loading.py +149 -0
  22. slmeval/cli/main.py +91 -0
  23. slmeval/cli/render.py +251 -0
  24. slmeval/cli/wrap.py +31 -0
  25. slmeval/config/__init__.py +47 -0
  26. slmeval/config/loader.py +363 -0
  27. slmeval/config/locate.py +97 -0
  28. slmeval/config/models.py +430 -0
  29. slmeval/config/templates.py +246 -0
  30. slmeval/core/__init__.py +78 -0
  31. slmeval/core/adapter.py +253 -0
  32. slmeval/core/context.py +418 -0
  33. slmeval/core/outcome.py +260 -0
  34. slmeval/core/router.py +319 -0
  35. slmeval/core/runner.py +1209 -0
  36. slmeval/core/trace.py +188 -0
  37. slmeval/core/verdict.py +127 -0
  38. slmeval/dataset/__init__.py +8 -0
  39. slmeval/dataset/loader.py +130 -0
  40. slmeval/dataset/models.py +108 -0
  41. slmeval/doctor/__init__.py +7 -0
  42. slmeval/doctor/checks.py +380 -0
  43. slmeval/errors.py +76 -0
  44. slmeval/eval/__init__.py +64 -0
  45. slmeval/eval/assertions.py +356 -0
  46. slmeval/eval/calibration.py +364 -0
  47. slmeval/eval/cost.py +291 -0
  48. slmeval/eval/pipeline.py +273 -0
  49. slmeval/eval/stats.py +376 -0
  50. slmeval/eval/tier1.py +510 -0
  51. slmeval/eval/tier2.py +582 -0
  52. slmeval/eval/tier3.py +415 -0
  53. slmeval/exit_codes.py +28 -0
  54. slmeval/gate.py +722 -0
  55. slmeval/providers/__init__.py +69 -0
  56. slmeval/providers/base.py +426 -0
  57. slmeval/providers/fake.py +760 -0
  58. slmeval/providers/jsonx.py +239 -0
  59. slmeval/providers/ladder.py +339 -0
  60. slmeval/providers/openai_compat.py +392 -0
  61. slmeval/providers/registry.py +185 -0
  62. slmeval/providers/retry.py +81 -0
  63. slmeval/providers/schema.py +136 -0
  64. slmeval/providers/tokens.py +150 -0
  65. slmeval/py.typed +0 -0
  66. slmeval/report/__init__.py +115 -0
  67. slmeval/report/attribution.py +197 -0
  68. slmeval/report/baseline.py +353 -0
  69. slmeval/report/drilldown.py +1395 -0
  70. slmeval/report/export.py +450 -0
  71. slmeval/report/model.py +562 -0
  72. slmeval/report/progress.py +257 -0
  73. slmeval/report/scorecard.py +367 -0
  74. slmeval/sandbox/__init__.py +145 -0
  75. slmeval/sandbox/diagnosis.py +690 -0
  76. slmeval/sandbox/envelope.py +486 -0
  77. slmeval/sandbox/faults.py +794 -0
  78. slmeval/sandbox/keys.py +458 -0
  79. slmeval/sandbox/normalizers.py +333 -0
  80. slmeval/sandbox/store.py +525 -0
  81. slmeval/sandbox/tool.py +685 -0
  82. slmeval/storage/__init__.py +73 -0
  83. slmeval/storage/blobs.py +308 -0
  84. slmeval/storage/codec.py +93 -0
  85. slmeval/storage/db.py +1334 -0
  86. slmeval/storage/ingest.py +346 -0
  87. slmeval/storage/migrations.py +358 -0
  88. slmeval/storage/models.py +554 -0
  89. slmeval/storage/provenance.py +201 -0
  90. slmeval_cli-0.1.0.dist-info/METADATA +484 -0
  91. slmeval_cli-0.1.0.dist-info/RECORD +94 -0
  92. slmeval_cli-0.1.0.dist-info/WHEEL +4 -0
  93. slmeval_cli-0.1.0.dist-info/entry_points.txt +2 -0
  94. slmeval_cli-0.1.0.dist-info/licenses/LICENSE +201 -0
@@ -0,0 +1,554 @@
1
+ """Row types for the trace store.
2
+
3
+ Traces are the asset. Reports, diffs, regressions and the eventual hosted
4
+ product are all queries over these rows, so the shape here outlives any one
5
+ command and is worth more care than the code that writes it.
6
+
7
+ These are frozen dataclasses rather than Pydantic models for the same reason
8
+ ``core.trace`` is: a 90-execution matrix over a 10-step agent writes tens of
9
+ thousands of them, and validating every field assignment on the way to SQLite
10
+ would be pure overhead. Validation belongs at the boundaries - config coming
11
+ in, report payloads going out.
12
+
13
+ Every enum stored in a column is a :class:`~enum.StrEnum`, so the database
14
+ holds a readable word rather than an integer whose meaning lives only in a
15
+ Python file. ``sqlite3 runs.sqlite "SELECT outcome_class, COUNT(*) ..."`` has
16
+ to be useful to a user debugging their own benchmark; that is most of why one
17
+ SQLite file was the right choice.
18
+ """
19
+
20
+ from __future__ import annotations
21
+
22
+ from dataclasses import dataclass, field
23
+ from enum import StrEnum
24
+ from typing import Any, Final
25
+
26
+ INLINE_LIMIT: Final = 1024
27
+ """Payloads at or below this many bytes stay in the row.
28
+
29
+ Content addressing pays for itself on tool outputs - the same recorded payload
30
+ is replayed by every candidate, so 1,500 executions share one file - but a
31
+ 20-byte payload in its own file costs a filesystem cluster plus an inode, and a
32
+ benchmark makes tens of thousands of them. Small payloads inline, large ones
33
+ shared; readers cannot tell the difference.
34
+ """
35
+
36
+
37
+ class RunStatus(StrEnum):
38
+ """Lifecycle of a whole run."""
39
+
40
+ RUNNING = "running"
41
+ """Written before the first execution. A run left in this state is a crash
42
+ or a Ctrl-C, which is exactly what ``--resume`` looks for."""
43
+
44
+ COMPLETED = "completed"
45
+
46
+ FAILED = "failed"
47
+ """The harness gave up - not a quality verdict. A run whose candidates all
48
+ scored zero is ``completed``."""
49
+
50
+ CANCELLED = "cancelled"
51
+
52
+
53
+ class ExecStatus(StrEnum):
54
+ """What mechanically happened to one execution.
55
+
56
+ Deliberately separate from :class:`Outcome`: this says whether the agent
57
+ returned, and that says whether what it returned was any good. Collapsing
58
+ the two is how a harness ends up reporting its own timeout as a model
59
+ failure.
60
+ """
61
+
62
+ OK = "ok"
63
+ """The target returned a result. It may still have failed every assertion."""
64
+
65
+ ERROR = "error"
66
+ """The target raised. ``error_class`` says what."""
67
+
68
+ TIMEOUT = "timeout"
69
+
70
+ SKIPPED = "skipped"
71
+ """Planned but never run - a unit left over when a run was interrupted.
72
+ Recorded rather than absent, so the shape of the matrix survives a crash."""
73
+
74
+
75
+ class Outcome(StrEnum):
76
+ """The classified verdict for one execution.
77
+
78
+ Filled by the evaluators in step 11, not by the runner. Day 8 only owns the
79
+ column and the vocabulary - but the vocabulary has to be settled now,
80
+ because the scorecard warnings are counts over these values and a rename
81
+ later is a migration.
82
+ """
83
+
84
+ UNCLASSIFIED = "unclassified"
85
+ """Stored but not yet judged. Never aggregated into a pass rate; a metric
86
+ that silently counts unjudged executions as failures is worse than a gap."""
87
+
88
+ PASSED = "passed"
89
+
90
+ FAILED_ASSERTION = "failed_assertion"
91
+ """Ran cleanly, produced the wrong answer or state. The ordinary failure."""
92
+
93
+ SCHEMA_FAILURE = "schema_failure"
94
+ """Could not produce output matching the requested schema. The headline SLM
95
+ failure mode, and the reason it is not folded into ``failed_assertion``."""
96
+
97
+ FORBIDDEN_TOOL = "forbidden_tool"
98
+ """Called a tool the case forbids. A hard fail that must never be averaged
99
+ away - the scorecard prints it above the table."""
100
+
101
+ CASSETTE_MISS = "cassette_miss"
102
+ """Asked for a tool call we never recorded, and the miss was diagnosed as
103
+ the model's doing: a hallucinated tool or parameter."""
104
+
105
+ UNCOVERED = "uncovered"
106
+ """Also a miss, but diagnosed as a valid path we simply never recorded.
107
+ Excluded from pass rates rather than counted against the candidate: marking
108
+ a model down for exploring a route the baseline never took would be our own
109
+ gap, reported as its failure."""
110
+
111
+ TOOL_ERROR = "tool_error"
112
+ """A tool raised - usually an injected fault the agent never recovered
113
+ from, which is the numerator of the recovery rate."""
114
+
115
+ PROVIDER_ERROR = "provider_error"
116
+ """The model server never produced a response: down, unreachable, key
117
+ rejected, model not pulled. Separate from every quality outcome, because
118
+ "Ollama was not running" must never be reported as a regression."""
119
+
120
+ STEP_BUDGET = "step_budget"
121
+ """Hit ``max_steps``. Usually a loop."""
122
+
123
+ TIMEOUT = "timeout"
124
+
125
+ CRASHED = "crashed"
126
+ """The target raised something unrelated to the model - a bug in the agent
127
+ under test. The agent's own code is the user's to fix, so this counts
128
+ against the run but is reported as the agent failing, not the model."""
129
+
130
+ HARNESS_ERROR = "harness_error"
131
+ """*We* broke. Counted separately and never attributed to the model or the
132
+ agent, because conflating our own bugs with model failures is exactly how a
133
+ benchmark loses the right to be believed. A scorecard prints these above
134
+ the table as a caveat on the whole run and leaves them out of every rate."""
135
+
136
+
137
+ TERMINAL_STATUSES: Final = frozenset({ExecStatus.OK, ExecStatus.ERROR, ExecStatus.TIMEOUT})
138
+ """Statuses meaning "this unit does not need running again".
139
+
140
+ ``--resume`` re-runs anything outside this set, which is why ``SKIPPED`` is not
141
+ in it: a unit the previous run never reached is work, not a result.
142
+ """
143
+
144
+
145
+ NOT_THE_MODELS_FAULT: Final = frozenset({Outcome.HARNESS_ERROR, Outcome.UNCOVERED})
146
+ """Outcomes excluded from a candidate's pass rate entirely.
147
+
148
+ Not scored as failures and not scored as passes - dropped from the denominator.
149
+ ``harness_error`` is our bug and ``uncovered`` is our missing recording; either
150
+ one counted against a candidate would be a regression the tool invented. Both
151
+ are surfaced as warnings above the table instead, where they belong.
152
+ """
153
+
154
+
155
+ @dataclass(frozen=True, slots=True)
156
+ class Payload:
157
+ """A JSON payload living either in the row or in the blob store.
158
+
159
+ Exactly one side is set, or neither for "nothing recorded". Readers go
160
+ through ``RunStore.load`` and never need to know which, so the inline/blob
161
+ threshold can change between versions without touching a query.
162
+ """
163
+
164
+ inline: str | None = None
165
+ digest: str | None = None
166
+
167
+ @property
168
+ def present(self) -> bool:
169
+ return self.inline is not None or self.digest is not None
170
+
171
+
172
+ EMPTY_PAYLOAD: Final = Payload()
173
+
174
+
175
+ @dataclass(frozen=True, slots=True)
176
+ class RunRecord:
177
+ """One invocation of ``slmeval run``, with its provenance.
178
+
179
+ The provenance fields are not bookkeeping. Without ``config_hash`` and
180
+ ``dataset_hash``, comparing two runs is guesswork dressed up as a table;
181
+ with them, ``slmeval compare`` can *refuse* to compare runs built on
182
+ different datasets unless forced. That refusal is a feature.
183
+ """
184
+
185
+ run_id: str
186
+ name: str
187
+ started_at: str
188
+ config_json: str
189
+ config_hash: str
190
+ dataset_hash: str
191
+ slmeval_version: str
192
+
193
+ mode: str
194
+ """``replay``, ``record`` or ``live`` - the tool mode the matrix ran under.
195
+ A live run and a replay run of the same config do not have comparable
196
+ latency numbers, and this is how a report knows not to put them in one
197
+ column."""
198
+
199
+ seed: int
200
+
201
+ scheduling: str = "throughput"
202
+ """``throughput`` or ``latency``. Throughput runs high concurrency and its
203
+ latency numbers are not reportable: requests queue inside the provider, and
204
+ a local model measured that way reads several times slower than it is.
205
+ Stored so the report refuses to print a p95 column rather than printing a
206
+ misleading one."""
207
+
208
+ status: RunStatus = RunStatus.RUNNING
209
+ finished_at: str | None = None
210
+ dataset_path: str = ""
211
+ git_sha: str = ""
212
+ host_info: str = "{}"
213
+ fault_rate: float = 0.0
214
+ runs_per_case: int = 1
215
+ error_message: str | None = None
216
+
217
+
218
+ @dataclass(frozen=True, slots=True)
219
+ class CandidateRecord:
220
+ """One model configuration under test in a run.
221
+
222
+ ``model_cfg_json`` is the whole resolved config, not a pointer at the file:
223
+ ``eval.toml`` will have been edited by the time anyone reads this row, and a
224
+ report that re-read the file would quietly describe a run that never
225
+ happened.
226
+ """
227
+
228
+ run_id: str
229
+ candidate_id: str
230
+ model_cfg_json: str
231
+
232
+ role: str = "candidate"
233
+ """``baseline`` or ``candidate``. The scorecard computes its delta column
234
+ against the baseline row, so which one that is has to survive the run."""
235
+
236
+ ordinal: int = 0
237
+ """Declaration order, so table row order is stable across reports rather
238
+ than whatever the query planner happens to return today."""
239
+
240
+ provider: str = ""
241
+ model: str = ""
242
+
243
+
244
+ @dataclass(frozen=True, slots=True)
245
+ class ExecutionRecord:
246
+ """One (candidate, case, repeat) cell of the matrix."""
247
+
248
+ exec_id: str
249
+ run_id: str
250
+ candidate_id: str
251
+ case_id: str
252
+ repeat_idx: int
253
+ status: ExecStatus
254
+ outcome_class: Outcome = Outcome.UNCLASSIFIED
255
+
256
+ attempt: int = 0
257
+ """Incremented when crash isolation re-runs a unit. The retry replaces the
258
+ row, but the count stays: "passed on the second attempt" is a finding about
259
+ the candidate, not an implementation detail of the runner."""
260
+
261
+ started_at: str = ""
262
+ duration_ms: float = 0.0
263
+ steps: int = 0
264
+ total_prompt_tokens: int | None = None
265
+ total_completion_tokens: int | None = None
266
+
267
+ tokens_estimated: bool = False
268
+ """True when any call here had its usage estimated rather than reported.
269
+ Any aggregate containing one must be marked ``~``, so the flag rides on the
270
+ execution and not only on the individual call."""
271
+
272
+ cost_usd: float | None = None
273
+ fault_vector_json: str = "[]"
274
+
275
+ fault_digest: str = ""
276
+ """Digest of the faults this execution actually met. Two candidates with
277
+ different digests were not tested under the same conditions, and
278
+ ``--require-fault-parity`` drops the pair rather than print both numbers as
279
+ if they were."""
280
+
281
+ trajectory_json: str = "[]"
282
+ """Ordered node visits. Stored even though ``node_traces`` implies most of
283
+ it, because a node can be entered without making a model call and the
284
+ trajectory-distance metric counts the visit, not the call."""
285
+
286
+ warnings_json: str = "[]"
287
+ output: Payload = EMPTY_PAYLOAD
288
+ state: Payload = EMPTY_PAYLOAD
289
+ error_class: str | None = None
290
+ error_message: str | None = None
291
+
292
+
293
+ @dataclass(frozen=True, slots=True)
294
+ class NodeTraceRow:
295
+ """One model call, as stored. Mirrors
296
+ :class:`~slmeval.core.trace.LLMCallTrace`."""
297
+
298
+ exec_id: str
299
+ seq: int
300
+ node_name: str = ""
301
+ node_path: str = "[]"
302
+
303
+ model_id: str = ""
304
+ """The candidate id, duplicated from the execution so per-node attribution
305
+ is one indexed query instead of a join across the whole matrix."""
306
+
307
+ provider: str = ""
308
+ model: str = ""
309
+ latency_ms: float = 0.0
310
+ ttft_ms: float | None = None
311
+ retries: int = 0
312
+ retry_ms: float = 0.0
313
+ prompt_tokens: int | None = None
314
+ completion_tokens: int | None = None
315
+ tokens_estimated: bool = False
316
+ schema_name: str | None = None
317
+ schema_state: str = "not_requested"
318
+ structured_requested: str = "auto"
319
+
320
+ structured_mode: str | None = None
321
+ """Which ladder rung actually worked. The most valuable field in this
322
+ table - "needed json_mode on 100% of calls" is a line in the scorecard."""
323
+
324
+ rungs_tried: str = "[]"
325
+ repair_attempts: int = 0
326
+ json_stage: str | None = None
327
+ error_class: str | None = None
328
+ error_message: str | None = None
329
+
330
+ raw: Payload = EMPTY_PAYLOAD
331
+ """The provider's raw response, kept for failing calls only. The failure
332
+ report needs it; keeping it for every call would balloon the store."""
333
+
334
+
335
+ @dataclass(frozen=True, slots=True)
336
+ class ToolTraceRow:
337
+ """One tool invocation, as stored. Mirrors
338
+ :class:`~slmeval.core.trace.ToolCallTrace`."""
339
+
340
+ exec_id: str
341
+ seq: int
342
+ tool_name: str
343
+ node_name: str = ""
344
+ node_path: str = "[]"
345
+ args: Payload = EMPTY_PAYLOAD
346
+ cassette_key: str = ""
347
+ outcome: str = "live"
348
+ match_tier: str | None = None
349
+ cache_hit: bool = False
350
+ fault: str | None = None
351
+ fault_injected: bool = False
352
+ miss_verdict: str | None = None
353
+ tool_version: str = ""
354
+ latency_ms: float = 0.0
355
+ output: Payload = EMPTY_PAYLOAD
356
+ error_message: str | None = None
357
+
358
+
359
+ @dataclass(frozen=True, slots=True)
360
+ class EvaluationRow:
361
+ """One metric for one execution.
362
+
363
+ Long and narrow rather than a column per metric, because week 3 keeps
364
+ adding evaluators and a schema where every new metric is a migration is a
365
+ schema that stops gaining metrics.
366
+ """
367
+
368
+ exec_id: str
369
+ tier: int
370
+ metric: str
371
+ value_num: float | None = None
372
+ value_text: str | None = None
373
+ detail_json: str | None = None
374
+
375
+
376
+ @dataclass(frozen=True, slots=True)
377
+ class CaseRollupRow:
378
+ """Consistency math for one (candidate, case) across its repeats.
379
+
380
+ Derived from ``executions``, and stored anyway: ``pass^k`` and the Wilson
381
+ interval are the headline numbers, a report must not recompute them and
382
+ risk disagreeing with the run that printed them, and a later change to the
383
+ estimator must not silently restate last month's results.
384
+ """
385
+
386
+ run_id: str
387
+ candidate_id: str
388
+ case_id: str
389
+
390
+ n: int
391
+ """Repeats that produced a judgeable result. Always printed - a pass rate
392
+ without its n is a number pretending to be evidence."""
393
+
394
+ c: int
395
+ """How many of them passed."""
396
+
397
+ pass_k: float | None = None
398
+ pass_hat_k: float | None = None
399
+ wilson_lo: float | None = None
400
+ wilson_hi: float | None = None
401
+
402
+
403
+ @dataclass(frozen=True, slots=True)
404
+ class MetricSample:
405
+ """One stored metric, with the cell it belongs to.
406
+
407
+ The join the report needs and the drill-down does not: a scorecard
408
+ aggregates thousands of metrics across a run, and doing it by reconstructing
409
+ each execution in turn would make rendering a table slower than producing
410
+ one.
411
+ """
412
+
413
+ candidate_id: str
414
+ case_id: str
415
+ repeat_idx: int
416
+ exec_id: str
417
+ tier: int
418
+ metric: str
419
+ value_num: float | None = None
420
+ value_text: str | None = None
421
+ detail_json: str | None = None
422
+ outcome: Outcome = Outcome.UNCLASSIFIED
423
+
424
+
425
+ @dataclass(frozen=True, slots=True)
426
+ class NodeSummaryRow:
427
+ """Model calls at one node, for one candidate, aggregated.
428
+
429
+ Per-node attribution is the table that turns "this candidate is 13 points
430
+ worse" into "this candidate breaks in ``policy_check``", which is the
431
+ difference between a benchmark and a recommendation.
432
+ """
433
+
434
+ candidate_id: str
435
+ node_name: str
436
+ provider: str = ""
437
+ model: str = ""
438
+ calls: int = 0
439
+ errors: int = 0
440
+ schema_valid: int = 0
441
+ schema_invalid: int = 0
442
+ repair_attempts: int = 0
443
+ latency_ms: float = 0.0
444
+ """Total across calls. The mean is the report's to compute, so a caller
445
+ that wants a different statistic is not stuck with ours."""
446
+ prompt_tokens: int = 0
447
+ completion_tokens: int = 0
448
+
449
+ @property
450
+ def mean_latency_ms(self) -> float | None:
451
+ return self.latency_ms / self.calls if self.calls else None
452
+
453
+ @property
454
+ def schema_valid_rate(self) -> float | None:
455
+ judged = self.schema_valid + self.schema_invalid
456
+ return self.schema_valid / judged if judged else None
457
+
458
+
459
+ @dataclass(frozen=True, slots=True)
460
+ class BlameRow:
461
+ """Where a candidate's failing executions went wrong, counted by node."""
462
+
463
+ candidate_id: str
464
+ node_name: str
465
+ kind: str
466
+ """``model`` or ``tool`` - which half of the step failed."""
467
+ error_class: str = ""
468
+ count: int = 0
469
+
470
+
471
+ @dataclass(frozen=True, slots=True)
472
+ class ExecutionDetail:
473
+ """One execution and everything recorded under it, payloads resolved.
474
+
475
+ What ``slmeval show exec <id>`` renders, and the proof of step 12's
476
+ acceptance test: built from rows alone, so dropping every in-memory object
477
+ changes nothing about what gets printed.
478
+ """
479
+
480
+ execution: ExecutionRecord
481
+ node_traces: tuple[NodeTraceRow, ...] = ()
482
+ tool_traces: tuple[ToolTraceRow, ...] = ()
483
+ evaluations: tuple[EvaluationRow, ...] = ()
484
+ output: Any = None
485
+ state: Any = None
486
+
487
+ tool_outputs: dict[int, Any] = field(default_factory=dict)
488
+ """Resolved tool return values, keyed by ``seq``. Loaded eagerly, because a
489
+ timeline that re-opened the blob store per row would make the drill-down
490
+ the slowest command in the tool."""
491
+
492
+ tool_arguments: dict[int, Any] = field(default_factory=dict)
493
+
494
+ @property
495
+ def trajectory(self) -> tuple[tuple[str, ...], ...]:
496
+ from slmeval.storage.codec import decode_paths
497
+
498
+ return decode_paths(self.execution.trajectory_json)
499
+
500
+
501
+ @dataclass(frozen=True, slots=True)
502
+ class RunDetail:
503
+ """A whole run, reconstructed. ``slmeval report`` reads exactly this."""
504
+
505
+ run: RunRecord
506
+ candidates: tuple[CandidateRecord, ...] = ()
507
+ executions: tuple[ExecutionRecord, ...] = ()
508
+ rollups: tuple[CaseRollupRow, ...] = ()
509
+
510
+ @property
511
+ def baseline(self) -> CandidateRecord | None:
512
+ return next((c for c in self.candidates if c.role == "baseline"), None)
513
+
514
+
515
+ @dataclass(frozen=True, slots=True)
516
+ class StoreStats:
517
+ """What ``slmeval cache stats`` prints."""
518
+
519
+ runs: int
520
+ running: int
521
+ executions: int
522
+ node_traces: int
523
+ tool_traces: int
524
+ evaluations: int
525
+ db_bytes: int
526
+ blobs: int
527
+ blob_bytes: int
528
+ oldest_run: str | None = None
529
+ newest_run: str | None = None
530
+
531
+
532
+ __all__ = [
533
+ "EMPTY_PAYLOAD",
534
+ "INLINE_LIMIT",
535
+ "NOT_THE_MODELS_FAULT",
536
+ "TERMINAL_STATUSES",
537
+ "BlameRow",
538
+ "CandidateRecord",
539
+ "CaseRollupRow",
540
+ "EvaluationRow",
541
+ "ExecStatus",
542
+ "ExecutionDetail",
543
+ "ExecutionRecord",
544
+ "MetricSample",
545
+ "NodeSummaryRow",
546
+ "NodeTraceRow",
547
+ "Outcome",
548
+ "Payload",
549
+ "RunDetail",
550
+ "RunRecord",
551
+ "RunStatus",
552
+ "StoreStats",
553
+ "ToolTraceRow",
554
+ ]