slmeval-cli 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (94) hide show
  1. slmeval/__init__.py +71 -0
  2. slmeval/__main__.py +14 -0
  3. slmeval/adapters/__init__.py +24 -0
  4. slmeval/adapters/instrument.py +135 -0
  5. slmeval/adapters/langgraph.py +237 -0
  6. slmeval/cli/__init__.py +15 -0
  7. slmeval/cli/commands/__init__.py +35 -0
  8. slmeval/cli/commands/baseline_cmd.py +201 -0
  9. slmeval/cli/commands/cache_cmd.py +249 -0
  10. slmeval/cli/commands/dataset_cmd.py +52 -0
  11. slmeval/cli/commands/doctor_cmd.py +68 -0
  12. slmeval/cli/commands/gate_cmd.py +602 -0
  13. slmeval/cli/commands/init_cmd.py +144 -0
  14. slmeval/cli/commands/judge_cmd.py +284 -0
  15. slmeval/cli/commands/record_cmd.py +154 -0
  16. slmeval/cli/commands/report_cmd.py +98 -0
  17. slmeval/cli/commands/run_cmd.py +247 -0
  18. slmeval/cli/commands/show_cmd.py +482 -0
  19. slmeval/cli/commands/stubs.py +57 -0
  20. slmeval/cli/commands/validate_cmd.py +296 -0
  21. slmeval/cli/loading.py +149 -0
  22. slmeval/cli/main.py +91 -0
  23. slmeval/cli/render.py +251 -0
  24. slmeval/cli/wrap.py +31 -0
  25. slmeval/config/__init__.py +47 -0
  26. slmeval/config/loader.py +363 -0
  27. slmeval/config/locate.py +97 -0
  28. slmeval/config/models.py +430 -0
  29. slmeval/config/templates.py +246 -0
  30. slmeval/core/__init__.py +78 -0
  31. slmeval/core/adapter.py +253 -0
  32. slmeval/core/context.py +418 -0
  33. slmeval/core/outcome.py +260 -0
  34. slmeval/core/router.py +319 -0
  35. slmeval/core/runner.py +1209 -0
  36. slmeval/core/trace.py +188 -0
  37. slmeval/core/verdict.py +127 -0
  38. slmeval/dataset/__init__.py +8 -0
  39. slmeval/dataset/loader.py +130 -0
  40. slmeval/dataset/models.py +108 -0
  41. slmeval/doctor/__init__.py +7 -0
  42. slmeval/doctor/checks.py +380 -0
  43. slmeval/errors.py +76 -0
  44. slmeval/eval/__init__.py +64 -0
  45. slmeval/eval/assertions.py +356 -0
  46. slmeval/eval/calibration.py +364 -0
  47. slmeval/eval/cost.py +291 -0
  48. slmeval/eval/pipeline.py +273 -0
  49. slmeval/eval/stats.py +376 -0
  50. slmeval/eval/tier1.py +510 -0
  51. slmeval/eval/tier2.py +582 -0
  52. slmeval/eval/tier3.py +415 -0
  53. slmeval/exit_codes.py +28 -0
  54. slmeval/gate.py +722 -0
  55. slmeval/providers/__init__.py +69 -0
  56. slmeval/providers/base.py +426 -0
  57. slmeval/providers/fake.py +760 -0
  58. slmeval/providers/jsonx.py +239 -0
  59. slmeval/providers/ladder.py +339 -0
  60. slmeval/providers/openai_compat.py +392 -0
  61. slmeval/providers/registry.py +185 -0
  62. slmeval/providers/retry.py +81 -0
  63. slmeval/providers/schema.py +136 -0
  64. slmeval/providers/tokens.py +150 -0
  65. slmeval/py.typed +0 -0
  66. slmeval/report/__init__.py +115 -0
  67. slmeval/report/attribution.py +197 -0
  68. slmeval/report/baseline.py +353 -0
  69. slmeval/report/drilldown.py +1395 -0
  70. slmeval/report/export.py +450 -0
  71. slmeval/report/model.py +562 -0
  72. slmeval/report/progress.py +257 -0
  73. slmeval/report/scorecard.py +367 -0
  74. slmeval/sandbox/__init__.py +145 -0
  75. slmeval/sandbox/diagnosis.py +690 -0
  76. slmeval/sandbox/envelope.py +486 -0
  77. slmeval/sandbox/faults.py +794 -0
  78. slmeval/sandbox/keys.py +458 -0
  79. slmeval/sandbox/normalizers.py +333 -0
  80. slmeval/sandbox/store.py +525 -0
  81. slmeval/sandbox/tool.py +685 -0
  82. slmeval/storage/__init__.py +73 -0
  83. slmeval/storage/blobs.py +308 -0
  84. slmeval/storage/codec.py +93 -0
  85. slmeval/storage/db.py +1334 -0
  86. slmeval/storage/ingest.py +346 -0
  87. slmeval/storage/migrations.py +358 -0
  88. slmeval/storage/models.py +554 -0
  89. slmeval/storage/provenance.py +201 -0
  90. slmeval_cli-0.1.0.dist-info/METADATA +484 -0
  91. slmeval_cli-0.1.0.dist-info/RECORD +94 -0
  92. slmeval_cli-0.1.0.dist-info/WHEEL +4 -0
  93. slmeval_cli-0.1.0.dist-info/entry_points.txt +2 -0
  94. slmeval_cli-0.1.0.dist-info/licenses/LICENSE +201 -0
@@ -0,0 +1,247 @@
1
+ """``slmeval run`` - execute the matrix and print the scorecard.
2
+
3
+ This is the command the whole tool exists for, so the flag surface is wide. The
4
+ defaults are the important part: ``replay`` tool mode, throughput scheduling,
5
+ no spend ceiling, and resume available by run id. A first run needs none of
6
+ them.
7
+
8
+ Two behaviours worth calling out because they are easy to get wrong:
9
+
10
+ **Rollups are written after the matrix, not during it.** ``pass^k`` and the
11
+ Wilson interval are properties of a case's whole set of repeats, and computing
12
+ them incrementally would mean storing a number that was briefly wrong - which
13
+ somebody would eventually read.
14
+
15
+ **A ``Ctrl-C`` is not an error.** It prints what was finished, names the run id
16
+ to resume, and exits. Treating an interrupt as a crash would throw away the
17
+ most valuable thing the user has at that moment: the id of the half-finished
18
+ run.
19
+ """
20
+
21
+ from __future__ import annotations
22
+
23
+ import asyncio
24
+ from pathlib import Path
25
+ from typing import Annotated
26
+
27
+ import typer
28
+ from rich.console import Console
29
+ from rich.text import Text
30
+
31
+ from slmeval import exit_codes
32
+ from slmeval.cli.loading import Project, open_project
33
+ from slmeval.cli.render import Symbols, console, symbols
34
+ from slmeval.cli.wrap import handle_errors
35
+ from slmeval.core.runner import MatrixRunner, RunOptions, RunSummary, encode_plan
36
+ from slmeval.eval.pipeline import case_rollups, evaluator_for
37
+ from slmeval.report.export import export
38
+ from slmeval.report.model import build_scorecard
39
+ from slmeval.report.progress import ProgressObserver
40
+ from slmeval.report.scorecard import render_scorecard, render_summary_line
41
+ from slmeval.storage.models import Outcome, RunStatus
42
+
43
+ _TOOL_MODES = ("replay", "auto", "record", "live")
44
+
45
+
46
+ @handle_errors
47
+ def run(
48
+ config: Annotated[
49
+ Path | None,
50
+ typer.Option("--config", "-c", help="Path to eval.toml. Defaults to the nearest one."),
51
+ ] = None,
52
+ candidate: Annotated[
53
+ list[str] | None,
54
+ typer.Option(
55
+ "--candidate",
56
+ help="Only these candidates. The baseline is always included - every "
57
+ "delta is measured against it.",
58
+ ),
59
+ ] = None,
60
+ case: Annotated[
61
+ list[str] | None,
62
+ typer.Option("--case", help="Only these case ids."),
63
+ ] = None,
64
+ runs: Annotated[
65
+ int | None,
66
+ typer.Option("--runs", min=1, max=50, help="Repeats per case. Overrides runs_per_case."),
67
+ ] = None,
68
+ tool_mode: Annotated[
69
+ str | None,
70
+ typer.Option(
71
+ "--tool-mode",
72
+ help="replay | auto | record | live. replay needs no network and no keys.",
73
+ ),
74
+ ] = None,
75
+ fault_rate: Annotated[
76
+ float | None,
77
+ typer.Option("--fault-rate", min=0.0, max=1.0, help="Chance any tool call is faulted."),
78
+ ] = None,
79
+ concurrency: Annotated[
80
+ int | None,
81
+ typer.Option("--concurrency", min=1, max=256, help="Executions in flight, at most."),
82
+ ] = None,
83
+ serial_latency: Annotated[
84
+ bool,
85
+ typer.Option(
86
+ "--serial-latency",
87
+ help="One request at a time per endpoint, after a warm-up, so latency "
88
+ "is reportable. Much slower, and the only mode whose p95 means anything.",
89
+ ),
90
+ ] = False,
91
+ max_spend: Annotated[
92
+ float | None,
93
+ typer.Option("--max-spend", min=0.0, help="Abort once estimated spend passes this (USD)."),
94
+ ] = None,
95
+ resume: Annotated[
96
+ str | None,
97
+ typer.Option("--resume", help="Finish an interrupted run. Takes a run id or a prefix."),
98
+ ] = None,
99
+ fail_fast: Annotated[
100
+ bool,
101
+ typer.Option("--fail-fast", help="Stop once a candidate fails three executions in a row."),
102
+ ] = False,
103
+ gpu_hourly: Annotated[
104
+ float | None,
105
+ typer.Option(
106
+ "--gpu-hourly",
107
+ min=0.0,
108
+ help="Your $/hour for GPU time, used to model local cost. Reported as "
109
+ "modelled, never as measured.",
110
+ ),
111
+ ] = None,
112
+ json_out: Annotated[
113
+ Path | None,
114
+ typer.Option("--json", help="Also write the versioned JSON report here."),
115
+ ] = None,
116
+ nodes: Annotated[
117
+ bool, typer.Option("--nodes/--no-nodes", help="Show the per-node attribution table.")
118
+ ] = True,
119
+ ascii_mode: Annotated[
120
+ bool, typer.Option("--ascii", help="ASCII only, for legacy Windows consoles.")
121
+ ] = False,
122
+ dry_run: Annotated[
123
+ bool, typer.Option("--dry-run", help="Print the plan as JSON and execute nothing.")
124
+ ] = False,
125
+ quiet: Annotated[
126
+ bool, typer.Option("--quiet", "-q", help="No progress bars; one line per failure.")
127
+ ] = False,
128
+ ) -> None:
129
+ """Run the candidate matrix and print the scorecard."""
130
+ if tool_mode is not None and tool_mode not in _TOOL_MODES:
131
+ raise typer.BadParameter(f"tool mode must be one of: {', '.join(_TOOL_MODES)}")
132
+
133
+ out = console()
134
+ sym = symbols(ascii_mode or None)
135
+
136
+ with open_project(config, gpu_hourly_usd=gpu_hourly) as project:
137
+ options = RunOptions.from_config(
138
+ project.config,
139
+ repeats=runs,
140
+ tool_mode=tool_mode,
141
+ fault_rate=fault_rate,
142
+ concurrency=concurrency,
143
+ serial_latency=serial_latency or None,
144
+ max_spend_usd=max_spend,
145
+ fail_fast=fail_fast or None,
146
+ case_ids=tuple(case) if case else None,
147
+ candidate_ids=tuple(candidate) if candidate else None,
148
+ gpu_hourly_usd=gpu_hourly,
149
+ )
150
+ runner = MatrixRunner(
151
+ config=project.config,
152
+ dataset=project.dataset,
153
+ target=project.target,
154
+ store=project.store,
155
+ options=options,
156
+ cassettes=project.cassettes,
157
+ evaluator=evaluator_for(project.config),
158
+ prices=project.prices,
159
+ root=project.root,
160
+ )
161
+
162
+ if dry_run:
163
+ # print, not rich-print: this is JSON on its way into a pipe, and
164
+ # Rich would wrap it at the terminal width into something no parser
165
+ # can read.
166
+ print(encode_plan(runner.plan(resume=resume)))
167
+ return
168
+
169
+ observer = ProgressObserver(console=out, ascii_mode=ascii_mode or None, quiet=quiet)
170
+ runner.observer = observer
171
+ try:
172
+ summary = asyncio.run(runner.run(resume=resume))
173
+ except KeyboardInterrupt:
174
+ # Everything finished is already in SQLite - that is the point of
175
+ # persisting per unit - so the only thing at risk here is the user
176
+ # knowing how to pick it back up.
177
+ observer.close()
178
+ _interrupted(out, sym, runner)
179
+ raise typer.Exit(exit_codes.USAGE) from None
180
+
181
+ _save_rollups(project, summary)
182
+ card = build_scorecard(
183
+ project.store, summary.run_id, prices_as_of=project.prices.as_of
184
+ )
185
+ out.print(render_scorecard(card, ascii_mode=ascii_mode or None, show_nodes=nodes))
186
+ line = render_summary_line(card, ascii_mode=ascii_mode or None)
187
+ if str(line).strip():
188
+ out.print(line)
189
+ out.print()
190
+
191
+ if json_out is not None:
192
+ json_out.write_text(export(card, "json"), encoding="utf-8")
193
+ out.print(Text(f" {sym.ok} wrote {json_out}", style="dim"))
194
+ out.print()
195
+
196
+ raise typer.Exit(_exit_code(summary))
197
+
198
+
199
+ def _save_rollups(project: Project, summary: RunSummary) -> None:
200
+ """Compute and store the consistency rollups for the whole run.
201
+
202
+ After the matrix, and from the *stored* executions rather than from the
203
+ results in memory: a resumed run's earlier half exists only in the
204
+ database, and rolling up what this process happened to execute would report
205
+ ``n=1`` for every case the first half already covered.
206
+ """
207
+ executions = project.store.executions(summary.run_id)
208
+ project.store.save_rollups(case_rollups(summary.run_id, executions))
209
+
210
+
211
+ def _interrupted(out: Console, sym: Symbols, runner: MatrixRunner) -> None:
212
+ out.print()
213
+ out.print(
214
+ Text(
215
+ f" {sym.warn} interrupted - everything finished is already stored",
216
+ style="yellow",
217
+ )
218
+ )
219
+ if runner.last_run_id:
220
+ out.print(
221
+ Text(
222
+ f" resume with: slmeval run --resume {runner.last_run_id}",
223
+ style="cyan",
224
+ )
225
+ )
226
+ out.print()
227
+
228
+
229
+ def _exit_code(summary: RunSummary) -> int:
230
+ """What the shell sees.
231
+
232
+ An unreachable provider exits 3, not 1. That distinction is the whole
233
+ reason the codes are a documented contract: a CI job that reports "Ollama
234
+ was not running" as a model regression teaches people to ignore the gate,
235
+ and once they do, the gate is worth nothing.
236
+ """
237
+ if summary.results and all(
238
+ r.outcome is Outcome.PROVIDER_ERROR for r in summary.results
239
+ ):
240
+ return exit_codes.ENVIRONMENT
241
+ if summary.status is RunStatus.CANCELLED:
242
+ return exit_codes.GATE_FAILED
243
+ return exit_codes.OK
244
+
245
+
246
+ def register(app: typer.Typer) -> None:
247
+ app.command(name="run", help="Run the candidate matrix and print the scorecard.")(run)