slmeval-cli 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (94) hide show
  1. slmeval/__init__.py +71 -0
  2. slmeval/__main__.py +14 -0
  3. slmeval/adapters/__init__.py +24 -0
  4. slmeval/adapters/instrument.py +135 -0
  5. slmeval/adapters/langgraph.py +237 -0
  6. slmeval/cli/__init__.py +15 -0
  7. slmeval/cli/commands/__init__.py +35 -0
  8. slmeval/cli/commands/baseline_cmd.py +201 -0
  9. slmeval/cli/commands/cache_cmd.py +249 -0
  10. slmeval/cli/commands/dataset_cmd.py +52 -0
  11. slmeval/cli/commands/doctor_cmd.py +68 -0
  12. slmeval/cli/commands/gate_cmd.py +602 -0
  13. slmeval/cli/commands/init_cmd.py +144 -0
  14. slmeval/cli/commands/judge_cmd.py +284 -0
  15. slmeval/cli/commands/record_cmd.py +154 -0
  16. slmeval/cli/commands/report_cmd.py +98 -0
  17. slmeval/cli/commands/run_cmd.py +247 -0
  18. slmeval/cli/commands/show_cmd.py +482 -0
  19. slmeval/cli/commands/stubs.py +57 -0
  20. slmeval/cli/commands/validate_cmd.py +296 -0
  21. slmeval/cli/loading.py +149 -0
  22. slmeval/cli/main.py +91 -0
  23. slmeval/cli/render.py +251 -0
  24. slmeval/cli/wrap.py +31 -0
  25. slmeval/config/__init__.py +47 -0
  26. slmeval/config/loader.py +363 -0
  27. slmeval/config/locate.py +97 -0
  28. slmeval/config/models.py +430 -0
  29. slmeval/config/templates.py +246 -0
  30. slmeval/core/__init__.py +78 -0
  31. slmeval/core/adapter.py +253 -0
  32. slmeval/core/context.py +418 -0
  33. slmeval/core/outcome.py +260 -0
  34. slmeval/core/router.py +319 -0
  35. slmeval/core/runner.py +1209 -0
  36. slmeval/core/trace.py +188 -0
  37. slmeval/core/verdict.py +127 -0
  38. slmeval/dataset/__init__.py +8 -0
  39. slmeval/dataset/loader.py +130 -0
  40. slmeval/dataset/models.py +108 -0
  41. slmeval/doctor/__init__.py +7 -0
  42. slmeval/doctor/checks.py +380 -0
  43. slmeval/errors.py +76 -0
  44. slmeval/eval/__init__.py +64 -0
  45. slmeval/eval/assertions.py +356 -0
  46. slmeval/eval/calibration.py +364 -0
  47. slmeval/eval/cost.py +291 -0
  48. slmeval/eval/pipeline.py +273 -0
  49. slmeval/eval/stats.py +376 -0
  50. slmeval/eval/tier1.py +510 -0
  51. slmeval/eval/tier2.py +582 -0
  52. slmeval/eval/tier3.py +415 -0
  53. slmeval/exit_codes.py +28 -0
  54. slmeval/gate.py +722 -0
  55. slmeval/providers/__init__.py +69 -0
  56. slmeval/providers/base.py +426 -0
  57. slmeval/providers/fake.py +760 -0
  58. slmeval/providers/jsonx.py +239 -0
  59. slmeval/providers/ladder.py +339 -0
  60. slmeval/providers/openai_compat.py +392 -0
  61. slmeval/providers/registry.py +185 -0
  62. slmeval/providers/retry.py +81 -0
  63. slmeval/providers/schema.py +136 -0
  64. slmeval/providers/tokens.py +150 -0
  65. slmeval/py.typed +0 -0
  66. slmeval/report/__init__.py +115 -0
  67. slmeval/report/attribution.py +197 -0
  68. slmeval/report/baseline.py +353 -0
  69. slmeval/report/drilldown.py +1395 -0
  70. slmeval/report/export.py +450 -0
  71. slmeval/report/model.py +562 -0
  72. slmeval/report/progress.py +257 -0
  73. slmeval/report/scorecard.py +367 -0
  74. slmeval/sandbox/__init__.py +145 -0
  75. slmeval/sandbox/diagnosis.py +690 -0
  76. slmeval/sandbox/envelope.py +486 -0
  77. slmeval/sandbox/faults.py +794 -0
  78. slmeval/sandbox/keys.py +458 -0
  79. slmeval/sandbox/normalizers.py +333 -0
  80. slmeval/sandbox/store.py +525 -0
  81. slmeval/sandbox/tool.py +685 -0
  82. slmeval/storage/__init__.py +73 -0
  83. slmeval/storage/blobs.py +308 -0
  84. slmeval/storage/codec.py +93 -0
  85. slmeval/storage/db.py +1334 -0
  86. slmeval/storage/ingest.py +346 -0
  87. slmeval/storage/migrations.py +358 -0
  88. slmeval/storage/models.py +554 -0
  89. slmeval/storage/provenance.py +201 -0
  90. slmeval_cli-0.1.0.dist-info/METADATA +484 -0
  91. slmeval_cli-0.1.0.dist-info/RECORD +94 -0
  92. slmeval_cli-0.1.0.dist-info/WHEEL +4 -0
  93. slmeval_cli-0.1.0.dist-info/entry_points.txt +2 -0
  94. slmeval_cli-0.1.0.dist-info/licenses/LICENSE +201 -0
@@ -0,0 +1,482 @@
1
+ """``slmeval show`` - the drill-down, from the scorecard to the exact step.
2
+
3
+ The scorecard is the headline; this is the audit trail under it. Four views,
4
+ each one a question somebody asks while staring at a red cell:
5
+
6
+ ``show run <id>``
7
+ Which executions exist and how did each end? Worst first.
8
+ ``show exec <id>``
9
+ What did this one actually do - trajectory, model calls, tool calls,
10
+ metrics, final state?
11
+ ``show case <id> --candidate <id>``
12
+ Where did it part company with the baseline? Side by side, divergence
13
+ marked, cassette-miss diagnosis inline.
14
+ ``show uncovered``
15
+ Which tool calls had no recording, so where does ``record --fill-gaps``
16
+ need to go?
17
+
18
+ Like ``report``, none of this loads the user's agent and none of it re-executes
19
+ anything. A checkout whose agent has since been refactored still has valid
20
+ results from last Tuesday, and a view that could not open them would be the
21
+ tool losing data it already holds.
22
+ """
23
+
24
+ from __future__ import annotations
25
+
26
+ import json
27
+ from collections.abc import Sequence
28
+ from pathlib import Path
29
+ from typing import Annotated, Any
30
+
31
+ import typer
32
+
33
+ from slmeval.cli.loading import LATEST, Project, open_project, resolve_run
34
+ from slmeval.cli.render import console
35
+ from slmeval.cli.wrap import handle_errors
36
+ from slmeval.errors import ConfigError
37
+ from slmeval.report.drilldown import (
38
+ Diff,
39
+ build_diff,
40
+ diff_payload,
41
+ execution_payload,
42
+ gaps,
43
+ gaps_payload,
44
+ known_tool_names,
45
+ rebuild_diagnosis,
46
+ render_diff,
47
+ render_execution,
48
+ render_gaps,
49
+ render_run,
50
+ tool_steps,
51
+ )
52
+ from slmeval.sandbox.diagnosis import MissDiagnosis
53
+ from slmeval.storage.db import RunStore
54
+ from slmeval.storage.models import ExecutionDetail, ExecutionRecord, Outcome
55
+
56
+ MISS_OUTCOMES = frozenset({Outcome.CASSETTE_MISS, Outcome.UNCOVERED})
57
+
58
+ show_app = typer.Typer(
59
+ no_args_is_help=True,
60
+ help="Drill into a run, an execution, or one case against the baseline.",
61
+ )
62
+
63
+ _ConfigOpt = Annotated[
64
+ Path | None,
65
+ typer.Option("--config", "-c", help="Path to eval.toml. Defaults to the nearest one."),
66
+ ]
67
+ _AsciiOpt = Annotated[
68
+ bool, typer.Option("--ascii", help="ASCII only, for legacy Windows consoles.")
69
+ ]
70
+ _JsonOpt = Annotated[
71
+ bool, typer.Option("--json", help="Emit the view as JSON instead of a table.")
72
+ ]
73
+ _RunOpt = Annotated[
74
+ str,
75
+ typer.Option("--run-id", help="A run id or prefix, or 'latest'."),
76
+ ]
77
+
78
+
79
+ # --------------------------------------------------------------------------- #
80
+ # show run
81
+ # --------------------------------------------------------------------------- #
82
+
83
+
84
+ @show_app.command("run")
85
+ @handle_errors
86
+ def show_run(
87
+ run_id: Annotated[
88
+ str, typer.Argument(help="A run id or prefix, or 'latest'.")
89
+ ] = LATEST,
90
+ config: _ConfigOpt = None,
91
+ candidate: Annotated[
92
+ str | None, typer.Option("--candidate", help="Only this candidate's cells.")
93
+ ] = None,
94
+ case: Annotated[str | None, typer.Option("--case", help="Only this case.")] = None,
95
+ failed: Annotated[
96
+ bool, typer.Option("--failed", help="Only the executions that did not pass.")
97
+ ] = False,
98
+ limit: Annotated[
99
+ int, typer.Option("--limit", min=0, help="Rows to show. 0 for all.")
100
+ ] = 40,
101
+ as_json: _JsonOpt = False,
102
+ ascii_mode: _AsciiOpt = False,
103
+ ) -> None:
104
+ """List a run's executions, worst first."""
105
+ with open_project(config, load_agent=False) as project:
106
+ resolved = resolve_run(project.store, run_id)
107
+ run = project.store.require_run(resolved)
108
+ candidates = project.store.candidates(resolved)
109
+ executions = _filtered(
110
+ project.store.executions(resolved, candidate_id=candidate, case_id=case),
111
+ failed=failed,
112
+ )
113
+ if as_json:
114
+ _print_json(
115
+ {
116
+ "run_id": run.run_id,
117
+ "name": run.name,
118
+ "status": run.status.value,
119
+ "mode": run.mode,
120
+ "seed": run.seed,
121
+ "runs_per_case": run.runs_per_case,
122
+ "config_hash": run.config_hash,
123
+ "dataset_hash": run.dataset_hash,
124
+ "candidates": [
125
+ {
126
+ "candidate_id": c.candidate_id,
127
+ "role": c.role,
128
+ "provider": c.provider,
129
+ "model": c.model,
130
+ }
131
+ for c in candidates
132
+ ],
133
+ "executions": [
134
+ {
135
+ "exec_id": e.exec_id,
136
+ "candidate_id": e.candidate_id,
137
+ "case_id": e.case_id,
138
+ "repeat_idx": e.repeat_idx,
139
+ "status": e.status.value,
140
+ "outcome": e.outcome_class.value,
141
+ "steps": e.steps,
142
+ "duration_ms": e.duration_ms,
143
+ "error_class": e.error_class,
144
+ "error_message": e.error_message,
145
+ }
146
+ for e in executions
147
+ ],
148
+ }
149
+ )
150
+ return
151
+ out = console()
152
+ out.print(
153
+ render_run(
154
+ run,
155
+ candidates,
156
+ executions,
157
+ ascii_mode=ascii_mode or None,
158
+ limit=limit,
159
+ )
160
+ )
161
+ out.print()
162
+
163
+
164
+ # --------------------------------------------------------------------------- #
165
+ # show exec
166
+ # --------------------------------------------------------------------------- #
167
+
168
+
169
+ @show_app.command("exec")
170
+ @handle_errors
171
+ def show_exec(
172
+ exec_id: Annotated[
173
+ str,
174
+ typer.Argument(
175
+ help="An execution id, or its readable tail: gemma3-4b.refund_207.r0"
176
+ ),
177
+ ],
178
+ config: _ConfigOpt = None,
179
+ run_id: _RunOpt = LATEST,
180
+ as_json: _JsonOpt = False,
181
+ ascii_mode: _AsciiOpt = False,
182
+ ) -> None:
183
+ """Show one execution's timeline, metrics and final state.
184
+
185
+ The id is resolved inside one run - the latest unless ``--run-id`` says
186
+ otherwise - because the readable tail is identical in every run of a
187
+ project: ``record`` alone leaves a second execution under every one of
188
+ them. A full execution id skips the question.
189
+ """
190
+ with open_project(config, load_agent=False) as project:
191
+ resolved = _resolve_exec(project.store, exec_id, run_id)
192
+ detail = project.store.reconstruct(resolved)
193
+ if as_json:
194
+ _print_json(execution_payload(detail))
195
+ return
196
+ out = console()
197
+ out.print(
198
+ render_execution(
199
+ detail,
200
+ ascii_mode=ascii_mode or None,
201
+ diagnoses=_diagnoses(detail, project),
202
+ )
203
+ )
204
+
205
+
206
+ # --------------------------------------------------------------------------- #
207
+ # show case
208
+ # --------------------------------------------------------------------------- #
209
+
210
+
211
+ @show_app.command("case")
212
+ @handle_errors
213
+ def show_case(
214
+ case_id: Annotated[str, typer.Argument(help="A case id from the dataset.")],
215
+ config: _ConfigOpt = None,
216
+ run_id: _RunOpt = LATEST,
217
+ candidate: Annotated[
218
+ list[str] | None,
219
+ typer.Option(
220
+ "--candidate",
221
+ help="Diff only these candidates. Defaults to every non-baseline one.",
222
+ ),
223
+ ] = None,
224
+ repeat: Annotated[
225
+ int | None,
226
+ typer.Option(
227
+ "--repeat",
228
+ min=0,
229
+ help="Which repeat to show. Defaults to the worst one, which is the "
230
+ "reason you are here.",
231
+ ),
232
+ ] = None,
233
+ as_json: _JsonOpt = False,
234
+ ascii_mode: _AsciiOpt = False,
235
+ ) -> None:
236
+ """Diff one case against the baseline, side by side."""
237
+ with open_project(config, load_agent=False) as project:
238
+ resolved = resolve_run(project.store, run_id)
239
+ store = project.store
240
+ executions = store.executions(resolved, case_id=case_id)
241
+ if not executions:
242
+ raise ConfigError(
243
+ f"run {resolved} has no execution for case {case_id!r}",
244
+ hint="`slmeval show run` lists the cases a run covered",
245
+ )
246
+
247
+ baseline_id = _baseline_id(store, resolved)
248
+ wanted = _wanted_candidates(executions, baseline_id, candidate)
249
+ diffs = [
250
+ _diff_for(
251
+ project,
252
+ case_id=case_id,
253
+ executions=executions,
254
+ baseline_id=baseline_id,
255
+ candidate_id=candidate_id,
256
+ repeat=repeat,
257
+ )
258
+ for candidate_id in wanted
259
+ ]
260
+
261
+ if as_json:
262
+ _print_json(
263
+ {
264
+ "run_id": resolved,
265
+ "case_id": case_id,
266
+ "diffs": [diff_payload(diff) for diff in diffs],
267
+ }
268
+ )
269
+ return
270
+
271
+ out = console()
272
+ for diff in diffs:
273
+ out.print(render_diff(diff, ascii_mode=ascii_mode or None))
274
+ out.print()
275
+
276
+
277
+ # --------------------------------------------------------------------------- #
278
+ # show uncovered
279
+ # --------------------------------------------------------------------------- #
280
+
281
+
282
+ @show_app.command("uncovered")
283
+ @handle_errors
284
+ def show_uncovered(
285
+ run_id: Annotated[
286
+ str, typer.Argument(help="A run id or prefix, or 'latest'.")
287
+ ] = LATEST,
288
+ config: _ConfigOpt = None,
289
+ candidate: Annotated[
290
+ str | None, typer.Option("--candidate", help="Only this candidate's misses.")
291
+ ] = None,
292
+ all_misses: Annotated[
293
+ bool,
294
+ typer.Option(
295
+ "--all",
296
+ help="Include hallucinated calls too, not only the coverage gaps.",
297
+ ),
298
+ ] = False,
299
+ as_json: _JsonOpt = False,
300
+ ascii_mode: _AsciiOpt = False,
301
+ ) -> None:
302
+ """List the tool calls no cassette covered - the input to `record --fill-gaps`.
303
+
304
+ Only the uncovered verdict by default. The other three verdicts are model
305
+ failures and belong in the failure report, not in a list of recordings to
306
+ go and make; mixing them would have a user re-recording a hallucinated
307
+ order id as if it were ground truth.
308
+ """
309
+ with open_project(config, load_agent=False) as project:
310
+ resolved = resolve_run(project.store, run_id)
311
+ executions = [
312
+ e
313
+ for e in project.store.executions(resolved, candidate_id=candidate)
314
+ if e.outcome_class in MISS_OUTCOMES
315
+ ]
316
+ details = [
317
+ (e.candidate_id, project.store.reconstruct(e.exec_id)) for e in executions
318
+ ]
319
+ found = gaps(details, uncovered_only=not all_misses)
320
+ if as_json:
321
+ _print_json({"run_id": resolved, **gaps_payload(found)})
322
+ return
323
+ out = console()
324
+ out.print(render_gaps(found, ascii_mode=ascii_mode or None, run_id=resolved))
325
+
326
+
327
+ # --------------------------------------------------------------------------- #
328
+ # Helpers
329
+ # --------------------------------------------------------------------------- #
330
+
331
+
332
+ def _resolve_exec(store: RunStore, exec_id: str, run_id: str) -> str:
333
+ """One execution id, resolved inside a run before the whole project.
334
+
335
+ Inside a run first, because ``candidate.case.rN`` is not unique across a
336
+ project and the id a user just read off ``show run`` belongs to the run
337
+ they were looking at. Falling back to a project-wide search afterwards is
338
+ still worth doing - an id pasted from an older run's table should not need
339
+ its run spelled out - and that search refuses ambiguity rather than
340
+ guessing.
341
+ """
342
+ if store.get_execution(exec_id) is not None:
343
+ return exec_id
344
+ scoped = f"{resolve_run(store, run_id)}."
345
+ within = f"{scoped}{exec_id.removeprefix(scoped)}"
346
+ if store.get_execution(within) is not None:
347
+ return within
348
+ return store.resolve_exec_id(exec_id)
349
+
350
+
351
+ def _filtered(
352
+ executions: Sequence[ExecutionRecord], *, failed: bool
353
+ ) -> list[ExecutionRecord]:
354
+ if not failed:
355
+ return list(executions)
356
+ return [e for e in executions if e.outcome_class is not Outcome.PASSED]
357
+
358
+
359
+ def _baseline_id(store: RunStore, run_id: str) -> str:
360
+ rows = store.candidates(run_id)
361
+ found = next((c for c in rows if c.role == "baseline"), None)
362
+ return found.candidate_id if found else ""
363
+
364
+
365
+ def _wanted_candidates(
366
+ executions: Sequence[ExecutionRecord], baseline_id: str, asked: list[str] | None
367
+ ) -> list[str]:
368
+ present = list(dict.fromkeys(e.candidate_id for e in executions))
369
+ if asked:
370
+ unknown = [c for c in asked if c not in present]
371
+ if unknown:
372
+ raise ConfigError(
373
+ f"no executions for candidate(s) {', '.join(unknown)} in this run",
374
+ hint=f"this run has: {', '.join(present)}",
375
+ )
376
+ return list(asked)
377
+ others = [c for c in present if c != baseline_id]
378
+ # With no candidates at all - a baseline-only run, which `record` produces -
379
+ # diffing the baseline against itself is the only honest thing left, and it
380
+ # still shows the timeline.
381
+ return others or present
382
+
383
+
384
+ def _diff_for(
385
+ project: Project,
386
+ *,
387
+ case_id: str,
388
+ executions: Sequence[ExecutionRecord],
389
+ baseline_id: str,
390
+ candidate_id: str,
391
+ repeat: int | None,
392
+ ) -> Diff:
393
+ mine = [e for e in executions if e.candidate_id == candidate_id]
394
+ chosen = _pick_repeat(mine, repeat)
395
+ if chosen is None:
396
+ raise ConfigError(
397
+ f"candidate {candidate_id!r} has no repeat {repeat} for case {case_id!r}",
398
+ hint=f"stored repeats: {', '.join(str(e.repeat_idx) for e in mine)}",
399
+ )
400
+ reference = _pick_baseline(
401
+ [e for e in executions if e.candidate_id == baseline_id], chosen.repeat_idx
402
+ )
403
+ return build_diff(
404
+ case_id=case_id,
405
+ candidate=project.store.reconstruct(chosen.exec_id),
406
+ baseline=(
407
+ project.store.reconstruct(reference.exec_id)
408
+ if reference is not None and reference.exec_id != chosen.exec_id
409
+ else None
410
+ ),
411
+ candidate_id=candidate_id,
412
+ baseline_id=baseline_id or "baseline",
413
+ cassettes=project.cassettes,
414
+ )
415
+
416
+
417
+ def _pick_repeat(
418
+ mine: Sequence[ExecutionRecord], repeat: int | None
419
+ ) -> ExecutionRecord | None:
420
+ """The repeat to show: the one the user asked for, or the worst one.
421
+
422
+ The worst one by default, because a user opening the drill-down for a case
423
+ that failed two runs in three does not want to be shown the pass. "Worst"
424
+ means failed before passed, and an earlier repeat before a later one so the
425
+ choice is reproducible.
426
+ """
427
+ if repeat is not None:
428
+ return next((e for e in mine if e.repeat_idx == repeat), None)
429
+ if not mine:
430
+ return None
431
+ return sorted(
432
+ mine,
433
+ key=lambda e: (e.outcome_class is Outcome.PASSED, e.repeat_idx),
434
+ )[0]
435
+
436
+
437
+ def _pick_baseline(
438
+ rows: Sequence[ExecutionRecord], repeat_idx: int
439
+ ) -> ExecutionRecord | None:
440
+ """The baseline execution to diff against: the matching repeat if it passed.
441
+
442
+ A failing baseline repeat is a poor reference - the left column would be
443
+ just as wrong as the right one, and the user would be comparing two
444
+ failures and seeing no divergence. So a passing repeat wins over the
445
+ matching index, and the matching index wins over anything else.
446
+ """
447
+ if not rows:
448
+ return None
449
+ passing = [e for e in rows if e.outcome_class is Outcome.PASSED]
450
+ pool = passing or list(rows)
451
+ exact = next((e for e in pool if e.repeat_idx == repeat_idx), None)
452
+ return exact or pool[0]
453
+
454
+
455
+ def _diagnoses(detail: ExecutionDetail, project: Project) -> list[MissDiagnosis]:
456
+ """Rebuild every cassette-miss block in one execution."""
457
+ known = known_tool_names([detail], project.cassettes)
458
+ found: list[MissDiagnosis] = []
459
+ for step in tool_steps(detail):
460
+ finding = rebuild_diagnosis(
461
+ step,
462
+ case_id=detail.execution.case_id,
463
+ candidate_id=detail.execution.candidate_id,
464
+ cassettes=project.cassettes,
465
+ known_tools=known,
466
+ )
467
+ if finding is not None:
468
+ found.append(finding)
469
+ return found
470
+
471
+
472
+ def _print_json(payload: dict[str, Any]) -> None:
473
+ """``print``, not rich-print: this is data on its way into a pipe, and Rich
474
+ would wrap and style it into something no parser can read."""
475
+ print(json.dumps(payload, indent=2, default=str))
476
+
477
+
478
+ def register(app: typer.Typer) -> None:
479
+ app.add_typer(show_app, name="show")
480
+
481
+
482
+ __all__ = ["register", "show_app"]
@@ -0,0 +1,57 @@
1
+ """Commands that are planned but not built yet.
2
+
3
+ They are hidden from `--help`, so nobody is offered a command that does not work.
4
+ They still accept whatever arguments you throw at them, so a scripted invocation
5
+ gets a useful message instead of a Click usage error.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ from collections.abc import Callable
11
+
12
+ import typer
13
+ from rich.text import Text
14
+
15
+ from slmeval import exit_codes
16
+ from slmeval.cli.render import console, symbols
17
+
18
+ PLANNED: list[tuple[str, str]] = [
19
+ ("compare", "Put two runs side by side."),
20
+ ("cassette", "Inspect, verify and prune recorded tool responses."),
21
+ ]
22
+
23
+ _CONTEXT = {"ignore_unknown_options": True, "allow_extra_args": True}
24
+
25
+
26
+ def _make(name: str, summary: str) -> Callable[..., None]:
27
+ def command(ctx: typer.Context) -> None:
28
+ out = console(stderr=True)
29
+ sym = symbols()
30
+ out.print()
31
+ out.print(Text(f" {sym.skip} `slmeval {name}` is not implemented yet.", style="yellow"))
32
+ out.print(Text(f" {summary}", style="dim"))
33
+ out.print(Text(" It is not part of this release.", style="dim"))
34
+ out.print()
35
+ out.print(
36
+ Text(
37
+ " Available now: init, doctor, validate, dataset validate, "
38
+ "record, run, report, show, gate, baseline save, cache stats",
39
+ style="dim",
40
+ )
41
+ )
42
+ out.print()
43
+ raise typer.Exit(exit_codes.USAGE)
44
+
45
+ command.__name__ = name
46
+ command.__doc__ = summary
47
+ return command
48
+
49
+
50
+ def register(app: typer.Typer) -> None:
51
+ for name, summary in PLANNED:
52
+ app.command(
53
+ name=name,
54
+ help=summary,
55
+ hidden=True,
56
+ context_settings=_CONTEXT,
57
+ )(_make(name, summary))