slmeval-cli 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (94) hide show
  1. slmeval/__init__.py +71 -0
  2. slmeval/__main__.py +14 -0
  3. slmeval/adapters/__init__.py +24 -0
  4. slmeval/adapters/instrument.py +135 -0
  5. slmeval/adapters/langgraph.py +237 -0
  6. slmeval/cli/__init__.py +15 -0
  7. slmeval/cli/commands/__init__.py +35 -0
  8. slmeval/cli/commands/baseline_cmd.py +201 -0
  9. slmeval/cli/commands/cache_cmd.py +249 -0
  10. slmeval/cli/commands/dataset_cmd.py +52 -0
  11. slmeval/cli/commands/doctor_cmd.py +68 -0
  12. slmeval/cli/commands/gate_cmd.py +602 -0
  13. slmeval/cli/commands/init_cmd.py +144 -0
  14. slmeval/cli/commands/judge_cmd.py +284 -0
  15. slmeval/cli/commands/record_cmd.py +154 -0
  16. slmeval/cli/commands/report_cmd.py +98 -0
  17. slmeval/cli/commands/run_cmd.py +247 -0
  18. slmeval/cli/commands/show_cmd.py +482 -0
  19. slmeval/cli/commands/stubs.py +57 -0
  20. slmeval/cli/commands/validate_cmd.py +296 -0
  21. slmeval/cli/loading.py +149 -0
  22. slmeval/cli/main.py +91 -0
  23. slmeval/cli/render.py +251 -0
  24. slmeval/cli/wrap.py +31 -0
  25. slmeval/config/__init__.py +47 -0
  26. slmeval/config/loader.py +363 -0
  27. slmeval/config/locate.py +97 -0
  28. slmeval/config/models.py +430 -0
  29. slmeval/config/templates.py +246 -0
  30. slmeval/core/__init__.py +78 -0
  31. slmeval/core/adapter.py +253 -0
  32. slmeval/core/context.py +418 -0
  33. slmeval/core/outcome.py +260 -0
  34. slmeval/core/router.py +319 -0
  35. slmeval/core/runner.py +1209 -0
  36. slmeval/core/trace.py +188 -0
  37. slmeval/core/verdict.py +127 -0
  38. slmeval/dataset/__init__.py +8 -0
  39. slmeval/dataset/loader.py +130 -0
  40. slmeval/dataset/models.py +108 -0
  41. slmeval/doctor/__init__.py +7 -0
  42. slmeval/doctor/checks.py +380 -0
  43. slmeval/errors.py +76 -0
  44. slmeval/eval/__init__.py +64 -0
  45. slmeval/eval/assertions.py +356 -0
  46. slmeval/eval/calibration.py +364 -0
  47. slmeval/eval/cost.py +291 -0
  48. slmeval/eval/pipeline.py +273 -0
  49. slmeval/eval/stats.py +376 -0
  50. slmeval/eval/tier1.py +510 -0
  51. slmeval/eval/tier2.py +582 -0
  52. slmeval/eval/tier3.py +415 -0
  53. slmeval/exit_codes.py +28 -0
  54. slmeval/gate.py +722 -0
  55. slmeval/providers/__init__.py +69 -0
  56. slmeval/providers/base.py +426 -0
  57. slmeval/providers/fake.py +760 -0
  58. slmeval/providers/jsonx.py +239 -0
  59. slmeval/providers/ladder.py +339 -0
  60. slmeval/providers/openai_compat.py +392 -0
  61. slmeval/providers/registry.py +185 -0
  62. slmeval/providers/retry.py +81 -0
  63. slmeval/providers/schema.py +136 -0
  64. slmeval/providers/tokens.py +150 -0
  65. slmeval/py.typed +0 -0
  66. slmeval/report/__init__.py +115 -0
  67. slmeval/report/attribution.py +197 -0
  68. slmeval/report/baseline.py +353 -0
  69. slmeval/report/drilldown.py +1395 -0
  70. slmeval/report/export.py +450 -0
  71. slmeval/report/model.py +562 -0
  72. slmeval/report/progress.py +257 -0
  73. slmeval/report/scorecard.py +367 -0
  74. slmeval/sandbox/__init__.py +145 -0
  75. slmeval/sandbox/diagnosis.py +690 -0
  76. slmeval/sandbox/envelope.py +486 -0
  77. slmeval/sandbox/faults.py +794 -0
  78. slmeval/sandbox/keys.py +458 -0
  79. slmeval/sandbox/normalizers.py +333 -0
  80. slmeval/sandbox/store.py +525 -0
  81. slmeval/sandbox/tool.py +685 -0
  82. slmeval/storage/__init__.py +73 -0
  83. slmeval/storage/blobs.py +308 -0
  84. slmeval/storage/codec.py +93 -0
  85. slmeval/storage/db.py +1334 -0
  86. slmeval/storage/ingest.py +346 -0
  87. slmeval/storage/migrations.py +358 -0
  88. slmeval/storage/models.py +554 -0
  89. slmeval/storage/provenance.py +201 -0
  90. slmeval_cli-0.1.0.dist-info/METADATA +484 -0
  91. slmeval_cli-0.1.0.dist-info/RECORD +94 -0
  92. slmeval_cli-0.1.0.dist-info/WHEEL +4 -0
  93. slmeval_cli-0.1.0.dist-info/entry_points.txt +2 -0
  94. slmeval_cli-0.1.0.dist-info/licenses/LICENSE +201 -0
slmeval/cli/render.py ADDED
@@ -0,0 +1,251 @@
1
+ """Terminal output.
2
+
3
+ Rendering lives in pure functions that take data and return Rich renderables,
4
+ so every view is snapshot-testable with ``Console(record=True)`` and nothing
5
+ deep in the engine ever calls ``print``.
6
+
7
+ The terminal table *is* the product for a CLI tool — it is what gets
8
+ screenshotted and pasted into a pull request — so this module gets the same
9
+ care as the measurement code.
10
+ """
11
+
12
+ from __future__ import annotations
13
+
14
+ import contextlib
15
+ import os
16
+ import sys
17
+ from dataclasses import dataclass
18
+ from pathlib import Path
19
+ from typing import TextIO
20
+
21
+ from rich.console import Console
22
+ from rich.table import Table
23
+ from rich.text import Text
24
+
25
+ from slmeval.doctor import Check, Status
26
+ from slmeval.errors import SlmevalError
27
+ from slmeval.sandbox.diagnosis import MissDiagnosis, Verdict
28
+
29
+
30
+ @dataclass(frozen=True)
31
+ class Symbols:
32
+ ok: str
33
+ warn: str
34
+ fail: str
35
+ skip: str
36
+ arrow: str
37
+ bar: str
38
+
39
+
40
+ UNICODE = Symbols(ok="✓", warn="!", fail="✗", skip="·", arrow="→", bar="│")
41
+ ASCII = Symbols(ok="OK", warn="!!", fail="XX", skip="--", arrow="->", bar="|")
42
+
43
+ STATUS_STYLE = {
44
+ Status.OK: "green",
45
+ Status.WARN: "yellow",
46
+ Status.FAIL: "red",
47
+ Status.SKIP: "dim",
48
+ }
49
+
50
+
51
+ def supports_unicode(stream: object | None = None) -> bool:
52
+ """Legacy Windows consoles use cp1252 and cannot encode box glyphs."""
53
+ encoding = getattr(stream or sys.stdout, "encoding", None) or ""
54
+ try:
55
+ UNICODE.ok.encode(encoding or "ascii")
56
+ except (LookupError, UnicodeEncodeError):
57
+ return False
58
+ return True
59
+
60
+
61
+ def symbols(ascii_mode: bool | None = None) -> Symbols:
62
+ if ascii_mode is None:
63
+ ascii_mode = not supports_unicode()
64
+ return ASCII if ascii_mode else UNICODE
65
+
66
+
67
+ def ascii_only(ascii_mode: bool | None = None) -> bool:
68
+ """Whether to avoid non-ASCII glyphs. ``None`` means "ask the terminal"."""
69
+ return not supports_unicode() if ascii_mode is None else ascii_mode
70
+
71
+
72
+ def dash(ascii_mode: bool | None = None) -> str:
73
+ """The "no value here" cell.
74
+
75
+ One helper rather than a constant per module, because an em dash written
76
+ straight to a legacy Windows codepage comes out as a replacement character
77
+ - and a table full of those is what a user screenshots.
78
+ """
79
+ return "-" if ascii_only(ascii_mode) else "—"
80
+
81
+
82
+ def make_encoding_safe(stream: object | None) -> None:
83
+ """Never let printing crash the tool.
84
+
85
+ On a legacy Windows codepage an unencodable character raises
86
+ UnicodeEncodeError from deep inside the writer. Our own messages are ASCII,
87
+ but the text we echo is not always ours: `render_error` prints the user's
88
+ own config line, and a case id or model name can contain anything. Degrading
89
+ one glyph to '?' is always better than a traceback over a success message.
90
+ """
91
+ if stream is None or supports_unicode(stream):
92
+ return
93
+ reconfigure = getattr(stream, "reconfigure", None)
94
+ if reconfigure is None:
95
+ return
96
+ # A detached or exotic stream may refuse; degrading is the whole point here.
97
+ with contextlib.suppress(OSError, ValueError):
98
+ reconfigure(errors="replace")
99
+
100
+
101
+ def console(*, stderr: bool = False, ascii_mode: bool | None = None) -> Console:
102
+ """Build a Console that behaves when piped.
103
+
104
+ Rich already honours NO_COLOR and non-tty detection; the explicit
105
+ ``no_color`` keeps behaviour identical when a caller hands us a recording
106
+ console in tests.
107
+
108
+ ``legacy_windows`` is switched off for a pipe or a file. Rich detects a
109
+ legacy console on Windows when there is no console at all, then renders one
110
+ column narrower, so the same report would wrap differently on Windows and
111
+ Linux at the same ``COLUMNS``.
112
+ """
113
+ stream = sys.stderr if stderr else sys.stdout
114
+ make_encoding_safe(stream)
115
+ return Console(
116
+ stderr=stderr,
117
+ legacy_windows=None if _is_terminal(stream) else False,
118
+ no_color=bool(os.environ.get("NO_COLOR")),
119
+ soft_wrap=False,
120
+ highlight=False,
121
+ emoji=False,
122
+ )
123
+
124
+
125
+ def _is_terminal(stream: TextIO) -> bool:
126
+ # A detached stream can be None or closed, and neither is a terminal.
127
+ try:
128
+ return stream.isatty()
129
+ except (AttributeError, ValueError):
130
+ return False
131
+
132
+
133
+ def status_cell(status: Status, sym: Symbols) -> Text:
134
+ glyph = {
135
+ Status.OK: sym.ok,
136
+ Status.WARN: sym.warn,
137
+ Status.FAIL: sym.fail,
138
+ Status.SKIP: sym.skip,
139
+ }[status]
140
+ return Text(glyph, style=STATUS_STYLE[status])
141
+
142
+
143
+ def render_checks(checks: list[Check], *, ascii_mode: bool | None = None) -> Table:
144
+ """The doctor report: one line per check, fix hints indented beneath."""
145
+ sym = symbols(ascii_mode)
146
+ grid = Table.grid(padding=(0, 2))
147
+ grid.add_column(width=2, justify="left")
148
+ grid.add_column(style="bold", no_wrap=True)
149
+ grid.add_column(overflow="fold")
150
+
151
+ for check in checks:
152
+ grid.add_row(status_cell(check.status, sym), check.name, check.detail)
153
+ if check.hint and check.status is not Status.OK:
154
+ grid.add_row("", "", Text(f"{sym.arrow} {check.hint}", style="dim"))
155
+ return grid
156
+
157
+
158
+ def checks_summary(checks: list[Check]) -> Text:
159
+ counts = {s: sum(1 for c in checks if c.status is s) for s in Status}
160
+ parts = []
161
+ if counts[Status.FAIL]:
162
+ parts.append(Text(f"{counts[Status.FAIL]} failed", style="bold red"))
163
+ if counts[Status.WARN]:
164
+ parts.append(Text(f"{counts[Status.WARN]} warning", style="yellow"))
165
+ if counts[Status.SKIP]:
166
+ parts.append(Text(f"{counts[Status.SKIP]} skipped", style="dim"))
167
+ parts.append(Text(f"{counts[Status.OK]} ok", style="green"))
168
+ out = Text()
169
+ for i, part in enumerate(parts):
170
+ if i:
171
+ out.append(", ", style="dim")
172
+ out.append(part)
173
+ return out
174
+
175
+
176
+ def render_error(err: SlmevalError, *, ascii_mode: bool | None = None) -> None:
177
+ """Print a user-fixable error: what is wrong, where, and the source line."""
178
+ sym = symbols(ascii_mode)
179
+ out = console(stderr=True, ascii_mode=ascii_mode)
180
+
181
+ out.print()
182
+ out.print(Text(f"{sym.fail} {err.message}", style="bold red"))
183
+
184
+ source_lines = _source_lines(err.source)
185
+ label = err.source.name if err.source else ""
186
+
187
+ for problem in err.problems:
188
+ out.print()
189
+ out.print(Text(f" {problem.path}", style="bold"), end="")
190
+ out.print(Text(f" {sym.arrow} {problem.message}"))
191
+ if problem.line and problem.line <= len(source_lines):
192
+ text = source_lines[problem.line - 1].rstrip()
193
+ prefix = f" {label}:{problem.line} {sym.bar} "
194
+ out.print(Text(prefix, style="dim"), end="")
195
+ out.print(Text(text, style="yellow"))
196
+ elif problem.value is not None:
197
+ out.print(Text(f" got {problem.value}", style="dim"))
198
+
199
+ if err.hint:
200
+ out.print()
201
+ out.print(Text(f" hint: {err.hint}", style="cyan"))
202
+ out.print()
203
+
204
+
205
+ VERDICT_STYLE = {
206
+ Verdict.HALLUCINATED_PARAMETER: "red",
207
+ Verdict.HALLUCINATED_TOOL: "red",
208
+ Verdict.WRONG_TOOL_FOR_STEP: "yellow",
209
+ # Not red, and that is the point: an uncovered path is a gap in our
210
+ # coverage, not a model failure, and colouring it like one would undo the
211
+ # whole distinction the diagnosis exists to draw.
212
+ Verdict.UNRECORDED_VALID_PATH: "cyan",
213
+ }
214
+
215
+
216
+ def render_miss(diagnosis: MissDiagnosis, *, ascii_mode: bool | None = None) -> Text:
217
+ """The cassette-miss diagnosis block, coloured for a terminal.
218
+
219
+ The layout comes from :meth:`MissDiagnosis.render` rather than being
220
+ rebuilt here, so the block a user screenshots from the terminal and the one
221
+ they paste out of a CI log are the same block. This function only adds
222
+ style, line by line.
223
+ """
224
+ if ascii_mode is None:
225
+ ascii_mode = not supports_unicode()
226
+ accent = VERDICT_STYLE[diagnosis.verdict]
227
+ out = Text()
228
+ lines = diagnosis.render(ascii_mode=ascii_mode).split("\n")
229
+ for index, line in enumerate(lines):
230
+ stripped = line.lstrip()
231
+ if index == 0 or stripped.startswith("verdict:"):
232
+ style = f"bold {accent}"
233
+ elif stripped.startswith(("diff:", "nearest recorded:", "did you mean:")):
234
+ style = "yellow"
235
+ elif stripped.startswith("called with:"):
236
+ style = "bold"
237
+ else:
238
+ style = "dim"
239
+ out.append(line, style=style)
240
+ if index < len(lines) - 1:
241
+ out.append("\n")
242
+ return out
243
+
244
+
245
+ def _source_lines(path: Path | None) -> list[str]:
246
+ if path is None or not path.exists():
247
+ return []
248
+ try:
249
+ return path.read_text(encoding="utf-8-sig").splitlines()
250
+ except OSError:
251
+ return []
slmeval/cli/wrap.py ADDED
@@ -0,0 +1,31 @@
1
+ """Turn user-fixable errors into readable output and a documented exit code.
2
+
3
+ Applied to every command, so the behaviour is identical whether the CLI is
4
+ invoked by a shell, by CI, or by `typer.testing.CliRunner` in the test suite.
5
+ A harness bug keeps its traceback — that distinction is deliberate.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ import functools
11
+ from collections.abc import Callable
12
+ from typing import Any, TypeVar
13
+
14
+ import typer
15
+
16
+ from slmeval.cli.render import render_error
17
+ from slmeval.errors import SlmevalError
18
+
19
+ F = TypeVar("F", bound=Callable[..., Any])
20
+
21
+
22
+ def handle_errors(func: F) -> F:
23
+ @functools.wraps(func)
24
+ def wrapper(*args: Any, **kwargs: Any) -> Any:
25
+ try:
26
+ return func(*args, **kwargs)
27
+ except SlmevalError as err:
28
+ render_error(err)
29
+ raise typer.Exit(err.exit_code) from None
30
+
31
+ return wrapper # type: ignore[return-value]
@@ -0,0 +1,47 @@
1
+ """Benchmark configuration: schema, loading, and source location."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from slmeval.config.loader import (
6
+ CACHE_DIR_NAME,
7
+ DEFAULT_CONFIG_NAME,
8
+ LoadedConfig,
9
+ find_config,
10
+ format_loc,
11
+ load_config,
12
+ require_dataset,
13
+ resolve_entrypoint,
14
+ )
15
+ from slmeval.config.models import (
16
+ BaselineCfg,
17
+ BenchmarkCfg,
18
+ EvalConfig,
19
+ GateCfg,
20
+ JudgeCfg,
21
+ ModelCfg,
22
+ Provider,
23
+ StructuredMode,
24
+ TargetCfg,
25
+ ToolMode,
26
+ )
27
+
28
+ __all__ = [
29
+ "CACHE_DIR_NAME",
30
+ "DEFAULT_CONFIG_NAME",
31
+ "BaselineCfg",
32
+ "BenchmarkCfg",
33
+ "EvalConfig",
34
+ "GateCfg",
35
+ "JudgeCfg",
36
+ "LoadedConfig",
37
+ "ModelCfg",
38
+ "Provider",
39
+ "StructuredMode",
40
+ "TargetCfg",
41
+ "ToolMode",
42
+ "find_config",
43
+ "format_loc",
44
+ "load_config",
45
+ "require_dataset",
46
+ "resolve_entrypoint",
47
+ ]
@@ -0,0 +1,363 @@
1
+ """Load, locate and validate `eval.toml`.
2
+
3
+ A typo in the config must fail in under a second with a line-level message —
4
+ never forty minutes into a benchmark run.
5
+ """
6
+
7
+ from __future__ import annotations
8
+
9
+ import importlib
10
+ import re
11
+ import sys
12
+ import sysconfig
13
+ import tomllib
14
+ from dataclasses import dataclass
15
+ from pathlib import Path
16
+ from typing import Any
17
+
18
+ from pydantic import ValidationError
19
+
20
+ from slmeval.config.locate import locate
21
+ from slmeval.config.models import EvalConfig, ModelCfg
22
+ from slmeval.errors import ConfigError, Problem
23
+
24
+ DEFAULT_CONFIG_NAME = "eval.toml"
25
+ CACHE_DIR_NAME = ".slmeval"
26
+
27
+ _TOML_POS_RE = re.compile(r"at line (\d+), column (\d+)")
28
+
29
+
30
+ @dataclass
31
+ class LoadedConfig:
32
+ """A validated config plus everything needed to report on it."""
33
+
34
+ config: EvalConfig
35
+ path: Path
36
+ text: str
37
+
38
+ @property
39
+ def root(self) -> Path:
40
+ """Directory the config lives in. All relative paths resolve against it."""
41
+ return self.path.parent
42
+
43
+ @property
44
+ def cache_dir(self) -> Path:
45
+ return self.root / CACHE_DIR_NAME
46
+
47
+ @property
48
+ def dataset_path(self) -> Path:
49
+ ds = self.config.benchmark.dataset
50
+ return ds if ds.is_absolute() else (self.root / ds).resolve()
51
+
52
+
53
+ def find_config(explicit: Path | None = None, *, start: Path | None = None) -> Path:
54
+ """Locate `eval.toml`, walking up from ``start`` like git does."""
55
+ if explicit is not None:
56
+ path = Path(explicit).expanduser()
57
+ if not path.exists():
58
+ raise ConfigError(
59
+ f"config file not found: {path}",
60
+ hint="run `slmeval init` to create one",
61
+ )
62
+ if path.is_dir():
63
+ candidate = path / DEFAULT_CONFIG_NAME
64
+ if not candidate.exists():
65
+ raise ConfigError(f"no {DEFAULT_CONFIG_NAME} in directory {path}")
66
+ return candidate.resolve()
67
+ return path.resolve()
68
+
69
+ here = (start or Path.cwd()).resolve()
70
+ for directory in (here, *here.parents):
71
+ candidate = directory / DEFAULT_CONFIG_NAME
72
+ if candidate.is_file():
73
+ return candidate
74
+ raise ConfigError(
75
+ f"no {DEFAULT_CONFIG_NAME} found in {here} or any parent directory",
76
+ hint="run `slmeval init` to create one, or pass --config PATH",
77
+ )
78
+
79
+
80
+ def load_config(path: Path) -> LoadedConfig:
81
+ """Parse and validate a config file. Raises :class:`ConfigError` on any problem."""
82
+ path = Path(path)
83
+ try:
84
+ # utf-8-sig: a file saved by Notepad carries a BOM, which tomllib
85
+ # otherwise reports as a syntax error on line 1.
86
+ text = path.read_text(encoding="utf-8-sig")
87
+ except OSError as exc:
88
+ raise ConfigError(f"could not read {path}: {exc}") from exc
89
+
90
+ try:
91
+ data: dict[str, Any] = tomllib.loads(text)
92
+ except tomllib.TOMLDecodeError as exc:
93
+ line = None
94
+ if m := _TOML_POS_RE.search(str(exc)):
95
+ line = int(m.group(1))
96
+ raise ConfigError(
97
+ "eval.toml is not valid TOML",
98
+ source=path,
99
+ problems=[Problem(path="<syntax>", message=str(exc), line=line)],
100
+ ) from exc
101
+
102
+ try:
103
+ config = EvalConfig.model_validate(data)
104
+ except ValidationError as exc:
105
+ raise ConfigError(
106
+ f"eval.toml is invalid ({exc.error_count()} "
107
+ f"problem{'s' if exc.error_count() != 1 else ''})",
108
+ source=path,
109
+ problems=_problems_from(exc, text),
110
+ ) from exc
111
+
112
+ loaded = LoadedConfig(config=config, path=path, text=text)
113
+ _settle_fake_scripts(loaded)
114
+ return loaded
115
+
116
+
117
+ def _settle_fake_scripts(loaded: LoadedConfig) -> None:
118
+ """Validate every `fake` script, and inline the fixture files it points at.
119
+
120
+ Two things happen here, both so that a scripted model behaves like a
121
+ configured one rather than a special case.
122
+
123
+ A ``script_file`` is read now and its table merged into the candidate's
124
+ ``extra_body``, then the key is dropped. That resolves the path against the
125
+ directory holding ``eval.toml`` - ``slmeval run -c ../other/eval.toml`` has
126
+ to mean what running from that directory means - and it puts the fixture's
127
+ *content* into :meth:`EvalConfig.config_hash` instead of an absolute path,
128
+ so the same benchmark hashes the same on two machines and an edited fixture
129
+ is correctly a different benchmark.
130
+
131
+ And every script is parsed here, while we still have the file and its line
132
+ numbers. Left to the provider, a typo would surface as a ``ValueError``
133
+ raised inside slmeval midway through a matrix, which the outcome classifier
134
+ reads - correctly, by its own rules - as a harness bug.
135
+ """
136
+ from slmeval.providers.fake import load_script_file, validate_extra_body
137
+
138
+ for loc, cfg in _model_locations(loaded.config):
139
+ if cfg.provider != "fake":
140
+ continue
141
+ extra = cfg.extra_body
142
+ key = "script_file" if "script_file" in extra else "script"
143
+ try:
144
+ if "script_file" in extra:
145
+ if "script" in extra:
146
+ raise ValueError(
147
+ "set extra_body.script or extra_body.script_file, not both - "
148
+ "two scripts for one model has no defined order"
149
+ )
150
+ fixture = Path(str(extra.pop("script_file"))).expanduser()
151
+ if not fixture.is_absolute():
152
+ fixture = (loaded.root / fixture).resolve()
153
+ table = load_script_file(fixture)
154
+ # Inline entries win over the fixture's, per case, so a file
155
+ # holding the whole dataset plus one override reads the way it
156
+ # looks.
157
+ by_case = dict(table.get("script_by_case") or {})
158
+ by_case.update(extra.get("script_by_case") or {})
159
+ if by_case:
160
+ extra["script_by_case"] = by_case
161
+ for name in ("script", "unsupported"):
162
+ if name in table and name not in extra:
163
+ extra[name] = table[name]
164
+ validate_extra_body(extra)
165
+ except ValueError as exc:
166
+ raise ConfigError(
167
+ f"{cfg.id}: invalid script for provider 'fake'",
168
+ source=loaded.path,
169
+ problems=[
170
+ Problem(
171
+ path=format_loc((*loc, "extra_body", key)),
172
+ message=str(exc),
173
+ # Either spelling: a dotted `extra_body.script = ...`
174
+ # line, or a `[baseline.extra_body]` table.
175
+ line=(
176
+ locate(loaded.text, (*loc, "extra_body", key))
177
+ or locate(loaded.text, (*loc, f"extra_body.{key}"))
178
+ ),
179
+ )
180
+ ],
181
+ hint=(
182
+ "a fake script is a list of turns: "
183
+ "extra_body.script = [{ kind = \"fenced_json\", text = '{}' }]"
184
+ ),
185
+ ) from exc
186
+
187
+
188
+ def _model_locations(config: EvalConfig) -> list[tuple[tuple[str | int, ...], ModelCfg]]:
189
+ """Every model with the config path it was written at, for error messages."""
190
+ out: list[tuple[tuple[str | int, ...], ModelCfg]] = [(("baseline",), config.baseline)]
191
+ out += [(("candidates", i), cfg) for i, cfg in enumerate(config.candidates)]
192
+ return out
193
+
194
+
195
+ def _problems_from(exc: ValidationError, text: str) -> list[Problem]:
196
+ problems: list[Problem] = []
197
+ for err in exc.errors():
198
+ loc = err["loc"]
199
+ message = err["msg"]
200
+ # Pydantic prefixes custom ValueError messages; the prefix adds nothing
201
+ # for a user reading their own config file.
202
+ message = message.removeprefix("Value error, ")
203
+ if err["type"] == "extra_forbidden":
204
+ message = "unknown key (check the spelling, or remove it)"
205
+ problems.append(
206
+ Problem(
207
+ path=format_loc(loc),
208
+ message=message,
209
+ line=locate(text, tuple(loc)),
210
+ value=_short(err.get("input")),
211
+ )
212
+ )
213
+ return problems
214
+
215
+
216
+ def format_loc(loc: tuple[int | str, ...]) -> str:
217
+ """``("candidates", 0, "provider")`` -> ``candidates[0].provider``."""
218
+ out = ""
219
+ for part in loc:
220
+ if isinstance(part, int):
221
+ out += f"[{part}]"
222
+ else:
223
+ out += f".{part}" if out else part
224
+ return out or "config"
225
+
226
+
227
+ def _short(value: Any, limit: int = 60) -> str | None:
228
+ if value is None or isinstance(value, dict):
229
+ return None
230
+ text = repr(value)
231
+ return text if len(text) <= limit else text[: limit - 1] + "..."
232
+
233
+
234
+ def require_dataset(loaded: LoadedConfig) -> Path:
235
+ """Resolve the dataset path, erroring clearly if it is not there."""
236
+ path = loaded.dataset_path
237
+ if not path.exists():
238
+ raise ConfigError(
239
+ f"dataset not found: {path}",
240
+ source=loaded.path,
241
+ problems=[
242
+ Problem(
243
+ path="benchmark.dataset",
244
+ message=f"resolved to {path} (relative to {loaded.root})",
245
+ line=locate(loaded.text, ("benchmark", "dataset")),
246
+ )
247
+ ],
248
+ hint="create the file, or point benchmark.dataset at an existing .jsonl",
249
+ )
250
+ return path
251
+
252
+
253
+ def _within(path: Path | str, root: Path) -> bool:
254
+ try:
255
+ return Path(path).resolve().is_relative_to(root.resolve())
256
+ except (OSError, ValueError): # pragma: no cover - unresolvable path
257
+ return False
258
+
259
+
260
+ def _is_installed(origin: str) -> bool:
261
+ """Did this module come from the standard library or site-packages?"""
262
+ locations = sysconfig.get_paths()
263
+ for key in ("purelib", "platlib", "stdlib", "platstdlib"):
264
+ location = locations.get(key)
265
+ if location and _within(origin, Path(location)):
266
+ return True
267
+ return False
268
+
269
+
270
+ _injected_roots: set[str] = set()
271
+
272
+
273
+ def _prepare_sys_path(root: str) -> None:
274
+ """Make the active project's root importable, and only that one.
275
+
276
+ The agent module lives next to eval.toml, which is not necessarily the
277
+ working directory, so the root has to go on `sys.path`. But `sys.path` is
278
+ global and append-only: left alone, a second config loaded in the same
279
+ process could still import the first project's agent and silently benchmark
280
+ the wrong code. So each resolve withdraws the roots a previous resolve
281
+ injected before adding its own.
282
+ """
283
+ for previous in _injected_roots:
284
+ while previous in sys.path:
285
+ sys.path.remove(previous)
286
+ _injected_roots.clear()
287
+
288
+ while root in sys.path:
289
+ sys.path.remove(root)
290
+ sys.path.insert(0, root)
291
+ _injected_roots.add(root)
292
+
293
+
294
+ def _import_target_module(module_name: str, root: Path) -> Any:
295
+ """Import a module, preferring the copy that belongs to this project.
296
+
297
+ `sys.modules` is global and sticky: once any module named `slmeval_target`
298
+ has been imported, a later import of that name returns the first one no
299
+ matter which directory we are now pointed at. That silently validates a
300
+ target that no longer exists, and in a monorepo it would benchmark the wrong
301
+ agent. So when the module we get back is neither inside this project nor an
302
+ installed package, drop it and resolve once more from scratch.
303
+ """
304
+ importlib.invalidate_caches()
305
+ module = importlib.import_module(module_name)
306
+ origin = getattr(module, "__file__", None)
307
+
308
+ if origin is None or _within(origin, root) or _is_installed(origin):
309
+ return module
310
+
311
+ stale = {
312
+ name: mod
313
+ for name, mod in list(sys.modules.items())
314
+ if name == module_name or name.startswith(f"{module_name}.")
315
+ }
316
+ for name in stale:
317
+ del sys.modules[name]
318
+
319
+ importlib.invalidate_caches()
320
+ try:
321
+ return importlib.import_module(module_name)
322
+ except ImportError:
323
+ # Nothing on the path can provide it any more. Put back what we removed
324
+ # so we do not corrupt the interpreter on the way out.
325
+ sys.modules.update(stale)
326
+ raise
327
+
328
+
329
+ def resolve_entrypoint(loaded: LoadedConfig) -> Any:
330
+ """Import the user's agent callable.
331
+
332
+ Only used by `validate --strict` and `run`: importing the target can be slow
333
+ and can execute arbitrary module-level code, so plain validation never does it.
334
+ """
335
+ target = loaded.config.target
336
+ _prepare_sys_path(str(loaded.root))
337
+
338
+ try:
339
+ module = _import_target_module(target.module, loaded.root)
340
+ except ImportError as exc:
341
+ raise ConfigError(
342
+ f"could not import {target.module!r} from {loaded.root}",
343
+ source=loaded.path,
344
+ problems=[Problem(path="target.entrypoint", message=str(exc))],
345
+ hint="check the module path, and that its own imports are installed",
346
+ ) from exc
347
+
348
+ try:
349
+ func = getattr(module, target.attr)
350
+ except AttributeError as exc:
351
+ available = ", ".join(n for n in dir(module) if not n.startswith("_")) or "nothing"
352
+ raise ConfigError(
353
+ f"{target.module!r} has no attribute {target.attr!r}",
354
+ source=loaded.path,
355
+ problems=[Problem(path="target.entrypoint", message=f"module defines: {available}")],
356
+ ) from exc
357
+
358
+ if not callable(func):
359
+ raise ConfigError(
360
+ f"{target.entrypoint} is not callable (it is a {type(func).__name__})",
361
+ source=loaded.path,
362
+ )
363
+ return func