slmeval-cli 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- slmeval/__init__.py +71 -0
- slmeval/__main__.py +14 -0
- slmeval/adapters/__init__.py +24 -0
- slmeval/adapters/instrument.py +135 -0
- slmeval/adapters/langgraph.py +237 -0
- slmeval/cli/__init__.py +15 -0
- slmeval/cli/commands/__init__.py +35 -0
- slmeval/cli/commands/baseline_cmd.py +201 -0
- slmeval/cli/commands/cache_cmd.py +249 -0
- slmeval/cli/commands/dataset_cmd.py +52 -0
- slmeval/cli/commands/doctor_cmd.py +68 -0
- slmeval/cli/commands/gate_cmd.py +602 -0
- slmeval/cli/commands/init_cmd.py +144 -0
- slmeval/cli/commands/judge_cmd.py +284 -0
- slmeval/cli/commands/record_cmd.py +154 -0
- slmeval/cli/commands/report_cmd.py +98 -0
- slmeval/cli/commands/run_cmd.py +247 -0
- slmeval/cli/commands/show_cmd.py +482 -0
- slmeval/cli/commands/stubs.py +57 -0
- slmeval/cli/commands/validate_cmd.py +296 -0
- slmeval/cli/loading.py +149 -0
- slmeval/cli/main.py +91 -0
- slmeval/cli/render.py +251 -0
- slmeval/cli/wrap.py +31 -0
- slmeval/config/__init__.py +47 -0
- slmeval/config/loader.py +363 -0
- slmeval/config/locate.py +97 -0
- slmeval/config/models.py +430 -0
- slmeval/config/templates.py +246 -0
- slmeval/core/__init__.py +78 -0
- slmeval/core/adapter.py +253 -0
- slmeval/core/context.py +418 -0
- slmeval/core/outcome.py +260 -0
- slmeval/core/router.py +319 -0
- slmeval/core/runner.py +1209 -0
- slmeval/core/trace.py +188 -0
- slmeval/core/verdict.py +127 -0
- slmeval/dataset/__init__.py +8 -0
- slmeval/dataset/loader.py +130 -0
- slmeval/dataset/models.py +108 -0
- slmeval/doctor/__init__.py +7 -0
- slmeval/doctor/checks.py +380 -0
- slmeval/errors.py +76 -0
- slmeval/eval/__init__.py +64 -0
- slmeval/eval/assertions.py +356 -0
- slmeval/eval/calibration.py +364 -0
- slmeval/eval/cost.py +291 -0
- slmeval/eval/pipeline.py +273 -0
- slmeval/eval/stats.py +376 -0
- slmeval/eval/tier1.py +510 -0
- slmeval/eval/tier2.py +582 -0
- slmeval/eval/tier3.py +415 -0
- slmeval/exit_codes.py +28 -0
- slmeval/gate.py +722 -0
- slmeval/providers/__init__.py +69 -0
- slmeval/providers/base.py +426 -0
- slmeval/providers/fake.py +760 -0
- slmeval/providers/jsonx.py +239 -0
- slmeval/providers/ladder.py +339 -0
- slmeval/providers/openai_compat.py +392 -0
- slmeval/providers/registry.py +185 -0
- slmeval/providers/retry.py +81 -0
- slmeval/providers/schema.py +136 -0
- slmeval/providers/tokens.py +150 -0
- slmeval/py.typed +0 -0
- slmeval/report/__init__.py +115 -0
- slmeval/report/attribution.py +197 -0
- slmeval/report/baseline.py +353 -0
- slmeval/report/drilldown.py +1395 -0
- slmeval/report/export.py +450 -0
- slmeval/report/model.py +562 -0
- slmeval/report/progress.py +257 -0
- slmeval/report/scorecard.py +367 -0
- slmeval/sandbox/__init__.py +145 -0
- slmeval/sandbox/diagnosis.py +690 -0
- slmeval/sandbox/envelope.py +486 -0
- slmeval/sandbox/faults.py +794 -0
- slmeval/sandbox/keys.py +458 -0
- slmeval/sandbox/normalizers.py +333 -0
- slmeval/sandbox/store.py +525 -0
- slmeval/sandbox/tool.py +685 -0
- slmeval/storage/__init__.py +73 -0
- slmeval/storage/blobs.py +308 -0
- slmeval/storage/codec.py +93 -0
- slmeval/storage/db.py +1334 -0
- slmeval/storage/ingest.py +346 -0
- slmeval/storage/migrations.py +358 -0
- slmeval/storage/models.py +554 -0
- slmeval/storage/provenance.py +201 -0
- slmeval_cli-0.1.0.dist-info/METADATA +484 -0
- slmeval_cli-0.1.0.dist-info/RECORD +94 -0
- slmeval_cli-0.1.0.dist-info/WHEEL +4 -0
- slmeval_cli-0.1.0.dist-info/entry_points.txt +2 -0
- slmeval_cli-0.1.0.dist-info/licenses/LICENSE +201 -0
slmeval/cli/render.py
ADDED
|
@@ -0,0 +1,251 @@
|
|
|
1
|
+
"""Terminal output.
|
|
2
|
+
|
|
3
|
+
Rendering lives in pure functions that take data and return Rich renderables,
|
|
4
|
+
so every view is snapshot-testable with ``Console(record=True)`` and nothing
|
|
5
|
+
deep in the engine ever calls ``print``.
|
|
6
|
+
|
|
7
|
+
The terminal table *is* the product for a CLI tool — it is what gets
|
|
8
|
+
screenshotted and pasted into a pull request — so this module gets the same
|
|
9
|
+
care as the measurement code.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
import contextlib
|
|
15
|
+
import os
|
|
16
|
+
import sys
|
|
17
|
+
from dataclasses import dataclass
|
|
18
|
+
from pathlib import Path
|
|
19
|
+
from typing import TextIO
|
|
20
|
+
|
|
21
|
+
from rich.console import Console
|
|
22
|
+
from rich.table import Table
|
|
23
|
+
from rich.text import Text
|
|
24
|
+
|
|
25
|
+
from slmeval.doctor import Check, Status
|
|
26
|
+
from slmeval.errors import SlmevalError
|
|
27
|
+
from slmeval.sandbox.diagnosis import MissDiagnosis, Verdict
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
@dataclass(frozen=True)
|
|
31
|
+
class Symbols:
|
|
32
|
+
ok: str
|
|
33
|
+
warn: str
|
|
34
|
+
fail: str
|
|
35
|
+
skip: str
|
|
36
|
+
arrow: str
|
|
37
|
+
bar: str
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
UNICODE = Symbols(ok="✓", warn="!", fail="✗", skip="·", arrow="→", bar="│")
|
|
41
|
+
ASCII = Symbols(ok="OK", warn="!!", fail="XX", skip="--", arrow="->", bar="|")
|
|
42
|
+
|
|
43
|
+
STATUS_STYLE = {
|
|
44
|
+
Status.OK: "green",
|
|
45
|
+
Status.WARN: "yellow",
|
|
46
|
+
Status.FAIL: "red",
|
|
47
|
+
Status.SKIP: "dim",
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def supports_unicode(stream: object | None = None) -> bool:
|
|
52
|
+
"""Legacy Windows consoles use cp1252 and cannot encode box glyphs."""
|
|
53
|
+
encoding = getattr(stream or sys.stdout, "encoding", None) or ""
|
|
54
|
+
try:
|
|
55
|
+
UNICODE.ok.encode(encoding or "ascii")
|
|
56
|
+
except (LookupError, UnicodeEncodeError):
|
|
57
|
+
return False
|
|
58
|
+
return True
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def symbols(ascii_mode: bool | None = None) -> Symbols:
|
|
62
|
+
if ascii_mode is None:
|
|
63
|
+
ascii_mode = not supports_unicode()
|
|
64
|
+
return ASCII if ascii_mode else UNICODE
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def ascii_only(ascii_mode: bool | None = None) -> bool:
|
|
68
|
+
"""Whether to avoid non-ASCII glyphs. ``None`` means "ask the terminal"."""
|
|
69
|
+
return not supports_unicode() if ascii_mode is None else ascii_mode
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def dash(ascii_mode: bool | None = None) -> str:
|
|
73
|
+
"""The "no value here" cell.
|
|
74
|
+
|
|
75
|
+
One helper rather than a constant per module, because an em dash written
|
|
76
|
+
straight to a legacy Windows codepage comes out as a replacement character
|
|
77
|
+
- and a table full of those is what a user screenshots.
|
|
78
|
+
"""
|
|
79
|
+
return "-" if ascii_only(ascii_mode) else "—"
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def make_encoding_safe(stream: object | None) -> None:
|
|
83
|
+
"""Never let printing crash the tool.
|
|
84
|
+
|
|
85
|
+
On a legacy Windows codepage an unencodable character raises
|
|
86
|
+
UnicodeEncodeError from deep inside the writer. Our own messages are ASCII,
|
|
87
|
+
but the text we echo is not always ours: `render_error` prints the user's
|
|
88
|
+
own config line, and a case id or model name can contain anything. Degrading
|
|
89
|
+
one glyph to '?' is always better than a traceback over a success message.
|
|
90
|
+
"""
|
|
91
|
+
if stream is None or supports_unicode(stream):
|
|
92
|
+
return
|
|
93
|
+
reconfigure = getattr(stream, "reconfigure", None)
|
|
94
|
+
if reconfigure is None:
|
|
95
|
+
return
|
|
96
|
+
# A detached or exotic stream may refuse; degrading is the whole point here.
|
|
97
|
+
with contextlib.suppress(OSError, ValueError):
|
|
98
|
+
reconfigure(errors="replace")
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def console(*, stderr: bool = False, ascii_mode: bool | None = None) -> Console:
|
|
102
|
+
"""Build a Console that behaves when piped.
|
|
103
|
+
|
|
104
|
+
Rich already honours NO_COLOR and non-tty detection; the explicit
|
|
105
|
+
``no_color`` keeps behaviour identical when a caller hands us a recording
|
|
106
|
+
console in tests.
|
|
107
|
+
|
|
108
|
+
``legacy_windows`` is switched off for a pipe or a file. Rich detects a
|
|
109
|
+
legacy console on Windows when there is no console at all, then renders one
|
|
110
|
+
column narrower, so the same report would wrap differently on Windows and
|
|
111
|
+
Linux at the same ``COLUMNS``.
|
|
112
|
+
"""
|
|
113
|
+
stream = sys.stderr if stderr else sys.stdout
|
|
114
|
+
make_encoding_safe(stream)
|
|
115
|
+
return Console(
|
|
116
|
+
stderr=stderr,
|
|
117
|
+
legacy_windows=None if _is_terminal(stream) else False,
|
|
118
|
+
no_color=bool(os.environ.get("NO_COLOR")),
|
|
119
|
+
soft_wrap=False,
|
|
120
|
+
highlight=False,
|
|
121
|
+
emoji=False,
|
|
122
|
+
)
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
def _is_terminal(stream: TextIO) -> bool:
|
|
126
|
+
# A detached stream can be None or closed, and neither is a terminal.
|
|
127
|
+
try:
|
|
128
|
+
return stream.isatty()
|
|
129
|
+
except (AttributeError, ValueError):
|
|
130
|
+
return False
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
def status_cell(status: Status, sym: Symbols) -> Text:
|
|
134
|
+
glyph = {
|
|
135
|
+
Status.OK: sym.ok,
|
|
136
|
+
Status.WARN: sym.warn,
|
|
137
|
+
Status.FAIL: sym.fail,
|
|
138
|
+
Status.SKIP: sym.skip,
|
|
139
|
+
}[status]
|
|
140
|
+
return Text(glyph, style=STATUS_STYLE[status])
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
def render_checks(checks: list[Check], *, ascii_mode: bool | None = None) -> Table:
|
|
144
|
+
"""The doctor report: one line per check, fix hints indented beneath."""
|
|
145
|
+
sym = symbols(ascii_mode)
|
|
146
|
+
grid = Table.grid(padding=(0, 2))
|
|
147
|
+
grid.add_column(width=2, justify="left")
|
|
148
|
+
grid.add_column(style="bold", no_wrap=True)
|
|
149
|
+
grid.add_column(overflow="fold")
|
|
150
|
+
|
|
151
|
+
for check in checks:
|
|
152
|
+
grid.add_row(status_cell(check.status, sym), check.name, check.detail)
|
|
153
|
+
if check.hint and check.status is not Status.OK:
|
|
154
|
+
grid.add_row("", "", Text(f"{sym.arrow} {check.hint}", style="dim"))
|
|
155
|
+
return grid
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
def checks_summary(checks: list[Check]) -> Text:
|
|
159
|
+
counts = {s: sum(1 for c in checks if c.status is s) for s in Status}
|
|
160
|
+
parts = []
|
|
161
|
+
if counts[Status.FAIL]:
|
|
162
|
+
parts.append(Text(f"{counts[Status.FAIL]} failed", style="bold red"))
|
|
163
|
+
if counts[Status.WARN]:
|
|
164
|
+
parts.append(Text(f"{counts[Status.WARN]} warning", style="yellow"))
|
|
165
|
+
if counts[Status.SKIP]:
|
|
166
|
+
parts.append(Text(f"{counts[Status.SKIP]} skipped", style="dim"))
|
|
167
|
+
parts.append(Text(f"{counts[Status.OK]} ok", style="green"))
|
|
168
|
+
out = Text()
|
|
169
|
+
for i, part in enumerate(parts):
|
|
170
|
+
if i:
|
|
171
|
+
out.append(", ", style="dim")
|
|
172
|
+
out.append(part)
|
|
173
|
+
return out
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
def render_error(err: SlmevalError, *, ascii_mode: bool | None = None) -> None:
|
|
177
|
+
"""Print a user-fixable error: what is wrong, where, and the source line."""
|
|
178
|
+
sym = symbols(ascii_mode)
|
|
179
|
+
out = console(stderr=True, ascii_mode=ascii_mode)
|
|
180
|
+
|
|
181
|
+
out.print()
|
|
182
|
+
out.print(Text(f"{sym.fail} {err.message}", style="bold red"))
|
|
183
|
+
|
|
184
|
+
source_lines = _source_lines(err.source)
|
|
185
|
+
label = err.source.name if err.source else ""
|
|
186
|
+
|
|
187
|
+
for problem in err.problems:
|
|
188
|
+
out.print()
|
|
189
|
+
out.print(Text(f" {problem.path}", style="bold"), end="")
|
|
190
|
+
out.print(Text(f" {sym.arrow} {problem.message}"))
|
|
191
|
+
if problem.line and problem.line <= len(source_lines):
|
|
192
|
+
text = source_lines[problem.line - 1].rstrip()
|
|
193
|
+
prefix = f" {label}:{problem.line} {sym.bar} "
|
|
194
|
+
out.print(Text(prefix, style="dim"), end="")
|
|
195
|
+
out.print(Text(text, style="yellow"))
|
|
196
|
+
elif problem.value is not None:
|
|
197
|
+
out.print(Text(f" got {problem.value}", style="dim"))
|
|
198
|
+
|
|
199
|
+
if err.hint:
|
|
200
|
+
out.print()
|
|
201
|
+
out.print(Text(f" hint: {err.hint}", style="cyan"))
|
|
202
|
+
out.print()
|
|
203
|
+
|
|
204
|
+
|
|
205
|
+
VERDICT_STYLE = {
|
|
206
|
+
Verdict.HALLUCINATED_PARAMETER: "red",
|
|
207
|
+
Verdict.HALLUCINATED_TOOL: "red",
|
|
208
|
+
Verdict.WRONG_TOOL_FOR_STEP: "yellow",
|
|
209
|
+
# Not red, and that is the point: an uncovered path is a gap in our
|
|
210
|
+
# coverage, not a model failure, and colouring it like one would undo the
|
|
211
|
+
# whole distinction the diagnosis exists to draw.
|
|
212
|
+
Verdict.UNRECORDED_VALID_PATH: "cyan",
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
|
|
216
|
+
def render_miss(diagnosis: MissDiagnosis, *, ascii_mode: bool | None = None) -> Text:
|
|
217
|
+
"""The cassette-miss diagnosis block, coloured for a terminal.
|
|
218
|
+
|
|
219
|
+
The layout comes from :meth:`MissDiagnosis.render` rather than being
|
|
220
|
+
rebuilt here, so the block a user screenshots from the terminal and the one
|
|
221
|
+
they paste out of a CI log are the same block. This function only adds
|
|
222
|
+
style, line by line.
|
|
223
|
+
"""
|
|
224
|
+
if ascii_mode is None:
|
|
225
|
+
ascii_mode = not supports_unicode()
|
|
226
|
+
accent = VERDICT_STYLE[diagnosis.verdict]
|
|
227
|
+
out = Text()
|
|
228
|
+
lines = diagnosis.render(ascii_mode=ascii_mode).split("\n")
|
|
229
|
+
for index, line in enumerate(lines):
|
|
230
|
+
stripped = line.lstrip()
|
|
231
|
+
if index == 0 or stripped.startswith("verdict:"):
|
|
232
|
+
style = f"bold {accent}"
|
|
233
|
+
elif stripped.startswith(("diff:", "nearest recorded:", "did you mean:")):
|
|
234
|
+
style = "yellow"
|
|
235
|
+
elif stripped.startswith("called with:"):
|
|
236
|
+
style = "bold"
|
|
237
|
+
else:
|
|
238
|
+
style = "dim"
|
|
239
|
+
out.append(line, style=style)
|
|
240
|
+
if index < len(lines) - 1:
|
|
241
|
+
out.append("\n")
|
|
242
|
+
return out
|
|
243
|
+
|
|
244
|
+
|
|
245
|
+
def _source_lines(path: Path | None) -> list[str]:
|
|
246
|
+
if path is None or not path.exists():
|
|
247
|
+
return []
|
|
248
|
+
try:
|
|
249
|
+
return path.read_text(encoding="utf-8-sig").splitlines()
|
|
250
|
+
except OSError:
|
|
251
|
+
return []
|
slmeval/cli/wrap.py
ADDED
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
"""Turn user-fixable errors into readable output and a documented exit code.
|
|
2
|
+
|
|
3
|
+
Applied to every command, so the behaviour is identical whether the CLI is
|
|
4
|
+
invoked by a shell, by CI, or by `typer.testing.CliRunner` in the test suite.
|
|
5
|
+
A harness bug keeps its traceback — that distinction is deliberate.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import functools
|
|
11
|
+
from collections.abc import Callable
|
|
12
|
+
from typing import Any, TypeVar
|
|
13
|
+
|
|
14
|
+
import typer
|
|
15
|
+
|
|
16
|
+
from slmeval.cli.render import render_error
|
|
17
|
+
from slmeval.errors import SlmevalError
|
|
18
|
+
|
|
19
|
+
F = TypeVar("F", bound=Callable[..., Any])
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def handle_errors(func: F) -> F:
|
|
23
|
+
@functools.wraps(func)
|
|
24
|
+
def wrapper(*args: Any, **kwargs: Any) -> Any:
|
|
25
|
+
try:
|
|
26
|
+
return func(*args, **kwargs)
|
|
27
|
+
except SlmevalError as err:
|
|
28
|
+
render_error(err)
|
|
29
|
+
raise typer.Exit(err.exit_code) from None
|
|
30
|
+
|
|
31
|
+
return wrapper # type: ignore[return-value]
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
"""Benchmark configuration: schema, loading, and source location."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from slmeval.config.loader import (
|
|
6
|
+
CACHE_DIR_NAME,
|
|
7
|
+
DEFAULT_CONFIG_NAME,
|
|
8
|
+
LoadedConfig,
|
|
9
|
+
find_config,
|
|
10
|
+
format_loc,
|
|
11
|
+
load_config,
|
|
12
|
+
require_dataset,
|
|
13
|
+
resolve_entrypoint,
|
|
14
|
+
)
|
|
15
|
+
from slmeval.config.models import (
|
|
16
|
+
BaselineCfg,
|
|
17
|
+
BenchmarkCfg,
|
|
18
|
+
EvalConfig,
|
|
19
|
+
GateCfg,
|
|
20
|
+
JudgeCfg,
|
|
21
|
+
ModelCfg,
|
|
22
|
+
Provider,
|
|
23
|
+
StructuredMode,
|
|
24
|
+
TargetCfg,
|
|
25
|
+
ToolMode,
|
|
26
|
+
)
|
|
27
|
+
|
|
28
|
+
__all__ = [
|
|
29
|
+
"CACHE_DIR_NAME",
|
|
30
|
+
"DEFAULT_CONFIG_NAME",
|
|
31
|
+
"BaselineCfg",
|
|
32
|
+
"BenchmarkCfg",
|
|
33
|
+
"EvalConfig",
|
|
34
|
+
"GateCfg",
|
|
35
|
+
"JudgeCfg",
|
|
36
|
+
"LoadedConfig",
|
|
37
|
+
"ModelCfg",
|
|
38
|
+
"Provider",
|
|
39
|
+
"StructuredMode",
|
|
40
|
+
"TargetCfg",
|
|
41
|
+
"ToolMode",
|
|
42
|
+
"find_config",
|
|
43
|
+
"format_loc",
|
|
44
|
+
"load_config",
|
|
45
|
+
"require_dataset",
|
|
46
|
+
"resolve_entrypoint",
|
|
47
|
+
]
|
slmeval/config/loader.py
ADDED
|
@@ -0,0 +1,363 @@
|
|
|
1
|
+
"""Load, locate and validate `eval.toml`.
|
|
2
|
+
|
|
3
|
+
A typo in the config must fail in under a second with a line-level message —
|
|
4
|
+
never forty minutes into a benchmark run.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import importlib
|
|
10
|
+
import re
|
|
11
|
+
import sys
|
|
12
|
+
import sysconfig
|
|
13
|
+
import tomllib
|
|
14
|
+
from dataclasses import dataclass
|
|
15
|
+
from pathlib import Path
|
|
16
|
+
from typing import Any
|
|
17
|
+
|
|
18
|
+
from pydantic import ValidationError
|
|
19
|
+
|
|
20
|
+
from slmeval.config.locate import locate
|
|
21
|
+
from slmeval.config.models import EvalConfig, ModelCfg
|
|
22
|
+
from slmeval.errors import ConfigError, Problem
|
|
23
|
+
|
|
24
|
+
DEFAULT_CONFIG_NAME = "eval.toml"
|
|
25
|
+
CACHE_DIR_NAME = ".slmeval"
|
|
26
|
+
|
|
27
|
+
_TOML_POS_RE = re.compile(r"at line (\d+), column (\d+)")
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
@dataclass
|
|
31
|
+
class LoadedConfig:
|
|
32
|
+
"""A validated config plus everything needed to report on it."""
|
|
33
|
+
|
|
34
|
+
config: EvalConfig
|
|
35
|
+
path: Path
|
|
36
|
+
text: str
|
|
37
|
+
|
|
38
|
+
@property
|
|
39
|
+
def root(self) -> Path:
|
|
40
|
+
"""Directory the config lives in. All relative paths resolve against it."""
|
|
41
|
+
return self.path.parent
|
|
42
|
+
|
|
43
|
+
@property
|
|
44
|
+
def cache_dir(self) -> Path:
|
|
45
|
+
return self.root / CACHE_DIR_NAME
|
|
46
|
+
|
|
47
|
+
@property
|
|
48
|
+
def dataset_path(self) -> Path:
|
|
49
|
+
ds = self.config.benchmark.dataset
|
|
50
|
+
return ds if ds.is_absolute() else (self.root / ds).resolve()
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def find_config(explicit: Path | None = None, *, start: Path | None = None) -> Path:
|
|
54
|
+
"""Locate `eval.toml`, walking up from ``start`` like git does."""
|
|
55
|
+
if explicit is not None:
|
|
56
|
+
path = Path(explicit).expanduser()
|
|
57
|
+
if not path.exists():
|
|
58
|
+
raise ConfigError(
|
|
59
|
+
f"config file not found: {path}",
|
|
60
|
+
hint="run `slmeval init` to create one",
|
|
61
|
+
)
|
|
62
|
+
if path.is_dir():
|
|
63
|
+
candidate = path / DEFAULT_CONFIG_NAME
|
|
64
|
+
if not candidate.exists():
|
|
65
|
+
raise ConfigError(f"no {DEFAULT_CONFIG_NAME} in directory {path}")
|
|
66
|
+
return candidate.resolve()
|
|
67
|
+
return path.resolve()
|
|
68
|
+
|
|
69
|
+
here = (start or Path.cwd()).resolve()
|
|
70
|
+
for directory in (here, *here.parents):
|
|
71
|
+
candidate = directory / DEFAULT_CONFIG_NAME
|
|
72
|
+
if candidate.is_file():
|
|
73
|
+
return candidate
|
|
74
|
+
raise ConfigError(
|
|
75
|
+
f"no {DEFAULT_CONFIG_NAME} found in {here} or any parent directory",
|
|
76
|
+
hint="run `slmeval init` to create one, or pass --config PATH",
|
|
77
|
+
)
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def load_config(path: Path) -> LoadedConfig:
|
|
81
|
+
"""Parse and validate a config file. Raises :class:`ConfigError` on any problem."""
|
|
82
|
+
path = Path(path)
|
|
83
|
+
try:
|
|
84
|
+
# utf-8-sig: a file saved by Notepad carries a BOM, which tomllib
|
|
85
|
+
# otherwise reports as a syntax error on line 1.
|
|
86
|
+
text = path.read_text(encoding="utf-8-sig")
|
|
87
|
+
except OSError as exc:
|
|
88
|
+
raise ConfigError(f"could not read {path}: {exc}") from exc
|
|
89
|
+
|
|
90
|
+
try:
|
|
91
|
+
data: dict[str, Any] = tomllib.loads(text)
|
|
92
|
+
except tomllib.TOMLDecodeError as exc:
|
|
93
|
+
line = None
|
|
94
|
+
if m := _TOML_POS_RE.search(str(exc)):
|
|
95
|
+
line = int(m.group(1))
|
|
96
|
+
raise ConfigError(
|
|
97
|
+
"eval.toml is not valid TOML",
|
|
98
|
+
source=path,
|
|
99
|
+
problems=[Problem(path="<syntax>", message=str(exc), line=line)],
|
|
100
|
+
) from exc
|
|
101
|
+
|
|
102
|
+
try:
|
|
103
|
+
config = EvalConfig.model_validate(data)
|
|
104
|
+
except ValidationError as exc:
|
|
105
|
+
raise ConfigError(
|
|
106
|
+
f"eval.toml is invalid ({exc.error_count()} "
|
|
107
|
+
f"problem{'s' if exc.error_count() != 1 else ''})",
|
|
108
|
+
source=path,
|
|
109
|
+
problems=_problems_from(exc, text),
|
|
110
|
+
) from exc
|
|
111
|
+
|
|
112
|
+
loaded = LoadedConfig(config=config, path=path, text=text)
|
|
113
|
+
_settle_fake_scripts(loaded)
|
|
114
|
+
return loaded
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def _settle_fake_scripts(loaded: LoadedConfig) -> None:
|
|
118
|
+
"""Validate every `fake` script, and inline the fixture files it points at.
|
|
119
|
+
|
|
120
|
+
Two things happen here, both so that a scripted model behaves like a
|
|
121
|
+
configured one rather than a special case.
|
|
122
|
+
|
|
123
|
+
A ``script_file`` is read now and its table merged into the candidate's
|
|
124
|
+
``extra_body``, then the key is dropped. That resolves the path against the
|
|
125
|
+
directory holding ``eval.toml`` - ``slmeval run -c ../other/eval.toml`` has
|
|
126
|
+
to mean what running from that directory means - and it puts the fixture's
|
|
127
|
+
*content* into :meth:`EvalConfig.config_hash` instead of an absolute path,
|
|
128
|
+
so the same benchmark hashes the same on two machines and an edited fixture
|
|
129
|
+
is correctly a different benchmark.
|
|
130
|
+
|
|
131
|
+
And every script is parsed here, while we still have the file and its line
|
|
132
|
+
numbers. Left to the provider, a typo would surface as a ``ValueError``
|
|
133
|
+
raised inside slmeval midway through a matrix, which the outcome classifier
|
|
134
|
+
reads - correctly, by its own rules - as a harness bug.
|
|
135
|
+
"""
|
|
136
|
+
from slmeval.providers.fake import load_script_file, validate_extra_body
|
|
137
|
+
|
|
138
|
+
for loc, cfg in _model_locations(loaded.config):
|
|
139
|
+
if cfg.provider != "fake":
|
|
140
|
+
continue
|
|
141
|
+
extra = cfg.extra_body
|
|
142
|
+
key = "script_file" if "script_file" in extra else "script"
|
|
143
|
+
try:
|
|
144
|
+
if "script_file" in extra:
|
|
145
|
+
if "script" in extra:
|
|
146
|
+
raise ValueError(
|
|
147
|
+
"set extra_body.script or extra_body.script_file, not both - "
|
|
148
|
+
"two scripts for one model has no defined order"
|
|
149
|
+
)
|
|
150
|
+
fixture = Path(str(extra.pop("script_file"))).expanduser()
|
|
151
|
+
if not fixture.is_absolute():
|
|
152
|
+
fixture = (loaded.root / fixture).resolve()
|
|
153
|
+
table = load_script_file(fixture)
|
|
154
|
+
# Inline entries win over the fixture's, per case, so a file
|
|
155
|
+
# holding the whole dataset plus one override reads the way it
|
|
156
|
+
# looks.
|
|
157
|
+
by_case = dict(table.get("script_by_case") or {})
|
|
158
|
+
by_case.update(extra.get("script_by_case") or {})
|
|
159
|
+
if by_case:
|
|
160
|
+
extra["script_by_case"] = by_case
|
|
161
|
+
for name in ("script", "unsupported"):
|
|
162
|
+
if name in table and name not in extra:
|
|
163
|
+
extra[name] = table[name]
|
|
164
|
+
validate_extra_body(extra)
|
|
165
|
+
except ValueError as exc:
|
|
166
|
+
raise ConfigError(
|
|
167
|
+
f"{cfg.id}: invalid script for provider 'fake'",
|
|
168
|
+
source=loaded.path,
|
|
169
|
+
problems=[
|
|
170
|
+
Problem(
|
|
171
|
+
path=format_loc((*loc, "extra_body", key)),
|
|
172
|
+
message=str(exc),
|
|
173
|
+
# Either spelling: a dotted `extra_body.script = ...`
|
|
174
|
+
# line, or a `[baseline.extra_body]` table.
|
|
175
|
+
line=(
|
|
176
|
+
locate(loaded.text, (*loc, "extra_body", key))
|
|
177
|
+
or locate(loaded.text, (*loc, f"extra_body.{key}"))
|
|
178
|
+
),
|
|
179
|
+
)
|
|
180
|
+
],
|
|
181
|
+
hint=(
|
|
182
|
+
"a fake script is a list of turns: "
|
|
183
|
+
"extra_body.script = [{ kind = \"fenced_json\", text = '{}' }]"
|
|
184
|
+
),
|
|
185
|
+
) from exc
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
def _model_locations(config: EvalConfig) -> list[tuple[tuple[str | int, ...], ModelCfg]]:
|
|
189
|
+
"""Every model with the config path it was written at, for error messages."""
|
|
190
|
+
out: list[tuple[tuple[str | int, ...], ModelCfg]] = [(("baseline",), config.baseline)]
|
|
191
|
+
out += [(("candidates", i), cfg) for i, cfg in enumerate(config.candidates)]
|
|
192
|
+
return out
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
def _problems_from(exc: ValidationError, text: str) -> list[Problem]:
|
|
196
|
+
problems: list[Problem] = []
|
|
197
|
+
for err in exc.errors():
|
|
198
|
+
loc = err["loc"]
|
|
199
|
+
message = err["msg"]
|
|
200
|
+
# Pydantic prefixes custom ValueError messages; the prefix adds nothing
|
|
201
|
+
# for a user reading their own config file.
|
|
202
|
+
message = message.removeprefix("Value error, ")
|
|
203
|
+
if err["type"] == "extra_forbidden":
|
|
204
|
+
message = "unknown key (check the spelling, or remove it)"
|
|
205
|
+
problems.append(
|
|
206
|
+
Problem(
|
|
207
|
+
path=format_loc(loc),
|
|
208
|
+
message=message,
|
|
209
|
+
line=locate(text, tuple(loc)),
|
|
210
|
+
value=_short(err.get("input")),
|
|
211
|
+
)
|
|
212
|
+
)
|
|
213
|
+
return problems
|
|
214
|
+
|
|
215
|
+
|
|
216
|
+
def format_loc(loc: tuple[int | str, ...]) -> str:
|
|
217
|
+
"""``("candidates", 0, "provider")`` -> ``candidates[0].provider``."""
|
|
218
|
+
out = ""
|
|
219
|
+
for part in loc:
|
|
220
|
+
if isinstance(part, int):
|
|
221
|
+
out += f"[{part}]"
|
|
222
|
+
else:
|
|
223
|
+
out += f".{part}" if out else part
|
|
224
|
+
return out or "config"
|
|
225
|
+
|
|
226
|
+
|
|
227
|
+
def _short(value: Any, limit: int = 60) -> str | None:
|
|
228
|
+
if value is None or isinstance(value, dict):
|
|
229
|
+
return None
|
|
230
|
+
text = repr(value)
|
|
231
|
+
return text if len(text) <= limit else text[: limit - 1] + "..."
|
|
232
|
+
|
|
233
|
+
|
|
234
|
+
def require_dataset(loaded: LoadedConfig) -> Path:
|
|
235
|
+
"""Resolve the dataset path, erroring clearly if it is not there."""
|
|
236
|
+
path = loaded.dataset_path
|
|
237
|
+
if not path.exists():
|
|
238
|
+
raise ConfigError(
|
|
239
|
+
f"dataset not found: {path}",
|
|
240
|
+
source=loaded.path,
|
|
241
|
+
problems=[
|
|
242
|
+
Problem(
|
|
243
|
+
path="benchmark.dataset",
|
|
244
|
+
message=f"resolved to {path} (relative to {loaded.root})",
|
|
245
|
+
line=locate(loaded.text, ("benchmark", "dataset")),
|
|
246
|
+
)
|
|
247
|
+
],
|
|
248
|
+
hint="create the file, or point benchmark.dataset at an existing .jsonl",
|
|
249
|
+
)
|
|
250
|
+
return path
|
|
251
|
+
|
|
252
|
+
|
|
253
|
+
def _within(path: Path | str, root: Path) -> bool:
|
|
254
|
+
try:
|
|
255
|
+
return Path(path).resolve().is_relative_to(root.resolve())
|
|
256
|
+
except (OSError, ValueError): # pragma: no cover - unresolvable path
|
|
257
|
+
return False
|
|
258
|
+
|
|
259
|
+
|
|
260
|
+
def _is_installed(origin: str) -> bool:
|
|
261
|
+
"""Did this module come from the standard library or site-packages?"""
|
|
262
|
+
locations = sysconfig.get_paths()
|
|
263
|
+
for key in ("purelib", "platlib", "stdlib", "platstdlib"):
|
|
264
|
+
location = locations.get(key)
|
|
265
|
+
if location and _within(origin, Path(location)):
|
|
266
|
+
return True
|
|
267
|
+
return False
|
|
268
|
+
|
|
269
|
+
|
|
270
|
+
_injected_roots: set[str] = set()
|
|
271
|
+
|
|
272
|
+
|
|
273
|
+
def _prepare_sys_path(root: str) -> None:
|
|
274
|
+
"""Make the active project's root importable, and only that one.
|
|
275
|
+
|
|
276
|
+
The agent module lives next to eval.toml, which is not necessarily the
|
|
277
|
+
working directory, so the root has to go on `sys.path`. But `sys.path` is
|
|
278
|
+
global and append-only: left alone, a second config loaded in the same
|
|
279
|
+
process could still import the first project's agent and silently benchmark
|
|
280
|
+
the wrong code. So each resolve withdraws the roots a previous resolve
|
|
281
|
+
injected before adding its own.
|
|
282
|
+
"""
|
|
283
|
+
for previous in _injected_roots:
|
|
284
|
+
while previous in sys.path:
|
|
285
|
+
sys.path.remove(previous)
|
|
286
|
+
_injected_roots.clear()
|
|
287
|
+
|
|
288
|
+
while root in sys.path:
|
|
289
|
+
sys.path.remove(root)
|
|
290
|
+
sys.path.insert(0, root)
|
|
291
|
+
_injected_roots.add(root)
|
|
292
|
+
|
|
293
|
+
|
|
294
|
+
def _import_target_module(module_name: str, root: Path) -> Any:
|
|
295
|
+
"""Import a module, preferring the copy that belongs to this project.
|
|
296
|
+
|
|
297
|
+
`sys.modules` is global and sticky: once any module named `slmeval_target`
|
|
298
|
+
has been imported, a later import of that name returns the first one no
|
|
299
|
+
matter which directory we are now pointed at. That silently validates a
|
|
300
|
+
target that no longer exists, and in a monorepo it would benchmark the wrong
|
|
301
|
+
agent. So when the module we get back is neither inside this project nor an
|
|
302
|
+
installed package, drop it and resolve once more from scratch.
|
|
303
|
+
"""
|
|
304
|
+
importlib.invalidate_caches()
|
|
305
|
+
module = importlib.import_module(module_name)
|
|
306
|
+
origin = getattr(module, "__file__", None)
|
|
307
|
+
|
|
308
|
+
if origin is None or _within(origin, root) or _is_installed(origin):
|
|
309
|
+
return module
|
|
310
|
+
|
|
311
|
+
stale = {
|
|
312
|
+
name: mod
|
|
313
|
+
for name, mod in list(sys.modules.items())
|
|
314
|
+
if name == module_name or name.startswith(f"{module_name}.")
|
|
315
|
+
}
|
|
316
|
+
for name in stale:
|
|
317
|
+
del sys.modules[name]
|
|
318
|
+
|
|
319
|
+
importlib.invalidate_caches()
|
|
320
|
+
try:
|
|
321
|
+
return importlib.import_module(module_name)
|
|
322
|
+
except ImportError:
|
|
323
|
+
# Nothing on the path can provide it any more. Put back what we removed
|
|
324
|
+
# so we do not corrupt the interpreter on the way out.
|
|
325
|
+
sys.modules.update(stale)
|
|
326
|
+
raise
|
|
327
|
+
|
|
328
|
+
|
|
329
|
+
def resolve_entrypoint(loaded: LoadedConfig) -> Any:
|
|
330
|
+
"""Import the user's agent callable.
|
|
331
|
+
|
|
332
|
+
Only used by `validate --strict` and `run`: importing the target can be slow
|
|
333
|
+
and can execute arbitrary module-level code, so plain validation never does it.
|
|
334
|
+
"""
|
|
335
|
+
target = loaded.config.target
|
|
336
|
+
_prepare_sys_path(str(loaded.root))
|
|
337
|
+
|
|
338
|
+
try:
|
|
339
|
+
module = _import_target_module(target.module, loaded.root)
|
|
340
|
+
except ImportError as exc:
|
|
341
|
+
raise ConfigError(
|
|
342
|
+
f"could not import {target.module!r} from {loaded.root}",
|
|
343
|
+
source=loaded.path,
|
|
344
|
+
problems=[Problem(path="target.entrypoint", message=str(exc))],
|
|
345
|
+
hint="check the module path, and that its own imports are installed",
|
|
346
|
+
) from exc
|
|
347
|
+
|
|
348
|
+
try:
|
|
349
|
+
func = getattr(module, target.attr)
|
|
350
|
+
except AttributeError as exc:
|
|
351
|
+
available = ", ".join(n for n in dir(module) if not n.startswith("_")) or "nothing"
|
|
352
|
+
raise ConfigError(
|
|
353
|
+
f"{target.module!r} has no attribute {target.attr!r}",
|
|
354
|
+
source=loaded.path,
|
|
355
|
+
problems=[Problem(path="target.entrypoint", message=f"module defines: {available}")],
|
|
356
|
+
) from exc
|
|
357
|
+
|
|
358
|
+
if not callable(func):
|
|
359
|
+
raise ConfigError(
|
|
360
|
+
f"{target.entrypoint} is not callable (it is a {type(func).__name__})",
|
|
361
|
+
source=loaded.path,
|
|
362
|
+
)
|
|
363
|
+
return func
|