slmeval-cli 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- slmeval/__init__.py +71 -0
- slmeval/__main__.py +14 -0
- slmeval/adapters/__init__.py +24 -0
- slmeval/adapters/instrument.py +135 -0
- slmeval/adapters/langgraph.py +237 -0
- slmeval/cli/__init__.py +15 -0
- slmeval/cli/commands/__init__.py +35 -0
- slmeval/cli/commands/baseline_cmd.py +201 -0
- slmeval/cli/commands/cache_cmd.py +249 -0
- slmeval/cli/commands/dataset_cmd.py +52 -0
- slmeval/cli/commands/doctor_cmd.py +68 -0
- slmeval/cli/commands/gate_cmd.py +602 -0
- slmeval/cli/commands/init_cmd.py +144 -0
- slmeval/cli/commands/judge_cmd.py +284 -0
- slmeval/cli/commands/record_cmd.py +154 -0
- slmeval/cli/commands/report_cmd.py +98 -0
- slmeval/cli/commands/run_cmd.py +247 -0
- slmeval/cli/commands/show_cmd.py +482 -0
- slmeval/cli/commands/stubs.py +57 -0
- slmeval/cli/commands/validate_cmd.py +296 -0
- slmeval/cli/loading.py +149 -0
- slmeval/cli/main.py +91 -0
- slmeval/cli/render.py +251 -0
- slmeval/cli/wrap.py +31 -0
- slmeval/config/__init__.py +47 -0
- slmeval/config/loader.py +363 -0
- slmeval/config/locate.py +97 -0
- slmeval/config/models.py +430 -0
- slmeval/config/templates.py +246 -0
- slmeval/core/__init__.py +78 -0
- slmeval/core/adapter.py +253 -0
- slmeval/core/context.py +418 -0
- slmeval/core/outcome.py +260 -0
- slmeval/core/router.py +319 -0
- slmeval/core/runner.py +1209 -0
- slmeval/core/trace.py +188 -0
- slmeval/core/verdict.py +127 -0
- slmeval/dataset/__init__.py +8 -0
- slmeval/dataset/loader.py +130 -0
- slmeval/dataset/models.py +108 -0
- slmeval/doctor/__init__.py +7 -0
- slmeval/doctor/checks.py +380 -0
- slmeval/errors.py +76 -0
- slmeval/eval/__init__.py +64 -0
- slmeval/eval/assertions.py +356 -0
- slmeval/eval/calibration.py +364 -0
- slmeval/eval/cost.py +291 -0
- slmeval/eval/pipeline.py +273 -0
- slmeval/eval/stats.py +376 -0
- slmeval/eval/tier1.py +510 -0
- slmeval/eval/tier2.py +582 -0
- slmeval/eval/tier3.py +415 -0
- slmeval/exit_codes.py +28 -0
- slmeval/gate.py +722 -0
- slmeval/providers/__init__.py +69 -0
- slmeval/providers/base.py +426 -0
- slmeval/providers/fake.py +760 -0
- slmeval/providers/jsonx.py +239 -0
- slmeval/providers/ladder.py +339 -0
- slmeval/providers/openai_compat.py +392 -0
- slmeval/providers/registry.py +185 -0
- slmeval/providers/retry.py +81 -0
- slmeval/providers/schema.py +136 -0
- slmeval/providers/tokens.py +150 -0
- slmeval/py.typed +0 -0
- slmeval/report/__init__.py +115 -0
- slmeval/report/attribution.py +197 -0
- slmeval/report/baseline.py +353 -0
- slmeval/report/drilldown.py +1395 -0
- slmeval/report/export.py +450 -0
- slmeval/report/model.py +562 -0
- slmeval/report/progress.py +257 -0
- slmeval/report/scorecard.py +367 -0
- slmeval/sandbox/__init__.py +145 -0
- slmeval/sandbox/diagnosis.py +690 -0
- slmeval/sandbox/envelope.py +486 -0
- slmeval/sandbox/faults.py +794 -0
- slmeval/sandbox/keys.py +458 -0
- slmeval/sandbox/normalizers.py +333 -0
- slmeval/sandbox/store.py +525 -0
- slmeval/sandbox/tool.py +685 -0
- slmeval/storage/__init__.py +73 -0
- slmeval/storage/blobs.py +308 -0
- slmeval/storage/codec.py +93 -0
- slmeval/storage/db.py +1334 -0
- slmeval/storage/ingest.py +346 -0
- slmeval/storage/migrations.py +358 -0
- slmeval/storage/models.py +554 -0
- slmeval/storage/provenance.py +201 -0
- slmeval_cli-0.1.0.dist-info/METADATA +484 -0
- slmeval_cli-0.1.0.dist-info/RECORD +94 -0
- slmeval_cli-0.1.0.dist-info/WHEEL +4 -0
- slmeval_cli-0.1.0.dist-info/entry_points.txt +2 -0
- slmeval_cli-0.1.0.dist-info/licenses/LICENSE +201 -0
slmeval/sandbox/tool.py
ADDED
|
@@ -0,0 +1,685 @@
|
|
|
1
|
+
"""``@sandbox_tool`` and the four interception modes.
|
|
2
|
+
|
|
3
|
+
This decorator is the first of the two touch points a user adds to their agent,
|
|
4
|
+
and the one that makes the whole benchmark possible: a recorded tool call is
|
|
5
|
+
comparable across models, fast, and safe to run 500 times, while a live one is
|
|
6
|
+
none of those things.
|
|
7
|
+
|
|
8
|
+
The modes:
|
|
9
|
+
|
|
10
|
+
======== ====================================================================
|
|
11
|
+
record call the real tool, store what it returned
|
|
12
|
+
replay serve from the store; a miss raises, it never falls through
|
|
13
|
+
auto replay when recorded, record when not
|
|
14
|
+
live no interception at all - the escape hatch, with a red banner
|
|
15
|
+
======== ====================================================================
|
|
16
|
+
|
|
17
|
+
The single most important rule here is that **no path falls back to the real
|
|
18
|
+
tool by accident**. In ``replay`` mode a miss is a finding and must surface as
|
|
19
|
+
one; silently calling the customer's live Stripe instead would spend their money
|
|
20
|
+
and invalidate the run, and it would do it without telling anyone. So every
|
|
21
|
+
degraded situation - no store, no case id, a lost context during a benchmark -
|
|
22
|
+
either raises or warns, and the only silent passthrough is the one case where
|
|
23
|
+
passing through is correct: there is no benchmark running at all, because the
|
|
24
|
+
user is in production.
|
|
25
|
+
|
|
26
|
+
Two things happen here that are the whole of Day 7:
|
|
27
|
+
|
|
28
|
+
**A miss is a diagnosis, not an exception string.** On a strict-replay miss the
|
|
29
|
+
store is searched for the nearest recorded call and the result is classified -
|
|
30
|
+
mangled argument, invented tool, wrong tool for the step, or a path nobody ever
|
|
31
|
+
recorded. The last of those is *our* gap, not the model's failure, and keeping
|
|
32
|
+
it separate is what stops the scorecard inflating itself. See
|
|
33
|
+
:mod:`slmeval.sandbox.diagnosis`.
|
|
34
|
+
|
|
35
|
+
**A fault is delivered after the response resolves, never before.** A cassette
|
|
36
|
+
miss is a finding about the model; a fault is a stimulus we chose. Resolving
|
|
37
|
+
first means an injected ``429`` can never mask a hallucinated argument - the
|
|
38
|
+
miss surfaces, and the fault is simply not reached. See
|
|
39
|
+
:mod:`slmeval.sandbox.faults`.
|
|
40
|
+
"""
|
|
41
|
+
|
|
42
|
+
from __future__ import annotations
|
|
43
|
+
|
|
44
|
+
import asyncio
|
|
45
|
+
import functools
|
|
46
|
+
import hashlib
|
|
47
|
+
import inspect
|
|
48
|
+
import os
|
|
49
|
+
import sys
|
|
50
|
+
import textwrap
|
|
51
|
+
import time
|
|
52
|
+
from collections.abc import Callable, Mapping
|
|
53
|
+
from dataclasses import dataclass, field
|
|
54
|
+
from typing import Any, TypeVar, cast
|
|
55
|
+
|
|
56
|
+
from slmeval.core.context import (
|
|
57
|
+
RunContext,
|
|
58
|
+
benchmark_running_now,
|
|
59
|
+
current_node,
|
|
60
|
+
current_node_path,
|
|
61
|
+
get_context,
|
|
62
|
+
warn_missing_context,
|
|
63
|
+
)
|
|
64
|
+
from slmeval.core.trace import ToolCallTrace
|
|
65
|
+
from slmeval.sandbox.diagnosis import MissDiagnosis, diagnose
|
|
66
|
+
from slmeval.sandbox.faults import Delivery, Fault, apply_fault
|
|
67
|
+
from slmeval.sandbox.keys import CassetteKeys, MatchPolicy, MatchTier, PolicyError, declare
|
|
68
|
+
from slmeval.sandbox.store import CassetteStore, Hit
|
|
69
|
+
|
|
70
|
+
F = TypeVar("F", bound=Callable[..., Any])
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
class CassetteMiss(Exception):
|
|
74
|
+
"""A tool was called in a way that was never recorded.
|
|
75
|
+
|
|
76
|
+
Not an :class:`~slmeval.errors.SlmevalError`, and deliberately not: in a
|
|
77
|
+
benchmark this is a *result*. The runner catches it, classifies the
|
|
78
|
+
execution as failed and keeps going, exactly as it would for any other way
|
|
79
|
+
an agent can fail. Aborting the process on it would throw away the other 89
|
|
80
|
+
executions and the finding itself.
|
|
81
|
+
|
|
82
|
+
Its message **is** the diagnosis block from
|
|
83
|
+
:mod:`slmeval.sandbox.diagnosis` - nearest recorded call, per-argument
|
|
84
|
+
diff, verdict - so one finding reads the same in a terminal, a log, a
|
|
85
|
+
pytest failure and a traceback, with nothing to re-parse out of a string.
|
|
86
|
+
:attr:`diagnosis` carries the same thing structured, for the runner to
|
|
87
|
+
persist and the scorecard to aggregate.
|
|
88
|
+
"""
|
|
89
|
+
|
|
90
|
+
def __init__(
|
|
91
|
+
self,
|
|
92
|
+
*,
|
|
93
|
+
case_id: str,
|
|
94
|
+
tool_name: str,
|
|
95
|
+
arguments: Mapping[str, Any],
|
|
96
|
+
keys: CassetteKeys,
|
|
97
|
+
diagnosis: MissDiagnosis | None = None,
|
|
98
|
+
recorded_calls: int = 0,
|
|
99
|
+
node: str | None = None,
|
|
100
|
+
candidate_id: str | None = None,
|
|
101
|
+
) -> None:
|
|
102
|
+
if diagnosis is not None:
|
|
103
|
+
super().__init__(diagnosis.render())
|
|
104
|
+
recorded_calls = diagnosis.recorded_calls
|
|
105
|
+
else:
|
|
106
|
+
# Only reachable when something builds this by hand: diagnosing
|
|
107
|
+
# needs a store to search, and a miss is still a miss without one.
|
|
108
|
+
where = " · ".join(
|
|
109
|
+
part for part in (case_id, candidate_id, f"node={node}" if node else None) if part
|
|
110
|
+
)
|
|
111
|
+
super().__init__(
|
|
112
|
+
f"{where} · CASSETTE_MISS\n"
|
|
113
|
+
f" tool: {tool_name}\n"
|
|
114
|
+
f" called with: {dict(arguments)!r}\n"
|
|
115
|
+
" no cassette store was available, so this miss could not be diagnosed"
|
|
116
|
+
)
|
|
117
|
+
self.case_id = case_id
|
|
118
|
+
self.tool_name = tool_name
|
|
119
|
+
self.arguments = dict(arguments)
|
|
120
|
+
self.keys = keys
|
|
121
|
+
self.diagnosis = diagnosis
|
|
122
|
+
self.recorded_calls = recorded_calls
|
|
123
|
+
self.node = node
|
|
124
|
+
self.candidate_id = candidate_id
|
|
125
|
+
|
|
126
|
+
@property
|
|
127
|
+
def verdict(self) -> str | None:
|
|
128
|
+
"""``hallucinated_parameter``, ``unrecorded_valid_path``, and so on."""
|
|
129
|
+
return self.diagnosis.verdict.value if self.diagnosis else None
|
|
130
|
+
|
|
131
|
+
@property
|
|
132
|
+
def uncovered(self) -> bool:
|
|
133
|
+
"""True when this miss is a hole in our coverage rather than a model
|
|
134
|
+
failure, so the gate can keep it out of the candidate's pass rate."""
|
|
135
|
+
return self.diagnosis is not None and self.diagnosis.uncovered
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
class SandboxNotReady(RuntimeError):
|
|
139
|
+
"""A benchmark is running but the sandbox cannot do its job.
|
|
140
|
+
|
|
141
|
+
A harness bug, so it keeps its traceback. It raises rather than degrading
|
|
142
|
+
because every degradation available here is worse than stopping.
|
|
143
|
+
"""
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
@dataclass(slots=True)
|
|
147
|
+
class SandboxedTool:
|
|
148
|
+
"""The registry entry for one decorated tool."""
|
|
149
|
+
|
|
150
|
+
name: str
|
|
151
|
+
policy: MatchPolicy
|
|
152
|
+
version: str
|
|
153
|
+
qualname: str
|
|
154
|
+
is_async: bool
|
|
155
|
+
func: Callable[..., Any] = field(repr=False)
|
|
156
|
+
signature: inspect.Signature | None = field(default=None, repr=False)
|
|
157
|
+
|
|
158
|
+
def bind(self, args: tuple[Any, ...], kwargs: Mapping[str, Any]) -> dict[str, Any]:
|
|
159
|
+
"""Positional arguments, resolved to names.
|
|
160
|
+
|
|
161
|
+
The key is computed from named arguments, so ``f("ORD-1")`` and
|
|
162
|
+
``f(order_id="ORD-1")`` must be the same call. Defaults are *not*
|
|
163
|
+
applied here - the policy fills them, and the trace should show what the
|
|
164
|
+
caller actually passed.
|
|
165
|
+
"""
|
|
166
|
+
if self.signature is None:
|
|
167
|
+
return dict(kwargs)
|
|
168
|
+
# A TypeError here is the caller invoking the tool wrongly, which the
|
|
169
|
+
# real tool would raise too. Let it through unchanged.
|
|
170
|
+
bound = self.signature.bind(*args, **kwargs)
|
|
171
|
+
return {
|
|
172
|
+
name: value
|
|
173
|
+
for name, value in bound.arguments.items()
|
|
174
|
+
if name not in {"self", "cls"}
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+
REGISTRY: dict[str, SandboxedTool] = {}
|
|
179
|
+
|
|
180
|
+
|
|
181
|
+
def registered_tools() -> Mapping[str, SandboxedTool]:
|
|
182
|
+
"""Every sandboxed tool known to this process.
|
|
183
|
+
|
|
184
|
+
The miss diagnosis needs it to tell `hallucinated_tool` (a name that is not
|
|
185
|
+
a tool at all) from `unrecorded_valid_path` (a real tool the baseline never
|
|
186
|
+
called). Only the decorator can populate it, so it lives here.
|
|
187
|
+
"""
|
|
188
|
+
return REGISTRY
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
def clear_registry() -> None:
|
|
192
|
+
"""For tests. A leaked registration would leak across test modules."""
|
|
193
|
+
REGISTRY.clear()
|
|
194
|
+
|
|
195
|
+
|
|
196
|
+
def _register(tool: SandboxedTool) -> None:
|
|
197
|
+
existing = REGISTRY.get(tool.name)
|
|
198
|
+
if existing is not None and existing.qualname != tool.qualname:
|
|
199
|
+
raise PolicyError(
|
|
200
|
+
f"two different tools are both named {tool.name!r}: "
|
|
201
|
+
f"{existing.qualname} and {tool.qualname}",
|
|
202
|
+
hint=(
|
|
203
|
+
"Cassettes are keyed by tool name, so these two would share - and "
|
|
204
|
+
"answer - each other's recordings. Pass name= to one of them."
|
|
205
|
+
),
|
|
206
|
+
)
|
|
207
|
+
REGISTRY[tool.name] = tool
|
|
208
|
+
|
|
209
|
+
|
|
210
|
+
def _source_version(fn: Callable[..., Any]) -> str:
|
|
211
|
+
"""A hash of the tool's source, used to flag stale cassettes.
|
|
212
|
+
|
|
213
|
+
Not part of the cassette key: an edited tool should be *reported* as stale
|
|
214
|
+
by a warning on replay, not turn into a wall of misses that reads
|
|
215
|
+
like the model fell apart. Unavailable source (a C function, a REPL) yields
|
|
216
|
+
an empty version, which disables staleness checking rather than guessing.
|
|
217
|
+
"""
|
|
218
|
+
try:
|
|
219
|
+
source = textwrap.dedent(inspect.getsource(fn))
|
|
220
|
+
except (OSError, TypeError):
|
|
221
|
+
return ""
|
|
222
|
+
return hashlib.sha256(source.encode("utf-8")).hexdigest()[:16]
|
|
223
|
+
|
|
224
|
+
|
|
225
|
+
_LIVE_BANNER_SHOWN = False
|
|
226
|
+
|
|
227
|
+
|
|
228
|
+
def _live_banner(tool_name: str) -> None:
|
|
229
|
+
"""Say out loud that real APIs are being hit.
|
|
230
|
+
|
|
231
|
+
``live`` mode exists because some tool genuinely cannot be recorded, but a
|
|
232
|
+
user who reaches for it by accident - or inherits it from a committed
|
|
233
|
+
``eval.toml`` - must not discover it from their cloud bill. Printed once,
|
|
234
|
+
to stderr, so it cannot be mistaken for part of the scorecard.
|
|
235
|
+
"""
|
|
236
|
+
global _LIVE_BANNER_SHOWN
|
|
237
|
+
if _LIVE_BANNER_SHOWN:
|
|
238
|
+
return
|
|
239
|
+
_LIVE_BANNER_SHOWN = True
|
|
240
|
+
message = (
|
|
241
|
+
f" tool_mode = \"live\": {tool_name} and every other sandboxed tool will hit "
|
|
242
|
+
"your REAL APIs.\n"
|
|
243
|
+
" Nothing is recorded, nothing is replayed, and side effects are real."
|
|
244
|
+
)
|
|
245
|
+
colour = not os.environ.get("NO_COLOR") and sys.stderr.isatty()
|
|
246
|
+
if colour:
|
|
247
|
+
message = f"\033[1;31m{message}\033[0m"
|
|
248
|
+
print(f"\n{message}\n", file=sys.stderr)
|
|
249
|
+
|
|
250
|
+
|
|
251
|
+
def reset_live_banner() -> None:
|
|
252
|
+
"""For tests, and for a second run in one process."""
|
|
253
|
+
global _LIVE_BANNER_SHOWN
|
|
254
|
+
_LIVE_BANNER_SHOWN = False
|
|
255
|
+
|
|
256
|
+
|
|
257
|
+
def _store_for(ctx: RunContext, tool_name: str) -> CassetteStore:
|
|
258
|
+
if ctx.cassettes is None:
|
|
259
|
+
raise SandboxNotReady(
|
|
260
|
+
f"tool {tool_name!r} was called in tool_mode={ctx.tool_mode!r} but no "
|
|
261
|
+
"cassette store was attached to the run context.\n"
|
|
262
|
+
" The sandbox will not fall through to the real tool to cover for "
|
|
263
|
+
"this: in a benchmark that means spending real money on an API call "
|
|
264
|
+
"whose result is then thrown away."
|
|
265
|
+
)
|
|
266
|
+
return ctx.cassettes
|
|
267
|
+
|
|
268
|
+
|
|
269
|
+
def _trace(
|
|
270
|
+
ctx: RunContext,
|
|
271
|
+
tool: SandboxedTool,
|
|
272
|
+
arguments: Mapping[str, Any],
|
|
273
|
+
) -> ToolCallTrace:
|
|
274
|
+
return ToolCallTrace(
|
|
275
|
+
seq=ctx.next_tool_seq(),
|
|
276
|
+
name=tool.name,
|
|
277
|
+
node=current_node(),
|
|
278
|
+
node_path=current_node_path(),
|
|
279
|
+
arguments=dict(arguments),
|
|
280
|
+
tool_version=tool.version or None,
|
|
281
|
+
)
|
|
282
|
+
|
|
283
|
+
|
|
284
|
+
def _note_degraded(ctx: RunContext, tool: SandboxedTool, hit: Hit) -> None:
|
|
285
|
+
"""Attach a warning for anything but an exact match.
|
|
286
|
+
|
|
287
|
+
A run whose numbers depend on our normalizers covering for the model must
|
|
288
|
+
not look clean in the scorecard.
|
|
289
|
+
"""
|
|
290
|
+
if hit.tier is MatchTier.NORMALIZED:
|
|
291
|
+
ctx.warn(
|
|
292
|
+
f"{tool.name}: matched a cassette only after normalizing arguments "
|
|
293
|
+
"(formatting differed from the recording)"
|
|
294
|
+
)
|
|
295
|
+
elif hit.tier is MatchTier.SUBSET:
|
|
296
|
+
ctx.warn(
|
|
297
|
+
f"{tool.name}: matched only on {', '.join(tool.policy.match_on or ())} - "
|
|
298
|
+
"the model got the other arguments wrong"
|
|
299
|
+
)
|
|
300
|
+
if (
|
|
301
|
+
tool.version
|
|
302
|
+
and hit.cassette.tool_version
|
|
303
|
+
and tool.version != hit.cassette.tool_version
|
|
304
|
+
):
|
|
305
|
+
ctx.warn(
|
|
306
|
+
f"{tool.name}: replayed a cassette recorded for a different version of "
|
|
307
|
+
"this tool - re-record it with `slmeval record`"
|
|
308
|
+
)
|
|
309
|
+
|
|
310
|
+
|
|
311
|
+
def _replay(
|
|
312
|
+
ctx: RunContext,
|
|
313
|
+
tool: SandboxedTool,
|
|
314
|
+
arguments: Mapping[str, Any],
|
|
315
|
+
trace: ToolCallTrace,
|
|
316
|
+
) -> tuple[bool, Any]:
|
|
317
|
+
"""Try the store. Returns ``(hit, response)``.
|
|
318
|
+
|
|
319
|
+
Raises :class:`CassetteMiss` only in strict ``replay`` mode; ``auto`` wants
|
|
320
|
+
the miss back so it can record.
|
|
321
|
+
"""
|
|
322
|
+
store = _store_for(ctx, tool.name)
|
|
323
|
+
keys = tool.policy.compute(ctx.scope, arguments)
|
|
324
|
+
hit = store.lookup(keys)
|
|
325
|
+
if hit is not None:
|
|
326
|
+
trace.cassette_key = hit.cassette.key
|
|
327
|
+
trace.match_tier = hit.tier.value
|
|
328
|
+
trace.outcome = "replayed"
|
|
329
|
+
_note_degraded(ctx, tool, hit)
|
|
330
|
+
return True, hit.response
|
|
331
|
+
|
|
332
|
+
if ctx.tool_mode == "replay":
|
|
333
|
+
trace.outcome = "miss"
|
|
334
|
+
trace.cassette_key = keys.exact
|
|
335
|
+
trace.error = "CASSETTE_MISS"
|
|
336
|
+
finding = diagnose(
|
|
337
|
+
case_id=ctx.scope,
|
|
338
|
+
tool_name=tool.name,
|
|
339
|
+
arguments=arguments,
|
|
340
|
+
store=store,
|
|
341
|
+
candidate_id=ctx.candidate_id,
|
|
342
|
+
node=current_node(),
|
|
343
|
+
node_path=current_node_path(),
|
|
344
|
+
)
|
|
345
|
+
trace.miss_verdict = finding.verdict.value
|
|
346
|
+
if finding.uncovered:
|
|
347
|
+
# An uncovered path is a coverage gap, so it is surfaced as a
|
|
348
|
+
# warning on the execution as well as in the block. Without this a
|
|
349
|
+
# user reading only the scorecard would see a failed case and
|
|
350
|
+
# assume the model broke.
|
|
351
|
+
ctx.warn(
|
|
352
|
+
f"{tool.name}: {finding.verdict.value} - the baseline never recorded this "
|
|
353
|
+
"call, so it is uncovered rather than wrong "
|
|
354
|
+
"(`slmeval record --fill-gaps`)"
|
|
355
|
+
)
|
|
356
|
+
raise CassetteMiss(
|
|
357
|
+
case_id=ctx.scope,
|
|
358
|
+
tool_name=tool.name,
|
|
359
|
+
arguments=arguments,
|
|
360
|
+
keys=keys,
|
|
361
|
+
diagnosis=finding,
|
|
362
|
+
node=current_node(),
|
|
363
|
+
candidate_id=ctx.candidate_id,
|
|
364
|
+
)
|
|
365
|
+
return False, None
|
|
366
|
+
|
|
367
|
+
|
|
368
|
+
def _after_real_call(
|
|
369
|
+
ctx: RunContext,
|
|
370
|
+
tool: SandboxedTool,
|
|
371
|
+
arguments: Mapping[str, Any],
|
|
372
|
+
trace: ToolCallTrace,
|
|
373
|
+
result: Any,
|
|
374
|
+
) -> Any:
|
|
375
|
+
"""Record a real call's result, in ``record`` and ``auto`` modes."""
|
|
376
|
+
store = _store_for(ctx, tool.name)
|
|
377
|
+
keys = tool.policy.compute(ctx.scope, arguments)
|
|
378
|
+
# An unserializable return raises out of here, which fails the run loudly at
|
|
379
|
+
# record time. That is the point: coercing it would replay a different type
|
|
380
|
+
# than it recorded, and the user would spend a day on it instead of a minute.
|
|
381
|
+
hit = store.record(
|
|
382
|
+
keys,
|
|
383
|
+
arguments=arguments,
|
|
384
|
+
response=result,
|
|
385
|
+
tool_version=tool.version,
|
|
386
|
+
dataset_hash=str(ctx.metadata.get("dataset_hash", "")),
|
|
387
|
+
)
|
|
388
|
+
trace.cassette_key = hit.cassette.key
|
|
389
|
+
trace.match_tier = MatchTier.EXACT.value
|
|
390
|
+
trace.outcome = "recorded"
|
|
391
|
+
return result
|
|
392
|
+
|
|
393
|
+
|
|
394
|
+
def _deliver_fault(
|
|
395
|
+
ctx: RunContext,
|
|
396
|
+
tool: SandboxedTool,
|
|
397
|
+
fault: Fault,
|
|
398
|
+
response: Any,
|
|
399
|
+
trace: ToolCallTrace,
|
|
400
|
+
) -> Any:
|
|
401
|
+
"""Deliver one fault: return a mangled payload, or raise.
|
|
402
|
+
|
|
403
|
+
Called *after* the response has been resolved, so an injected fault can
|
|
404
|
+
never mask a cassette miss - the miss is a finding about the model and the
|
|
405
|
+
fault is a stimulus we chose, and when both apply to one call the finding
|
|
406
|
+
wins.
|
|
407
|
+
|
|
408
|
+
The delay happened at the call site, because sleeping correctly is the one
|
|
409
|
+
thing this function cannot do for both the sync and the async wrapper.
|
|
410
|
+
"""
|
|
411
|
+
trace.fault = fault.label()
|
|
412
|
+
trace.outcome = "fault"
|
|
413
|
+
node = current_node()
|
|
414
|
+
|
|
415
|
+
if fault.delivery is Delivery.RAISE:
|
|
416
|
+
error = fault.error()
|
|
417
|
+
ctx.record_fault(fault, node=node)
|
|
418
|
+
# Set before raising, so the wrapper's `except BaseException` sees an
|
|
419
|
+
# error already attributed to the fault and does not overwrite it with
|
|
420
|
+
# a generic one.
|
|
421
|
+
trace.error = f"{type(error).__name__}: {error}"
|
|
422
|
+
raise error
|
|
423
|
+
|
|
424
|
+
payload, applied = apply_fault(fault, response)
|
|
425
|
+
ctx.record_fault(fault, node=node, applied=applied)
|
|
426
|
+
if not applied:
|
|
427
|
+
# Recorded as delivered-but-ineffective rather than dropped: a vector
|
|
428
|
+
# that quietly omitted it would compare equal to a run that never
|
|
429
|
+
# scheduled it, and the parity check would pass on a real difference.
|
|
430
|
+
ctx.warn(
|
|
431
|
+
f"{tool.name}: {fault.kind.value} could not be shaped onto a "
|
|
432
|
+
f"{type(response).__name__} response, so the call was counted as faulted "
|
|
433
|
+
"but the payload is unchanged"
|
|
434
|
+
)
|
|
435
|
+
return payload
|
|
436
|
+
|
|
437
|
+
|
|
438
|
+
async def _fault_async(
|
|
439
|
+
ctx: RunContext,
|
|
440
|
+
tool: SandboxedTool,
|
|
441
|
+
fault: Fault,
|
|
442
|
+
response: Any,
|
|
443
|
+
trace: ToolCallTrace,
|
|
444
|
+
) -> Any:
|
|
445
|
+
if fault.delay_s:
|
|
446
|
+
# A real wait, because a timeout that takes no time never trips the
|
|
447
|
+
# agent's own timeout and the measurement would be a lie. Scaled by
|
|
448
|
+
# the schedule's `delay_scale`, which the test suite sets to zero.
|
|
449
|
+
await asyncio.sleep(fault.delay_s)
|
|
450
|
+
return _deliver_fault(ctx, tool, fault, response, trace)
|
|
451
|
+
|
|
452
|
+
|
|
453
|
+
def _fault_sync(
|
|
454
|
+
ctx: RunContext,
|
|
455
|
+
tool: SandboxedTool,
|
|
456
|
+
fault: Fault,
|
|
457
|
+
response: Any,
|
|
458
|
+
trace: ToolCallTrace,
|
|
459
|
+
) -> Any:
|
|
460
|
+
if fault.delay_s:
|
|
461
|
+
time.sleep(fault.delay_s)
|
|
462
|
+
return _deliver_fault(ctx, tool, fault, response, trace)
|
|
463
|
+
|
|
464
|
+
|
|
465
|
+
def sandbox_tool(
|
|
466
|
+
fn: F | None = None,
|
|
467
|
+
*,
|
|
468
|
+
match_on: list[str] | tuple[str, ...] | None = None,
|
|
469
|
+
normalize: Mapping[str, str] | None = None,
|
|
470
|
+
ignore_args: list[str] | tuple[str, ...] | None = None,
|
|
471
|
+
version: str | None = None,
|
|
472
|
+
name: str | None = None,
|
|
473
|
+
) -> Any:
|
|
474
|
+
"""Record this tool once, then replay it forever.
|
|
475
|
+
|
|
476
|
+
@sandbox_tool(
|
|
477
|
+
match_on=["order_id"],
|
|
478
|
+
normalize={"order_id": "upper_strip", "query": "sql"},
|
|
479
|
+
ignore_args=["request_id", "timestamp"],
|
|
480
|
+
)
|
|
481
|
+
async def fetch_order_from_erp(order_id: str, request_id: str) -> dict:
|
|
482
|
+
...
|
|
483
|
+
|
|
484
|
+
``match_on``
|
|
485
|
+
The arguments that identify the call. They form the last rung of the
|
|
486
|
+
lookup cascade, so a call that gets *these* right and something else
|
|
487
|
+
wrong is reported as such instead of passing or missing outright.
|
|
488
|
+
``normalize``
|
|
489
|
+
Named normalizers, per argument, so a formatting difference is not
|
|
490
|
+
reported as a hallucination. See
|
|
491
|
+
:mod:`slmeval.sandbox.normalizers`.
|
|
492
|
+
``ignore_args``
|
|
493
|
+
Arguments that never identify a call - a request id, a timestamp. Left
|
|
494
|
+
in, they would make every call unique and every replay a miss.
|
|
495
|
+
``version``
|
|
496
|
+
Declare it when you want to invalidate cassettes deliberately.
|
|
497
|
+
Otherwise a hash of the tool's source is used to *flag* staleness.
|
|
498
|
+
|
|
499
|
+
A no-op outside a benchmark run: with no run in progress the real tool is
|
|
500
|
+
called, untouched, so this is safe to leave in production code.
|
|
501
|
+
"""
|
|
502
|
+
|
|
503
|
+
def decorate(func: F) -> F:
|
|
504
|
+
tool_name = name or func.__name__
|
|
505
|
+
policy = declare(
|
|
506
|
+
func,
|
|
507
|
+
tool_name=tool_name,
|
|
508
|
+
match_on=match_on,
|
|
509
|
+
normalize=normalize,
|
|
510
|
+
ignore_args=ignore_args,
|
|
511
|
+
tool_version=version or "",
|
|
512
|
+
)
|
|
513
|
+
try:
|
|
514
|
+
signature: inspect.Signature | None = inspect.signature(func)
|
|
515
|
+
except (TypeError, ValueError):
|
|
516
|
+
signature = None
|
|
517
|
+
|
|
518
|
+
tool = SandboxedTool(
|
|
519
|
+
name=tool_name,
|
|
520
|
+
policy=policy,
|
|
521
|
+
version=version or _source_version(func),
|
|
522
|
+
qualname=f"{func.__module__}.{func.__qualname__}",
|
|
523
|
+
is_async=inspect.iscoroutinefunction(func),
|
|
524
|
+
func=func,
|
|
525
|
+
signature=signature,
|
|
526
|
+
)
|
|
527
|
+
_register(tool)
|
|
528
|
+
|
|
529
|
+
if tool.is_async:
|
|
530
|
+
|
|
531
|
+
@functools.wraps(func)
|
|
532
|
+
async def async_wrapper(*args: Any, **kwargs: Any) -> Any:
|
|
533
|
+
ctx = get_context()
|
|
534
|
+
if ctx is None:
|
|
535
|
+
_warn_if_benchmarking(tool_name)
|
|
536
|
+
return await func(*args, **kwargs)
|
|
537
|
+
arguments = tool.bind(args, kwargs)
|
|
538
|
+
ctx.note_step()
|
|
539
|
+
if ctx.tool_mode == "live":
|
|
540
|
+
_live_banner(tool_name)
|
|
541
|
+
return await _run_traced(ctx, tool, arguments, func(*args, **kwargs))
|
|
542
|
+
|
|
543
|
+
trace = _trace(ctx, tool, arguments)
|
|
544
|
+
# This tool's own call index, not a global counter, and the
|
|
545
|
+
# fault is looked up before the call resolves so a raise-
|
|
546
|
+
# delivered one can be applied to a replayed response without
|
|
547
|
+
# a second trip through the schedule.
|
|
548
|
+
fault = ctx.fault_for(tool.name, ctx.next_tool_index(tool.name))
|
|
549
|
+
try:
|
|
550
|
+
if ctx.tool_mode != "record":
|
|
551
|
+
# `record` does not consult the store: it executes the
|
|
552
|
+
# real tool and overwrites. Looking first would make
|
|
553
|
+
# `record` silently behave like `auto`, so re-recording
|
|
554
|
+
# a case after changing a fixture would appear to do
|
|
555
|
+
# nothing.
|
|
556
|
+
replayed, response = _replay(ctx, tool, arguments, trace)
|
|
557
|
+
if replayed:
|
|
558
|
+
if fault is not None:
|
|
559
|
+
return await _fault_async(ctx, tool, fault, response, trace)
|
|
560
|
+
return response
|
|
561
|
+
result = await func(*args, **kwargs)
|
|
562
|
+
except BaseException as exc:
|
|
563
|
+
if trace.error is None:
|
|
564
|
+
trace.outcome = "error"
|
|
565
|
+
trace.error = f"{type(exc).__name__}: {exc}"
|
|
566
|
+
raise
|
|
567
|
+
else:
|
|
568
|
+
recorded = _after_real_call(ctx, tool, arguments, trace, result)
|
|
569
|
+
if fault is not None:
|
|
570
|
+
# The cassette already holds the clean response - it was
|
|
571
|
+
# written above - so `auto` mode faults what the agent
|
|
572
|
+
# sees without poisoning what the store keeps.
|
|
573
|
+
return await _fault_async(ctx, tool, fault, recorded, trace)
|
|
574
|
+
return recorded
|
|
575
|
+
finally:
|
|
576
|
+
# In `finally`: every path leaves exactly one trace, or the
|
|
577
|
+
# denominator of every per-tool rate in the report is wrong.
|
|
578
|
+
ctx.record_tool(trace)
|
|
579
|
+
|
|
580
|
+
wrapper: Callable[..., Any] = async_wrapper
|
|
581
|
+
else:
|
|
582
|
+
|
|
583
|
+
@functools.wraps(func)
|
|
584
|
+
def sync_wrapper(*args: Any, **kwargs: Any) -> Any:
|
|
585
|
+
ctx = get_context()
|
|
586
|
+
if ctx is None:
|
|
587
|
+
_warn_if_benchmarking(tool_name)
|
|
588
|
+
return func(*args, **kwargs)
|
|
589
|
+
arguments = tool.bind(args, kwargs)
|
|
590
|
+
ctx.note_step()
|
|
591
|
+
if ctx.tool_mode == "live":
|
|
592
|
+
_live_banner(tool_name)
|
|
593
|
+
return _run_traced_sync(ctx, tool, arguments, func, args, kwargs)
|
|
594
|
+
|
|
595
|
+
trace = _trace(ctx, tool, arguments)
|
|
596
|
+
fault = ctx.fault_for(tool.name, ctx.next_tool_index(tool.name))
|
|
597
|
+
try:
|
|
598
|
+
if ctx.tool_mode != "record":
|
|
599
|
+
replayed, response = _replay(ctx, tool, arguments, trace)
|
|
600
|
+
if replayed:
|
|
601
|
+
if fault is not None:
|
|
602
|
+
return _fault_sync(ctx, tool, fault, response, trace)
|
|
603
|
+
return response
|
|
604
|
+
result = func(*args, **kwargs)
|
|
605
|
+
except BaseException as exc:
|
|
606
|
+
if trace.error is None:
|
|
607
|
+
trace.outcome = "error"
|
|
608
|
+
trace.error = f"{type(exc).__name__}: {exc}"
|
|
609
|
+
raise
|
|
610
|
+
else:
|
|
611
|
+
recorded = _after_real_call(ctx, tool, arguments, trace, result)
|
|
612
|
+
if fault is not None:
|
|
613
|
+
return _fault_sync(ctx, tool, fault, recorded, trace)
|
|
614
|
+
return recorded
|
|
615
|
+
finally:
|
|
616
|
+
ctx.record_tool(trace)
|
|
617
|
+
|
|
618
|
+
wrapper = sync_wrapper
|
|
619
|
+
|
|
620
|
+
wrapper.slmeval_tool = tool # type: ignore[attr-defined]
|
|
621
|
+
return cast("F", wrapper)
|
|
622
|
+
|
|
623
|
+
if fn is not None:
|
|
624
|
+
return decorate(fn)
|
|
625
|
+
return decorate
|
|
626
|
+
|
|
627
|
+
|
|
628
|
+
def _warn_if_benchmarking(tool_name: str) -> None:
|
|
629
|
+
"""No context: production passthrough, or a context lost crossing a thread?
|
|
630
|
+
|
|
631
|
+
Only the second is a problem, and only it gets a warning - a warning on
|
|
632
|
+
every production tool call is a warning nobody reads.
|
|
633
|
+
"""
|
|
634
|
+
if benchmark_running_now():
|
|
635
|
+
warn_missing_context(f"@sandbox_tool {tool_name!r}")
|
|
636
|
+
|
|
637
|
+
|
|
638
|
+
async def _run_traced(
|
|
639
|
+
ctx: RunContext,
|
|
640
|
+
tool: SandboxedTool,
|
|
641
|
+
arguments: Mapping[str, Any],
|
|
642
|
+
awaitable: Any,
|
|
643
|
+
) -> Any:
|
|
644
|
+
"""`live` mode: no interception, but still traced and still counted."""
|
|
645
|
+
trace = _trace(ctx, tool, arguments)
|
|
646
|
+
trace.outcome = "live"
|
|
647
|
+
try:
|
|
648
|
+
return await awaitable
|
|
649
|
+
except BaseException as exc:
|
|
650
|
+
trace.error = f"{type(exc).__name__}: {exc}"
|
|
651
|
+
raise
|
|
652
|
+
finally:
|
|
653
|
+
ctx.record_tool(trace)
|
|
654
|
+
|
|
655
|
+
|
|
656
|
+
def _run_traced_sync(
|
|
657
|
+
ctx: RunContext,
|
|
658
|
+
tool: SandboxedTool,
|
|
659
|
+
arguments: Mapping[str, Any],
|
|
660
|
+
func: Callable[..., Any],
|
|
661
|
+
args: tuple[Any, ...],
|
|
662
|
+
kwargs: Mapping[str, Any],
|
|
663
|
+
) -> Any:
|
|
664
|
+
trace = _trace(ctx, tool, arguments)
|
|
665
|
+
trace.outcome = "live"
|
|
666
|
+
try:
|
|
667
|
+
return func(*args, **dict(kwargs))
|
|
668
|
+
except BaseException as exc:
|
|
669
|
+
trace.error = f"{type(exc).__name__}: {exc}"
|
|
670
|
+
raise
|
|
671
|
+
finally:
|
|
672
|
+
ctx.record_tool(trace)
|
|
673
|
+
|
|
674
|
+
|
|
675
|
+
__all__ = [
|
|
676
|
+
"REGISTRY",
|
|
677
|
+
"CassetteMiss",
|
|
678
|
+
"MissDiagnosis",
|
|
679
|
+
"SandboxNotReady",
|
|
680
|
+
"SandboxedTool",
|
|
681
|
+
"clear_registry",
|
|
682
|
+
"registered_tools",
|
|
683
|
+
"reset_live_banner",
|
|
684
|
+
"sandbox_tool",
|
|
685
|
+
]
|