slmeval-cli 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (94) hide show
  1. slmeval/__init__.py +71 -0
  2. slmeval/__main__.py +14 -0
  3. slmeval/adapters/__init__.py +24 -0
  4. slmeval/adapters/instrument.py +135 -0
  5. slmeval/adapters/langgraph.py +237 -0
  6. slmeval/cli/__init__.py +15 -0
  7. slmeval/cli/commands/__init__.py +35 -0
  8. slmeval/cli/commands/baseline_cmd.py +201 -0
  9. slmeval/cli/commands/cache_cmd.py +249 -0
  10. slmeval/cli/commands/dataset_cmd.py +52 -0
  11. slmeval/cli/commands/doctor_cmd.py +68 -0
  12. slmeval/cli/commands/gate_cmd.py +602 -0
  13. slmeval/cli/commands/init_cmd.py +144 -0
  14. slmeval/cli/commands/judge_cmd.py +284 -0
  15. slmeval/cli/commands/record_cmd.py +154 -0
  16. slmeval/cli/commands/report_cmd.py +98 -0
  17. slmeval/cli/commands/run_cmd.py +247 -0
  18. slmeval/cli/commands/show_cmd.py +482 -0
  19. slmeval/cli/commands/stubs.py +57 -0
  20. slmeval/cli/commands/validate_cmd.py +296 -0
  21. slmeval/cli/loading.py +149 -0
  22. slmeval/cli/main.py +91 -0
  23. slmeval/cli/render.py +251 -0
  24. slmeval/cli/wrap.py +31 -0
  25. slmeval/config/__init__.py +47 -0
  26. slmeval/config/loader.py +363 -0
  27. slmeval/config/locate.py +97 -0
  28. slmeval/config/models.py +430 -0
  29. slmeval/config/templates.py +246 -0
  30. slmeval/core/__init__.py +78 -0
  31. slmeval/core/adapter.py +253 -0
  32. slmeval/core/context.py +418 -0
  33. slmeval/core/outcome.py +260 -0
  34. slmeval/core/router.py +319 -0
  35. slmeval/core/runner.py +1209 -0
  36. slmeval/core/trace.py +188 -0
  37. slmeval/core/verdict.py +127 -0
  38. slmeval/dataset/__init__.py +8 -0
  39. slmeval/dataset/loader.py +130 -0
  40. slmeval/dataset/models.py +108 -0
  41. slmeval/doctor/__init__.py +7 -0
  42. slmeval/doctor/checks.py +380 -0
  43. slmeval/errors.py +76 -0
  44. slmeval/eval/__init__.py +64 -0
  45. slmeval/eval/assertions.py +356 -0
  46. slmeval/eval/calibration.py +364 -0
  47. slmeval/eval/cost.py +291 -0
  48. slmeval/eval/pipeline.py +273 -0
  49. slmeval/eval/stats.py +376 -0
  50. slmeval/eval/tier1.py +510 -0
  51. slmeval/eval/tier2.py +582 -0
  52. slmeval/eval/tier3.py +415 -0
  53. slmeval/exit_codes.py +28 -0
  54. slmeval/gate.py +722 -0
  55. slmeval/providers/__init__.py +69 -0
  56. slmeval/providers/base.py +426 -0
  57. slmeval/providers/fake.py +760 -0
  58. slmeval/providers/jsonx.py +239 -0
  59. slmeval/providers/ladder.py +339 -0
  60. slmeval/providers/openai_compat.py +392 -0
  61. slmeval/providers/registry.py +185 -0
  62. slmeval/providers/retry.py +81 -0
  63. slmeval/providers/schema.py +136 -0
  64. slmeval/providers/tokens.py +150 -0
  65. slmeval/py.typed +0 -0
  66. slmeval/report/__init__.py +115 -0
  67. slmeval/report/attribution.py +197 -0
  68. slmeval/report/baseline.py +353 -0
  69. slmeval/report/drilldown.py +1395 -0
  70. slmeval/report/export.py +450 -0
  71. slmeval/report/model.py +562 -0
  72. slmeval/report/progress.py +257 -0
  73. slmeval/report/scorecard.py +367 -0
  74. slmeval/sandbox/__init__.py +145 -0
  75. slmeval/sandbox/diagnosis.py +690 -0
  76. slmeval/sandbox/envelope.py +486 -0
  77. slmeval/sandbox/faults.py +794 -0
  78. slmeval/sandbox/keys.py +458 -0
  79. slmeval/sandbox/normalizers.py +333 -0
  80. slmeval/sandbox/store.py +525 -0
  81. slmeval/sandbox/tool.py +685 -0
  82. slmeval/storage/__init__.py +73 -0
  83. slmeval/storage/blobs.py +308 -0
  84. slmeval/storage/codec.py +93 -0
  85. slmeval/storage/db.py +1334 -0
  86. slmeval/storage/ingest.py +346 -0
  87. slmeval/storage/migrations.py +358 -0
  88. slmeval/storage/models.py +554 -0
  89. slmeval/storage/provenance.py +201 -0
  90. slmeval_cli-0.1.0.dist-info/METADATA +484 -0
  91. slmeval_cli-0.1.0.dist-info/RECORD +94 -0
  92. slmeval_cli-0.1.0.dist-info/WHEEL +4 -0
  93. slmeval_cli-0.1.0.dist-info/entry_points.txt +2 -0
  94. slmeval_cli-0.1.0.dist-info/licenses/LICENSE +201 -0
@@ -0,0 +1,685 @@
1
+ """``@sandbox_tool`` and the four interception modes.
2
+
3
+ This decorator is the first of the two touch points a user adds to their agent,
4
+ and the one that makes the whole benchmark possible: a recorded tool call is
5
+ comparable across models, fast, and safe to run 500 times, while a live one is
6
+ none of those things.
7
+
8
+ The modes:
9
+
10
+ ======== ====================================================================
11
+ record call the real tool, store what it returned
12
+ replay serve from the store; a miss raises, it never falls through
13
+ auto replay when recorded, record when not
14
+ live no interception at all - the escape hatch, with a red banner
15
+ ======== ====================================================================
16
+
17
+ The single most important rule here is that **no path falls back to the real
18
+ tool by accident**. In ``replay`` mode a miss is a finding and must surface as
19
+ one; silently calling the customer's live Stripe instead would spend their money
20
+ and invalidate the run, and it would do it without telling anyone. So every
21
+ degraded situation - no store, no case id, a lost context during a benchmark -
22
+ either raises or warns, and the only silent passthrough is the one case where
23
+ passing through is correct: there is no benchmark running at all, because the
24
+ user is in production.
25
+
26
+ Two things happen here that are the whole of Day 7:
27
+
28
+ **A miss is a diagnosis, not an exception string.** On a strict-replay miss the
29
+ store is searched for the nearest recorded call and the result is classified -
30
+ mangled argument, invented tool, wrong tool for the step, or a path nobody ever
31
+ recorded. The last of those is *our* gap, not the model's failure, and keeping
32
+ it separate is what stops the scorecard inflating itself. See
33
+ :mod:`slmeval.sandbox.diagnosis`.
34
+
35
+ **A fault is delivered after the response resolves, never before.** A cassette
36
+ miss is a finding about the model; a fault is a stimulus we chose. Resolving
37
+ first means an injected ``429`` can never mask a hallucinated argument - the
38
+ miss surfaces, and the fault is simply not reached. See
39
+ :mod:`slmeval.sandbox.faults`.
40
+ """
41
+
42
+ from __future__ import annotations
43
+
44
+ import asyncio
45
+ import functools
46
+ import hashlib
47
+ import inspect
48
+ import os
49
+ import sys
50
+ import textwrap
51
+ import time
52
+ from collections.abc import Callable, Mapping
53
+ from dataclasses import dataclass, field
54
+ from typing import Any, TypeVar, cast
55
+
56
+ from slmeval.core.context import (
57
+ RunContext,
58
+ benchmark_running_now,
59
+ current_node,
60
+ current_node_path,
61
+ get_context,
62
+ warn_missing_context,
63
+ )
64
+ from slmeval.core.trace import ToolCallTrace
65
+ from slmeval.sandbox.diagnosis import MissDiagnosis, diagnose
66
+ from slmeval.sandbox.faults import Delivery, Fault, apply_fault
67
+ from slmeval.sandbox.keys import CassetteKeys, MatchPolicy, MatchTier, PolicyError, declare
68
+ from slmeval.sandbox.store import CassetteStore, Hit
69
+
70
+ F = TypeVar("F", bound=Callable[..., Any])
71
+
72
+
73
+ class CassetteMiss(Exception):
74
+ """A tool was called in a way that was never recorded.
75
+
76
+ Not an :class:`~slmeval.errors.SlmevalError`, and deliberately not: in a
77
+ benchmark this is a *result*. The runner catches it, classifies the
78
+ execution as failed and keeps going, exactly as it would for any other way
79
+ an agent can fail. Aborting the process on it would throw away the other 89
80
+ executions and the finding itself.
81
+
82
+ Its message **is** the diagnosis block from
83
+ :mod:`slmeval.sandbox.diagnosis` - nearest recorded call, per-argument
84
+ diff, verdict - so one finding reads the same in a terminal, a log, a
85
+ pytest failure and a traceback, with nothing to re-parse out of a string.
86
+ :attr:`diagnosis` carries the same thing structured, for the runner to
87
+ persist and the scorecard to aggregate.
88
+ """
89
+
90
+ def __init__(
91
+ self,
92
+ *,
93
+ case_id: str,
94
+ tool_name: str,
95
+ arguments: Mapping[str, Any],
96
+ keys: CassetteKeys,
97
+ diagnosis: MissDiagnosis | None = None,
98
+ recorded_calls: int = 0,
99
+ node: str | None = None,
100
+ candidate_id: str | None = None,
101
+ ) -> None:
102
+ if diagnosis is not None:
103
+ super().__init__(diagnosis.render())
104
+ recorded_calls = diagnosis.recorded_calls
105
+ else:
106
+ # Only reachable when something builds this by hand: diagnosing
107
+ # needs a store to search, and a miss is still a miss without one.
108
+ where = " · ".join(
109
+ part for part in (case_id, candidate_id, f"node={node}" if node else None) if part
110
+ )
111
+ super().__init__(
112
+ f"{where} · CASSETTE_MISS\n"
113
+ f" tool: {tool_name}\n"
114
+ f" called with: {dict(arguments)!r}\n"
115
+ " no cassette store was available, so this miss could not be diagnosed"
116
+ )
117
+ self.case_id = case_id
118
+ self.tool_name = tool_name
119
+ self.arguments = dict(arguments)
120
+ self.keys = keys
121
+ self.diagnosis = diagnosis
122
+ self.recorded_calls = recorded_calls
123
+ self.node = node
124
+ self.candidate_id = candidate_id
125
+
126
+ @property
127
+ def verdict(self) -> str | None:
128
+ """``hallucinated_parameter``, ``unrecorded_valid_path``, and so on."""
129
+ return self.diagnosis.verdict.value if self.diagnosis else None
130
+
131
+ @property
132
+ def uncovered(self) -> bool:
133
+ """True when this miss is a hole in our coverage rather than a model
134
+ failure, so the gate can keep it out of the candidate's pass rate."""
135
+ return self.diagnosis is not None and self.diagnosis.uncovered
136
+
137
+
138
+ class SandboxNotReady(RuntimeError):
139
+ """A benchmark is running but the sandbox cannot do its job.
140
+
141
+ A harness bug, so it keeps its traceback. It raises rather than degrading
142
+ because every degradation available here is worse than stopping.
143
+ """
144
+
145
+
146
+ @dataclass(slots=True)
147
+ class SandboxedTool:
148
+ """The registry entry for one decorated tool."""
149
+
150
+ name: str
151
+ policy: MatchPolicy
152
+ version: str
153
+ qualname: str
154
+ is_async: bool
155
+ func: Callable[..., Any] = field(repr=False)
156
+ signature: inspect.Signature | None = field(default=None, repr=False)
157
+
158
+ def bind(self, args: tuple[Any, ...], kwargs: Mapping[str, Any]) -> dict[str, Any]:
159
+ """Positional arguments, resolved to names.
160
+
161
+ The key is computed from named arguments, so ``f("ORD-1")`` and
162
+ ``f(order_id="ORD-1")`` must be the same call. Defaults are *not*
163
+ applied here - the policy fills them, and the trace should show what the
164
+ caller actually passed.
165
+ """
166
+ if self.signature is None:
167
+ return dict(kwargs)
168
+ # A TypeError here is the caller invoking the tool wrongly, which the
169
+ # real tool would raise too. Let it through unchanged.
170
+ bound = self.signature.bind(*args, **kwargs)
171
+ return {
172
+ name: value
173
+ for name, value in bound.arguments.items()
174
+ if name not in {"self", "cls"}
175
+ }
176
+
177
+
178
+ REGISTRY: dict[str, SandboxedTool] = {}
179
+
180
+
181
+ def registered_tools() -> Mapping[str, SandboxedTool]:
182
+ """Every sandboxed tool known to this process.
183
+
184
+ The miss diagnosis needs it to tell `hallucinated_tool` (a name that is not
185
+ a tool at all) from `unrecorded_valid_path` (a real tool the baseline never
186
+ called). Only the decorator can populate it, so it lives here.
187
+ """
188
+ return REGISTRY
189
+
190
+
191
+ def clear_registry() -> None:
192
+ """For tests. A leaked registration would leak across test modules."""
193
+ REGISTRY.clear()
194
+
195
+
196
+ def _register(tool: SandboxedTool) -> None:
197
+ existing = REGISTRY.get(tool.name)
198
+ if existing is not None and existing.qualname != tool.qualname:
199
+ raise PolicyError(
200
+ f"two different tools are both named {tool.name!r}: "
201
+ f"{existing.qualname} and {tool.qualname}",
202
+ hint=(
203
+ "Cassettes are keyed by tool name, so these two would share - and "
204
+ "answer - each other's recordings. Pass name= to one of them."
205
+ ),
206
+ )
207
+ REGISTRY[tool.name] = tool
208
+
209
+
210
+ def _source_version(fn: Callable[..., Any]) -> str:
211
+ """A hash of the tool's source, used to flag stale cassettes.
212
+
213
+ Not part of the cassette key: an edited tool should be *reported* as stale
214
+ by a warning on replay, not turn into a wall of misses that reads
215
+ like the model fell apart. Unavailable source (a C function, a REPL) yields
216
+ an empty version, which disables staleness checking rather than guessing.
217
+ """
218
+ try:
219
+ source = textwrap.dedent(inspect.getsource(fn))
220
+ except (OSError, TypeError):
221
+ return ""
222
+ return hashlib.sha256(source.encode("utf-8")).hexdigest()[:16]
223
+
224
+
225
+ _LIVE_BANNER_SHOWN = False
226
+
227
+
228
+ def _live_banner(tool_name: str) -> None:
229
+ """Say out loud that real APIs are being hit.
230
+
231
+ ``live`` mode exists because some tool genuinely cannot be recorded, but a
232
+ user who reaches for it by accident - or inherits it from a committed
233
+ ``eval.toml`` - must not discover it from their cloud bill. Printed once,
234
+ to stderr, so it cannot be mistaken for part of the scorecard.
235
+ """
236
+ global _LIVE_BANNER_SHOWN
237
+ if _LIVE_BANNER_SHOWN:
238
+ return
239
+ _LIVE_BANNER_SHOWN = True
240
+ message = (
241
+ f" tool_mode = \"live\": {tool_name} and every other sandboxed tool will hit "
242
+ "your REAL APIs.\n"
243
+ " Nothing is recorded, nothing is replayed, and side effects are real."
244
+ )
245
+ colour = not os.environ.get("NO_COLOR") and sys.stderr.isatty()
246
+ if colour:
247
+ message = f"\033[1;31m{message}\033[0m"
248
+ print(f"\n{message}\n", file=sys.stderr)
249
+
250
+
251
+ def reset_live_banner() -> None:
252
+ """For tests, and for a second run in one process."""
253
+ global _LIVE_BANNER_SHOWN
254
+ _LIVE_BANNER_SHOWN = False
255
+
256
+
257
+ def _store_for(ctx: RunContext, tool_name: str) -> CassetteStore:
258
+ if ctx.cassettes is None:
259
+ raise SandboxNotReady(
260
+ f"tool {tool_name!r} was called in tool_mode={ctx.tool_mode!r} but no "
261
+ "cassette store was attached to the run context.\n"
262
+ " The sandbox will not fall through to the real tool to cover for "
263
+ "this: in a benchmark that means spending real money on an API call "
264
+ "whose result is then thrown away."
265
+ )
266
+ return ctx.cassettes
267
+
268
+
269
+ def _trace(
270
+ ctx: RunContext,
271
+ tool: SandboxedTool,
272
+ arguments: Mapping[str, Any],
273
+ ) -> ToolCallTrace:
274
+ return ToolCallTrace(
275
+ seq=ctx.next_tool_seq(),
276
+ name=tool.name,
277
+ node=current_node(),
278
+ node_path=current_node_path(),
279
+ arguments=dict(arguments),
280
+ tool_version=tool.version or None,
281
+ )
282
+
283
+
284
+ def _note_degraded(ctx: RunContext, tool: SandboxedTool, hit: Hit) -> None:
285
+ """Attach a warning for anything but an exact match.
286
+
287
+ A run whose numbers depend on our normalizers covering for the model must
288
+ not look clean in the scorecard.
289
+ """
290
+ if hit.tier is MatchTier.NORMALIZED:
291
+ ctx.warn(
292
+ f"{tool.name}: matched a cassette only after normalizing arguments "
293
+ "(formatting differed from the recording)"
294
+ )
295
+ elif hit.tier is MatchTier.SUBSET:
296
+ ctx.warn(
297
+ f"{tool.name}: matched only on {', '.join(tool.policy.match_on or ())} - "
298
+ "the model got the other arguments wrong"
299
+ )
300
+ if (
301
+ tool.version
302
+ and hit.cassette.tool_version
303
+ and tool.version != hit.cassette.tool_version
304
+ ):
305
+ ctx.warn(
306
+ f"{tool.name}: replayed a cassette recorded for a different version of "
307
+ "this tool - re-record it with `slmeval record`"
308
+ )
309
+
310
+
311
+ def _replay(
312
+ ctx: RunContext,
313
+ tool: SandboxedTool,
314
+ arguments: Mapping[str, Any],
315
+ trace: ToolCallTrace,
316
+ ) -> tuple[bool, Any]:
317
+ """Try the store. Returns ``(hit, response)``.
318
+
319
+ Raises :class:`CassetteMiss` only in strict ``replay`` mode; ``auto`` wants
320
+ the miss back so it can record.
321
+ """
322
+ store = _store_for(ctx, tool.name)
323
+ keys = tool.policy.compute(ctx.scope, arguments)
324
+ hit = store.lookup(keys)
325
+ if hit is not None:
326
+ trace.cassette_key = hit.cassette.key
327
+ trace.match_tier = hit.tier.value
328
+ trace.outcome = "replayed"
329
+ _note_degraded(ctx, tool, hit)
330
+ return True, hit.response
331
+
332
+ if ctx.tool_mode == "replay":
333
+ trace.outcome = "miss"
334
+ trace.cassette_key = keys.exact
335
+ trace.error = "CASSETTE_MISS"
336
+ finding = diagnose(
337
+ case_id=ctx.scope,
338
+ tool_name=tool.name,
339
+ arguments=arguments,
340
+ store=store,
341
+ candidate_id=ctx.candidate_id,
342
+ node=current_node(),
343
+ node_path=current_node_path(),
344
+ )
345
+ trace.miss_verdict = finding.verdict.value
346
+ if finding.uncovered:
347
+ # An uncovered path is a coverage gap, so it is surfaced as a
348
+ # warning on the execution as well as in the block. Without this a
349
+ # user reading only the scorecard would see a failed case and
350
+ # assume the model broke.
351
+ ctx.warn(
352
+ f"{tool.name}: {finding.verdict.value} - the baseline never recorded this "
353
+ "call, so it is uncovered rather than wrong "
354
+ "(`slmeval record --fill-gaps`)"
355
+ )
356
+ raise CassetteMiss(
357
+ case_id=ctx.scope,
358
+ tool_name=tool.name,
359
+ arguments=arguments,
360
+ keys=keys,
361
+ diagnosis=finding,
362
+ node=current_node(),
363
+ candidate_id=ctx.candidate_id,
364
+ )
365
+ return False, None
366
+
367
+
368
+ def _after_real_call(
369
+ ctx: RunContext,
370
+ tool: SandboxedTool,
371
+ arguments: Mapping[str, Any],
372
+ trace: ToolCallTrace,
373
+ result: Any,
374
+ ) -> Any:
375
+ """Record a real call's result, in ``record`` and ``auto`` modes."""
376
+ store = _store_for(ctx, tool.name)
377
+ keys = tool.policy.compute(ctx.scope, arguments)
378
+ # An unserializable return raises out of here, which fails the run loudly at
379
+ # record time. That is the point: coercing it would replay a different type
380
+ # than it recorded, and the user would spend a day on it instead of a minute.
381
+ hit = store.record(
382
+ keys,
383
+ arguments=arguments,
384
+ response=result,
385
+ tool_version=tool.version,
386
+ dataset_hash=str(ctx.metadata.get("dataset_hash", "")),
387
+ )
388
+ trace.cassette_key = hit.cassette.key
389
+ trace.match_tier = MatchTier.EXACT.value
390
+ trace.outcome = "recorded"
391
+ return result
392
+
393
+
394
+ def _deliver_fault(
395
+ ctx: RunContext,
396
+ tool: SandboxedTool,
397
+ fault: Fault,
398
+ response: Any,
399
+ trace: ToolCallTrace,
400
+ ) -> Any:
401
+ """Deliver one fault: return a mangled payload, or raise.
402
+
403
+ Called *after* the response has been resolved, so an injected fault can
404
+ never mask a cassette miss - the miss is a finding about the model and the
405
+ fault is a stimulus we chose, and when both apply to one call the finding
406
+ wins.
407
+
408
+ The delay happened at the call site, because sleeping correctly is the one
409
+ thing this function cannot do for both the sync and the async wrapper.
410
+ """
411
+ trace.fault = fault.label()
412
+ trace.outcome = "fault"
413
+ node = current_node()
414
+
415
+ if fault.delivery is Delivery.RAISE:
416
+ error = fault.error()
417
+ ctx.record_fault(fault, node=node)
418
+ # Set before raising, so the wrapper's `except BaseException` sees an
419
+ # error already attributed to the fault and does not overwrite it with
420
+ # a generic one.
421
+ trace.error = f"{type(error).__name__}: {error}"
422
+ raise error
423
+
424
+ payload, applied = apply_fault(fault, response)
425
+ ctx.record_fault(fault, node=node, applied=applied)
426
+ if not applied:
427
+ # Recorded as delivered-but-ineffective rather than dropped: a vector
428
+ # that quietly omitted it would compare equal to a run that never
429
+ # scheduled it, and the parity check would pass on a real difference.
430
+ ctx.warn(
431
+ f"{tool.name}: {fault.kind.value} could not be shaped onto a "
432
+ f"{type(response).__name__} response, so the call was counted as faulted "
433
+ "but the payload is unchanged"
434
+ )
435
+ return payload
436
+
437
+
438
+ async def _fault_async(
439
+ ctx: RunContext,
440
+ tool: SandboxedTool,
441
+ fault: Fault,
442
+ response: Any,
443
+ trace: ToolCallTrace,
444
+ ) -> Any:
445
+ if fault.delay_s:
446
+ # A real wait, because a timeout that takes no time never trips the
447
+ # agent's own timeout and the measurement would be a lie. Scaled by
448
+ # the schedule's `delay_scale`, which the test suite sets to zero.
449
+ await asyncio.sleep(fault.delay_s)
450
+ return _deliver_fault(ctx, tool, fault, response, trace)
451
+
452
+
453
+ def _fault_sync(
454
+ ctx: RunContext,
455
+ tool: SandboxedTool,
456
+ fault: Fault,
457
+ response: Any,
458
+ trace: ToolCallTrace,
459
+ ) -> Any:
460
+ if fault.delay_s:
461
+ time.sleep(fault.delay_s)
462
+ return _deliver_fault(ctx, tool, fault, response, trace)
463
+
464
+
465
+ def sandbox_tool(
466
+ fn: F | None = None,
467
+ *,
468
+ match_on: list[str] | tuple[str, ...] | None = None,
469
+ normalize: Mapping[str, str] | None = None,
470
+ ignore_args: list[str] | tuple[str, ...] | None = None,
471
+ version: str | None = None,
472
+ name: str | None = None,
473
+ ) -> Any:
474
+ """Record this tool once, then replay it forever.
475
+
476
+ @sandbox_tool(
477
+ match_on=["order_id"],
478
+ normalize={"order_id": "upper_strip", "query": "sql"},
479
+ ignore_args=["request_id", "timestamp"],
480
+ )
481
+ async def fetch_order_from_erp(order_id: str, request_id: str) -> dict:
482
+ ...
483
+
484
+ ``match_on``
485
+ The arguments that identify the call. They form the last rung of the
486
+ lookup cascade, so a call that gets *these* right and something else
487
+ wrong is reported as such instead of passing or missing outright.
488
+ ``normalize``
489
+ Named normalizers, per argument, so a formatting difference is not
490
+ reported as a hallucination. See
491
+ :mod:`slmeval.sandbox.normalizers`.
492
+ ``ignore_args``
493
+ Arguments that never identify a call - a request id, a timestamp. Left
494
+ in, they would make every call unique and every replay a miss.
495
+ ``version``
496
+ Declare it when you want to invalidate cassettes deliberately.
497
+ Otherwise a hash of the tool's source is used to *flag* staleness.
498
+
499
+ A no-op outside a benchmark run: with no run in progress the real tool is
500
+ called, untouched, so this is safe to leave in production code.
501
+ """
502
+
503
+ def decorate(func: F) -> F:
504
+ tool_name = name or func.__name__
505
+ policy = declare(
506
+ func,
507
+ tool_name=tool_name,
508
+ match_on=match_on,
509
+ normalize=normalize,
510
+ ignore_args=ignore_args,
511
+ tool_version=version or "",
512
+ )
513
+ try:
514
+ signature: inspect.Signature | None = inspect.signature(func)
515
+ except (TypeError, ValueError):
516
+ signature = None
517
+
518
+ tool = SandboxedTool(
519
+ name=tool_name,
520
+ policy=policy,
521
+ version=version or _source_version(func),
522
+ qualname=f"{func.__module__}.{func.__qualname__}",
523
+ is_async=inspect.iscoroutinefunction(func),
524
+ func=func,
525
+ signature=signature,
526
+ )
527
+ _register(tool)
528
+
529
+ if tool.is_async:
530
+
531
+ @functools.wraps(func)
532
+ async def async_wrapper(*args: Any, **kwargs: Any) -> Any:
533
+ ctx = get_context()
534
+ if ctx is None:
535
+ _warn_if_benchmarking(tool_name)
536
+ return await func(*args, **kwargs)
537
+ arguments = tool.bind(args, kwargs)
538
+ ctx.note_step()
539
+ if ctx.tool_mode == "live":
540
+ _live_banner(tool_name)
541
+ return await _run_traced(ctx, tool, arguments, func(*args, **kwargs))
542
+
543
+ trace = _trace(ctx, tool, arguments)
544
+ # This tool's own call index, not a global counter, and the
545
+ # fault is looked up before the call resolves so a raise-
546
+ # delivered one can be applied to a replayed response without
547
+ # a second trip through the schedule.
548
+ fault = ctx.fault_for(tool.name, ctx.next_tool_index(tool.name))
549
+ try:
550
+ if ctx.tool_mode != "record":
551
+ # `record` does not consult the store: it executes the
552
+ # real tool and overwrites. Looking first would make
553
+ # `record` silently behave like `auto`, so re-recording
554
+ # a case after changing a fixture would appear to do
555
+ # nothing.
556
+ replayed, response = _replay(ctx, tool, arguments, trace)
557
+ if replayed:
558
+ if fault is not None:
559
+ return await _fault_async(ctx, tool, fault, response, trace)
560
+ return response
561
+ result = await func(*args, **kwargs)
562
+ except BaseException as exc:
563
+ if trace.error is None:
564
+ trace.outcome = "error"
565
+ trace.error = f"{type(exc).__name__}: {exc}"
566
+ raise
567
+ else:
568
+ recorded = _after_real_call(ctx, tool, arguments, trace, result)
569
+ if fault is not None:
570
+ # The cassette already holds the clean response - it was
571
+ # written above - so `auto` mode faults what the agent
572
+ # sees without poisoning what the store keeps.
573
+ return await _fault_async(ctx, tool, fault, recorded, trace)
574
+ return recorded
575
+ finally:
576
+ # In `finally`: every path leaves exactly one trace, or the
577
+ # denominator of every per-tool rate in the report is wrong.
578
+ ctx.record_tool(trace)
579
+
580
+ wrapper: Callable[..., Any] = async_wrapper
581
+ else:
582
+
583
+ @functools.wraps(func)
584
+ def sync_wrapper(*args: Any, **kwargs: Any) -> Any:
585
+ ctx = get_context()
586
+ if ctx is None:
587
+ _warn_if_benchmarking(tool_name)
588
+ return func(*args, **kwargs)
589
+ arguments = tool.bind(args, kwargs)
590
+ ctx.note_step()
591
+ if ctx.tool_mode == "live":
592
+ _live_banner(tool_name)
593
+ return _run_traced_sync(ctx, tool, arguments, func, args, kwargs)
594
+
595
+ trace = _trace(ctx, tool, arguments)
596
+ fault = ctx.fault_for(tool.name, ctx.next_tool_index(tool.name))
597
+ try:
598
+ if ctx.tool_mode != "record":
599
+ replayed, response = _replay(ctx, tool, arguments, trace)
600
+ if replayed:
601
+ if fault is not None:
602
+ return _fault_sync(ctx, tool, fault, response, trace)
603
+ return response
604
+ result = func(*args, **kwargs)
605
+ except BaseException as exc:
606
+ if trace.error is None:
607
+ trace.outcome = "error"
608
+ trace.error = f"{type(exc).__name__}: {exc}"
609
+ raise
610
+ else:
611
+ recorded = _after_real_call(ctx, tool, arguments, trace, result)
612
+ if fault is not None:
613
+ return _fault_sync(ctx, tool, fault, recorded, trace)
614
+ return recorded
615
+ finally:
616
+ ctx.record_tool(trace)
617
+
618
+ wrapper = sync_wrapper
619
+
620
+ wrapper.slmeval_tool = tool # type: ignore[attr-defined]
621
+ return cast("F", wrapper)
622
+
623
+ if fn is not None:
624
+ return decorate(fn)
625
+ return decorate
626
+
627
+
628
+ def _warn_if_benchmarking(tool_name: str) -> None:
629
+ """No context: production passthrough, or a context lost crossing a thread?
630
+
631
+ Only the second is a problem, and only it gets a warning - a warning on
632
+ every production tool call is a warning nobody reads.
633
+ """
634
+ if benchmark_running_now():
635
+ warn_missing_context(f"@sandbox_tool {tool_name!r}")
636
+
637
+
638
+ async def _run_traced(
639
+ ctx: RunContext,
640
+ tool: SandboxedTool,
641
+ arguments: Mapping[str, Any],
642
+ awaitable: Any,
643
+ ) -> Any:
644
+ """`live` mode: no interception, but still traced and still counted."""
645
+ trace = _trace(ctx, tool, arguments)
646
+ trace.outcome = "live"
647
+ try:
648
+ return await awaitable
649
+ except BaseException as exc:
650
+ trace.error = f"{type(exc).__name__}: {exc}"
651
+ raise
652
+ finally:
653
+ ctx.record_tool(trace)
654
+
655
+
656
+ def _run_traced_sync(
657
+ ctx: RunContext,
658
+ tool: SandboxedTool,
659
+ arguments: Mapping[str, Any],
660
+ func: Callable[..., Any],
661
+ args: tuple[Any, ...],
662
+ kwargs: Mapping[str, Any],
663
+ ) -> Any:
664
+ trace = _trace(ctx, tool, arguments)
665
+ trace.outcome = "live"
666
+ try:
667
+ return func(*args, **dict(kwargs))
668
+ except BaseException as exc:
669
+ trace.error = f"{type(exc).__name__}: {exc}"
670
+ raise
671
+ finally:
672
+ ctx.record_tool(trace)
673
+
674
+
675
+ __all__ = [
676
+ "REGISTRY",
677
+ "CassetteMiss",
678
+ "MissDiagnosis",
679
+ "SandboxNotReady",
680
+ "SandboxedTool",
681
+ "clear_registry",
682
+ "registered_tools",
683
+ "reset_live_banner",
684
+ "sandbox_tool",
685
+ ]