slmeval-cli 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- slmeval/__init__.py +71 -0
- slmeval/__main__.py +14 -0
- slmeval/adapters/__init__.py +24 -0
- slmeval/adapters/instrument.py +135 -0
- slmeval/adapters/langgraph.py +237 -0
- slmeval/cli/__init__.py +15 -0
- slmeval/cli/commands/__init__.py +35 -0
- slmeval/cli/commands/baseline_cmd.py +201 -0
- slmeval/cli/commands/cache_cmd.py +249 -0
- slmeval/cli/commands/dataset_cmd.py +52 -0
- slmeval/cli/commands/doctor_cmd.py +68 -0
- slmeval/cli/commands/gate_cmd.py +602 -0
- slmeval/cli/commands/init_cmd.py +144 -0
- slmeval/cli/commands/judge_cmd.py +284 -0
- slmeval/cli/commands/record_cmd.py +154 -0
- slmeval/cli/commands/report_cmd.py +98 -0
- slmeval/cli/commands/run_cmd.py +247 -0
- slmeval/cli/commands/show_cmd.py +482 -0
- slmeval/cli/commands/stubs.py +57 -0
- slmeval/cli/commands/validate_cmd.py +296 -0
- slmeval/cli/loading.py +149 -0
- slmeval/cli/main.py +91 -0
- slmeval/cli/render.py +251 -0
- slmeval/cli/wrap.py +31 -0
- slmeval/config/__init__.py +47 -0
- slmeval/config/loader.py +363 -0
- slmeval/config/locate.py +97 -0
- slmeval/config/models.py +430 -0
- slmeval/config/templates.py +246 -0
- slmeval/core/__init__.py +78 -0
- slmeval/core/adapter.py +253 -0
- slmeval/core/context.py +418 -0
- slmeval/core/outcome.py +260 -0
- slmeval/core/router.py +319 -0
- slmeval/core/runner.py +1209 -0
- slmeval/core/trace.py +188 -0
- slmeval/core/verdict.py +127 -0
- slmeval/dataset/__init__.py +8 -0
- slmeval/dataset/loader.py +130 -0
- slmeval/dataset/models.py +108 -0
- slmeval/doctor/__init__.py +7 -0
- slmeval/doctor/checks.py +380 -0
- slmeval/errors.py +76 -0
- slmeval/eval/__init__.py +64 -0
- slmeval/eval/assertions.py +356 -0
- slmeval/eval/calibration.py +364 -0
- slmeval/eval/cost.py +291 -0
- slmeval/eval/pipeline.py +273 -0
- slmeval/eval/stats.py +376 -0
- slmeval/eval/tier1.py +510 -0
- slmeval/eval/tier2.py +582 -0
- slmeval/eval/tier3.py +415 -0
- slmeval/exit_codes.py +28 -0
- slmeval/gate.py +722 -0
- slmeval/providers/__init__.py +69 -0
- slmeval/providers/base.py +426 -0
- slmeval/providers/fake.py +760 -0
- slmeval/providers/jsonx.py +239 -0
- slmeval/providers/ladder.py +339 -0
- slmeval/providers/openai_compat.py +392 -0
- slmeval/providers/registry.py +185 -0
- slmeval/providers/retry.py +81 -0
- slmeval/providers/schema.py +136 -0
- slmeval/providers/tokens.py +150 -0
- slmeval/py.typed +0 -0
- slmeval/report/__init__.py +115 -0
- slmeval/report/attribution.py +197 -0
- slmeval/report/baseline.py +353 -0
- slmeval/report/drilldown.py +1395 -0
- slmeval/report/export.py +450 -0
- slmeval/report/model.py +562 -0
- slmeval/report/progress.py +257 -0
- slmeval/report/scorecard.py +367 -0
- slmeval/sandbox/__init__.py +145 -0
- slmeval/sandbox/diagnosis.py +690 -0
- slmeval/sandbox/envelope.py +486 -0
- slmeval/sandbox/faults.py +794 -0
- slmeval/sandbox/keys.py +458 -0
- slmeval/sandbox/normalizers.py +333 -0
- slmeval/sandbox/store.py +525 -0
- slmeval/sandbox/tool.py +685 -0
- slmeval/storage/__init__.py +73 -0
- slmeval/storage/blobs.py +308 -0
- slmeval/storage/codec.py +93 -0
- slmeval/storage/db.py +1334 -0
- slmeval/storage/ingest.py +346 -0
- slmeval/storage/migrations.py +358 -0
- slmeval/storage/models.py +554 -0
- slmeval/storage/provenance.py +201 -0
- slmeval_cli-0.1.0.dist-info/METADATA +484 -0
- slmeval_cli-0.1.0.dist-info/RECORD +94 -0
- slmeval_cli-0.1.0.dist-info/WHEEL +4 -0
- slmeval_cli-0.1.0.dist-info/entry_points.txt +2 -0
- slmeval_cli-0.1.0.dist-info/licenses/LICENSE +201 -0
|
@@ -0,0 +1,145 @@
|
|
|
1
|
+
"""The VCR tool sandbox.
|
|
2
|
+
|
|
3
|
+
Recording a tool call once and replaying it is what makes a benchmark possible
|
|
4
|
+
at all: you cannot run 500 trajectories against a customer's live Stripe, ERP or
|
|
5
|
+
production database, and you cannot compare two models against an API that
|
|
6
|
+
returns different data every call.
|
|
7
|
+
|
|
8
|
+
Three pieces, and they are separable on purpose:
|
|
9
|
+
|
|
10
|
+
:mod:`slmeval.sandbox.keys`
|
|
11
|
+
What identifies a recorded call, and the declared per-tool policy that
|
|
12
|
+
decides when two calls are the same call.
|
|
13
|
+
:mod:`slmeval.sandbox.normalizers`
|
|
14
|
+
The named normalizers that policy refers to.
|
|
15
|
+
:mod:`slmeval.sandbox.envelope`
|
|
16
|
+
How a return value is stored so the agent sees the same *types* on replay
|
|
17
|
+
that it saw on record - and how recording refuses, loudly, when it cannot
|
|
18
|
+
promise that.
|
|
19
|
+
|
|
20
|
+
:mod:`slmeval.sandbox.store`
|
|
21
|
+
Where recordings live: SQLite with an LRU in front, so the second replay of
|
|
22
|
+
a case never touches the disk.
|
|
23
|
+
:mod:`slmeval.sandbox.tool`
|
|
24
|
+
``@sandbox_tool`` and the four modes - ``record``, ``replay``, ``auto``,
|
|
25
|
+
``live``.
|
|
26
|
+
:mod:`slmeval.sandbox.diagnosis`
|
|
27
|
+
What a miss *means*: the nearest recorded call, a per-argument diff, and a
|
|
28
|
+
verdict that separates "the model got it wrong" from "we never recorded
|
|
29
|
+
this path".
|
|
30
|
+
:mod:`slmeval.sandbox.faults`
|
|
31
|
+
Deterministic fault injection, scheduled per tool per call index so one
|
|
32
|
+
extra call cannot shift every later fault onto a different step.
|
|
33
|
+
|
|
34
|
+
The one invariant across all of it: no path falls back to the real tool by
|
|
35
|
+
accident. A replay miss is a finding and surfaces as one, because silently
|
|
36
|
+
calling a customer's live API instead would spend their money and invalidate
|
|
37
|
+
the run without telling anyone.
|
|
38
|
+
"""
|
|
39
|
+
|
|
40
|
+
from __future__ import annotations
|
|
41
|
+
|
|
42
|
+
from slmeval.sandbox.diagnosis import (
|
|
43
|
+
ArgDiff,
|
|
44
|
+
MissDiagnosis,
|
|
45
|
+
Verdict,
|
|
46
|
+
argument_distance,
|
|
47
|
+
diagnose,
|
|
48
|
+
diff_arguments,
|
|
49
|
+
)
|
|
50
|
+
from slmeval.sandbox.envelope import (
|
|
51
|
+
EnvelopeDecodeError,
|
|
52
|
+
UnserializableResponse,
|
|
53
|
+
decode,
|
|
54
|
+
decode_json,
|
|
55
|
+
encode,
|
|
56
|
+
encode_json,
|
|
57
|
+
)
|
|
58
|
+
from slmeval.sandbox.faults import (
|
|
59
|
+
Delivery,
|
|
60
|
+
Fault,
|
|
61
|
+
FaultError,
|
|
62
|
+
FaultKind,
|
|
63
|
+
FaultSchedule,
|
|
64
|
+
FaultVector,
|
|
65
|
+
InjectedFault,
|
|
66
|
+
ParityReport,
|
|
67
|
+
RateLimited,
|
|
68
|
+
ToolTimeout,
|
|
69
|
+
apply_fault,
|
|
70
|
+
compare_vectors,
|
|
71
|
+
fault_schedule,
|
|
72
|
+
)
|
|
73
|
+
from slmeval.sandbox.keys import (
|
|
74
|
+
CASCADE,
|
|
75
|
+
CassetteKeys,
|
|
76
|
+
KeyEntry,
|
|
77
|
+
MatchPolicy,
|
|
78
|
+
MatchTier,
|
|
79
|
+
PolicyError,
|
|
80
|
+
canonicalize,
|
|
81
|
+
declare,
|
|
82
|
+
)
|
|
83
|
+
from slmeval.sandbox.normalizers import (
|
|
84
|
+
Normalizer,
|
|
85
|
+
UnknownNormalizer,
|
|
86
|
+
resolve,
|
|
87
|
+
sql_backend,
|
|
88
|
+
)
|
|
89
|
+
from slmeval.sandbox.store import Cassette, CassetteStore, Hit, StoreStats
|
|
90
|
+
from slmeval.sandbox.tool import (
|
|
91
|
+
CassetteMiss,
|
|
92
|
+
SandboxedTool,
|
|
93
|
+
SandboxNotReady,
|
|
94
|
+
registered_tools,
|
|
95
|
+
sandbox_tool,
|
|
96
|
+
)
|
|
97
|
+
|
|
98
|
+
__all__ = [
|
|
99
|
+
"CASCADE",
|
|
100
|
+
"ArgDiff",
|
|
101
|
+
"Cassette",
|
|
102
|
+
"CassetteKeys",
|
|
103
|
+
"CassetteMiss",
|
|
104
|
+
"CassetteStore",
|
|
105
|
+
"Delivery",
|
|
106
|
+
"EnvelopeDecodeError",
|
|
107
|
+
"Fault",
|
|
108
|
+
"FaultError",
|
|
109
|
+
"FaultKind",
|
|
110
|
+
"FaultSchedule",
|
|
111
|
+
"FaultVector",
|
|
112
|
+
"Hit",
|
|
113
|
+
"InjectedFault",
|
|
114
|
+
"KeyEntry",
|
|
115
|
+
"MatchPolicy",
|
|
116
|
+
"MatchTier",
|
|
117
|
+
"MissDiagnosis",
|
|
118
|
+
"Normalizer",
|
|
119
|
+
"ParityReport",
|
|
120
|
+
"PolicyError",
|
|
121
|
+
"RateLimited",
|
|
122
|
+
"SandboxNotReady",
|
|
123
|
+
"SandboxedTool",
|
|
124
|
+
"StoreStats",
|
|
125
|
+
"ToolTimeout",
|
|
126
|
+
"UnknownNormalizer",
|
|
127
|
+
"UnserializableResponse",
|
|
128
|
+
"Verdict",
|
|
129
|
+
"apply_fault",
|
|
130
|
+
"argument_distance",
|
|
131
|
+
"canonicalize",
|
|
132
|
+
"compare_vectors",
|
|
133
|
+
"declare",
|
|
134
|
+
"decode",
|
|
135
|
+
"decode_json",
|
|
136
|
+
"diagnose",
|
|
137
|
+
"diff_arguments",
|
|
138
|
+
"encode",
|
|
139
|
+
"encode_json",
|
|
140
|
+
"fault_schedule",
|
|
141
|
+
"registered_tools",
|
|
142
|
+
"resolve",
|
|
143
|
+
"sandbox_tool",
|
|
144
|
+
"sql_backend",
|
|
145
|
+
]
|