slmeval-cli 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- slmeval/__init__.py +71 -0
- slmeval/__main__.py +14 -0
- slmeval/adapters/__init__.py +24 -0
- slmeval/adapters/instrument.py +135 -0
- slmeval/adapters/langgraph.py +237 -0
- slmeval/cli/__init__.py +15 -0
- slmeval/cli/commands/__init__.py +35 -0
- slmeval/cli/commands/baseline_cmd.py +201 -0
- slmeval/cli/commands/cache_cmd.py +249 -0
- slmeval/cli/commands/dataset_cmd.py +52 -0
- slmeval/cli/commands/doctor_cmd.py +68 -0
- slmeval/cli/commands/gate_cmd.py +602 -0
- slmeval/cli/commands/init_cmd.py +144 -0
- slmeval/cli/commands/judge_cmd.py +284 -0
- slmeval/cli/commands/record_cmd.py +154 -0
- slmeval/cli/commands/report_cmd.py +98 -0
- slmeval/cli/commands/run_cmd.py +247 -0
- slmeval/cli/commands/show_cmd.py +482 -0
- slmeval/cli/commands/stubs.py +57 -0
- slmeval/cli/commands/validate_cmd.py +296 -0
- slmeval/cli/loading.py +149 -0
- slmeval/cli/main.py +91 -0
- slmeval/cli/render.py +251 -0
- slmeval/cli/wrap.py +31 -0
- slmeval/config/__init__.py +47 -0
- slmeval/config/loader.py +363 -0
- slmeval/config/locate.py +97 -0
- slmeval/config/models.py +430 -0
- slmeval/config/templates.py +246 -0
- slmeval/core/__init__.py +78 -0
- slmeval/core/adapter.py +253 -0
- slmeval/core/context.py +418 -0
- slmeval/core/outcome.py +260 -0
- slmeval/core/router.py +319 -0
- slmeval/core/runner.py +1209 -0
- slmeval/core/trace.py +188 -0
- slmeval/core/verdict.py +127 -0
- slmeval/dataset/__init__.py +8 -0
- slmeval/dataset/loader.py +130 -0
- slmeval/dataset/models.py +108 -0
- slmeval/doctor/__init__.py +7 -0
- slmeval/doctor/checks.py +380 -0
- slmeval/errors.py +76 -0
- slmeval/eval/__init__.py +64 -0
- slmeval/eval/assertions.py +356 -0
- slmeval/eval/calibration.py +364 -0
- slmeval/eval/cost.py +291 -0
- slmeval/eval/pipeline.py +273 -0
- slmeval/eval/stats.py +376 -0
- slmeval/eval/tier1.py +510 -0
- slmeval/eval/tier2.py +582 -0
- slmeval/eval/tier3.py +415 -0
- slmeval/exit_codes.py +28 -0
- slmeval/gate.py +722 -0
- slmeval/providers/__init__.py +69 -0
- slmeval/providers/base.py +426 -0
- slmeval/providers/fake.py +760 -0
- slmeval/providers/jsonx.py +239 -0
- slmeval/providers/ladder.py +339 -0
- slmeval/providers/openai_compat.py +392 -0
- slmeval/providers/registry.py +185 -0
- slmeval/providers/retry.py +81 -0
- slmeval/providers/schema.py +136 -0
- slmeval/providers/tokens.py +150 -0
- slmeval/py.typed +0 -0
- slmeval/report/__init__.py +115 -0
- slmeval/report/attribution.py +197 -0
- slmeval/report/baseline.py +353 -0
- slmeval/report/drilldown.py +1395 -0
- slmeval/report/export.py +450 -0
- slmeval/report/model.py +562 -0
- slmeval/report/progress.py +257 -0
- slmeval/report/scorecard.py +367 -0
- slmeval/sandbox/__init__.py +145 -0
- slmeval/sandbox/diagnosis.py +690 -0
- slmeval/sandbox/envelope.py +486 -0
- slmeval/sandbox/faults.py +794 -0
- slmeval/sandbox/keys.py +458 -0
- slmeval/sandbox/normalizers.py +333 -0
- slmeval/sandbox/store.py +525 -0
- slmeval/sandbox/tool.py +685 -0
- slmeval/storage/__init__.py +73 -0
- slmeval/storage/blobs.py +308 -0
- slmeval/storage/codec.py +93 -0
- slmeval/storage/db.py +1334 -0
- slmeval/storage/ingest.py +346 -0
- slmeval/storage/migrations.py +358 -0
- slmeval/storage/models.py +554 -0
- slmeval/storage/provenance.py +201 -0
- slmeval_cli-0.1.0.dist-info/METADATA +484 -0
- slmeval_cli-0.1.0.dist-info/RECORD +94 -0
- slmeval_cli-0.1.0.dist-info/WHEEL +4 -0
- slmeval_cli-0.1.0.dist-info/entry_points.txt +2 -0
- slmeval_cli-0.1.0.dist-info/licenses/LICENSE +201 -0
|
@@ -0,0 +1,486 @@
|
|
|
1
|
+
"""The typed envelope: how a tool's return value survives a round trip.
|
|
2
|
+
|
|
3
|
+
A cassette is only useful if the agent sees on replay exactly what it saw on
|
|
4
|
+
record. The obvious implementation - ``json.dumps(response, default=str)`` -
|
|
5
|
+
quietly breaks that. A tool returning a ``datetime`` records as a string and
|
|
6
|
+
replays as a string, so on record the agent's next line does
|
|
7
|
+
``order.created.year`` and on replay it does the same thing to a ``str`` and
|
|
8
|
+
raises. The benchmark then reports a model failure that is entirely our fault,
|
|
9
|
+
and the day you spend finding it is a day spent doubting your own tool.
|
|
10
|
+
|
|
11
|
+
The same hole swallows ``Decimal`` (replays as ``str``, so arithmetic breaks or
|
|
12
|
+
silently loses precision), tuples (replay as lists), non-string dict keys
|
|
13
|
+
(``{1: "a"}`` replays as ``{"1": "a"}``), sets, and - worst of the lot -
|
|
14
|
+
``StrEnum`` and ``IntEnum`` members, which *are* strings and ints, so a naive
|
|
15
|
+
encoder flattens them without even noticing.
|
|
16
|
+
|
|
17
|
+
So every value is tagged::
|
|
18
|
+
|
|
19
|
+
{"__t": "json", "v": {...}}
|
|
20
|
+
{"__t": "datetime", "v": "2026-10-28T10:00:00+00:00"}
|
|
21
|
+
{"__t": "pydantic", "cls": "myapp.models.Order", "v": {...}}
|
|
22
|
+
|
|
23
|
+
Two rules make this trustworthy:
|
|
24
|
+
|
|
25
|
+
1. **Refuse, never coerce.** A value with no faithful representation - an open
|
|
26
|
+
database connection, a file handle, a socket - stops the recording with a
|
|
27
|
+
message naming the tool, the path to the value and its type. A loud failure
|
|
28
|
+
at record time costs a user two minutes. A silent divergence at replay time
|
|
29
|
+
costs them an afternoon and costs us their trust.
|
|
30
|
+
2. **Nothing is escaped by accident.** A tool that legitimately returns a dict
|
|
31
|
+
containing the key ``"__t"`` round-trips intact, because such dicts are
|
|
32
|
+
re-encoded as explicit key/value pairs rather than left to collide with our
|
|
33
|
+
own tag.
|
|
34
|
+
"""
|
|
35
|
+
|
|
36
|
+
from __future__ import annotations
|
|
37
|
+
|
|
38
|
+
import base64
|
|
39
|
+
import dataclasses
|
|
40
|
+
import importlib
|
|
41
|
+
import json
|
|
42
|
+
import math
|
|
43
|
+
from collections.abc import Mapping, Sequence
|
|
44
|
+
from datetime import date, datetime, time, timedelta
|
|
45
|
+
from decimal import Decimal
|
|
46
|
+
from enum import Enum
|
|
47
|
+
from pathlib import PurePath
|
|
48
|
+
from typing import Any, Final, cast
|
|
49
|
+
from uuid import UUID
|
|
50
|
+
|
|
51
|
+
from pydantic import BaseModel
|
|
52
|
+
|
|
53
|
+
from slmeval.errors import SlmevalError
|
|
54
|
+
|
|
55
|
+
TAG: Final = "__t"
|
|
56
|
+
"""The reserved key. Short because it appears once per nested value, and a
|
|
57
|
+
cassette store holds hundreds of thousands of them."""
|
|
58
|
+
|
|
59
|
+
VALUE: Final = "v"
|
|
60
|
+
CLS: Final = "cls"
|
|
61
|
+
|
|
62
|
+
SUPPORTED: Final = (
|
|
63
|
+
"dict, list, tuple, set, frozenset, str, int, float, bool, None, "
|
|
64
|
+
"datetime, date, time, timedelta, Decimal, UUID, bytes, bytearray, "
|
|
65
|
+
"Path, Enum, Pydantic models and dataclasses"
|
|
66
|
+
)
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
class UnserializableResponse(SlmevalError):
|
|
70
|
+
"""A tool returned something that cannot be recorded faithfully.
|
|
71
|
+
|
|
72
|
+
The user's to fix: their tool hands back a live object, and the harness
|
|
73
|
+
refuses to pretend otherwise.
|
|
74
|
+
"""
|
|
75
|
+
|
|
76
|
+
def __init__(
|
|
77
|
+
self, *, what: str, path: str, python_type: str, reason: str | None = None
|
|
78
|
+
) -> None:
|
|
79
|
+
detail = reason or "it has no JSON representation"
|
|
80
|
+
super().__init__(
|
|
81
|
+
f"cannot record {what}: the value at {path} is a {python_type}, and {detail}.",
|
|
82
|
+
hint=(
|
|
83
|
+
"Return something representable and slmeval will preserve its type exactly.\n"
|
|
84
|
+
f"Supported: {SUPPORTED}.\n"
|
|
85
|
+
"slmeval refuses to coerce it with str() on purpose: that would record a "
|
|
86
|
+
"string and replay a string, so your agent would see a different type on "
|
|
87
|
+
"replay than it saw on record - a divergence that looks like a model bug."
|
|
88
|
+
),
|
|
89
|
+
)
|
|
90
|
+
self.what = what
|
|
91
|
+
self.path = path
|
|
92
|
+
self.python_type = python_type
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
class EnvelopeDecodeError(SlmevalError):
|
|
96
|
+
"""A recorded envelope cannot be rebuilt.
|
|
97
|
+
|
|
98
|
+
Almost always a code change since recording: the class a cassette names has
|
|
99
|
+
been moved, renamed or deleted. That is staleness, not corruption, so the
|
|
100
|
+
message points at the cassette rather than at the payload.
|
|
101
|
+
"""
|
|
102
|
+
|
|
103
|
+
def __init__(self, message: str, *, hint: str | None = None) -> None:
|
|
104
|
+
super().__init__(
|
|
105
|
+
message,
|
|
106
|
+
hint=hint
|
|
107
|
+
or (
|
|
108
|
+
"This cassette was recorded against code that has since changed. "
|
|
109
|
+
"Re-record the affected cases with `slmeval record --case <id>`."
|
|
110
|
+
),
|
|
111
|
+
)
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
# --------------------------------------------------------------------------- #
|
|
115
|
+
# Encoding
|
|
116
|
+
# --------------------------------------------------------------------------- #
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
def _qualified(cls: type) -> str:
|
|
120
|
+
return f"{cls.__module__}.{cls.__qualname__}"
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def _check_importable(cls: type, *, what: str, path: str) -> str:
|
|
124
|
+
"""Reject classes that could not be imported back on replay.
|
|
125
|
+
|
|
126
|
+
A class defined in ``__main__`` or inside a function body has a name nobody
|
|
127
|
+
can resolve in a fresh process. Replay happens in a fresh process - usually
|
|
128
|
+
in CI - so recording one guarantees a decode failure later. Say so now.
|
|
129
|
+
"""
|
|
130
|
+
name = _qualified(cls)
|
|
131
|
+
if cls.__module__ in {"__main__", "builtins"} or "<locals>" in cls.__qualname__:
|
|
132
|
+
raise UnserializableResponse(
|
|
133
|
+
what=what,
|
|
134
|
+
path=path,
|
|
135
|
+
python_type=name,
|
|
136
|
+
reason=(
|
|
137
|
+
"its class cannot be imported back on replay (it is defined in "
|
|
138
|
+
f"{cls.__module__!r}"
|
|
139
|
+
+ (" inside a function body" if "<locals>" in cls.__qualname__ else "")
|
|
140
|
+
+ "), and replay runs in a fresh process"
|
|
141
|
+
),
|
|
142
|
+
)
|
|
143
|
+
return name
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
def to_jsonable(value: Any, *, what: str = "value", path: str = "$") -> Any:
|
|
147
|
+
"""Recursively convert to a JSON-safe tree, tagging anything non-native.
|
|
148
|
+
|
|
149
|
+
Returns a node, not a complete envelope - see :func:`encode`.
|
|
150
|
+
"""
|
|
151
|
+
return _encode(value, what=what, path=path, seen=())
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
def _encode(value: Any, *, what: str, path: str, seen: tuple[int, ...]) -> Any:
|
|
155
|
+
# Enum first: StrEnum and IntEnum members pass `isinstance(x, str)` and
|
|
156
|
+
# `isinstance(x, int)`, so any later branch would flatten them to their
|
|
157
|
+
# value and the agent would see a bare str on replay.
|
|
158
|
+
if isinstance(value, Enum):
|
|
159
|
+
cls = _check_importable(type(value), what=what, path=path)
|
|
160
|
+
return {TAG: "enum", CLS: cls, VALUE: _encode(value.value, what=what, path=path, seen=seen)}
|
|
161
|
+
|
|
162
|
+
if value is None or isinstance(value, (bool, int, str)):
|
|
163
|
+
return value
|
|
164
|
+
|
|
165
|
+
if isinstance(value, float):
|
|
166
|
+
if math.isfinite(value):
|
|
167
|
+
return value
|
|
168
|
+
# json.dumps emits bare NaN/Infinity, which is not JSON and which a
|
|
169
|
+
# strict reader on the other side of the store will reject.
|
|
170
|
+
marker = "nan" if math.isnan(value) else ("inf" if value > 0 else "-inf")
|
|
171
|
+
return {TAG: "float", VALUE: marker}
|
|
172
|
+
|
|
173
|
+
if isinstance(value, (bytes, bytearray)):
|
|
174
|
+
tag = "bytes" if isinstance(value, bytes) else "bytearray"
|
|
175
|
+
return {TAG: tag, VALUE: base64.b64encode(bytes(value)).decode("ascii")}
|
|
176
|
+
|
|
177
|
+
# datetime before date: datetime is a date subclass.
|
|
178
|
+
if isinstance(value, datetime):
|
|
179
|
+
return {TAG: "datetime", VALUE: value.isoformat()}
|
|
180
|
+
if isinstance(value, date):
|
|
181
|
+
return {TAG: "date", VALUE: value.isoformat()}
|
|
182
|
+
if isinstance(value, time):
|
|
183
|
+
return {TAG: "time", VALUE: value.isoformat()}
|
|
184
|
+
if isinstance(value, timedelta):
|
|
185
|
+
return {TAG: "timedelta", VALUE: value.total_seconds()}
|
|
186
|
+
if isinstance(value, Decimal):
|
|
187
|
+
# str(), not float(): Decimal exists precisely because float loses
|
|
188
|
+
# digits, and a money amount is the likeliest thing a tool returns.
|
|
189
|
+
return {TAG: "decimal", VALUE: str(value)}
|
|
190
|
+
if isinstance(value, UUID):
|
|
191
|
+
return {TAG: "uuid", VALUE: str(value)}
|
|
192
|
+
if isinstance(value, PurePath):
|
|
193
|
+
return {TAG: "path", VALUE: str(value)}
|
|
194
|
+
|
|
195
|
+
if id(value) in seen:
|
|
196
|
+
raise UnserializableResponse(
|
|
197
|
+
what=what,
|
|
198
|
+
path=path,
|
|
199
|
+
python_type=_qualified(type(value)),
|
|
200
|
+
reason="it contains a circular reference back to itself",
|
|
201
|
+
)
|
|
202
|
+
nested = (*seen, id(value))
|
|
203
|
+
|
|
204
|
+
if isinstance(value, BaseModel):
|
|
205
|
+
cls = _check_importable(type(value), what=what, path=path)
|
|
206
|
+
# model_dump(mode="python") then recurse, rather than mode="json":
|
|
207
|
+
# pydantic's JSON mode would str()-coerce the very types this module
|
|
208
|
+
# exists to preserve, and would hide an unserializable field instead of
|
|
209
|
+
# refusing it.
|
|
210
|
+
return {
|
|
211
|
+
TAG: "pydantic",
|
|
212
|
+
CLS: cls,
|
|
213
|
+
VALUE: _encode(value.model_dump(mode="python"), what=what, path=path, seen=nested),
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
if dataclasses.is_dataclass(value) and not isinstance(value, type):
|
|
217
|
+
cls = _check_importable(type(value), what=what, path=path)
|
|
218
|
+
fields = {f.name: getattr(value, f.name) for f in dataclasses.fields(value) if f.init}
|
|
219
|
+
return {
|
|
220
|
+
TAG: "dataclass",
|
|
221
|
+
CLS: cls,
|
|
222
|
+
VALUE: {
|
|
223
|
+
name: _encode(item, what=what, path=f"{path}.{name}", seen=nested)
|
|
224
|
+
for name, item in fields.items()
|
|
225
|
+
},
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
if isinstance(value, Mapping):
|
|
229
|
+
return _encode_mapping(value, what=what, path=path, seen=nested)
|
|
230
|
+
|
|
231
|
+
if isinstance(value, (set, frozenset)):
|
|
232
|
+
tag = "set" if isinstance(value, set) else "frozenset"
|
|
233
|
+
items = [
|
|
234
|
+
_encode(item, what=what, path=f"{path}{{{index}}}", seen=nested)
|
|
235
|
+
for index, item in enumerate(sorted(value, key=_stable_repr))
|
|
236
|
+
]
|
|
237
|
+
return {TAG: tag, VALUE: items}
|
|
238
|
+
|
|
239
|
+
if isinstance(value, tuple):
|
|
240
|
+
# Tagged, because JSON has one array type and a tuple replaying as a
|
|
241
|
+
# list breaks unpacking and any `isinstance` check downstream.
|
|
242
|
+
return {
|
|
243
|
+
TAG: "tuple",
|
|
244
|
+
VALUE: [
|
|
245
|
+
_encode(item, what=what, path=f"{path}[{index}]", seen=nested)
|
|
246
|
+
for index, item in enumerate(value)
|
|
247
|
+
],
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
if isinstance(value, Sequence):
|
|
251
|
+
return [
|
|
252
|
+
_encode(item, what=what, path=f"{path}[{index}]", seen=nested)
|
|
253
|
+
for index, item in enumerate(value)
|
|
254
|
+
]
|
|
255
|
+
|
|
256
|
+
raise UnserializableResponse(what=what, path=path, python_type=_qualified(type(value)))
|
|
257
|
+
|
|
258
|
+
|
|
259
|
+
def _stable_repr(item: Any) -> str:
|
|
260
|
+
"""Ordering key for sets, so a recorded set has a deterministic layout and
|
|
261
|
+
the cassette key for an identical call does not change between runs."""
|
|
262
|
+
return f"{type(item).__name__}:{item!r}"
|
|
263
|
+
|
|
264
|
+
|
|
265
|
+
def _encode_mapping(
|
|
266
|
+
value: Mapping[Any, Any], *, what: str, path: str, seen: tuple[int, ...]
|
|
267
|
+
) -> Any:
|
|
268
|
+
plain = all(isinstance(key, str) for key in value) and TAG not in value
|
|
269
|
+
if plain:
|
|
270
|
+
return {
|
|
271
|
+
str(key): _encode(item, what=what, path=f"{path}.{key}", seen=seen)
|
|
272
|
+
for key, item in value.items()
|
|
273
|
+
}
|
|
274
|
+
# Non-string keys would be stringified by JSON ({1: "a"} -> {"1": "a"}), and
|
|
275
|
+
# a dict that genuinely contains our tag key would be mistaken for an
|
|
276
|
+
# envelope on the way back. Both become explicit pairs. Insertion order is
|
|
277
|
+
# preserved: dict order is observable, so it is part of the payload.
|
|
278
|
+
pairs = [
|
|
279
|
+
[
|
|
280
|
+
_encode(key, what=what, path=f"{path}.<key {index}>", seen=seen),
|
|
281
|
+
_encode(item, what=what, path=f"{path}.<value {index}>", seen=seen),
|
|
282
|
+
]
|
|
283
|
+
for index, (key, item) in enumerate(value.items())
|
|
284
|
+
]
|
|
285
|
+
return {TAG: "map", VALUE: pairs}
|
|
286
|
+
|
|
287
|
+
|
|
288
|
+
def encode(value: Any, *, what: str = "tool return value") -> dict[str, Any]:
|
|
289
|
+
"""Wrap a value in a complete envelope, ready to store.
|
|
290
|
+
|
|
291
|
+
Always a tagged dict, so the stored column has one shape whatever the tool
|
|
292
|
+
returned. Raises :class:`UnserializableResponse` rather than coercing.
|
|
293
|
+
"""
|
|
294
|
+
node = _encode(value, what=what, path="$", seen=())
|
|
295
|
+
if isinstance(node, dict) and TAG in node:
|
|
296
|
+
return node
|
|
297
|
+
return {TAG: "json", VALUE: node}
|
|
298
|
+
|
|
299
|
+
|
|
300
|
+
def encode_json(value: Any, *, what: str = "tool return value") -> str:
|
|
301
|
+
"""Encode and serialize in one step - what the store writes."""
|
|
302
|
+
return json.dumps(
|
|
303
|
+
encode(value, what=what), sort_keys=False, separators=(",", ":"), allow_nan=False
|
|
304
|
+
)
|
|
305
|
+
|
|
306
|
+
|
|
307
|
+
# --------------------------------------------------------------------------- #
|
|
308
|
+
# Decoding
|
|
309
|
+
# --------------------------------------------------------------------------- #
|
|
310
|
+
|
|
311
|
+
|
|
312
|
+
def _import_class(name: str, *, tag: str) -> Any:
|
|
313
|
+
module_name, _, attribute = name.rpartition(".")
|
|
314
|
+
if not module_name:
|
|
315
|
+
raise EnvelopeDecodeError(f"{tag} envelope names {name!r}, which is not a dotted path")
|
|
316
|
+
try:
|
|
317
|
+
module = importlib.import_module(module_name)
|
|
318
|
+
except ImportError as exc:
|
|
319
|
+
raise EnvelopeDecodeError(
|
|
320
|
+
f"cannot import {module_name!r} to rebuild the recorded {tag} {name!r}: {exc}"
|
|
321
|
+
) from exc
|
|
322
|
+
try:
|
|
323
|
+
return getattr(module, attribute)
|
|
324
|
+
except AttributeError:
|
|
325
|
+
raise EnvelopeDecodeError(
|
|
326
|
+
f"{module_name!r} no longer defines {attribute!r}, "
|
|
327
|
+
f"which a cassette recorded as a {tag}"
|
|
328
|
+
) from None
|
|
329
|
+
|
|
330
|
+
|
|
331
|
+
_FLOATS: Final = {"nan": math.nan, "inf": math.inf, "-inf": -math.inf}
|
|
332
|
+
|
|
333
|
+
|
|
334
|
+
def decode(envelope: Any) -> Any:
|
|
335
|
+
"""Rebuild the original value from a complete envelope."""
|
|
336
|
+
if not isinstance(envelope, dict) or TAG not in envelope:
|
|
337
|
+
raise EnvelopeDecodeError(
|
|
338
|
+
f"expected a typed envelope with a {TAG!r} tag, got {type(envelope).__name__}",
|
|
339
|
+
hint="Envelopes are written by slmeval; this payload was not.",
|
|
340
|
+
)
|
|
341
|
+
return _decode(envelope)
|
|
342
|
+
|
|
343
|
+
|
|
344
|
+
def decode_json(text: str) -> Any:
|
|
345
|
+
try:
|
|
346
|
+
payload = json.loads(text)
|
|
347
|
+
except ValueError as exc:
|
|
348
|
+
raise EnvelopeDecodeError(f"stored envelope is not valid JSON: {exc}") from exc
|
|
349
|
+
return decode(payload)
|
|
350
|
+
|
|
351
|
+
|
|
352
|
+
def _decode(node: Any) -> Any:
|
|
353
|
+
if isinstance(node, list):
|
|
354
|
+
return [_decode(item) for item in node]
|
|
355
|
+
if not isinstance(node, dict):
|
|
356
|
+
return node
|
|
357
|
+
if TAG not in node:
|
|
358
|
+
return {key: _decode(item) for key, item in node.items()}
|
|
359
|
+
|
|
360
|
+
tag = node[TAG]
|
|
361
|
+
raw = node.get(VALUE)
|
|
362
|
+
|
|
363
|
+
if tag == "json":
|
|
364
|
+
return _decode(raw)
|
|
365
|
+
if tag == "map":
|
|
366
|
+
if not isinstance(raw, list):
|
|
367
|
+
raise EnvelopeDecodeError("a 'map' envelope must hold a list of key/value pairs")
|
|
368
|
+
return {_hashable(_decode(pair[0])): _decode(pair[1]) for pair in raw}
|
|
369
|
+
if tag == "tuple":
|
|
370
|
+
return tuple(_decode(item) for item in raw or [])
|
|
371
|
+
if tag == "set":
|
|
372
|
+
return {_hashable(_decode(item)) for item in raw or []}
|
|
373
|
+
if tag == "frozenset":
|
|
374
|
+
return frozenset(_hashable(_decode(item)) for item in raw or [])
|
|
375
|
+
if tag == "float":
|
|
376
|
+
try:
|
|
377
|
+
return _FLOATS[str(raw)]
|
|
378
|
+
except KeyError:
|
|
379
|
+
raise EnvelopeDecodeError(f"unknown non-finite float marker {raw!r}") from None
|
|
380
|
+
if tag == "bytes":
|
|
381
|
+
return base64.b64decode(str(raw))
|
|
382
|
+
if tag == "bytearray":
|
|
383
|
+
return bytearray(base64.b64decode(str(raw)))
|
|
384
|
+
if tag == "datetime":
|
|
385
|
+
return datetime.fromisoformat(str(raw))
|
|
386
|
+
if tag == "date":
|
|
387
|
+
return date.fromisoformat(str(raw))
|
|
388
|
+
if tag == "time":
|
|
389
|
+
return time.fromisoformat(str(raw))
|
|
390
|
+
if tag == "timedelta":
|
|
391
|
+
return timedelta(seconds=float(raw)) # type: ignore[arg-type]
|
|
392
|
+
if tag == "decimal":
|
|
393
|
+
return Decimal(str(raw))
|
|
394
|
+
if tag == "uuid":
|
|
395
|
+
return UUID(str(raw))
|
|
396
|
+
if tag == "path":
|
|
397
|
+
from pathlib import Path
|
|
398
|
+
|
|
399
|
+
return Path(str(raw))
|
|
400
|
+
if tag in {"pydantic", "dataclass", "enum"}:
|
|
401
|
+
return _decode_class(tag, node, raw)
|
|
402
|
+
|
|
403
|
+
raise EnvelopeDecodeError(
|
|
404
|
+
f"unknown envelope tag {tag!r}",
|
|
405
|
+
hint=(
|
|
406
|
+
"This cassette was written by a newer slmeval than the one reading it. "
|
|
407
|
+
"Upgrade slmeval, or re-record."
|
|
408
|
+
),
|
|
409
|
+
)
|
|
410
|
+
|
|
411
|
+
|
|
412
|
+
def _decode_class(tag: str, node: dict[str, Any], raw: Any) -> Any:
|
|
413
|
+
name = node.get(CLS)
|
|
414
|
+
if not isinstance(name, str):
|
|
415
|
+
raise EnvelopeDecodeError(f"a {tag!r} envelope must carry a {CLS!r} class path")
|
|
416
|
+
cls = _import_class(name, tag=tag)
|
|
417
|
+
|
|
418
|
+
if tag == "enum":
|
|
419
|
+
member = _decode(raw)
|
|
420
|
+
try:
|
|
421
|
+
return cls(member)
|
|
422
|
+
except (ValueError, TypeError) as exc:
|
|
423
|
+
raise EnvelopeDecodeError(
|
|
424
|
+
f"{name} no longer has a member with value {member!r}: {exc}"
|
|
425
|
+
) from exc
|
|
426
|
+
|
|
427
|
+
decoded = _decode(raw)
|
|
428
|
+
if not isinstance(decoded, dict):
|
|
429
|
+
raise EnvelopeDecodeError(
|
|
430
|
+
f"a {tag!r} envelope must hold an object, got {type(decoded).__name__}"
|
|
431
|
+
)
|
|
432
|
+
|
|
433
|
+
if tag == "pydantic":
|
|
434
|
+
if not (isinstance(cls, type) and issubclass(cls, BaseModel)):
|
|
435
|
+
raise EnvelopeDecodeError(
|
|
436
|
+
f"{name} was recorded as a Pydantic model but no longer is one"
|
|
437
|
+
)
|
|
438
|
+
try:
|
|
439
|
+
return cls.model_validate(decoded)
|
|
440
|
+
except Exception as exc: # pydantic ValidationError, and anything a validator raises
|
|
441
|
+
raise EnvelopeDecodeError(
|
|
442
|
+
f"a recorded {name} no longer validates against its current definition: {exc}"
|
|
443
|
+
) from exc
|
|
444
|
+
|
|
445
|
+
if not dataclasses.is_dataclass(cls):
|
|
446
|
+
raise EnvelopeDecodeError(f"{name} was recorded as a dataclass but no longer is one")
|
|
447
|
+
known = {f.name for f in dataclasses.fields(cls) if f.init}
|
|
448
|
+
unknown = sorted(set(decoded) - known)
|
|
449
|
+
if unknown:
|
|
450
|
+
raise EnvelopeDecodeError(
|
|
451
|
+
f"a recorded {name} carries field(s) {', '.join(unknown)} "
|
|
452
|
+
"that its current definition does not accept"
|
|
453
|
+
)
|
|
454
|
+
try:
|
|
455
|
+
return cast("Any", cls)(**decoded)
|
|
456
|
+
except TypeError as exc:
|
|
457
|
+
raise EnvelopeDecodeError(f"cannot rebuild a recorded {name}: {exc}") from exc
|
|
458
|
+
|
|
459
|
+
|
|
460
|
+
def _hashable(value: Any) -> Any:
|
|
461
|
+
"""Dict keys and set members have to be hashable.
|
|
462
|
+
|
|
463
|
+
A decoded list cannot be either, which means the encoder produced something
|
|
464
|
+
the decoder cannot place - a harness bug, so it keeps its traceback.
|
|
465
|
+
"""
|
|
466
|
+
if isinstance(value, (list, dict, set, bytearray)):
|
|
467
|
+
raise TypeError(
|
|
468
|
+
f"decoded an unhashable {type(value).__name__} where a key or set member "
|
|
469
|
+
"was expected - this is an slmeval bug, please report the cassette"
|
|
470
|
+
)
|
|
471
|
+
return value
|
|
472
|
+
|
|
473
|
+
|
|
474
|
+
__all__ = [
|
|
475
|
+
"CLS",
|
|
476
|
+
"SUPPORTED",
|
|
477
|
+
"TAG",
|
|
478
|
+
"VALUE",
|
|
479
|
+
"EnvelopeDecodeError",
|
|
480
|
+
"UnserializableResponse",
|
|
481
|
+
"decode",
|
|
482
|
+
"decode_json",
|
|
483
|
+
"encode",
|
|
484
|
+
"encode_json",
|
|
485
|
+
"to_jsonable",
|
|
486
|
+
]
|