slmeval-cli 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (94) hide show
  1. slmeval/__init__.py +71 -0
  2. slmeval/__main__.py +14 -0
  3. slmeval/adapters/__init__.py +24 -0
  4. slmeval/adapters/instrument.py +135 -0
  5. slmeval/adapters/langgraph.py +237 -0
  6. slmeval/cli/__init__.py +15 -0
  7. slmeval/cli/commands/__init__.py +35 -0
  8. slmeval/cli/commands/baseline_cmd.py +201 -0
  9. slmeval/cli/commands/cache_cmd.py +249 -0
  10. slmeval/cli/commands/dataset_cmd.py +52 -0
  11. slmeval/cli/commands/doctor_cmd.py +68 -0
  12. slmeval/cli/commands/gate_cmd.py +602 -0
  13. slmeval/cli/commands/init_cmd.py +144 -0
  14. slmeval/cli/commands/judge_cmd.py +284 -0
  15. slmeval/cli/commands/record_cmd.py +154 -0
  16. slmeval/cli/commands/report_cmd.py +98 -0
  17. slmeval/cli/commands/run_cmd.py +247 -0
  18. slmeval/cli/commands/show_cmd.py +482 -0
  19. slmeval/cli/commands/stubs.py +57 -0
  20. slmeval/cli/commands/validate_cmd.py +296 -0
  21. slmeval/cli/loading.py +149 -0
  22. slmeval/cli/main.py +91 -0
  23. slmeval/cli/render.py +251 -0
  24. slmeval/cli/wrap.py +31 -0
  25. slmeval/config/__init__.py +47 -0
  26. slmeval/config/loader.py +363 -0
  27. slmeval/config/locate.py +97 -0
  28. slmeval/config/models.py +430 -0
  29. slmeval/config/templates.py +246 -0
  30. slmeval/core/__init__.py +78 -0
  31. slmeval/core/adapter.py +253 -0
  32. slmeval/core/context.py +418 -0
  33. slmeval/core/outcome.py +260 -0
  34. slmeval/core/router.py +319 -0
  35. slmeval/core/runner.py +1209 -0
  36. slmeval/core/trace.py +188 -0
  37. slmeval/core/verdict.py +127 -0
  38. slmeval/dataset/__init__.py +8 -0
  39. slmeval/dataset/loader.py +130 -0
  40. slmeval/dataset/models.py +108 -0
  41. slmeval/doctor/__init__.py +7 -0
  42. slmeval/doctor/checks.py +380 -0
  43. slmeval/errors.py +76 -0
  44. slmeval/eval/__init__.py +64 -0
  45. slmeval/eval/assertions.py +356 -0
  46. slmeval/eval/calibration.py +364 -0
  47. slmeval/eval/cost.py +291 -0
  48. slmeval/eval/pipeline.py +273 -0
  49. slmeval/eval/stats.py +376 -0
  50. slmeval/eval/tier1.py +510 -0
  51. slmeval/eval/tier2.py +582 -0
  52. slmeval/eval/tier3.py +415 -0
  53. slmeval/exit_codes.py +28 -0
  54. slmeval/gate.py +722 -0
  55. slmeval/providers/__init__.py +69 -0
  56. slmeval/providers/base.py +426 -0
  57. slmeval/providers/fake.py +760 -0
  58. slmeval/providers/jsonx.py +239 -0
  59. slmeval/providers/ladder.py +339 -0
  60. slmeval/providers/openai_compat.py +392 -0
  61. slmeval/providers/registry.py +185 -0
  62. slmeval/providers/retry.py +81 -0
  63. slmeval/providers/schema.py +136 -0
  64. slmeval/providers/tokens.py +150 -0
  65. slmeval/py.typed +0 -0
  66. slmeval/report/__init__.py +115 -0
  67. slmeval/report/attribution.py +197 -0
  68. slmeval/report/baseline.py +353 -0
  69. slmeval/report/drilldown.py +1395 -0
  70. slmeval/report/export.py +450 -0
  71. slmeval/report/model.py +562 -0
  72. slmeval/report/progress.py +257 -0
  73. slmeval/report/scorecard.py +367 -0
  74. slmeval/sandbox/__init__.py +145 -0
  75. slmeval/sandbox/diagnosis.py +690 -0
  76. slmeval/sandbox/envelope.py +486 -0
  77. slmeval/sandbox/faults.py +794 -0
  78. slmeval/sandbox/keys.py +458 -0
  79. slmeval/sandbox/normalizers.py +333 -0
  80. slmeval/sandbox/store.py +525 -0
  81. slmeval/sandbox/tool.py +685 -0
  82. slmeval/storage/__init__.py +73 -0
  83. slmeval/storage/blobs.py +308 -0
  84. slmeval/storage/codec.py +93 -0
  85. slmeval/storage/db.py +1334 -0
  86. slmeval/storage/ingest.py +346 -0
  87. slmeval/storage/migrations.py +358 -0
  88. slmeval/storage/models.py +554 -0
  89. slmeval/storage/provenance.py +201 -0
  90. slmeval_cli-0.1.0.dist-info/METADATA +484 -0
  91. slmeval_cli-0.1.0.dist-info/RECORD +94 -0
  92. slmeval_cli-0.1.0.dist-info/WHEEL +4 -0
  93. slmeval_cli-0.1.0.dist-info/entry_points.txt +2 -0
  94. slmeval_cli-0.1.0.dist-info/licenses/LICENSE +201 -0
@@ -0,0 +1,73 @@
1
+ """Trace storage: one SQLite file plus a content-addressed blob directory.
2
+
3
+ Traces are the asset. Everything the tool prints - the scorecard, the
4
+ drill-down diff, the CI gate - is a query over these rows, so the contract is
5
+ that a completed run is fully reconstructible from storage with every in-memory
6
+ object thrown away.
7
+
8
+ from slmeval.storage import RunStore, new_run_id
9
+
10
+ with RunStore.for_project(root) as store:
11
+ store.save_run(record)
12
+ ...
13
+ detail = store.reconstruct(exec_id)
14
+ """
15
+
16
+ from __future__ import annotations
17
+
18
+ from slmeval.storage.blobs import BlobStore, MissingBlob
19
+ from slmeval.storage.db import DB_NAME, RunNotFound, RunStore, make_exec_id
20
+ from slmeval.storage.migrations import SCHEMA_VERSION
21
+ from slmeval.storage.models import (
22
+ NOT_THE_MODELS_FAULT,
23
+ BlameRow,
24
+ CandidateRecord,
25
+ CaseRollupRow,
26
+ EvaluationRow,
27
+ ExecStatus,
28
+ ExecutionDetail,
29
+ ExecutionRecord,
30
+ MetricSample,
31
+ NodeSummaryRow,
32
+ NodeTraceRow,
33
+ Outcome,
34
+ Payload,
35
+ RunDetail,
36
+ RunRecord,
37
+ RunStatus,
38
+ StoreStats,
39
+ ToolTraceRow,
40
+ )
41
+ from slmeval.storage.provenance import Divergence, Provenance, divergences, new_run_id
42
+
43
+ __all__ = [
44
+ "DB_NAME",
45
+ "NOT_THE_MODELS_FAULT",
46
+ "SCHEMA_VERSION",
47
+ "BlameRow",
48
+ "BlobStore",
49
+ "CandidateRecord",
50
+ "CaseRollupRow",
51
+ "Divergence",
52
+ "EvaluationRow",
53
+ "ExecStatus",
54
+ "ExecutionDetail",
55
+ "ExecutionRecord",
56
+ "MetricSample",
57
+ "MissingBlob",
58
+ "NodeSummaryRow",
59
+ "NodeTraceRow",
60
+ "Outcome",
61
+ "Payload",
62
+ "Provenance",
63
+ "RunDetail",
64
+ "RunNotFound",
65
+ "RunRecord",
66
+ "RunStatus",
67
+ "RunStore",
68
+ "StoreStats",
69
+ "ToolTraceRow",
70
+ "divergences",
71
+ "make_exec_id",
72
+ "new_run_id",
73
+ ]
@@ -0,0 +1,308 @@
1
+ """Content-addressed storage for large payloads.
2
+
3
+ Full tool outputs and raw model responses do not belong in a database row. A
4
+ single recorded payload is replayed by every candidate on every repeat, so a
5
+ 10-case x 3-candidate x 3-repeat matrix would store the same JSON ninety times;
6
+ addressed by its SHA-256 it is stored once, and the rows hold a 64-character
7
+ hash. On a real benchmark that is the difference between a 40 MB file and a
8
+ 400 MB one, and the saving grows with exactly the thing we want users to do
9
+ more of - run the matrix again.
10
+
11
+ Four things here are deliberate rather than incidental.
12
+
13
+ **Writes are atomic.** Content is written to a temporary file and moved into
14
+ place with ``os.replace``. A Ctrl-C halfway through a write would otherwise
15
+ leave a truncated file *under the hash of its full contents* - a corrupt
16
+ payload that every future lookup trusts, because the name is the checksum and
17
+ we would never check it again.
18
+
19
+ **A digest is validated before it touches the filesystem.** A hash read back
20
+ from a row is still input: a hand-edited database with ``../../../etc/passwd``
21
+ in ``output_hash`` must get a refusal, not a file read. Cheap, and the kind of
22
+ thing that is embarrassing to add later.
23
+
24
+ **Files are sharded by the first two hex characters.** A benchmark produces
25
+ tens of thousands of blobs, and a single directory with 60,000 entries is slow
26
+ to enumerate on NTFS and hostile to anyone who opens it in a file manager.
27
+
28
+ **Garbage collection is mark-and-sweep with a grace period.** The database is
29
+ the only authority on what is referenced; the sweep deletes what it cannot find
30
+ a reference to. The grace period covers the gap between writing a blob and
31
+ committing the row that points at it - without it, pruning while a run is in
32
+ flight can delete a payload the run is about to reference.
33
+ """
34
+
35
+ from __future__ import annotations
36
+
37
+ import contextlib
38
+ import hashlib
39
+ import os
40
+ import re
41
+ import tempfile
42
+ from collections.abc import Iterable, Iterator
43
+ from dataclasses import dataclass
44
+ from datetime import UTC, datetime, timedelta
45
+ from pathlib import Path
46
+ from typing import Any, Final
47
+
48
+ from slmeval.errors import EnvironmentError_, SlmevalError
49
+ from slmeval.storage import codec
50
+
51
+ DIGEST_RE: Final = re.compile(r"^[0-9a-f]{64}$")
52
+
53
+ SWEEP_GRACE: Final = timedelta(hours=1)
54
+ """How recently a blob must have been touched to be spared by a sweep.
55
+
56
+ Generous on purpose. The cost of being wrong in one direction is a few
57
+ kilobytes of dead payload that the next prune collects; in the other it is a
58
+ report that cannot render a trace it has a row for. An hour is long enough to
59
+ cover any plausible gap between a blob write and its commit, including a run
60
+ paused by a sleeping laptop.
61
+ """
62
+
63
+
64
+ class MissingBlob(SlmevalError):
65
+ """A row points at a payload the store does not have.
66
+
67
+ Its own type because it has a specific cause worth naming in the message:
68
+ someone deleted ``.slmeval/blobs`` while keeping ``runs.sqlite``, or pruned
69
+ with a clock skewed far enough to defeat the grace period. Either way the
70
+ run's shape is intact and only this payload is gone, so the caller can
71
+ degrade to "payload unavailable" instead of failing the whole report.
72
+ """
73
+
74
+
75
+ @dataclass(frozen=True, slots=True)
76
+ class BlobStats:
77
+ count: int
78
+ size_bytes: int
79
+
80
+
81
+ @dataclass(frozen=True, slots=True)
82
+ class SweepResult:
83
+ """What a garbage collection pass did, and what it chose not to touch."""
84
+
85
+ deleted: int
86
+ freed_bytes: int
87
+ kept_referenced: int
88
+ kept_recent: int
89
+ """Unreferenced blobs spared by the grace period. A non-zero count here
90
+ while a run is in flight is the mechanism working, not a leak."""
91
+
92
+
93
+ class BlobStore:
94
+ """Payloads addressed by the SHA-256 of their bytes.
95
+
96
+ Thread-safe without a lock, which is the quiet benefit of content
97
+ addressing: two threads writing the same payload write identical bytes to
98
+ different temporary files and race only to rename, and either winner leaves
99
+ the correct content in place.
100
+ """
101
+
102
+ def __init__(self, root: Path | str) -> None:
103
+ self.root = Path(root)
104
+ self.writes = 0
105
+ self.deduped = 0
106
+ """Writes that found the payload already present. The ratio of these to
107
+ ``writes`` is what ``cache stats`` reports as the dedupe rate, and it is
108
+ the number that justifies this module existing."""
109
+
110
+ @classmethod
111
+ def for_project(cls, root: Path | str) -> BlobStore:
112
+ return cls(Path(root) / ".slmeval" / "blobs")
113
+
114
+ # ------------------------------------------------------------------ paths
115
+
116
+ def path_for(self, digest: str) -> Path:
117
+ """Where a digest lives. Validates first, always."""
118
+ if not DIGEST_RE.match(digest):
119
+ raise MissingBlob(
120
+ f"not a payload hash: {digest[:80]!r}",
121
+ hint="the trace store may be corrupt; `slmeval cache stats` reports its state",
122
+ )
123
+ return self.root / digest[:2] / digest[2:]
124
+
125
+ def exists(self, digest: str) -> bool:
126
+ return self.path_for(digest).is_file()
127
+
128
+ # ---------------------------------------------------------------- writing
129
+
130
+ def put(self, data: bytes) -> str:
131
+ """Store bytes, return their digest. A no-op if already present."""
132
+ digest = hashlib.sha256(data).hexdigest()
133
+ target = self.path_for(digest)
134
+ self.writes += 1
135
+ if target.is_file():
136
+ self.deduped += 1
137
+ # Touch it. The sweep spares recently-written blobs, and a payload
138
+ # re-referenced by a new run has to look new - otherwise pruning
139
+ # the old run that first wrote it can collect a blob the new run's
140
+ # uncommitted row already points at.
141
+ with contextlib.suppress(OSError):
142
+ os.utime(target)
143
+ return digest
144
+
145
+ try:
146
+ target.parent.mkdir(parents=True, exist_ok=True)
147
+ # Same directory as the target, so the move is a rename within one
148
+ # filesystem and therefore atomic. A temp file in the system temp
149
+ # directory would make this a copy across devices, which is neither
150
+ # atomic nor fast.
151
+ fd, tmp_name = tempfile.mkstemp(dir=target.parent, prefix=".tmp-")
152
+ tmp = Path(tmp_name)
153
+ try:
154
+ with os.fdopen(fd, "wb") as handle:
155
+ handle.write(data)
156
+ # Windows refuses to replace an open file, and will refuse the
157
+ # rename if another thread won the race to the same digest.
158
+ # Identical content, so that loser is a success.
159
+ os.replace(tmp, target)
160
+ except OSError:
161
+ tmp.unlink(missing_ok=True)
162
+ if target.is_file():
163
+ self.deduped += 1
164
+ return digest
165
+ raise
166
+ except OSError as exc:
167
+ raise EnvironmentError_(
168
+ f"cannot write to the payload store at {self.root}: {exc}",
169
+ hint="run `slmeval doctor` to check the cache directory is writable",
170
+ ) from exc
171
+ return digest
172
+
173
+ def put_text(self, text: str) -> str:
174
+ return self.put(text.encode("utf-8"))
175
+
176
+ def put_json(self, value: Any) -> str:
177
+ """Store a JSON-able value through the shared codec, so the same value
178
+ always hashes to the same digest."""
179
+ return self.put_text(codec.dumps(value))
180
+
181
+ # ---------------------------------------------------------------- reading
182
+
183
+ def get(self, digest: str) -> bytes:
184
+ path = self.path_for(digest)
185
+ try:
186
+ return path.read_bytes()
187
+ except FileNotFoundError as exc:
188
+ raise MissingBlob(
189
+ f"payload {digest[:12]} is referenced by a trace but not in the store",
190
+ hint="it was pruned or deleted; re-run the benchmark to regenerate it",
191
+ ) from exc
192
+ except OSError as exc:
193
+ raise EnvironmentError_(f"cannot read payload {digest[:12]}: {exc}") from exc
194
+
195
+ def get_text(self, digest: str) -> str:
196
+ return self.get(digest).decode("utf-8")
197
+
198
+ def get_json(self, digest: str) -> Any:
199
+ return codec.loads(self.get_text(digest))
200
+
201
+ def size_of(self, digest: str) -> int:
202
+ try:
203
+ return self.path_for(digest).stat().st_size
204
+ except OSError:
205
+ return 0
206
+
207
+ # ------------------------------------------------------- enumerate & stats
208
+
209
+ def iter_digests(self) -> Iterator[tuple[str, Path]]:
210
+ """Every stored payload. Skips temporary files from an interrupted
211
+ write and anything whose name is not a digest, so a stray file in the
212
+ blob directory is ignored rather than swept or reported as a payload."""
213
+ if not self.root.is_dir():
214
+ return
215
+ for shard in sorted(self.root.iterdir()):
216
+ if not shard.is_dir() or len(shard.name) != 2:
217
+ continue
218
+ for entry in shard.iterdir():
219
+ if not entry.is_file():
220
+ continue
221
+ digest = shard.name + entry.name
222
+ if DIGEST_RE.match(digest):
223
+ yield digest, entry
224
+
225
+ def stats(self) -> BlobStats:
226
+ count = 0
227
+ size = 0
228
+ for _digest, path in self.iter_digests():
229
+ count += 1
230
+ with contextlib.suppress(OSError):
231
+ size += path.stat().st_size
232
+ return BlobStats(count=count, size_bytes=size)
233
+
234
+ # ------------------------------------------------------------------ sweep
235
+
236
+ def sweep(
237
+ self,
238
+ referenced: Iterable[str],
239
+ *,
240
+ grace: timedelta = SWEEP_GRACE,
241
+ dry_run: bool = False,
242
+ ) -> SweepResult:
243
+ """Delete payloads nothing references.
244
+
245
+ ``referenced`` must be every hash in the database, not the hashes of
246
+ one run - a payload shared by three runs is referenced until the last
247
+ of them is gone, and that is the entire point of content addressing.
248
+ Passing a partial set deletes live data, so the only caller is
249
+ :meth:`RunStore.prune <slmeval.storage.db.RunStore.prune>`, which reads
250
+ the whole store.
251
+ """
252
+ keep = set(referenced)
253
+ # No grace means nothing is spared for being recent, so the comparison is
254
+ # skipped. Python's clock ticks every 15.6 ms on Windows before 3.13, and a
255
+ # file written just now can carry a later timestamp than that clock reads.
256
+ cutoff = datetime.now(UTC) - grace if grace > timedelta(0) else None
257
+ deleted = 0
258
+ freed = 0
259
+ kept_recent = 0
260
+ kept_referenced = 0
261
+
262
+ for digest, path in self.iter_digests():
263
+ if digest in keep:
264
+ kept_referenced += 1
265
+ continue
266
+ try:
267
+ stat = path.stat()
268
+ except OSError:
269
+ continue
270
+ if cutoff is not None and datetime.fromtimestamp(stat.st_mtime, UTC) > cutoff:
271
+ kept_recent += 1
272
+ continue
273
+ if not dry_run:
274
+ try:
275
+ path.unlink()
276
+ except OSError:
277
+ # A payload another process is reading, or already gone.
278
+ # Neither is a reason to abandon the rest of the sweep.
279
+ continue
280
+ deleted += 1
281
+ freed += stat.st_size
282
+
283
+ if not dry_run:
284
+ self._drop_empty_shards()
285
+ return SweepResult(
286
+ deleted=deleted,
287
+ freed_bytes=freed,
288
+ kept_referenced=kept_referenced,
289
+ kept_recent=kept_recent,
290
+ )
291
+
292
+ def _drop_empty_shards(self) -> None:
293
+ if not self.root.is_dir():
294
+ return
295
+ for shard in self.root.iterdir():
296
+ if shard.is_dir() and len(shard.name) == 2:
297
+ with contextlib.suppress(OSError):
298
+ shard.rmdir()
299
+
300
+
301
+ __all__ = [
302
+ "DIGEST_RE",
303
+ "SWEEP_GRACE",
304
+ "BlobStats",
305
+ "BlobStore",
306
+ "MissingBlob",
307
+ "SweepResult",
308
+ ]
@@ -0,0 +1,93 @@
1
+ """JSON in and out of columns, one way only.
2
+
3
+ Every column that holds JSON is written through these helpers rather than with
4
+ a bare ``json.dumps``, for two reasons. Compact separators, because at ~40
5
+ JSON columns per execution the whitespace in the default separators is real
6
+ megabytes across a benchmark. And ``sort_keys``, because a column written
7
+ differently by two code paths breaks the one thing content addressing buys us:
8
+ identical payloads have to hash identically or the blob store stops deduping.
9
+
10
+ Reads are forgiving by design. A row written by an older build, or hand-edited
11
+ by a user poking at their own SQLite file, must not take down a report - a
12
+ malformed JSON column degrades to the empty value and the rest of the timeline
13
+ still prints.
14
+ """
15
+
16
+ from __future__ import annotations
17
+
18
+ import json
19
+ from collections.abc import Iterable, Sequence
20
+ from typing import Any
21
+
22
+ _SEPARATORS = (",", ":")
23
+
24
+
25
+ def dumps(value: Any) -> str:
26
+ """Compact, key-sorted JSON. The only writer for a JSON column."""
27
+ return json.dumps(value, separators=_SEPARATORS, sort_keys=True, default=str)
28
+
29
+
30
+ def loads(text: str | None, default: Any = None) -> Any:
31
+ """Parse a JSON column, falling back rather than raising."""
32
+ if not text:
33
+ return default
34
+ try:
35
+ return json.loads(text)
36
+ except (TypeError, ValueError):
37
+ return default
38
+
39
+
40
+ def encode_paths(paths: Iterable[Sequence[str] | str] | None) -> str:
41
+ """Serialize node paths - a sequence of name tuples.
42
+
43
+ A bare string is accepted as a one-element path, because a nested node's
44
+ full path and a top-level node's name are the same thing at different
45
+ depths and callers should not have to wrap one of them.
46
+ """
47
+ if not paths:
48
+ return "[]"
49
+ out: list[list[str]] = []
50
+ for path in paths:
51
+ if isinstance(path, str):
52
+ out.append([path])
53
+ else:
54
+ out.append([str(part) for part in path])
55
+ return dumps(out)
56
+
57
+
58
+ def decode_paths(text: str | None) -> tuple[tuple[str, ...], ...]:
59
+ """Rebuild node paths. Tolerates the flat ``["a", "b"]`` shape an older
60
+ build wrote, reading each entry as a one-element path."""
61
+ raw = loads(text, [])
62
+ if not isinstance(raw, list):
63
+ return ()
64
+ out: list[tuple[str, ...]] = []
65
+ for item in raw:
66
+ if isinstance(item, str):
67
+ out.append((item,))
68
+ elif isinstance(item, list):
69
+ out.append(tuple(str(part) for part in item))
70
+ return tuple(out)
71
+
72
+
73
+ def encode_strings(values: Iterable[object] | None) -> str:
74
+ if not values:
75
+ return "[]"
76
+ return dumps([str(v) for v in values])
77
+
78
+
79
+ def decode_strings(text: str | None) -> tuple[str, ...]:
80
+ raw = loads(text, [])
81
+ if not isinstance(raw, list):
82
+ return ()
83
+ return tuple(str(v) for v in raw)
84
+
85
+
86
+ __all__ = [
87
+ "decode_paths",
88
+ "decode_strings",
89
+ "dumps",
90
+ "encode_paths",
91
+ "encode_strings",
92
+ "loads",
93
+ ]