slmeval-cli 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- slmeval/__init__.py +71 -0
- slmeval/__main__.py +14 -0
- slmeval/adapters/__init__.py +24 -0
- slmeval/adapters/instrument.py +135 -0
- slmeval/adapters/langgraph.py +237 -0
- slmeval/cli/__init__.py +15 -0
- slmeval/cli/commands/__init__.py +35 -0
- slmeval/cli/commands/baseline_cmd.py +201 -0
- slmeval/cli/commands/cache_cmd.py +249 -0
- slmeval/cli/commands/dataset_cmd.py +52 -0
- slmeval/cli/commands/doctor_cmd.py +68 -0
- slmeval/cli/commands/gate_cmd.py +602 -0
- slmeval/cli/commands/init_cmd.py +144 -0
- slmeval/cli/commands/judge_cmd.py +284 -0
- slmeval/cli/commands/record_cmd.py +154 -0
- slmeval/cli/commands/report_cmd.py +98 -0
- slmeval/cli/commands/run_cmd.py +247 -0
- slmeval/cli/commands/show_cmd.py +482 -0
- slmeval/cli/commands/stubs.py +57 -0
- slmeval/cli/commands/validate_cmd.py +296 -0
- slmeval/cli/loading.py +149 -0
- slmeval/cli/main.py +91 -0
- slmeval/cli/render.py +251 -0
- slmeval/cli/wrap.py +31 -0
- slmeval/config/__init__.py +47 -0
- slmeval/config/loader.py +363 -0
- slmeval/config/locate.py +97 -0
- slmeval/config/models.py +430 -0
- slmeval/config/templates.py +246 -0
- slmeval/core/__init__.py +78 -0
- slmeval/core/adapter.py +253 -0
- slmeval/core/context.py +418 -0
- slmeval/core/outcome.py +260 -0
- slmeval/core/router.py +319 -0
- slmeval/core/runner.py +1209 -0
- slmeval/core/trace.py +188 -0
- slmeval/core/verdict.py +127 -0
- slmeval/dataset/__init__.py +8 -0
- slmeval/dataset/loader.py +130 -0
- slmeval/dataset/models.py +108 -0
- slmeval/doctor/__init__.py +7 -0
- slmeval/doctor/checks.py +380 -0
- slmeval/errors.py +76 -0
- slmeval/eval/__init__.py +64 -0
- slmeval/eval/assertions.py +356 -0
- slmeval/eval/calibration.py +364 -0
- slmeval/eval/cost.py +291 -0
- slmeval/eval/pipeline.py +273 -0
- slmeval/eval/stats.py +376 -0
- slmeval/eval/tier1.py +510 -0
- slmeval/eval/tier2.py +582 -0
- slmeval/eval/tier3.py +415 -0
- slmeval/exit_codes.py +28 -0
- slmeval/gate.py +722 -0
- slmeval/providers/__init__.py +69 -0
- slmeval/providers/base.py +426 -0
- slmeval/providers/fake.py +760 -0
- slmeval/providers/jsonx.py +239 -0
- slmeval/providers/ladder.py +339 -0
- slmeval/providers/openai_compat.py +392 -0
- slmeval/providers/registry.py +185 -0
- slmeval/providers/retry.py +81 -0
- slmeval/providers/schema.py +136 -0
- slmeval/providers/tokens.py +150 -0
- slmeval/py.typed +0 -0
- slmeval/report/__init__.py +115 -0
- slmeval/report/attribution.py +197 -0
- slmeval/report/baseline.py +353 -0
- slmeval/report/drilldown.py +1395 -0
- slmeval/report/export.py +450 -0
- slmeval/report/model.py +562 -0
- slmeval/report/progress.py +257 -0
- slmeval/report/scorecard.py +367 -0
- slmeval/sandbox/__init__.py +145 -0
- slmeval/sandbox/diagnosis.py +690 -0
- slmeval/sandbox/envelope.py +486 -0
- slmeval/sandbox/faults.py +794 -0
- slmeval/sandbox/keys.py +458 -0
- slmeval/sandbox/normalizers.py +333 -0
- slmeval/sandbox/store.py +525 -0
- slmeval/sandbox/tool.py +685 -0
- slmeval/storage/__init__.py +73 -0
- slmeval/storage/blobs.py +308 -0
- slmeval/storage/codec.py +93 -0
- slmeval/storage/db.py +1334 -0
- slmeval/storage/ingest.py +346 -0
- slmeval/storage/migrations.py +358 -0
- slmeval/storage/models.py +554 -0
- slmeval/storage/provenance.py +201 -0
- slmeval_cli-0.1.0.dist-info/METADATA +484 -0
- slmeval_cli-0.1.0.dist-info/RECORD +94 -0
- slmeval_cli-0.1.0.dist-info/WHEEL +4 -0
- slmeval_cli-0.1.0.dist-info/entry_points.txt +2 -0
- slmeval_cli-0.1.0.dist-info/licenses/LICENSE +201 -0
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
"""Trace storage: one SQLite file plus a content-addressed blob directory.
|
|
2
|
+
|
|
3
|
+
Traces are the asset. Everything the tool prints - the scorecard, the
|
|
4
|
+
drill-down diff, the CI gate - is a query over these rows, so the contract is
|
|
5
|
+
that a completed run is fully reconstructible from storage with every in-memory
|
|
6
|
+
object thrown away.
|
|
7
|
+
|
|
8
|
+
from slmeval.storage import RunStore, new_run_id
|
|
9
|
+
|
|
10
|
+
with RunStore.for_project(root) as store:
|
|
11
|
+
store.save_run(record)
|
|
12
|
+
...
|
|
13
|
+
detail = store.reconstruct(exec_id)
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
from __future__ import annotations
|
|
17
|
+
|
|
18
|
+
from slmeval.storage.blobs import BlobStore, MissingBlob
|
|
19
|
+
from slmeval.storage.db import DB_NAME, RunNotFound, RunStore, make_exec_id
|
|
20
|
+
from slmeval.storage.migrations import SCHEMA_VERSION
|
|
21
|
+
from slmeval.storage.models import (
|
|
22
|
+
NOT_THE_MODELS_FAULT,
|
|
23
|
+
BlameRow,
|
|
24
|
+
CandidateRecord,
|
|
25
|
+
CaseRollupRow,
|
|
26
|
+
EvaluationRow,
|
|
27
|
+
ExecStatus,
|
|
28
|
+
ExecutionDetail,
|
|
29
|
+
ExecutionRecord,
|
|
30
|
+
MetricSample,
|
|
31
|
+
NodeSummaryRow,
|
|
32
|
+
NodeTraceRow,
|
|
33
|
+
Outcome,
|
|
34
|
+
Payload,
|
|
35
|
+
RunDetail,
|
|
36
|
+
RunRecord,
|
|
37
|
+
RunStatus,
|
|
38
|
+
StoreStats,
|
|
39
|
+
ToolTraceRow,
|
|
40
|
+
)
|
|
41
|
+
from slmeval.storage.provenance import Divergence, Provenance, divergences, new_run_id
|
|
42
|
+
|
|
43
|
+
__all__ = [
|
|
44
|
+
"DB_NAME",
|
|
45
|
+
"NOT_THE_MODELS_FAULT",
|
|
46
|
+
"SCHEMA_VERSION",
|
|
47
|
+
"BlameRow",
|
|
48
|
+
"BlobStore",
|
|
49
|
+
"CandidateRecord",
|
|
50
|
+
"CaseRollupRow",
|
|
51
|
+
"Divergence",
|
|
52
|
+
"EvaluationRow",
|
|
53
|
+
"ExecStatus",
|
|
54
|
+
"ExecutionDetail",
|
|
55
|
+
"ExecutionRecord",
|
|
56
|
+
"MetricSample",
|
|
57
|
+
"MissingBlob",
|
|
58
|
+
"NodeSummaryRow",
|
|
59
|
+
"NodeTraceRow",
|
|
60
|
+
"Outcome",
|
|
61
|
+
"Payload",
|
|
62
|
+
"Provenance",
|
|
63
|
+
"RunDetail",
|
|
64
|
+
"RunNotFound",
|
|
65
|
+
"RunRecord",
|
|
66
|
+
"RunStatus",
|
|
67
|
+
"RunStore",
|
|
68
|
+
"StoreStats",
|
|
69
|
+
"ToolTraceRow",
|
|
70
|
+
"divergences",
|
|
71
|
+
"make_exec_id",
|
|
72
|
+
"new_run_id",
|
|
73
|
+
]
|
slmeval/storage/blobs.py
ADDED
|
@@ -0,0 +1,308 @@
|
|
|
1
|
+
"""Content-addressed storage for large payloads.
|
|
2
|
+
|
|
3
|
+
Full tool outputs and raw model responses do not belong in a database row. A
|
|
4
|
+
single recorded payload is replayed by every candidate on every repeat, so a
|
|
5
|
+
10-case x 3-candidate x 3-repeat matrix would store the same JSON ninety times;
|
|
6
|
+
addressed by its SHA-256 it is stored once, and the rows hold a 64-character
|
|
7
|
+
hash. On a real benchmark that is the difference between a 40 MB file and a
|
|
8
|
+
400 MB one, and the saving grows with exactly the thing we want users to do
|
|
9
|
+
more of - run the matrix again.
|
|
10
|
+
|
|
11
|
+
Four things here are deliberate rather than incidental.
|
|
12
|
+
|
|
13
|
+
**Writes are atomic.** Content is written to a temporary file and moved into
|
|
14
|
+
place with ``os.replace``. A Ctrl-C halfway through a write would otherwise
|
|
15
|
+
leave a truncated file *under the hash of its full contents* - a corrupt
|
|
16
|
+
payload that every future lookup trusts, because the name is the checksum and
|
|
17
|
+
we would never check it again.
|
|
18
|
+
|
|
19
|
+
**A digest is validated before it touches the filesystem.** A hash read back
|
|
20
|
+
from a row is still input: a hand-edited database with ``../../../etc/passwd``
|
|
21
|
+
in ``output_hash`` must get a refusal, not a file read. Cheap, and the kind of
|
|
22
|
+
thing that is embarrassing to add later.
|
|
23
|
+
|
|
24
|
+
**Files are sharded by the first two hex characters.** A benchmark produces
|
|
25
|
+
tens of thousands of blobs, and a single directory with 60,000 entries is slow
|
|
26
|
+
to enumerate on NTFS and hostile to anyone who opens it in a file manager.
|
|
27
|
+
|
|
28
|
+
**Garbage collection is mark-and-sweep with a grace period.** The database is
|
|
29
|
+
the only authority on what is referenced; the sweep deletes what it cannot find
|
|
30
|
+
a reference to. The grace period covers the gap between writing a blob and
|
|
31
|
+
committing the row that points at it - without it, pruning while a run is in
|
|
32
|
+
flight can delete a payload the run is about to reference.
|
|
33
|
+
"""
|
|
34
|
+
|
|
35
|
+
from __future__ import annotations
|
|
36
|
+
|
|
37
|
+
import contextlib
|
|
38
|
+
import hashlib
|
|
39
|
+
import os
|
|
40
|
+
import re
|
|
41
|
+
import tempfile
|
|
42
|
+
from collections.abc import Iterable, Iterator
|
|
43
|
+
from dataclasses import dataclass
|
|
44
|
+
from datetime import UTC, datetime, timedelta
|
|
45
|
+
from pathlib import Path
|
|
46
|
+
from typing import Any, Final
|
|
47
|
+
|
|
48
|
+
from slmeval.errors import EnvironmentError_, SlmevalError
|
|
49
|
+
from slmeval.storage import codec
|
|
50
|
+
|
|
51
|
+
DIGEST_RE: Final = re.compile(r"^[0-9a-f]{64}$")
|
|
52
|
+
|
|
53
|
+
SWEEP_GRACE: Final = timedelta(hours=1)
|
|
54
|
+
"""How recently a blob must have been touched to be spared by a sweep.
|
|
55
|
+
|
|
56
|
+
Generous on purpose. The cost of being wrong in one direction is a few
|
|
57
|
+
kilobytes of dead payload that the next prune collects; in the other it is a
|
|
58
|
+
report that cannot render a trace it has a row for. An hour is long enough to
|
|
59
|
+
cover any plausible gap between a blob write and its commit, including a run
|
|
60
|
+
paused by a sleeping laptop.
|
|
61
|
+
"""
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
class MissingBlob(SlmevalError):
|
|
65
|
+
"""A row points at a payload the store does not have.
|
|
66
|
+
|
|
67
|
+
Its own type because it has a specific cause worth naming in the message:
|
|
68
|
+
someone deleted ``.slmeval/blobs`` while keeping ``runs.sqlite``, or pruned
|
|
69
|
+
with a clock skewed far enough to defeat the grace period. Either way the
|
|
70
|
+
run's shape is intact and only this payload is gone, so the caller can
|
|
71
|
+
degrade to "payload unavailable" instead of failing the whole report.
|
|
72
|
+
"""
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
@dataclass(frozen=True, slots=True)
|
|
76
|
+
class BlobStats:
|
|
77
|
+
count: int
|
|
78
|
+
size_bytes: int
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
@dataclass(frozen=True, slots=True)
|
|
82
|
+
class SweepResult:
|
|
83
|
+
"""What a garbage collection pass did, and what it chose not to touch."""
|
|
84
|
+
|
|
85
|
+
deleted: int
|
|
86
|
+
freed_bytes: int
|
|
87
|
+
kept_referenced: int
|
|
88
|
+
kept_recent: int
|
|
89
|
+
"""Unreferenced blobs spared by the grace period. A non-zero count here
|
|
90
|
+
while a run is in flight is the mechanism working, not a leak."""
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
class BlobStore:
|
|
94
|
+
"""Payloads addressed by the SHA-256 of their bytes.
|
|
95
|
+
|
|
96
|
+
Thread-safe without a lock, which is the quiet benefit of content
|
|
97
|
+
addressing: two threads writing the same payload write identical bytes to
|
|
98
|
+
different temporary files and race only to rename, and either winner leaves
|
|
99
|
+
the correct content in place.
|
|
100
|
+
"""
|
|
101
|
+
|
|
102
|
+
def __init__(self, root: Path | str) -> None:
|
|
103
|
+
self.root = Path(root)
|
|
104
|
+
self.writes = 0
|
|
105
|
+
self.deduped = 0
|
|
106
|
+
"""Writes that found the payload already present. The ratio of these to
|
|
107
|
+
``writes`` is what ``cache stats`` reports as the dedupe rate, and it is
|
|
108
|
+
the number that justifies this module existing."""
|
|
109
|
+
|
|
110
|
+
@classmethod
|
|
111
|
+
def for_project(cls, root: Path | str) -> BlobStore:
|
|
112
|
+
return cls(Path(root) / ".slmeval" / "blobs")
|
|
113
|
+
|
|
114
|
+
# ------------------------------------------------------------------ paths
|
|
115
|
+
|
|
116
|
+
def path_for(self, digest: str) -> Path:
|
|
117
|
+
"""Where a digest lives. Validates first, always."""
|
|
118
|
+
if not DIGEST_RE.match(digest):
|
|
119
|
+
raise MissingBlob(
|
|
120
|
+
f"not a payload hash: {digest[:80]!r}",
|
|
121
|
+
hint="the trace store may be corrupt; `slmeval cache stats` reports its state",
|
|
122
|
+
)
|
|
123
|
+
return self.root / digest[:2] / digest[2:]
|
|
124
|
+
|
|
125
|
+
def exists(self, digest: str) -> bool:
|
|
126
|
+
return self.path_for(digest).is_file()
|
|
127
|
+
|
|
128
|
+
# ---------------------------------------------------------------- writing
|
|
129
|
+
|
|
130
|
+
def put(self, data: bytes) -> str:
|
|
131
|
+
"""Store bytes, return their digest. A no-op if already present."""
|
|
132
|
+
digest = hashlib.sha256(data).hexdigest()
|
|
133
|
+
target = self.path_for(digest)
|
|
134
|
+
self.writes += 1
|
|
135
|
+
if target.is_file():
|
|
136
|
+
self.deduped += 1
|
|
137
|
+
# Touch it. The sweep spares recently-written blobs, and a payload
|
|
138
|
+
# re-referenced by a new run has to look new - otherwise pruning
|
|
139
|
+
# the old run that first wrote it can collect a blob the new run's
|
|
140
|
+
# uncommitted row already points at.
|
|
141
|
+
with contextlib.suppress(OSError):
|
|
142
|
+
os.utime(target)
|
|
143
|
+
return digest
|
|
144
|
+
|
|
145
|
+
try:
|
|
146
|
+
target.parent.mkdir(parents=True, exist_ok=True)
|
|
147
|
+
# Same directory as the target, so the move is a rename within one
|
|
148
|
+
# filesystem and therefore atomic. A temp file in the system temp
|
|
149
|
+
# directory would make this a copy across devices, which is neither
|
|
150
|
+
# atomic nor fast.
|
|
151
|
+
fd, tmp_name = tempfile.mkstemp(dir=target.parent, prefix=".tmp-")
|
|
152
|
+
tmp = Path(tmp_name)
|
|
153
|
+
try:
|
|
154
|
+
with os.fdopen(fd, "wb") as handle:
|
|
155
|
+
handle.write(data)
|
|
156
|
+
# Windows refuses to replace an open file, and will refuse the
|
|
157
|
+
# rename if another thread won the race to the same digest.
|
|
158
|
+
# Identical content, so that loser is a success.
|
|
159
|
+
os.replace(tmp, target)
|
|
160
|
+
except OSError:
|
|
161
|
+
tmp.unlink(missing_ok=True)
|
|
162
|
+
if target.is_file():
|
|
163
|
+
self.deduped += 1
|
|
164
|
+
return digest
|
|
165
|
+
raise
|
|
166
|
+
except OSError as exc:
|
|
167
|
+
raise EnvironmentError_(
|
|
168
|
+
f"cannot write to the payload store at {self.root}: {exc}",
|
|
169
|
+
hint="run `slmeval doctor` to check the cache directory is writable",
|
|
170
|
+
) from exc
|
|
171
|
+
return digest
|
|
172
|
+
|
|
173
|
+
def put_text(self, text: str) -> str:
|
|
174
|
+
return self.put(text.encode("utf-8"))
|
|
175
|
+
|
|
176
|
+
def put_json(self, value: Any) -> str:
|
|
177
|
+
"""Store a JSON-able value through the shared codec, so the same value
|
|
178
|
+
always hashes to the same digest."""
|
|
179
|
+
return self.put_text(codec.dumps(value))
|
|
180
|
+
|
|
181
|
+
# ---------------------------------------------------------------- reading
|
|
182
|
+
|
|
183
|
+
def get(self, digest: str) -> bytes:
|
|
184
|
+
path = self.path_for(digest)
|
|
185
|
+
try:
|
|
186
|
+
return path.read_bytes()
|
|
187
|
+
except FileNotFoundError as exc:
|
|
188
|
+
raise MissingBlob(
|
|
189
|
+
f"payload {digest[:12]} is referenced by a trace but not in the store",
|
|
190
|
+
hint="it was pruned or deleted; re-run the benchmark to regenerate it",
|
|
191
|
+
) from exc
|
|
192
|
+
except OSError as exc:
|
|
193
|
+
raise EnvironmentError_(f"cannot read payload {digest[:12]}: {exc}") from exc
|
|
194
|
+
|
|
195
|
+
def get_text(self, digest: str) -> str:
|
|
196
|
+
return self.get(digest).decode("utf-8")
|
|
197
|
+
|
|
198
|
+
def get_json(self, digest: str) -> Any:
|
|
199
|
+
return codec.loads(self.get_text(digest))
|
|
200
|
+
|
|
201
|
+
def size_of(self, digest: str) -> int:
|
|
202
|
+
try:
|
|
203
|
+
return self.path_for(digest).stat().st_size
|
|
204
|
+
except OSError:
|
|
205
|
+
return 0
|
|
206
|
+
|
|
207
|
+
# ------------------------------------------------------- enumerate & stats
|
|
208
|
+
|
|
209
|
+
def iter_digests(self) -> Iterator[tuple[str, Path]]:
|
|
210
|
+
"""Every stored payload. Skips temporary files from an interrupted
|
|
211
|
+
write and anything whose name is not a digest, so a stray file in the
|
|
212
|
+
blob directory is ignored rather than swept or reported as a payload."""
|
|
213
|
+
if not self.root.is_dir():
|
|
214
|
+
return
|
|
215
|
+
for shard in sorted(self.root.iterdir()):
|
|
216
|
+
if not shard.is_dir() or len(shard.name) != 2:
|
|
217
|
+
continue
|
|
218
|
+
for entry in shard.iterdir():
|
|
219
|
+
if not entry.is_file():
|
|
220
|
+
continue
|
|
221
|
+
digest = shard.name + entry.name
|
|
222
|
+
if DIGEST_RE.match(digest):
|
|
223
|
+
yield digest, entry
|
|
224
|
+
|
|
225
|
+
def stats(self) -> BlobStats:
|
|
226
|
+
count = 0
|
|
227
|
+
size = 0
|
|
228
|
+
for _digest, path in self.iter_digests():
|
|
229
|
+
count += 1
|
|
230
|
+
with contextlib.suppress(OSError):
|
|
231
|
+
size += path.stat().st_size
|
|
232
|
+
return BlobStats(count=count, size_bytes=size)
|
|
233
|
+
|
|
234
|
+
# ------------------------------------------------------------------ sweep
|
|
235
|
+
|
|
236
|
+
def sweep(
|
|
237
|
+
self,
|
|
238
|
+
referenced: Iterable[str],
|
|
239
|
+
*,
|
|
240
|
+
grace: timedelta = SWEEP_GRACE,
|
|
241
|
+
dry_run: bool = False,
|
|
242
|
+
) -> SweepResult:
|
|
243
|
+
"""Delete payloads nothing references.
|
|
244
|
+
|
|
245
|
+
``referenced`` must be every hash in the database, not the hashes of
|
|
246
|
+
one run - a payload shared by three runs is referenced until the last
|
|
247
|
+
of them is gone, and that is the entire point of content addressing.
|
|
248
|
+
Passing a partial set deletes live data, so the only caller is
|
|
249
|
+
:meth:`RunStore.prune <slmeval.storage.db.RunStore.prune>`, which reads
|
|
250
|
+
the whole store.
|
|
251
|
+
"""
|
|
252
|
+
keep = set(referenced)
|
|
253
|
+
# No grace means nothing is spared for being recent, so the comparison is
|
|
254
|
+
# skipped. Python's clock ticks every 15.6 ms on Windows before 3.13, and a
|
|
255
|
+
# file written just now can carry a later timestamp than that clock reads.
|
|
256
|
+
cutoff = datetime.now(UTC) - grace if grace > timedelta(0) else None
|
|
257
|
+
deleted = 0
|
|
258
|
+
freed = 0
|
|
259
|
+
kept_recent = 0
|
|
260
|
+
kept_referenced = 0
|
|
261
|
+
|
|
262
|
+
for digest, path in self.iter_digests():
|
|
263
|
+
if digest in keep:
|
|
264
|
+
kept_referenced += 1
|
|
265
|
+
continue
|
|
266
|
+
try:
|
|
267
|
+
stat = path.stat()
|
|
268
|
+
except OSError:
|
|
269
|
+
continue
|
|
270
|
+
if cutoff is not None and datetime.fromtimestamp(stat.st_mtime, UTC) > cutoff:
|
|
271
|
+
kept_recent += 1
|
|
272
|
+
continue
|
|
273
|
+
if not dry_run:
|
|
274
|
+
try:
|
|
275
|
+
path.unlink()
|
|
276
|
+
except OSError:
|
|
277
|
+
# A payload another process is reading, or already gone.
|
|
278
|
+
# Neither is a reason to abandon the rest of the sweep.
|
|
279
|
+
continue
|
|
280
|
+
deleted += 1
|
|
281
|
+
freed += stat.st_size
|
|
282
|
+
|
|
283
|
+
if not dry_run:
|
|
284
|
+
self._drop_empty_shards()
|
|
285
|
+
return SweepResult(
|
|
286
|
+
deleted=deleted,
|
|
287
|
+
freed_bytes=freed,
|
|
288
|
+
kept_referenced=kept_referenced,
|
|
289
|
+
kept_recent=kept_recent,
|
|
290
|
+
)
|
|
291
|
+
|
|
292
|
+
def _drop_empty_shards(self) -> None:
|
|
293
|
+
if not self.root.is_dir():
|
|
294
|
+
return
|
|
295
|
+
for shard in self.root.iterdir():
|
|
296
|
+
if shard.is_dir() and len(shard.name) == 2:
|
|
297
|
+
with contextlib.suppress(OSError):
|
|
298
|
+
shard.rmdir()
|
|
299
|
+
|
|
300
|
+
|
|
301
|
+
__all__ = [
|
|
302
|
+
"DIGEST_RE",
|
|
303
|
+
"SWEEP_GRACE",
|
|
304
|
+
"BlobStats",
|
|
305
|
+
"BlobStore",
|
|
306
|
+
"MissingBlob",
|
|
307
|
+
"SweepResult",
|
|
308
|
+
]
|
slmeval/storage/codec.py
ADDED
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
"""JSON in and out of columns, one way only.
|
|
2
|
+
|
|
3
|
+
Every column that holds JSON is written through these helpers rather than with
|
|
4
|
+
a bare ``json.dumps``, for two reasons. Compact separators, because at ~40
|
|
5
|
+
JSON columns per execution the whitespace in the default separators is real
|
|
6
|
+
megabytes across a benchmark. And ``sort_keys``, because a column written
|
|
7
|
+
differently by two code paths breaks the one thing content addressing buys us:
|
|
8
|
+
identical payloads have to hash identically or the blob store stops deduping.
|
|
9
|
+
|
|
10
|
+
Reads are forgiving by design. A row written by an older build, or hand-edited
|
|
11
|
+
by a user poking at their own SQLite file, must not take down a report - a
|
|
12
|
+
malformed JSON column degrades to the empty value and the rest of the timeline
|
|
13
|
+
still prints.
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
from __future__ import annotations
|
|
17
|
+
|
|
18
|
+
import json
|
|
19
|
+
from collections.abc import Iterable, Sequence
|
|
20
|
+
from typing import Any
|
|
21
|
+
|
|
22
|
+
_SEPARATORS = (",", ":")
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def dumps(value: Any) -> str:
|
|
26
|
+
"""Compact, key-sorted JSON. The only writer for a JSON column."""
|
|
27
|
+
return json.dumps(value, separators=_SEPARATORS, sort_keys=True, default=str)
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def loads(text: str | None, default: Any = None) -> Any:
|
|
31
|
+
"""Parse a JSON column, falling back rather than raising."""
|
|
32
|
+
if not text:
|
|
33
|
+
return default
|
|
34
|
+
try:
|
|
35
|
+
return json.loads(text)
|
|
36
|
+
except (TypeError, ValueError):
|
|
37
|
+
return default
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def encode_paths(paths: Iterable[Sequence[str] | str] | None) -> str:
|
|
41
|
+
"""Serialize node paths - a sequence of name tuples.
|
|
42
|
+
|
|
43
|
+
A bare string is accepted as a one-element path, because a nested node's
|
|
44
|
+
full path and a top-level node's name are the same thing at different
|
|
45
|
+
depths and callers should not have to wrap one of them.
|
|
46
|
+
"""
|
|
47
|
+
if not paths:
|
|
48
|
+
return "[]"
|
|
49
|
+
out: list[list[str]] = []
|
|
50
|
+
for path in paths:
|
|
51
|
+
if isinstance(path, str):
|
|
52
|
+
out.append([path])
|
|
53
|
+
else:
|
|
54
|
+
out.append([str(part) for part in path])
|
|
55
|
+
return dumps(out)
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def decode_paths(text: str | None) -> tuple[tuple[str, ...], ...]:
|
|
59
|
+
"""Rebuild node paths. Tolerates the flat ``["a", "b"]`` shape an older
|
|
60
|
+
build wrote, reading each entry as a one-element path."""
|
|
61
|
+
raw = loads(text, [])
|
|
62
|
+
if not isinstance(raw, list):
|
|
63
|
+
return ()
|
|
64
|
+
out: list[tuple[str, ...]] = []
|
|
65
|
+
for item in raw:
|
|
66
|
+
if isinstance(item, str):
|
|
67
|
+
out.append((item,))
|
|
68
|
+
elif isinstance(item, list):
|
|
69
|
+
out.append(tuple(str(part) for part in item))
|
|
70
|
+
return tuple(out)
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def encode_strings(values: Iterable[object] | None) -> str:
|
|
74
|
+
if not values:
|
|
75
|
+
return "[]"
|
|
76
|
+
return dumps([str(v) for v in values])
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def decode_strings(text: str | None) -> tuple[str, ...]:
|
|
80
|
+
raw = loads(text, [])
|
|
81
|
+
if not isinstance(raw, list):
|
|
82
|
+
return ()
|
|
83
|
+
return tuple(str(v) for v in raw)
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
__all__ = [
|
|
87
|
+
"decode_paths",
|
|
88
|
+
"decode_strings",
|
|
89
|
+
"dumps",
|
|
90
|
+
"encode_paths",
|
|
91
|
+
"encode_strings",
|
|
92
|
+
"loads",
|
|
93
|
+
]
|