skillbench-cli 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- skillbench/__init__.py +1 -0
- skillbench/assertions.py +250 -0
- skillbench/claude.py +277 -0
- skillbench/cli.py +323 -0
- skillbench/config.py +269 -0
- skillbench/dashboard.py +543 -0
- skillbench/progress.py +219 -0
- skillbench/runner.py +471 -0
- skillbench/scaffold.py +106 -0
- skillbench/skills.py +168 -0
- skillbench/workspace.py +124 -0
- skillbench_cli-0.1.0.dist-info/METADATA +218 -0
- skillbench_cli-0.1.0.dist-info/RECORD +16 -0
- skillbench_cli-0.1.0.dist-info/WHEEL +4 -0
- skillbench_cli-0.1.0.dist-info/entry_points.txt +2 -0
- skillbench_cli-0.1.0.dist-info/licenses/LICENSE +21 -0
skillbench/dashboard.py
ADDED
|
@@ -0,0 +1,543 @@
|
|
|
1
|
+
"""The skillbench/runs/index.html dashboard: every run, newest first, each selectable
|
|
2
|
+
for its full timeline.
|
|
3
|
+
|
|
4
|
+
One static file with every run's data embedded and no scripts, so it opens from disk. Selecting
|
|
5
|
+
a run is a link to its detail section, which CSS `:target` reveals.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
import json
|
|
9
|
+
from collections import Counter
|
|
10
|
+
from dataclasses import dataclass
|
|
11
|
+
from datetime import datetime
|
|
12
|
+
from html import escape
|
|
13
|
+
from pathlib import Path
|
|
14
|
+
from typing import Any
|
|
15
|
+
|
|
16
|
+
INDEX_FILE_NAME = "index.html"
|
|
17
|
+
_RUN_RECORD_FILE_NAME = "run.json"
|
|
18
|
+
_EMPTY_STATE = (
|
|
19
|
+
'<section class="empty-state"><h2>No runs yet</h2>'
|
|
20
|
+
"<p>Run <code>skillbench skill</code> or <code>skillbench agent</code>. Each run is added "
|
|
21
|
+
"here with its full timeline.</p></section>"
|
|
22
|
+
)
|
|
23
|
+
_COLUMNS = ("date", "test", "backend", "model", "status", "duration", "turns", "cost / usage")
|
|
24
|
+
|
|
25
|
+
# Master-detail: the runs table, restyled as a pinned list, beside the selected run. A run is
|
|
26
|
+
# selected by its link (CSS :target); with none selected the newest run shows.
|
|
27
|
+
_DASHBOARD_STYLE = """
|
|
28
|
+
.dashboard { display: grid; grid-template-columns: minmax(16rem, 21rem) 1fr; min-height: 100vh; }
|
|
29
|
+
.runs { position: sticky; top: 0; height: 100vh; overflow-y: auto; background: var(--panel);
|
|
30
|
+
border-right: 1px solid var(--rule); }
|
|
31
|
+
.runs header { padding: 1.5rem 1.25rem 1rem; border-bottom: 1px solid var(--rule); }
|
|
32
|
+
.runs h1 { font-size: 1.6rem; font-weight: 700; letter-spacing: -.01em; }
|
|
33
|
+
.runs header p { margin: .2rem 0 0; color: var(--muted); font-size: .85rem; }
|
|
34
|
+
.runs table, .runs tbody { display: block; border-collapse: collapse; }
|
|
35
|
+
.runs thead { position: absolute; width: 1px; height: 1px; overflow: hidden;
|
|
36
|
+
clip-path: inset(50%); }
|
|
37
|
+
.runs tr { position: relative; display: grid; grid-template-columns: auto auto 1fr auto;
|
|
38
|
+
grid-template-areas: "test test test status" "backend duration . ."
|
|
39
|
+
"date date date date";
|
|
40
|
+
gap: 0 .75rem; padding: .7rem 1.25rem .7rem 1rem; border-left: 4px solid transparent;
|
|
41
|
+
border-bottom: 1px solid var(--rule); }
|
|
42
|
+
.runs tr:hover { background: var(--code); }
|
|
43
|
+
.runs tr:focus-within { outline: 2px solid var(--focus); outline-offset: -2px; }
|
|
44
|
+
.runs td { padding: 0; font-size: .8rem; color: var(--muted); }
|
|
45
|
+
.runs td:nth-child(1) { grid-area: date; font-variant-numeric: tabular-nums; }
|
|
46
|
+
.runs td:nth-child(2) { grid-area: test; font-size: .95rem; font-weight: 600; color: var(--ink);
|
|
47
|
+
overflow-wrap: anywhere; }
|
|
48
|
+
.runs td:nth-child(2) a { text-decoration: none; outline: none; }
|
|
49
|
+
.runs td:nth-child(2) a::after { content: ""; position: absolute; inset: 0; }
|
|
50
|
+
.runs td:nth-child(3) { grid-area: backend; }
|
|
51
|
+
.runs td:nth-child(5) { grid-area: status; font-family: var(--display); font-weight: 600;
|
|
52
|
+
font-size: .9rem; }
|
|
53
|
+
.runs td:nth-child(6) { grid-area: duration; font-variant-numeric: tabular-nums; }
|
|
54
|
+
.runs td:nth-child(4), .runs td:nth-child(7), .runs td:nth-child(8) { display: none; }
|
|
55
|
+
.runs .empty { padding: 1.25rem; color: var(--muted); font-size: .9rem; }
|
|
56
|
+
.details { padding: 2.5rem clamp(1rem, 4vw, 3.5rem) 4rem; min-width: 0; max-width: 72rem; }
|
|
57
|
+
.run-detail { display: none; scroll-margin-top: 1rem; }
|
|
58
|
+
.run-detail:target,
|
|
59
|
+
body:not(:has(.run-detail:target)) .run-detail:first-child { display: block; }
|
|
60
|
+
body:not(:has(.run-detail:target)) .runs tbody tr:first-child { border-left-color: var(--ink);
|
|
61
|
+
background: var(--bench); }
|
|
62
|
+
.empty-state { max-width: 34rem; }
|
|
63
|
+
.empty-state h2 { margin-bottom: .5rem; }
|
|
64
|
+
@media (max-width: 760px) {
|
|
65
|
+
.dashboard { display: block; }
|
|
66
|
+
.runs { position: static; height: auto; max-height: 45vh; border-right: 0;
|
|
67
|
+
border-bottom: 1px solid var(--rule); }
|
|
68
|
+
}
|
|
69
|
+
"""
|
|
70
|
+
# Longer texts show this many characters, with the full text behind a collapsed <details>,
|
|
71
|
+
# so a run with megabytes of tool output still renders quickly.
|
|
72
|
+
_PREVIEW_CHARS = 2000
|
|
73
|
+
_UNKNOWN_VALUE = "-"
|
|
74
|
+
_COST_DECIMALS = 4
|
|
75
|
+
_DATE_FORMAT = "%Y-%m-%d %H:%M:%S"
|
|
76
|
+
# A tool call's summary line quotes this much of its most telling input.
|
|
77
|
+
_INPUT_HINT_CHARS = 90
|
|
78
|
+
_INPUT_HINT_KEYS = (
|
|
79
|
+
"command",
|
|
80
|
+
"skill",
|
|
81
|
+
"pattern",
|
|
82
|
+
"file_path",
|
|
83
|
+
"path",
|
|
84
|
+
"url",
|
|
85
|
+
"query",
|
|
86
|
+
"description",
|
|
87
|
+
"prompt",
|
|
88
|
+
)
|
|
89
|
+
|
|
90
|
+
# System fonts only: the page must open from disk.
|
|
91
|
+
PAGE_STYLE = """
|
|
92
|
+
:root {
|
|
93
|
+
--bench: #eef2f4; --panel: #ffffff; --ink: #1e2a33; --muted: #5b6b76; --rule: #d3dbe0;
|
|
94
|
+
--code: #f5f8f9; --pass: #0f7b6c; --fail: #c8431f; --warn: #a15c00; --user: #4b5ba6;
|
|
95
|
+
--user-tint: #eceffa; --focus: #2f6fd0;
|
|
96
|
+
--display: Bahnschrift, "DIN Alternate", "DIN 2014", "Barlow Semi Condensed",
|
|
97
|
+
"Arial Narrow", sans-serif;
|
|
98
|
+
--body: system-ui, -apple-system, "Segoe UI", Roboto, sans-serif;
|
|
99
|
+
--mono: "Cascadia Code", ui-monospace, "SF Mono", Consolas, monospace;
|
|
100
|
+
color-scheme: light dark;
|
|
101
|
+
}
|
|
102
|
+
@media (prefers-color-scheme: dark) {
|
|
103
|
+
:root {
|
|
104
|
+
--bench: #141b20; --panel: #1c252b; --ink: #e3eaee; --muted: #93a3ad; --rule: #2e3a42;
|
|
105
|
+
--code: #172026; --pass: #3cc2aa; --fail: #f07a55; --warn: #e0a04a; --user: #9aa8f0;
|
|
106
|
+
--user-tint: #232a45; --focus: #7fb0ff;
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
* { box-sizing: border-box; }
|
|
110
|
+
body { margin: 0; background: var(--bench); color: var(--ink); font: 15px/1.5 var(--body); }
|
|
111
|
+
a { color: inherit; }
|
|
112
|
+
:focus-visible { outline: 2px solid var(--focus); outline-offset: 2px; }
|
|
113
|
+
h1, h2, h3 { font-family: var(--display); font-weight: 600; line-height: 1.2; margin: 0; }
|
|
114
|
+
h2 { font-size: 1.75rem; overflow-wrap: anywhere; }
|
|
115
|
+
h2 small { display: block; margin-top: .25rem; font: 400 .8rem var(--mono); color: var(--muted); }
|
|
116
|
+
h3 { font-size: 1.1rem; margin: 2rem 0 .75rem; }
|
|
117
|
+
h4 { margin: .75rem 0 .25rem; font-size: .8rem; font-weight: 600; color: var(--muted); }
|
|
118
|
+
code, pre, summary { font-family: var(--mono); font-size: .82rem; }
|
|
119
|
+
pre { margin: 0; padding: .5rem .65rem; background: var(--code); border-radius: 3px;
|
|
120
|
+
white-space: pre-wrap; word-break: break-word; max-height: 28rem; overflow: auto; }
|
|
121
|
+
|
|
122
|
+
.status-pass { color: var(--pass); }
|
|
123
|
+
.status-fail { color: var(--fail); }
|
|
124
|
+
.status-error, .status-timeout { color: var(--warn); }
|
|
125
|
+
.status { font-family: var(--display); font-weight: 600; font-size: 1rem; padding: .1rem .55rem;
|
|
126
|
+
border: 2px solid currentColor; border-radius: 3px; }
|
|
127
|
+
|
|
128
|
+
.meter { display: flex; flex-wrap: wrap; gap: .75rem 2rem; margin: 1.25rem 0 0; padding: 0; }
|
|
129
|
+
.meter div { min-width: 0; }
|
|
130
|
+
.meter dt { font-size: .78rem; color: var(--muted); }
|
|
131
|
+
.meter dd { margin: 0; font-family: var(--display); font-size: 1.15rem;
|
|
132
|
+
font-variant-numeric: tabular-nums; overflow-wrap: anywhere; }
|
|
133
|
+
.workspace { margin: .9rem 0 0; color: var(--muted); font-size: .85rem; overflow-wrap: anywhere; }
|
|
134
|
+
.workspace code { color: var(--ink); }
|
|
135
|
+
.run-error, .error { color: var(--fail); overflow-wrap: anywhere; }
|
|
136
|
+
.run-error { margin: 1rem 0 0; padding: .5rem .75rem; border-left: 3px solid var(--fail);
|
|
137
|
+
background: var(--panel); white-space: pre-wrap; }
|
|
138
|
+
|
|
139
|
+
.assertions { list-style: none; margin: 0; padding: 0; background: var(--panel);
|
|
140
|
+
border: 1px solid var(--rule); border-radius: 4px; }
|
|
141
|
+
.assertions li { display: grid; grid-template-columns: 1.4rem 1fr; gap: 0 .5rem;
|
|
142
|
+
padding: .5rem .75rem; border-top: 1px solid var(--rule); }
|
|
143
|
+
.assertions li:first-child { border-top: 0; }
|
|
144
|
+
.assertions .verdict { grid-row: span 2; font-family: var(--display); font-weight: 700;
|
|
145
|
+
font-size: 0; }
|
|
146
|
+
.assertions .verdict::before { font-size: 1.1rem; }
|
|
147
|
+
.assertions .pass .verdict { color: var(--pass); }
|
|
148
|
+
.assertions .pass .verdict::before { content: "\\2713"; }
|
|
149
|
+
.assertions .fail .verdict { color: var(--fail); }
|
|
150
|
+
.assertions .fail .verdict::before { content: "\\2717"; }
|
|
151
|
+
.assertions .reason { color: var(--muted); font-size: .88rem; }
|
|
152
|
+
.assertions .fail .reason { color: var(--ink); }
|
|
153
|
+
.tally { font-family: var(--body); font-weight: 400; font-size: .85rem; color: var(--muted);
|
|
154
|
+
margin-left: .5rem; }
|
|
155
|
+
|
|
156
|
+
/* The timeline: one spine, a numbered stop per conversation turn. */
|
|
157
|
+
.turns { list-style: none; margin: 0; padding: 0 0 0 2.6rem; position: relative; }
|
|
158
|
+
.turns::before { content: ""; position: absolute; left: .95rem; top: .4rem; bottom: .4rem;
|
|
159
|
+
width: 2px; background: var(--rule); }
|
|
160
|
+
.turn { position: relative; padding-bottom: 1.5rem; }
|
|
161
|
+
.turn-head { display: flex; align-items: baseline; gap: .5rem; margin: 0 0 .6rem;
|
|
162
|
+
font-family: var(--display); font-size: 1rem; font-weight: 600;
|
|
163
|
+
color: var(--ink); }
|
|
164
|
+
.turn-no { position: absolute; left: -2.6rem; top: -.15rem; width: 2rem; height: 2rem;
|
|
165
|
+
display: grid; place-items: center; border-radius: 50%; background: var(--ink);
|
|
166
|
+
color: var(--bench); font: 600 1rem var(--display); }
|
|
167
|
+
.turn-answer .turn-no { background: var(--user); }
|
|
168
|
+
.item { margin: .4rem 0; }
|
|
169
|
+
.text { white-space: pre-wrap; background: var(--panel); padding: .6rem .8rem;
|
|
170
|
+
border-radius: 4px; border: 1px solid var(--rule); max-width: 75ch; }
|
|
171
|
+
.user { white-space: pre-wrap; background: var(--user-tint); color: var(--ink);
|
|
172
|
+
padding: .6rem .8rem; border-radius: 4px; border-left: 3px solid var(--user);
|
|
173
|
+
max-width: 75ch; }
|
|
174
|
+
.unknown { color: var(--muted); font-size: .8rem; }
|
|
175
|
+
details { background: var(--panel); border: 1px solid var(--rule); border-radius: 4px;
|
|
176
|
+
padding: .35rem .7rem; }
|
|
177
|
+
details[open] { padding-bottom: .6rem; }
|
|
178
|
+
summary { cursor: pointer; display: flex; gap: .6rem; align-items: baseline; min-width: 0; }
|
|
179
|
+
summary::marker { color: var(--muted); }
|
|
180
|
+
.tool { font-weight: 700; }
|
|
181
|
+
.hint { color: var(--muted); overflow: hidden; text-overflow: ellipsis; white-space: nowrap;
|
|
182
|
+
min-width: 0; flex: 1; }
|
|
183
|
+
.outcome { margin-left: auto; font-size: .75rem; color: var(--pass); }
|
|
184
|
+
.outcome.error, .outcome.none { color: var(--fail); }
|
|
185
|
+
.outcome.none { color: var(--muted); }
|
|
186
|
+
details details { border-style: dashed; margin-top: .4rem; }
|
|
187
|
+
.children { margin: .7rem 0 0 .3rem; padding-left: .9rem; border-left: 2px solid var(--user); }
|
|
188
|
+
|
|
189
|
+
@media (prefers-reduced-motion: no-preference) { html { scroll-behavior: smooth; } }
|
|
190
|
+
"""
|
|
191
|
+
|
|
192
|
+
|
|
193
|
+
@dataclass(frozen=True)
|
|
194
|
+
class _RunView:
|
|
195
|
+
"""One run as the dashboard shows it: a table row and a detail section."""
|
|
196
|
+
|
|
197
|
+
run_id: str
|
|
198
|
+
status: str
|
|
199
|
+
sort_time: datetime
|
|
200
|
+
# Runs of one test in one invocation (its repeats) share a group.
|
|
201
|
+
group: str
|
|
202
|
+
row_html: str
|
|
203
|
+
detail_html: str
|
|
204
|
+
|
|
205
|
+
|
|
206
|
+
def write_dashboard(runs_root: Path) -> None:
|
|
207
|
+
"""(Re)write runs_root/index.html from every run directory found under runs_root."""
|
|
208
|
+
runs_root.mkdir(parents=True, exist_ok=True)
|
|
209
|
+
run_dirs = [entry for entry in runs_root.iterdir() if entry.is_dir()]
|
|
210
|
+
views = [view for run_dir in run_dirs if (view := _view_run(run_dir)) is not None]
|
|
211
|
+
views = _newest_first_by_group(views)
|
|
212
|
+
(runs_root / INDEX_FILE_NAME).write_text(_render_page(views), encoding="utf-8")
|
|
213
|
+
|
|
214
|
+
|
|
215
|
+
def _newest_first_by_group(views: list[_RunView]) -> list[_RunView]:
|
|
216
|
+
"""Groups ordered by their newest run, newest first; within a group, newest first too."""
|
|
217
|
+
group_times: dict[str, datetime] = {}
|
|
218
|
+
for view in views:
|
|
219
|
+
group_times[view.group] = max(view.sort_time, group_times.get(view.group, view.sort_time))
|
|
220
|
+
return sorted(
|
|
221
|
+
views,
|
|
222
|
+
key=lambda view: (group_times[view.group], view.group, view.sort_time),
|
|
223
|
+
reverse=True,
|
|
224
|
+
)
|
|
225
|
+
|
|
226
|
+
|
|
227
|
+
def _view_run(run_dir: Path) -> _RunView | None:
|
|
228
|
+
"""The run's view, or None if its run.json is missing or can't be shown (it may still be
|
|
229
|
+
running, or was interrupted)."""
|
|
230
|
+
try:
|
|
231
|
+
record = json.loads((run_dir / _RUN_RECORD_FILE_NAME).read_text(encoding="utf-8"))
|
|
232
|
+
return _render_run(run_dir.name, record)
|
|
233
|
+
except (OSError, ValueError, KeyError, TypeError, AttributeError):
|
|
234
|
+
return None
|
|
235
|
+
|
|
236
|
+
|
|
237
|
+
def _render_run(run_id: str, record: dict[str, Any]) -> _RunView:
|
|
238
|
+
"""Raises KeyError, TypeError or ValueError if the record is malformed."""
|
|
239
|
+
cells = (
|
|
240
|
+
format_start_time(record["start_time"]),
|
|
241
|
+
_detail_link(run_id, _run_label(record)),
|
|
242
|
+
escape(record["backend"]),
|
|
243
|
+
escape(format_optional(record["model"])),
|
|
244
|
+
_status_cell(record["status"]),
|
|
245
|
+
format_duration(record["duration_s"]),
|
|
246
|
+
escape(format_optional(record["trace"]["num_turns"])),
|
|
247
|
+
escape(format_cost_and_usage(record["trace"])),
|
|
248
|
+
)
|
|
249
|
+
eval_id = record.get("eval_id")
|
|
250
|
+
return _RunView(
|
|
251
|
+
run_id=run_id,
|
|
252
|
+
status=record["status"],
|
|
253
|
+
sort_time=datetime.fromisoformat(record["start_time"]).astimezone(),
|
|
254
|
+
group=f"{eval_id}/{record['test_name']}" if eval_id else run_id,
|
|
255
|
+
row_html=_row(cells),
|
|
256
|
+
detail_html=_detail_section(run_id, record["test_name"], _render_run_detail(record)),
|
|
257
|
+
)
|
|
258
|
+
|
|
259
|
+
|
|
260
|
+
def _run_label(record: dict[str, Any]) -> str:
|
|
261
|
+
"""The test name, plus which repeat this was when the test ran more than once."""
|
|
262
|
+
repeat_count = record.get("repeat_count") or 1
|
|
263
|
+
if repeat_count > 1:
|
|
264
|
+
return f"{record['test_name']} #{record.get('repeat_index', 1)}/{repeat_count}"
|
|
265
|
+
return record["test_name"]
|
|
266
|
+
|
|
267
|
+
|
|
268
|
+
def _detail_id(run_id: str) -> str:
|
|
269
|
+
return f"run-{run_id}"
|
|
270
|
+
|
|
271
|
+
|
|
272
|
+
def _detail_link(run_id: str, label: str) -> str:
|
|
273
|
+
return f'<a href="#{escape(_detail_id(run_id))}">{escape(label)}</a>'
|
|
274
|
+
|
|
275
|
+
|
|
276
|
+
def _status_cell(status: str) -> str:
|
|
277
|
+
return f'<span class="status-{escape(status)}">{escape(status)}</span>'
|
|
278
|
+
|
|
279
|
+
|
|
280
|
+
def _row(cells: tuple[str, ...]) -> str:
|
|
281
|
+
return "<tr>" + "".join(f"<td>{cell}</td>" for cell in cells) + "</tr>"
|
|
282
|
+
|
|
283
|
+
|
|
284
|
+
def _detail_section(run_id: str, title: str, body_html: str) -> str:
|
|
285
|
+
run_id_html = "" if title == run_id else f" <small>{escape(run_id)}</small>"
|
|
286
|
+
return (
|
|
287
|
+
f'<section class="run-detail" id="{escape(_detail_id(run_id))}">'
|
|
288
|
+
f"<h2>{escape(title)}{run_id_html}</h2>{body_html}</section>"
|
|
289
|
+
)
|
|
290
|
+
|
|
291
|
+
|
|
292
|
+
def _render_page(views: list[_RunView]) -> str:
|
|
293
|
+
header = "".join(f"<th>{escape(column)}</th>" for column in _COLUMNS)
|
|
294
|
+
rows = "".join(view.row_html for view in views)
|
|
295
|
+
details = "".join(view.detail_html for view in views) or _EMPTY_STATE
|
|
296
|
+
selected_rows = "".join(_selected_row_style(view.run_id) for view in views)
|
|
297
|
+
return (
|
|
298
|
+
"<!doctype html>\n"
|
|
299
|
+
'<html lang="en"><head><meta charset="utf-8">'
|
|
300
|
+
'<meta name="viewport" content="width=device-width, initial-scale=1">'
|
|
301
|
+
"<title>skillbench runs</title>"
|
|
302
|
+
f"<style>{PAGE_STYLE}{_DASHBOARD_STYLE}</style></head>"
|
|
303
|
+
'<body class="dashboard">'
|
|
304
|
+
'<nav class="runs" aria-label="Runs">'
|
|
305
|
+
f"<header><h1>skillbench</h1><p>{escape(_tally(views))}</p></header>"
|
|
306
|
+
f'<table aria-label="Runs, newest first"><thead><tr>{header}</tr></thead>'
|
|
307
|
+
f"<tbody>{rows}</tbody></table></nav>"
|
|
308
|
+
f'<main class="details">{details}</main>'
|
|
309
|
+
# After the list, not in <head>: these rules name every run.
|
|
310
|
+
f"<style>{selected_rows}</style>"
|
|
311
|
+
"</body></html>\n"
|
|
312
|
+
)
|
|
313
|
+
|
|
314
|
+
|
|
315
|
+
def _tally(views: list[_RunView]) -> str:
|
|
316
|
+
if not views:
|
|
317
|
+
return "No runs"
|
|
318
|
+
runs = f"{len(views)} run" + ("" if len(views) == 1 else "s")
|
|
319
|
+
counts: dict[str, int] = {}
|
|
320
|
+
for view in views:
|
|
321
|
+
counts[view.status] = counts.get(view.status, 0) + 1
|
|
322
|
+
labels = {"pass": "passed", "fail": "failed", "error": "errored", "timeout": "timed out"}
|
|
323
|
+
parts = [f"{count} {labels.get(status, status)}" for status, count in counts.items()]
|
|
324
|
+
return f"{runs}: " + ", ".join(parts)
|
|
325
|
+
|
|
326
|
+
|
|
327
|
+
def _selected_row_style(run_id: str) -> str:
|
|
328
|
+
"""Marks the list row of the run whose detail is showing."""
|
|
329
|
+
detail_id = _css_string(_detail_id(run_id))
|
|
330
|
+
return (
|
|
331
|
+
f'body:has([id="{detail_id}"]:target) .runs tr:has(a[href="#{detail_id}"])'
|
|
332
|
+
"{border-left-color:var(--ink);background:var(--bench)}"
|
|
333
|
+
)
|
|
334
|
+
|
|
335
|
+
|
|
336
|
+
def _css_string(value: str) -> str:
|
|
337
|
+
"""value escaped for a double-quoted CSS string inside <style>."""
|
|
338
|
+
return "".join(
|
|
339
|
+
char if char.isascii() and (char.isalnum() or char in "-_.") else f"\\{ord(char):x} "
|
|
340
|
+
for char in value
|
|
341
|
+
)
|
|
342
|
+
|
|
343
|
+
|
|
344
|
+
# --- one run's detail: readout, assertions and timeline --------------------------------
|
|
345
|
+
|
|
346
|
+
|
|
347
|
+
def _render_run_detail(record: dict[str, Any]) -> str:
|
|
348
|
+
"""Readout, assertion results and the full timeline of one run."""
|
|
349
|
+
return (
|
|
350
|
+
f"{_render_readout(record)}"
|
|
351
|
+
f"{_render_assertions(record['assertions'])}"
|
|
352
|
+
"<h3>Timeline</h3>"
|
|
353
|
+
f"{_render_timeline(record['trace']['items'])}"
|
|
354
|
+
)
|
|
355
|
+
|
|
356
|
+
|
|
357
|
+
def format_duration(duration_s: float) -> str:
|
|
358
|
+
return f"{duration_s:.1f}s"
|
|
359
|
+
|
|
360
|
+
|
|
361
|
+
def format_optional(value: Any) -> str:
|
|
362
|
+
return _UNKNOWN_VALUE if value is None else str(value)
|
|
363
|
+
|
|
364
|
+
|
|
365
|
+
def format_start_time(start_time: str) -> str:
|
|
366
|
+
return datetime.fromisoformat(start_time).astimezone().strftime(_DATE_FORMAT)
|
|
367
|
+
|
|
368
|
+
|
|
369
|
+
def format_cost_and_usage(trace: dict[str, Any]) -> str:
|
|
370
|
+
usage = trace["usage"]
|
|
371
|
+
parts = []
|
|
372
|
+
if trace["cost_usd"] is not None:
|
|
373
|
+
parts.append(f"${trace['cost_usd']:.{_COST_DECIMALS}f}")
|
|
374
|
+
if "input_tokens" in usage or "output_tokens" in usage:
|
|
375
|
+
input_tokens = format_optional(usage.get("input_tokens"))
|
|
376
|
+
output_tokens = format_optional(usage.get("output_tokens"))
|
|
377
|
+
parts.append(f"{input_tokens} in / {output_tokens} out tokens")
|
|
378
|
+
return ", ".join(parts) if parts else format_optional(None)
|
|
379
|
+
|
|
380
|
+
|
|
381
|
+
def _render_readout(record: dict[str, Any]) -> str:
|
|
382
|
+
trace = record["trace"]
|
|
383
|
+
readings = (
|
|
384
|
+
("Backend", record["backend"]),
|
|
385
|
+
("Model", format_optional(record["model"])),
|
|
386
|
+
("Duration", format_duration(record["duration_s"])),
|
|
387
|
+
("Turns", format_optional(trace["num_turns"])),
|
|
388
|
+
("Cost and usage", format_cost_and_usage(trace)),
|
|
389
|
+
("Started", format_start_time(record["start_time"])),
|
|
390
|
+
)
|
|
391
|
+
status = record["status"]
|
|
392
|
+
cells = "".join(
|
|
393
|
+
f"<div><dt>{escape(label)}</dt><dd>{escape(value)}</dd></div>" for label, value in readings
|
|
394
|
+
)
|
|
395
|
+
error = record.get("error")
|
|
396
|
+
error_html = f'<p class="run-error">{escape(str(error))}</p>' if error else ""
|
|
397
|
+
return (
|
|
398
|
+
f'<dl class="meter"><div><dt>Status</dt><dd><span class="status status-{escape(status)}">'
|
|
399
|
+
f"{escape(status)}</span></dd></div>{cells}</dl>{error_html}{_render_location(record)}"
|
|
400
|
+
)
|
|
401
|
+
|
|
402
|
+
|
|
403
|
+
def _render_location(record: dict[str, Any]) -> str:
|
|
404
|
+
"""Where the run happened: its workspace, and the fixture (and commit) it started from."""
|
|
405
|
+
html = ""
|
|
406
|
+
if record.get("workspace"):
|
|
407
|
+
html += f'<p class="workspace">Workspace <code>{escape(record["workspace"])}</code></p>'
|
|
408
|
+
if record.get("fixture"):
|
|
409
|
+
commit = record.get("fixture_commit")
|
|
410
|
+
at_commit = f" at <code>{escape(commit)}</code>" if commit else ""
|
|
411
|
+
fixture = escape(record["fixture"])
|
|
412
|
+
html += f'<p class="workspace">Fixture <code>{fixture}</code>{at_commit}</p>'
|
|
413
|
+
return html
|
|
414
|
+
|
|
415
|
+
|
|
416
|
+
def _render_assertions(results: list[dict[str, Any]]) -> str:
|
|
417
|
+
if not results:
|
|
418
|
+
return ""
|
|
419
|
+
passed = sum(1 for result in results if result["passed"])
|
|
420
|
+
entries = "".join(_render_assertion(result) for result in results)
|
|
421
|
+
return (
|
|
422
|
+
f'<h3>Assertions<span class="tally">{passed} of {len(results)} passed</span></h3>'
|
|
423
|
+
f'<ul class="assertions">{entries}</ul>'
|
|
424
|
+
)
|
|
425
|
+
|
|
426
|
+
|
|
427
|
+
def _render_assertion(result: dict[str, Any]) -> str:
|
|
428
|
+
outcome = "pass" if result["passed"] else "fail"
|
|
429
|
+
expected = json.dumps(result["expected"])
|
|
430
|
+
return (
|
|
431
|
+
f'<li class="{outcome}"><span class="verdict">{outcome}</span>'
|
|
432
|
+
f"<code>{escape(result['type'])}: {escape(expected)}</code>"
|
|
433
|
+
f'<span class="reason">{escape(result["reason"])}</span></li>'
|
|
434
|
+
)
|
|
435
|
+
|
|
436
|
+
|
|
437
|
+
def _render_timeline(items: list[dict[str, Any]]) -> str:
|
|
438
|
+
"""Items split into turns: the first starts with the test prompt, each later one with an
|
|
439
|
+
answer."""
|
|
440
|
+
turns: list[tuple[dict[str, Any] | None, list[dict[str, Any]]]] = [(None, [])]
|
|
441
|
+
for item in items:
|
|
442
|
+
if item["kind"] == "user_message":
|
|
443
|
+
turns.append((item, []))
|
|
444
|
+
else:
|
|
445
|
+
turns[-1][1].append(item)
|
|
446
|
+
return '<ol class="turns">' + "".join(_render_turn(*turn) for turn in turns) + "</ol>"
|
|
447
|
+
|
|
448
|
+
|
|
449
|
+
def _render_turn(answer: dict[str, Any] | None, items: list[dict[str, Any]]) -> str:
|
|
450
|
+
body = _render_items(items) or '<p class="unknown">No events recorded.</p>'
|
|
451
|
+
if answer is None:
|
|
452
|
+
return (
|
|
453
|
+
'<li class="turn"><span class="turn-no">1</span>'
|
|
454
|
+
f'<p class="turn-head">Test prompt</p>{body}</li>'
|
|
455
|
+
)
|
|
456
|
+
label = f"{answer['source'].capitalize()} answer"
|
|
457
|
+
return (
|
|
458
|
+
f'<li class="turn turn-answer"><span class="turn-no">{escape(str(answer["turn"]))}</span>'
|
|
459
|
+
f'<p class="turn-head">{escape(label)}</p>'
|
|
460
|
+
f'<div class="item"><div class="user">{escape(answer["text"])}</div></div>{body}</li>'
|
|
461
|
+
)
|
|
462
|
+
|
|
463
|
+
|
|
464
|
+
def _render_items(items: list[dict[str, Any]]) -> str:
|
|
465
|
+
"""Each item in order; runs of events skillbench doesn't display collapse to one line."""
|
|
466
|
+
parts: list[str] = []
|
|
467
|
+
skipped: list[str] = []
|
|
468
|
+
for item in items:
|
|
469
|
+
if item["kind"] == "unknown":
|
|
470
|
+
skipped.append(item["raw_type"])
|
|
471
|
+
continue
|
|
472
|
+
if skipped:
|
|
473
|
+
parts.append(_render_skipped(skipped))
|
|
474
|
+
skipped = []
|
|
475
|
+
parts.append(f'<div class="item">{_render_item(item)}</div>')
|
|
476
|
+
if skipped:
|
|
477
|
+
parts.append(_render_skipped(skipped))
|
|
478
|
+
return "".join(parts)
|
|
479
|
+
|
|
480
|
+
|
|
481
|
+
def _render_skipped(raw_types: list[str]) -> str:
|
|
482
|
+
counts = Counter(raw_types)
|
|
483
|
+
listed = ", ".join(
|
|
484
|
+
name if count == 1 else f"{name} ({count})" for name, count in counts.items()
|
|
485
|
+
)
|
|
486
|
+
return f'<div class="item unknown">Other events: {escape(listed)}</div>'
|
|
487
|
+
|
|
488
|
+
|
|
489
|
+
def _render_item(item: dict[str, Any]) -> str:
|
|
490
|
+
if item["kind"] == "assistant_text":
|
|
491
|
+
return f'<div class="text">{escape(item["text"])}</div>'
|
|
492
|
+
if item["kind"] == "tool_call":
|
|
493
|
+
return _render_tool_call(item)
|
|
494
|
+
if item["kind"] == "user_message":
|
|
495
|
+
label = f"{item['source'].capitalize()} answer"
|
|
496
|
+
return f'<div class="user"><strong>{escape(label)}:</strong> {escape(item["text"])}</div>'
|
|
497
|
+
return f'<div class="unknown">Other event: {escape(item["raw_type"])}</div>'
|
|
498
|
+
|
|
499
|
+
|
|
500
|
+
def _render_tool_call(tool_call: dict[str, Any]) -> str:
|
|
501
|
+
outcome, outcome_class = {
|
|
502
|
+
True: ("ok", "ok"),
|
|
503
|
+
False: ("error", "error"),
|
|
504
|
+
None: ("no result", "none"),
|
|
505
|
+
}[tool_call["ok"]]
|
|
506
|
+
children = tool_call["children"]
|
|
507
|
+
children_html = f'<div class="children">{_render_items(children)}</div>' if children else ""
|
|
508
|
+
output = tool_call["output"] if tool_call["output"] is not None else ""
|
|
509
|
+
return (
|
|
510
|
+
"<details open>"
|
|
511
|
+
f'<summary><span class="tool">{escape(tool_call["name"])}</span>'
|
|
512
|
+
f'<span class="hint">{escape(_input_hint(tool_call["input"]))}</span>'
|
|
513
|
+
f'<span class="outcome {outcome_class}">{outcome}</span></summary>'
|
|
514
|
+
f"<h4>Input</h4>{_render_long_text(json.dumps(tool_call['input'], indent=2))}"
|
|
515
|
+
f"<h4>Output</h4>{_render_long_text(output)}"
|
|
516
|
+
f"{children_html}"
|
|
517
|
+
"</details>"
|
|
518
|
+
)
|
|
519
|
+
|
|
520
|
+
|
|
521
|
+
def _input_hint(tool_input: Any) -> str:
|
|
522
|
+
"""The input value that says most about a call (its command, pattern, path...), one line."""
|
|
523
|
+
if not isinstance(tool_input, dict):
|
|
524
|
+
return ""
|
|
525
|
+
keys = [key for key in _INPUT_HINT_KEYS if key in tool_input] + list(tool_input)
|
|
526
|
+
for key in keys:
|
|
527
|
+
value = tool_input[key]
|
|
528
|
+
if isinstance(value, str) and value.strip():
|
|
529
|
+
line = " ".join(value.split())
|
|
530
|
+
if len(line) > _INPUT_HINT_CHARS:
|
|
531
|
+
return line[:_INPUT_HINT_CHARS] + "…"
|
|
532
|
+
return line
|
|
533
|
+
return ""
|
|
534
|
+
|
|
535
|
+
|
|
536
|
+
def _render_long_text(text: str) -> str:
|
|
537
|
+
if len(text) <= _PREVIEW_CHARS:
|
|
538
|
+
return f"<pre>{escape(text)}</pre>"
|
|
539
|
+
return (
|
|
540
|
+
f"<pre>{escape(text[:_PREVIEW_CHARS])}…</pre>"
|
|
541
|
+
f"<details><summary>Show all {len(text):,} characters</summary>"
|
|
542
|
+
f"<pre>{escape(text)}</pre></details>"
|
|
543
|
+
)
|