benchkit-cli 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- benchkit/__init__.py +1 -0
- benchkit/_pi_proxy.py +219 -0
- benchkit/answer_key_guard.ts +49 -0
- benchkit/artifacts.py +87 -0
- benchkit/benchmarks/__init__.py +105 -0
- benchkit/benchmarks/aider_polyglot.py +405 -0
- benchkit/benchmarks/arc.py +52 -0
- benchkit/benchmarks/banking77.py +149 -0
- benchkit/benchmarks/base.py +19 -0
- benchkit/benchmarks/boolq.py +49 -0
- benchkit/benchmarks/chartqa.py +133 -0
- benchkit/benchmarks/code_verifier.py +77 -0
- benchkit/benchmarks/downloads.py +166 -0
- benchkit/benchmarks/evalplus.py +288 -0
- benchkit/benchmarks/git_surgery.py +485 -0
- benchkit/benchmarks/gpqa.py +56 -0
- benchkit/benchmarks/gsm8k.py +97 -0
- benchkit/benchmarks/hellaswag.py +49 -0
- benchkit/benchmarks/humaneval.py +103 -0
- benchkit/benchmarks/ifeval.py +56 -0
- benchkit/benchmarks/ifeval_instructions.py +477 -0
- benchkit/benchmarks/mbpp.py +60 -0
- benchkit/benchmarks/mc_arena.py +479 -0
- benchkit/benchmarks/mcq.py +94 -0
- benchkit/benchmarks/medfailbench.py +537 -0
- benchkit/benchmarks/mmlu.py +54 -0
- benchkit/benchmarks/mmlu_pro.py +62 -0
- benchkit/benchmarks/mmvp.py +115 -0
- benchkit/benchmarks/ocrbench.py +96 -0
- benchkit/benchmarks/openbookqa.py +52 -0
- benchkit/benchmarks/patcheval.py +929 -0
- benchkit/benchmarks/piqa.py +52 -0
- benchkit/benchmarks/ruler.py +172 -0
- benchkit/benchmarks/ruler_generator.py +535 -0
- benchkit/benchmarks/sanity.py +174 -0
- benchkit/benchmarks/treejs_arena.py +268 -0
- benchkit/benchmarks/truthfulqa.py +52 -0
- benchkit/benchmarks/utils.py +25 -0
- benchkit/benchmarks/vision.py +230 -0
- benchkit/benchmarks/vstar.py +108 -0
- benchkit/benchmarks/winogrande.py +52 -0
- benchkit/benchmarks/xstest.py +116 -0
- benchkit/browser.py +429 -0
- benchkit/catalog.py +121 -0
- benchkit/cli.py +1000 -0
- benchkit/client.py +1413 -0
- benchkit/controls.py +87 -0
- benchkit/datasets/MMLU_LICENSE.txt +21 -0
- benchkit/datasets/README.md +142 -0
- benchkit/datasets/arc_challenge.jsonl +1172 -0
- benchkit/datasets/banking77.jsonl +3080 -0
- benchkit/datasets/banking77_intents.json +79 -0
- benchkit/datasets/boolq.jsonl +3270 -0
- benchkit/datasets/code_verifier.jsonl +500 -0
- benchkit/datasets/gpqa_diamond.jsonl +198 -0
- benchkit/datasets/gsm8k.jsonl +1319 -0
- benchkit/datasets/hellaswag.jsonl +1000 -0
- benchkit/datasets/humaneval.jsonl +164 -0
- benchkit/datasets/ifeval.jsonl +541 -0
- benchkit/datasets/mbpp.jsonl +500 -0
- benchkit/datasets/mc_blocks_1_20_1.jsonl +1003 -0
- benchkit/datasets/medfailbench_hard30.jsonl +30 -0
- benchkit/datasets/medfailbench_hard30.md +13 -0
- benchkit/datasets/openbookqa.jsonl +500 -0
- benchkit/datasets/piqa.jsonl +1838 -0
- benchkit/datasets/sanity_code.jsonl +6 -0
- benchkit/datasets/truthfulqa.jsonl +817 -0
- benchkit/datasets/winogrande.jsonl +1267 -0
- benchkit/datasets/xstest.csv +451 -0
- benchkit/decision.py +194 -0
- benchkit/demo.py +451 -0
- benchkit/dockerfiles.py +247 -0
- benchkit/engine.py +1270 -0
- benchkit/evaluation.py +195 -0
- benchkit/events.py +141 -0
- benchkit/executor.py +96 -0
- benchkit/git_surgery/backport-release-stack/setup.sh +328 -0
- benchkit/git_surgery/backport-release-stack/verify.sh +119 -0
- benchkit/git_surgery/bisect-the-regression/setup.sh +49 -0
- benchkit/git_surgery/bisect-the-regression/verify.sh +34 -0
- benchkit/git_surgery/rebase-conflict-chain/setup.sh +65 -0
- benchkit/git_surgery/rebase-conflict-chain/verify.sh +43 -0
- benchkit/git_surgery/recover-complex-stash/setup.sh +152 -0
- benchkit/git_surgery/recover-complex-stash/verify.sh +95 -0
- benchkit/git_surgery/recover-lost-work/setup.sh +40 -0
- benchkit/git_surgery/recover-lost-work/verify.sh +36 -0
- benchkit/git_surgery/repair-force-pushed-remote/setup.sh +171 -0
- benchkit/git_surgery/repair-force-pushed-remote/verify.sh +83 -0
- benchkit/git_surgery/revert-merge-with-followups/setup.sh +198 -0
- benchkit/git_surgery/revert-merge-with-followups/verify.sh +98 -0
- benchkit/git_surgery/secret-in-history/README.md +27 -0
- benchkit/git_surgery/secret-in-history/setup.sh +107 -0
- benchkit/git_surgery/secret-in-history/verify.sh +106 -0
- benchkit/git_surgery/split-the-mega-commit/setup.sh +50 -0
- benchkit/git_surgery/split-the-mega-commit/verify.sh +48 -0
- benchkit/git_surgery/untangle-nested-submodules/setup.sh +193 -0
- benchkit/git_surgery/untangle-nested-submodules/verify.sh +90 -0
- benchkit/history.py +474 -0
- benchkit/jobs.py +332 -0
- benchkit/leakage.py +51 -0
- benchkit/looping.py +873 -0
- benchkit/mc_render.py +274 -0
- benchkit/mc_viewer/dist/atlas.png +0 -0
- benchkit/mc_viewer/dist/blockStates.json.gz +0 -0
- benchkit/mc_viewer/dist/version.json +4 -0
- benchkit/mc_viewer/dist/viewer.html +23 -0
- benchkit/mc_viewer/dist/viewer.js +2 -0
- benchkit/mc_viewer/dist/viewer.js.LICENSE.txt +25 -0
- benchkit/mc_viewer/dist/worker.js +2 -0
- benchkit/mc_viewer/dist/worker.js.LICENSE.txt +19 -0
- benchkit/metrics.py +100 -0
- benchkit/ollama.py +60 -0
- benchkit/perf.py +353 -0
- benchkit/perf_report.py +207 -0
- benchkit/perturbations.py +208 -0
- benchkit/pi_agent.py +476 -0
- benchkit/pi_package/package-lock.json +1827 -0
- benchkit/pi_package/package.json +8 -0
- benchkit/report.py +142 -0
- benchkit/report_markdown.py +645 -0
- benchkit/results.py +688 -0
- benchkit/retry.py +163 -0
- benchkit/runner.py +769 -0
- benchkit/sandbox.py +1167 -0
- benchkit/template_check.py +291 -0
- benchkit/templates/arena.html +504 -0
- benchkit/templates/history.html +58 -0
- benchkit/templates/perf.html +509 -0
- benchkit/templates/report.html +769 -0
- benchkit/tui/__init__.py +5 -0
- benchkit/tui/app.py +87 -0
- benchkit/tui/app.tcss +512 -0
- benchkit/tui/formatting.py +84 -0
- benchkit/tui/screens/__init__.py +1 -0
- benchkit/tui/screens/connect.py +191 -0
- benchkit/tui/screens/detail.py +288 -0
- benchkit/tui/screens/help.py +84 -0
- benchkit/tui/screens/modals.py +544 -0
- benchkit/tui/screens/results.py +361 -0
- benchkit/tui/screens/run.py +988 -0
- benchkit/tui/screens/setup.py +733 -0
- benchkit/tui/theme.py +135 -0
- benchkit/tui/widgets.py +120 -0
- benchkit_cli-0.1.0.dist-info/METADATA +294 -0
- benchkit_cli-0.1.0.dist-info/RECORD +149 -0
- benchkit_cli-0.1.0.dist-info/WHEEL +5 -0
- benchkit_cli-0.1.0.dist-info/entry_points.txt +3 -0
- benchkit_cli-0.1.0.dist-info/licenses/LICENSE +201 -0
- benchkit_cli-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,769 @@
|
|
|
1
|
+
<!doctype html>
|
|
2
|
+
<html lang="en">
|
|
3
|
+
<head>
|
|
4
|
+
<meta charset="utf-8">
|
|
5
|
+
<meta name="viewport" content="width=device-width, initial-scale=1">
|
|
6
|
+
<meta name="color-scheme" content="light dark">
|
|
7
|
+
<title>BenchKit Results</title>
|
|
8
|
+
<script>
|
|
9
|
+
// Applied before paint so the report never flashes the wrong theme.
|
|
10
|
+
// Dogi Light is the default; an explicit choice is remembered.
|
|
11
|
+
(() => {
|
|
12
|
+
let stored = null;
|
|
13
|
+
try { stored = localStorage.getItem("benchkit-theme"); } catch (error) { stored = null; }
|
|
14
|
+
document.documentElement.dataset.theme = stored === "dark" ? "dark" : "light";
|
|
15
|
+
})();
|
|
16
|
+
</script>
|
|
17
|
+
<style>
|
|
18
|
+
/* Dogi (https://github.com/DogukanUrker/DogiZed): flat black or white,
|
|
19
|
+
colour only where it carries meaning. */
|
|
20
|
+
:root,
|
|
21
|
+
:root[data-theme="light"] {
|
|
22
|
+
color-scheme: light;
|
|
23
|
+
--page: #ffffff;
|
|
24
|
+
--card: #ffffff;
|
|
25
|
+
--ink: #0b0b0b;
|
|
26
|
+
--muted: #6b7280;
|
|
27
|
+
--line: rgba(11, 11, 11, .12);
|
|
28
|
+
--raise: rgba(11, 11, 11, .04);
|
|
29
|
+
--track: rgba(11, 11, 11, .09);
|
|
30
|
+
--green: #059669;
|
|
31
|
+
--green-soft: #10b981;
|
|
32
|
+
--red: #dc2626;
|
|
33
|
+
--blue: #2563eb;
|
|
34
|
+
--amber: #d97706;
|
|
35
|
+
--bar: #6b7280;
|
|
36
|
+
--bar-light: rgba(11, 11, 11, .09);
|
|
37
|
+
--shade-1: #dbeafe;
|
|
38
|
+
--shade-2: #bfdbfe;
|
|
39
|
+
--shade-3: #93c5fd;
|
|
40
|
+
--shade-4: #60a5fa;
|
|
41
|
+
--shade-5: #3b82f6;
|
|
42
|
+
--shade-6: #2563eb;
|
|
43
|
+
--shade-7: #1d4ed8;
|
|
44
|
+
--series-1: #059669;
|
|
45
|
+
--series-2: #2563eb;
|
|
46
|
+
--series-3: #db2777;
|
|
47
|
+
--series-4: #0891b2;
|
|
48
|
+
--series-5: #d97706;
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
:root[data-theme="dark"] {
|
|
52
|
+
color-scheme: dark;
|
|
53
|
+
--page: #000000;
|
|
54
|
+
--card: #000000;
|
|
55
|
+
--ink: #ffffff;
|
|
56
|
+
--muted: #6b7280;
|
|
57
|
+
--line: rgba(255, 255, 255, .14);
|
|
58
|
+
--raise: rgba(255, 255, 255, .06);
|
|
59
|
+
--track: rgba(255, 255, 255, .12);
|
|
60
|
+
--green: #34d399;
|
|
61
|
+
--green-soft: #6ee7b7;
|
|
62
|
+
--red: #ef4444;
|
|
63
|
+
--blue: #60a5fa;
|
|
64
|
+
--amber: #facc15;
|
|
65
|
+
--bar: #9aa1ad;
|
|
66
|
+
--bar-light: rgba(255, 255, 255, .12);
|
|
67
|
+
--shade-1: #14304f;
|
|
68
|
+
--shade-2: #1b4470;
|
|
69
|
+
--shade-3: #235891;
|
|
70
|
+
--shade-4: #2c6cb2;
|
|
71
|
+
--shade-5: #3f86d4;
|
|
72
|
+
--shade-6: #4f95e8;
|
|
73
|
+
--shade-7: #60a5fa;
|
|
74
|
+
--series-1: #34d399;
|
|
75
|
+
--series-2: #60a5fa;
|
|
76
|
+
--series-3: #f472b6;
|
|
77
|
+
--series-4: #22d3ee;
|
|
78
|
+
--series-5: #facc15;
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
:root {
|
|
82
|
+
--sans: "Avenir Next", Avenir, "Century Gothic", sans-serif;
|
|
83
|
+
--mono: "SFMono-Regular", "Cascadia Mono", Menlo, monospace;
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
* { box-sizing: border-box; }
|
|
87
|
+
html { background: var(--page); }
|
|
88
|
+
body {
|
|
89
|
+
margin: 0;
|
|
90
|
+
min-height: 100vh;
|
|
91
|
+
color: var(--ink);
|
|
92
|
+
background: var(--page);
|
|
93
|
+
font-family: var(--sans);
|
|
94
|
+
-webkit-font-smoothing: antialiased;
|
|
95
|
+
}
|
|
96
|
+
button { font: inherit; }
|
|
97
|
+
.page { width: min(1180px, calc(100% - 40px)); margin: 0 auto; padding: 54px 0 70px; }
|
|
98
|
+
|
|
99
|
+
.report-head {
|
|
100
|
+
display: flex;
|
|
101
|
+
justify-content: space-between;
|
|
102
|
+
gap: 32px;
|
|
103
|
+
align-items: end;
|
|
104
|
+
padding-bottom: 30px;
|
|
105
|
+
border-bottom: 1px solid var(--line);
|
|
106
|
+
}
|
|
107
|
+
h1, h2, h3, p { margin-top: 0; }
|
|
108
|
+
h1 { margin-bottom: 7px; font-size: 42px; line-height: 1; letter-spacing: -.045em; }
|
|
109
|
+
.meta { margin: 0; color: var(--muted); font: 11px/1.55 var(--mono); }
|
|
110
|
+
.head-right { display: flex; flex-direction: column; align-items: flex-end; gap: 18px; }
|
|
111
|
+
.best { text-align: right; }
|
|
112
|
+
|
|
113
|
+
.theme-toggle {
|
|
114
|
+
display: inline-flex;
|
|
115
|
+
align-items: center;
|
|
116
|
+
gap: 8px;
|
|
117
|
+
padding: 7px 14px;
|
|
118
|
+
border: 1px solid var(--line);
|
|
119
|
+
border-radius: 999px;
|
|
120
|
+
background: var(--raise);
|
|
121
|
+
color: var(--ink);
|
|
122
|
+
cursor: pointer;
|
|
123
|
+
font: 700 9px/1 var(--mono);
|
|
124
|
+
letter-spacing: .09em;
|
|
125
|
+
text-transform: uppercase;
|
|
126
|
+
}
|
|
127
|
+
.theme-toggle:hover { border-color: var(--blue); color: var(--blue); }
|
|
128
|
+
.theme-dot { width: 9px; height: 9px; border-radius: 50%; background: var(--blue); }
|
|
129
|
+
.best-score { color: var(--green); font-size: 42px; font-weight: 800; line-height: 1; letter-spacing: -.045em; }
|
|
130
|
+
.best-model { margin-top: 6px; font-size: 12px; font-weight: 800; }
|
|
131
|
+
|
|
132
|
+
.summary {
|
|
133
|
+
display: grid;
|
|
134
|
+
grid-template-columns: repeat(6, 1fr);
|
|
135
|
+
margin: 28px 0 64px;
|
|
136
|
+
background: var(--card);
|
|
137
|
+
border: 1px solid var(--line);
|
|
138
|
+
}
|
|
139
|
+
.stat { padding: 21px 24px; }
|
|
140
|
+
.stat + .stat { border-left: 1px solid var(--line); }
|
|
141
|
+
.stat-label { color: var(--muted); font: 9px/1 var(--mono); letter-spacing: .07em; text-transform: uppercase; }
|
|
142
|
+
.stat-value { margin-top: 8px; font-size: 25px; font-weight: 800; letter-spacing: -.035em; }
|
|
143
|
+
|
|
144
|
+
.section { margin-top: 64px; }
|
|
145
|
+
.section-head {
|
|
146
|
+
display: flex;
|
|
147
|
+
align-items: center;
|
|
148
|
+
justify-content: space-between;
|
|
149
|
+
gap: 20px;
|
|
150
|
+
margin-bottom: 18px;
|
|
151
|
+
}
|
|
152
|
+
h2 { margin: 0; font-size: 24px; letter-spacing: -.035em; }
|
|
153
|
+
.section-note { margin: 0; color: var(--muted); font-size: 10px; }
|
|
154
|
+
.charts { display: grid; gap: 30px; }
|
|
155
|
+
|
|
156
|
+
.chart-shell {
|
|
157
|
+
position: relative;
|
|
158
|
+
width: 100%;
|
|
159
|
+
min-width: 0;
|
|
160
|
+
padding: 42px 52px 36px;
|
|
161
|
+
overflow: hidden;
|
|
162
|
+
border: 1px solid var(--line);
|
|
163
|
+
border-radius: 22px;
|
|
164
|
+
background: var(--card);
|
|
165
|
+
}
|
|
166
|
+
.chart-header { display: flex; align-items: baseline; gap: 8px; }
|
|
167
|
+
.chart-title { margin: 0; font-size: 24px; font-weight: 800; letter-spacing: -.035em; text-transform: none; }
|
|
168
|
+
.chart-subtitle { color: var(--muted); font-size: 22px; font-weight: 700; }
|
|
169
|
+
.bar-plot {
|
|
170
|
+
display: flex;
|
|
171
|
+
align-items: stretch;
|
|
172
|
+
gap: clamp(18px, 4vw, 54px);
|
|
173
|
+
height: 510px;
|
|
174
|
+
margin-top: 32px;
|
|
175
|
+
padding: 0 24px;
|
|
176
|
+
border-bottom: 2px solid var(--line);
|
|
177
|
+
}
|
|
178
|
+
.bar-item {
|
|
179
|
+
display: grid;
|
|
180
|
+
grid-template-rows: 54px 1fr;
|
|
181
|
+
flex: 1 1 0;
|
|
182
|
+
min-width: 0;
|
|
183
|
+
text-align: center;
|
|
184
|
+
}
|
|
185
|
+
.bar-value {
|
|
186
|
+
align-self: end;
|
|
187
|
+
padding-bottom: 12px;
|
|
188
|
+
color: var(--bar);
|
|
189
|
+
font-size: clamp(24px, 3.4vw, 42px);
|
|
190
|
+
font-weight: 800;
|
|
191
|
+
line-height: 1;
|
|
192
|
+
letter-spacing: -.05em;
|
|
193
|
+
}
|
|
194
|
+
.bar-value small { font-size: .56em; letter-spacing: -.02em; }
|
|
195
|
+
.bar-item.winner .bar-value { color: var(--green); }
|
|
196
|
+
.bar-slot { display: flex; align-items: end; min-height: 0; }
|
|
197
|
+
.bar {
|
|
198
|
+
width: 100%;
|
|
199
|
+
height: calc(var(--value) * 1%);
|
|
200
|
+
min-height: 3px;
|
|
201
|
+
border-radius: 11px 11px 0 0;
|
|
202
|
+
background: var(--shade, var(--bar));
|
|
203
|
+
transform-origin: bottom;
|
|
204
|
+
animation: rise .55s cubic-bezier(.25,.7,.2,1) both;
|
|
205
|
+
}
|
|
206
|
+
.bar-item.winner .bar { background: linear-gradient(var(--green-soft), var(--green)); }
|
|
207
|
+
@keyframes rise { from { transform: scaleY(0); } }
|
|
208
|
+
|
|
209
|
+
.bar-labels {
|
|
210
|
+
display: flex;
|
|
211
|
+
gap: clamp(18px, 4vw, 54px);
|
|
212
|
+
padding: 17px 24px 0;
|
|
213
|
+
}
|
|
214
|
+
.bar-label { flex: 1 1 0; min-width: 0; text-align: center; }
|
|
215
|
+
.model-name {
|
|
216
|
+
overflow: hidden;
|
|
217
|
+
font-size: clamp(10px, 1.5vw, 14px);
|
|
218
|
+
font-weight: 800;
|
|
219
|
+
text-overflow: ellipsis;
|
|
220
|
+
white-space: nowrap;
|
|
221
|
+
}
|
|
222
|
+
.model-meta { margin-top: 5px; color: var(--muted); font-size: clamp(8px, 1vw, 10px); font-weight: 700; }
|
|
223
|
+
|
|
224
|
+
.suite-plot {
|
|
225
|
+
display: flex;
|
|
226
|
+
align-items: end;
|
|
227
|
+
gap: 34px;
|
|
228
|
+
height: 515px;
|
|
229
|
+
margin-top: 32px;
|
|
230
|
+
padding: 0 16px;
|
|
231
|
+
border-bottom: 2px solid var(--line);
|
|
232
|
+
}
|
|
233
|
+
.suite-group { display: grid; grid-template-rows: 1fr 42px; flex: 1 1 0; min-width: 0; height: 100%; }
|
|
234
|
+
.suite-bars { display: flex; align-items: end; justify-content: center; gap: 8px; min-height: 0; }
|
|
235
|
+
.suite-bar { width: min(34px, 28%); height: calc(var(--value) * 1%); min-height: 3px; border-radius: 6px 6px 0 0; background: var(--shade); }
|
|
236
|
+
.suite-name { align-self: center; overflow: hidden; font-size: 10px; font-weight: 800; text-align: center; text-overflow: ellipsis; text-transform: capitalize; white-space: nowrap; }
|
|
237
|
+
.legend { display: flex; flex-wrap: wrap; gap: 10px 20px; margin-top: 18px; }
|
|
238
|
+
.legend-item { display: flex; align-items: center; gap: 7px; font-size: 9px; font-weight: 700; }
|
|
239
|
+
.legend-swatch { width: 10px; height: 10px; border-radius: 2px; background: var(--shade); }
|
|
240
|
+
|
|
241
|
+
.scatter {
|
|
242
|
+
display: block;
|
|
243
|
+
width: 100%;
|
|
244
|
+
height: 490px;
|
|
245
|
+
margin-top: 18px;
|
|
246
|
+
overflow: visible;
|
|
247
|
+
}
|
|
248
|
+
.grid-line { stroke: var(--line); stroke-width: 1; }
|
|
249
|
+
.axis-label { fill: var(--muted); font: 9px var(--mono); }
|
|
250
|
+
.scatter-dot { fill: var(--bar); stroke: var(--card); stroke-width: 3; }
|
|
251
|
+
.scatter-dot.winner { fill: var(--green); }
|
|
252
|
+
.scatter-label { fill: var(--ink); font: 800 10px var(--sans); }
|
|
253
|
+
|
|
254
|
+
.stack-list { display: grid; gap: 24px; margin-top: 58px; }
|
|
255
|
+
.stack-row { display: grid; grid-template-columns: 190px 1fr 70px; gap: 18px; align-items: center; }
|
|
256
|
+
.stack-name { overflow: hidden; font-size: 12px; font-weight: 800; text-overflow: ellipsis; white-space: nowrap; }
|
|
257
|
+
.stack-track { display: flex; height: 28px; overflow: hidden; border-radius: 7px; background: var(--track); }
|
|
258
|
+
.stack-pass { width: calc(var(--value) * 1%); background: var(--green); }
|
|
259
|
+
.stack-score { font-size: 17px; font-weight: 800; text-align: right; }
|
|
260
|
+
.stack-key { display: flex; gap: 18px; margin-top: 26px; color: var(--muted); font-size: 9px; }
|
|
261
|
+
.stack-key span::before { content: ""; display: inline-block; width: 9px; height: 9px; margin-right: 6px; border-radius: 2px; background: var(--green); vertical-align: -1px; }
|
|
262
|
+
.stack-key span:last-child::before { background: var(--bar-light); }
|
|
263
|
+
|
|
264
|
+
.horizontal-list { display: grid; gap: 20px; margin-top: 46px; }
|
|
265
|
+
.horizontal-row { display: grid; grid-template-columns: 190px 1fr 88px; gap: 18px; align-items: center; }
|
|
266
|
+
.horizontal-name { overflow: hidden; font-size: 12px; font-weight: 800; text-overflow: ellipsis; white-space: nowrap; }
|
|
267
|
+
.horizontal-track { height: 18px; border-radius: 5px; background: var(--track); overflow: hidden; }
|
|
268
|
+
.horizontal-fill { width: calc(var(--value) * 1%); height: 100%; border-radius: inherit; background: var(--bar); }
|
|
269
|
+
.horizontal-row.winner .horizontal-fill { background: var(--green); }
|
|
270
|
+
.horizontal-fill.loop-fill { background: var(--red); }
|
|
271
|
+
.horizontal-row.clean .horizontal-value { color: var(--green); }
|
|
272
|
+
.horizontal-value { color: var(--bar); font-size: 15px; font-weight: 800; text-align: right; }
|
|
273
|
+
.horizontal-row.winner .horizontal-value { color: var(--green); }
|
|
274
|
+
|
|
275
|
+
.matrix-card { display: flex; flex-direction: column; }
|
|
276
|
+
.matrix-wrap { margin-top: 34px; overflow: hidden; border: 1px solid var(--line); border-radius: 10px; background: var(--card); }
|
|
277
|
+
table { width: 100%; border-collapse: collapse; }
|
|
278
|
+
th, td { padding: 13px 14px; border-bottom: 1px solid var(--line); font-size: 10px; text-align: right; white-space: nowrap; }
|
|
279
|
+
tr:last-child td { border-bottom: 0; }
|
|
280
|
+
th { color: var(--muted); background: var(--raise); font: 8px/1 var(--mono); letter-spacing: .07em; text-transform: uppercase; }
|
|
281
|
+
th:first-child, td:first-child { position: sticky; left: 0; text-align: left; }
|
|
282
|
+
th:first-child { z-index: 2; background: var(--raise); }
|
|
283
|
+
td:first-child { z-index: 1; max-width: 240px; background: var(--card); font-weight: 800; overflow: hidden; text-overflow: ellipsis; }
|
|
284
|
+
.score-cell { color: var(--green); font-size: 14px; font-weight: 800; }
|
|
285
|
+
|
|
286
|
+
.details { display: grid; gap: 8px; }
|
|
287
|
+
details { border: 1px solid var(--line); border-radius: 10px; background: var(--card); overflow: hidden; }
|
|
288
|
+
summary {
|
|
289
|
+
display: grid;
|
|
290
|
+
grid-template-columns: 1fr auto auto;
|
|
291
|
+
gap: 18px;
|
|
292
|
+
align-items: center;
|
|
293
|
+
padding: 17px 19px;
|
|
294
|
+
list-style: none;
|
|
295
|
+
cursor: pointer;
|
|
296
|
+
font-size: 10px;
|
|
297
|
+
}
|
|
298
|
+
summary::-webkit-details-marker { display: none; }
|
|
299
|
+
summary::after { content: "+"; color: var(--muted); font: 15px/1 var(--mono); }
|
|
300
|
+
details[open] summary::after { content: "−"; }
|
|
301
|
+
.detail-title { font-weight: 800; overflow-wrap: anywhere; }
|
|
302
|
+
.detail-score { color: var(--muted); }
|
|
303
|
+
.tasks { padding: 0 10px 10px; }
|
|
304
|
+
.task { padding: 15px 8px; border-top: 1px solid var(--line); }
|
|
305
|
+
.task-head { display: flex; justify-content: space-between; gap: 16px; margin-bottom: 10px; font: 9px/1 var(--mono); }
|
|
306
|
+
.status { color: var(--red); font-weight: 900; }
|
|
307
|
+
.status.pass { color: var(--green); }
|
|
308
|
+
.status.partial { color: var(--amber); }
|
|
309
|
+
.loop-status { margin-left: 10px; color: var(--muted); font-weight: 900; }
|
|
310
|
+
.loop-status.looping { color: var(--red); }
|
|
311
|
+
.loop-status.suspected { color: var(--amber); }
|
|
312
|
+
.exchange { display: grid; grid-template-columns: 1fr 1fr; gap: 9px; }
|
|
313
|
+
.thinking-code { grid-column: 1 / -1; }
|
|
314
|
+
.code { min-width: 0; border: 1px solid var(--line); border-radius: 7px; background: var(--raise); overflow: hidden; }
|
|
315
|
+
.code-label a { color: var(--blue); }
|
|
316
|
+
.code-label { padding: 8px 10px; border-bottom: 1px solid var(--line); color: var(--muted); font: 8px/1 var(--mono); text-transform: uppercase; }
|
|
317
|
+
pre { max-height: 300px; margin: 0; padding: 11px; overflow: auto; white-space: pre-wrap; word-break: break-word; font: 9px/1.55 var(--mono); }
|
|
318
|
+
.arena-link-row { display: flex; align-items: baseline; gap: 14px; flex-wrap: wrap; margin: 0 0 54px; }
|
|
319
|
+
.arena-link { color: var(--blue); font: 800 11px/1 var(--mono); text-decoration: none; }
|
|
320
|
+
.arena-link:hover { text-decoration: underline; }
|
|
321
|
+
.arena-note { color: var(--muted); font: 9px/1.4 var(--mono); }
|
|
322
|
+
footer { display: flex; justify-content: space-between; gap: 20px; margin-top: 70px; padding-top: 24px; border-top: 1px solid var(--line); color: var(--muted); font: 8px/1.4 var(--mono); }
|
|
323
|
+
|
|
324
|
+
@media (max-width: 720px) {
|
|
325
|
+
.page { width: min(100% - 24px, 600px); padding-top: 34px; }
|
|
326
|
+
.report-head { display: block; }
|
|
327
|
+
.head-right { align-items: flex-start; margin-top: 24px; }
|
|
328
|
+
.best { text-align: left; }
|
|
329
|
+
.summary { grid-template-columns: repeat(2, 1fr); }
|
|
330
|
+
.stat:nth-child(3) { border-top: 1px solid var(--line); border-left: 0; }
|
|
331
|
+
.stat:nth-child(4) { border-top: 1px solid var(--line); }
|
|
332
|
+
.stat:nth-child(5) { border-top: 1px solid var(--line); border-left: 0; }
|
|
333
|
+
.chart-shell { padding: 25px 18px 24px; border-radius: 14px; }
|
|
334
|
+
.chart-title { font-size: 18px; }
|
|
335
|
+
.chart-subtitle { font-size: 17px; }
|
|
336
|
+
.bar-plot { height: 380px; gap: 12px; padding: 0 4px; }
|
|
337
|
+
.bar-labels { gap: 12px; padding: 14px 4px 0; }
|
|
338
|
+
.stack-row, .horizontal-row { grid-template-columns: 110px 1fr 58px; gap: 10px; }
|
|
339
|
+
.exchange { grid-template-columns: 1fr; }
|
|
340
|
+
}
|
|
341
|
+
</style>
|
|
342
|
+
</head>
|
|
343
|
+
<body>
|
|
344
|
+
<main class="page">
|
|
345
|
+
<header class="report-head">
|
|
346
|
+
<div>
|
|
347
|
+
<h1>BenchKit results</h1>
|
|
348
|
+
<p class="meta" id="reportMeta"></p>
|
|
349
|
+
</div>
|
|
350
|
+
<div class="head-right">
|
|
351
|
+
<button class="theme-toggle" id="themeToggle" type="button" aria-label="Switch colour theme">
|
|
352
|
+
<span class="theme-dot"></span><span id="themeLabel">Dark</span>
|
|
353
|
+
</button>
|
|
354
|
+
<div class="best" id="best"></div>
|
|
355
|
+
</div>
|
|
356
|
+
</header>
|
|
357
|
+
|
|
358
|
+
<section class="summary" id="summary"></section>
|
|
359
|
+
<section id="contamination"></section>
|
|
360
|
+
|
|
361
|
+
<section class="section">
|
|
362
|
+
<div class="section-head"><h2>Chart gallery</h2><p class="section-note">Inspect and capture any chart directly</p></div>
|
|
363
|
+
<div class="charts">
|
|
364
|
+
<div id="overallChart"></div>
|
|
365
|
+
<div id="harnessChart"></div>
|
|
366
|
+
<div id="repairChart"></div>
|
|
367
|
+
<div id="robustnessChart"></div>
|
|
368
|
+
<div id="loopChart"></div>
|
|
369
|
+
<div id="benchmarkCharts" class="charts"></div>
|
|
370
|
+
<div id="suiteChart"></div>
|
|
371
|
+
<div id="efficiencyChart"></div>
|
|
372
|
+
<div id="speedChart"></div>
|
|
373
|
+
<div id="compositionChart"></div>
|
|
374
|
+
<div id="runtimeChart"></div>
|
|
375
|
+
<div id="matrix"></div>
|
|
376
|
+
</div>
|
|
377
|
+
</section>
|
|
378
|
+
|
|
379
|
+
<div class="arena-link-row" id="arenaLink"></div>
|
|
380
|
+
|
|
381
|
+
<section class="section">
|
|
382
|
+
<div class="section-head"><h2>Task details</h2></div>
|
|
383
|
+
<div class="details" id="details"></div>
|
|
384
|
+
</section>
|
|
385
|
+
|
|
386
|
+
<footer><span id="footerMeta"></span><span>Generated by BenchKit</span></footer>
|
|
387
|
+
</main>
|
|
388
|
+
|
|
389
|
+
<script id="reportData" type="application/json">__BENCHKIT_REPORT_DATA__</script>
|
|
390
|
+
<script>
|
|
391
|
+
const root = document.documentElement;
|
|
392
|
+
const themeLabel = document.getElementById("themeLabel");
|
|
393
|
+
const setTheme = mode => {
|
|
394
|
+
root.dataset.theme = mode;
|
|
395
|
+
themeLabel.textContent = mode === "dark" ? "Light" : "Dark";
|
|
396
|
+
};
|
|
397
|
+
setTheme(root.dataset.theme === "dark" ? "dark" : "light");
|
|
398
|
+
document.getElementById("themeToggle").addEventListener("click", () => {
|
|
399
|
+
const next = root.dataset.theme === "dark" ? "light" : "dark";
|
|
400
|
+
setTheme(next);
|
|
401
|
+
try { localStorage.setItem("benchkit-theme", next); } catch (error) { /* file:// */ }
|
|
402
|
+
});
|
|
403
|
+
|
|
404
|
+
const report = JSON.parse(document.getElementById("reportData").textContent);
|
|
405
|
+
const rows = report.results.map(row => ({
|
|
406
|
+
...row,
|
|
407
|
+
model: `${row.model} · ${row.harness_label || (row.harness === "pi" ? "Pi agent" : "Direct")}`,
|
|
408
|
+
}));
|
|
409
|
+
const capabilityRows = rows.filter(row => !row.perturbation);
|
|
410
|
+
const harnessRows = rows.filter(row => row.harness_paired_total != null);
|
|
411
|
+
const repairRows = rows.filter(row => Number(row.repair_attempts) > 0);
|
|
412
|
+
const robustnessRows = rows.filter(row => row.perturbation && row.paired_total);
|
|
413
|
+
const escapeHtml = value => String(value ?? "").replace(/[&<>"']/g, char => ({"&":"&","<":"<",">":">",'"':""","'":"'"}[char]));
|
|
414
|
+
const number = value => Number(value) || 0;
|
|
415
|
+
const renderDiagnostics = task => [
|
|
416
|
+
...(task.workspace?.page_errors || []).map(item => `[uncaught] ${item}`),
|
|
417
|
+
...(task.workspace?.console_errors || []).map(item => `[console] ${item}`),
|
|
418
|
+
...(task.workspace?.blocked_requests || []).map(item => `[blocked] ${item}`),
|
|
419
|
+
...(task.workspace?.failed_requests || []).map(item => `[network] ${item}`),
|
|
420
|
+
].join("\n");
|
|
421
|
+
const benchmarkLabel = row => row.benchmark_label || row.benchmark;
|
|
422
|
+
const robustnessMeta = row => row.perturbation
|
|
423
|
+
? ` · ${number(row.score_delta_pp) >= 0 ? "+" : ""}${number(row.score_delta_pp).toFixed(1)}pp · ${number(row.regressions)}/${number(row.paired_total)} regressions`
|
|
424
|
+
: "";
|
|
425
|
+
const streamSpeed = row => number(row.tok_s_per_stream ?? row.tok_s);
|
|
426
|
+
const effectiveConcurrency = row => row.concurrency_eff != null
|
|
427
|
+
? number(row.concurrency_eff)
|
|
428
|
+
: number(row.total_time) > 0
|
|
429
|
+
? number(row.avg_response_time) * number(row.total) / number(row.total_time)
|
|
430
|
+
: 0;
|
|
431
|
+
const aggregateSpeed = row => {
|
|
432
|
+
if (row.tok_s_aggregate != null) return number(row.tok_s_aggregate);
|
|
433
|
+
const tasksWithUsage = (row.tasks || []).filter(task => task.output_tokens != null);
|
|
434
|
+
if (tasksWithUsage.length && number(row.total_time) > 0) {
|
|
435
|
+
return tasksWithUsage.reduce((sum, task) => sum + number(task.output_tokens), 0) / number(row.total_time);
|
|
436
|
+
}
|
|
437
|
+
return streamSpeed(row) * effectiveConcurrency(row);
|
|
438
|
+
};
|
|
439
|
+
const loopLabel = task => task.recovered_cycle
|
|
440
|
+
? "RECOVERED"
|
|
441
|
+
: task.loop_state === "clear" && task.loop_source === "answer" && task.trace_status === "unavailable"
|
|
442
|
+
? "NO TRACE"
|
|
443
|
+
: (task.loop_state || "unavailable").toUpperCase();
|
|
444
|
+
const formatScore = value => value == null ? "—" : `${number(value).toFixed(1)}<small>%</small>`;
|
|
445
|
+
const clamp = value => Math.max(0, Math.min(100, number(value)));
|
|
446
|
+
const formatTime = value => {
|
|
447
|
+
const seconds = Math.round(number(value));
|
|
448
|
+
if (seconds >= 3600) return `${Math.floor(seconds / 3600)}h ${Math.floor(seconds % 3600 / 60)}m`;
|
|
449
|
+
if (seconds >= 60) return `${Math.floor(seconds / 60)}m ${seconds % 60}s`;
|
|
450
|
+
return `${seconds}s`;
|
|
451
|
+
};
|
|
452
|
+
const date = report.generated_at.replace("_", " · ").replaceAll("-", ".");
|
|
453
|
+
const benchmarks = [...new Set(capabilityRows.map(benchmarkLabel))];
|
|
454
|
+
const totalTasks = rows.reduce((sum, row) => sum + number(row.total), 0);
|
|
455
|
+
const contaminatedTasks = rows.flatMap(row => row.contaminated_tasks || []);
|
|
456
|
+
const contaminationByLanguage = rows.reduce((counts, row) => {
|
|
457
|
+
Object.entries(row.contamination_by_language || {}).forEach(([language, count]) => {
|
|
458
|
+
counts[language] = number(counts[language]) + number(count);
|
|
459
|
+
});
|
|
460
|
+
return counts;
|
|
461
|
+
}, {});
|
|
462
|
+
const totalScored = rows.reduce((sum, row) => sum + number(row.scored_total ?? row.total), 0);
|
|
463
|
+
const totalLoops = rows.reduce((sum, row) => sum + number(row.loops), 0);
|
|
464
|
+
const totalRuntime = rows.reduce((sum, row) => sum + number(row.total_time), 0);
|
|
465
|
+
const environment = [report.provider, report.hardware].filter(Boolean).join(" · ") || "Local inference";
|
|
466
|
+
|
|
467
|
+
const grouped = new Map();
|
|
468
|
+
rows.forEach(row => {
|
|
469
|
+
const item = grouped.get(row.model) || { model: row.model, passed: 0, total: 0, taskTotal: 0, loops: 0, outputTokens: 0, generationTime: 0, requestTime: 0, runtime: 0 };
|
|
470
|
+
const knownOutputTokens = row.throughput_output_tokens != null
|
|
471
|
+
? number(row.throughput_output_tokens)
|
|
472
|
+
: row.total_output_tokens != null
|
|
473
|
+
? number(row.total_output_tokens)
|
|
474
|
+
: 0;
|
|
475
|
+
const reportedRuntime = number(row.throughput_wall_time ?? row.total_time);
|
|
476
|
+
const runtime = reportedRuntime > 0
|
|
477
|
+
? reportedRuntime
|
|
478
|
+
: knownOutputTokens > 0 && aggregateSpeed(row) > 0
|
|
479
|
+
? knownOutputTokens / aggregateSpeed(row)
|
|
480
|
+
: 0;
|
|
481
|
+
const outputTokens = knownOutputTokens || aggregateSpeed(row) * runtime;
|
|
482
|
+
if (row.include_in_overall !== false) {
|
|
483
|
+
item.passed += number(row.passed);
|
|
484
|
+
item.total += number(row.scored_total ?? row.total);
|
|
485
|
+
}
|
|
486
|
+
item.taskTotal += number(row.total);
|
|
487
|
+
item.loops += number(row.loops);
|
|
488
|
+
item.outputTokens += outputTokens;
|
|
489
|
+
item.generationTime += row.sum_generation_time != null
|
|
490
|
+
? number(row.sum_generation_time)
|
|
491
|
+
: streamSpeed(row) > 0 ? outputTokens / streamSpeed(row) : 0;
|
|
492
|
+
item.requestTime += row.sum_request_time != null
|
|
493
|
+
? number(row.sum_request_time)
|
|
494
|
+
: effectiveConcurrency(row) * runtime;
|
|
495
|
+
item.runtime += runtime;
|
|
496
|
+
grouped.set(row.model, item);
|
|
497
|
+
});
|
|
498
|
+
const models = [...grouped.values()].map(item => ({
|
|
499
|
+
...item,
|
|
500
|
+
score: item.total ? item.passed / item.total * 100 : null,
|
|
501
|
+
loopRate: item.taskTotal ? item.loops / item.taskTotal * 100 : 0,
|
|
502
|
+
speed: item.runtime ? item.outputTokens / item.runtime : 0,
|
|
503
|
+
streamSpeed: item.generationTime ? item.outputTokens / item.generationTime : 0,
|
|
504
|
+
effective: item.runtime ? item.requestTime / item.runtime : 0,
|
|
505
|
+
})).sort((a, b) => number(b.score) - number(a.score) || b.speed - a.speed);
|
|
506
|
+
const scoredModels = models.filter(model => model.score != null);
|
|
507
|
+
const best = scoredModels[0] || null;
|
|
508
|
+
|
|
509
|
+
const chartItems = items => [...items].sort((a, b) => b.value - a.value).slice(0, 7).sort((a, b) => a.value - b.value);
|
|
510
|
+
const shades = [1, 2, 3, 4, 5, 6, 7].map(step => `var(--shade-${step})`);
|
|
511
|
+
const chart = ({ title, subtitle, items, scale = 100, suffix = "%", meta }) => {
|
|
512
|
+
const visible = chartItems(items);
|
|
513
|
+
const max = Math.max(
|
|
514
|
+
1,
|
|
515
|
+
scale === "max"
|
|
516
|
+
? Math.max(...visible.map(item => item.value))
|
|
517
|
+
: number(scale)
|
|
518
|
+
);
|
|
519
|
+
const bestValue = Math.max(0, ...visible.map(item => number(item.value)));
|
|
520
|
+
return `<article class="chart-shell">
|
|
521
|
+
<div class="chart-header"><h3 class="chart-title">${escapeHtml(title)}</h3><span class="chart-subtitle">· ${escapeHtml(subtitle)}</span></div>
|
|
522
|
+
<div class="bar-plot">${visible.map((item, index) => {
|
|
523
|
+
const winner = number(item.value) === bestValue;
|
|
524
|
+
const height = Math.max(0, Math.min(100, item.value / max * 100));
|
|
525
|
+
const shown = suffix === "%" ? formatScore(item.value) : `${number(item.value).toFixed(1)}`;
|
|
526
|
+
return `<div class="bar-item ${winner ? "winner" : ""}">
|
|
527
|
+
<div class="bar-value">${shown}</div>
|
|
528
|
+
<div class="bar-slot"><div class="bar" style="--value:${height};--shade:${shades[Math.min(index, shades.length - 1)]};animation-delay:${index * 45}ms"></div></div>
|
|
529
|
+
</div>`;
|
|
530
|
+
}).join("")}</div>
|
|
531
|
+
<div class="bar-labels">${visible.map(item => `<div class="bar-label">
|
|
532
|
+
<div class="model-name" title="${escapeHtml(item.model)}">${escapeHtml(item.model)}</div>
|
|
533
|
+
<div class="model-meta">${escapeHtml(meta(item))}</div>
|
|
534
|
+
</div>`).join("")}</div>
|
|
535
|
+
</article>`;
|
|
536
|
+
};
|
|
537
|
+
|
|
538
|
+
const overallItems = scoredModels.map(model => ({ ...model, value: model.score }));
|
|
539
|
+
const speedItems = models.map(model => ({ ...model, value: model.speed }));
|
|
540
|
+
const benchmarkItems = benchmark => rows
|
|
541
|
+
.filter(row => benchmarkLabel(row) === benchmark)
|
|
542
|
+
.map(row => ({ model: row.model, value: number(row.score), passed: row.passed, scoredTotal: number(row.scored_total ?? row.total), speed: aggregateSpeed(row) }));
|
|
543
|
+
|
|
544
|
+
const chartConfigs = new Map();
|
|
545
|
+
chartConfigs.set("overall", {
|
|
546
|
+
title: "Overall", subtitle: "accuracy", items: overallItems,
|
|
547
|
+
meta: item => `${item.passed}/${number(item.scored_total ?? item.total)} scored`,
|
|
548
|
+
});
|
|
549
|
+
chartConfigs.set("speed", {
|
|
550
|
+
title: "Aggregate throughput", subtitle: "output tok/s · wall clock", items: speedItems, scale: "max", suffix: "",
|
|
551
|
+
meta: item => `${item.speed.toFixed(1)} tok/s · ${item.streamSpeed.toFixed(1)} stream · ${item.effective.toFixed(2)}x eff`,
|
|
552
|
+
});
|
|
553
|
+
benchmarks.forEach((benchmark, index) => chartConfigs.set(`benchmark-${index}`, {
|
|
554
|
+
title: benchmark, subtitle: "accuracy", items: benchmarkItems(benchmark),
|
|
555
|
+
meta: item => `${item.passed}/${item.scoredTotal} scored`,
|
|
556
|
+
}));
|
|
557
|
+
|
|
558
|
+
document.getElementById("reportMeta").textContent = [date, environment, report.host].filter(Boolean).join(" · ");
|
|
559
|
+
document.getElementById("best").innerHTML = best
|
|
560
|
+
? `<div class="best-score">${formatScore(best.score)}</div><div class="best-model">${escapeHtml(best.model)} · best overall</div>`
|
|
561
|
+
: `<div class="best-score">—</div><div class="best-model">Per-context scores · no RULER average</div>`;
|
|
562
|
+
document.getElementById("summary").innerHTML = [
|
|
563
|
+
["Configurations", models.length],
|
|
564
|
+
["Benchmarks", benchmarks.length],
|
|
565
|
+
["Tasks scored", `${totalScored} / ${totalTasks} attempted`],
|
|
566
|
+
["Loops detected", `${totalLoops} · ${totalTasks ? (totalLoops / totalTasks * 100).toFixed(1) : "0.0"}%`],
|
|
567
|
+
["Peak throughput", `${Math.max(0, ...models.map(model => model.speed)).toFixed(1)} tok/s`],
|
|
568
|
+
["Runtime", formatTime(totalRuntime)],
|
|
569
|
+
].map(([label, value]) => `<div class="stat"><div class="stat-label">${label}</div><div class="stat-value">${value}</div></div>`).join("");
|
|
570
|
+
document.getElementById("contamination").innerHTML = contaminatedTasks.length
|
|
571
|
+
? `<article class="chart-shell"><div class="chart-header"><h3 class="chart-title">⚠️ Contamination detected</h3><span class="chart-subtitle">· excluded from headline score</span></div><p>${contaminatedTasks.length} task(s) · ${escapeHtml(Object.entries(contaminationByLanguage).sort().map(([language, count]) => `${language} ${count}`).join(" · "))}</p><p>${escapeHtml(contaminatedTasks.map(task => task.task_id).join(", "))}</p></article>`
|
|
572
|
+
: "";
|
|
573
|
+
document.getElementById("overallChart").innerHTML = scoredModels.length
|
|
574
|
+
? chart(chartConfigs.get("overall"))
|
|
575
|
+
: `<article class="chart-shell"><div class="chart-header"><h3 class="chart-title">Overall</h3><span class="chart-subtitle">· not applicable</span></div><p class="section-note">RULER is reported independently at each context length.</p></article>`;
|
|
576
|
+
document.getElementById("harnessChart").innerHTML = harnessRows.length
|
|
577
|
+
? `<article class="chart-shell matrix-card">
|
|
578
|
+
<div class="chart-header"><h3 class="chart-title">Harness score comparison</h3><span class="chart-subtitle">· paired direct vs stock Pi</span></div>
|
|
579
|
+
<div class="matrix-wrap"><table>
|
|
580
|
+
<thead><tr><th>Model</th><th>Benchmark</th><th>Feedback</th><th>Paired</th><th>Direct first</th><th>Pi first</th><th>Initial score Δ</th><th>Direct final</th><th>Pi final</th><th>Final score Δ</th><th>Direct loop kill</th><th>Pi loop kill</th><th>Loop-kill Δ</th><th>Gains</th><th>Regressions</th><th>Pi version</th></tr></thead>
|
|
581
|
+
<tbody>${harnessRows.map(row => `<tr>
|
|
582
|
+
<td>${escapeHtml(row.model.replace(/ · Pi agent(?: \+ repair)?$/, ""))}</td>
|
|
583
|
+
<td>${escapeHtml(benchmarkLabel(row))}</td>
|
|
584
|
+
<td>${number(row.repair_attempts)} repair turn(s)</td>
|
|
585
|
+
<td>${number(row.harness_paired_total)}</td>
|
|
586
|
+
<td>${number(row.direct_first_score).toFixed(1)}%</td>
|
|
587
|
+
<td>${number(row.harness_first_score).toFixed(1)}%</td>
|
|
588
|
+
<td>${number(row.harness_first_score_delta_pp) >= 0 ? "+" : ""}${number(row.harness_first_score_delta_pp).toFixed(1)}pp</td>
|
|
589
|
+
<td>${number(row.direct_score).toFixed(1)}%</td>
|
|
590
|
+
<td>${number(row.harness_score).toFixed(1)}%</td>
|
|
591
|
+
<td>${number(row.harness_score_delta_pp) >= 0 ? "+" : ""}${number(row.harness_score_delta_pp).toFixed(1)}pp</td>
|
|
592
|
+
<td>${number(row.direct_loop_kill_rate).toFixed(1)}%</td>
|
|
593
|
+
<td>${number(row.harness_loop_kill_rate).toFixed(1)}%</td>
|
|
594
|
+
<td>${number(row.loop_kill_delta_pp) >= 0 ? "+" : ""}${number(row.loop_kill_delta_pp).toFixed(1)}pp</td>
|
|
595
|
+
<td>${number(row.harness_gains)}</td>
|
|
596
|
+
<td>${number(row.harness_regressions)}</td>
|
|
597
|
+
<td>${escapeHtml(row.harness_version || "latest")}</td>
|
|
598
|
+
</tr>`).join("")}</tbody>
|
|
599
|
+
</table></div>
|
|
600
|
+
</article>`
|
|
601
|
+
: "";
|
|
602
|
+
document.getElementById("repairChart").innerHTML = repairRows.length
|
|
603
|
+
? `<article class="chart-shell matrix-card">
|
|
604
|
+
<div class="chart-header"><h3 class="chart-title">Verifier repair effect</h3><span class="chart-subtitle">· first answer vs final answer</span></div>
|
|
605
|
+
<div class="matrix-wrap"><table>
|
|
606
|
+
<thead><tr><th>Configuration</th><th>Benchmark</th><th>First</th><th>Final</th><th>Delta</th><th>Retried</th><th>Repaired</th></tr></thead>
|
|
607
|
+
<tbody>${repairRows.map(row => `<tr>
|
|
608
|
+
<td>${escapeHtml(row.model)}</td>
|
|
609
|
+
<td>${escapeHtml(benchmarkLabel(row))}</td>
|
|
610
|
+
<td>${number(row.first_attempt_score).toFixed(1)}%</td>
|
|
611
|
+
<td>${number(row.score).toFixed(1)}%</td>
|
|
612
|
+
<td>${number(row.repair_delta_pp) >= 0 ? "+" : ""}${number(row.repair_delta_pp).toFixed(1)}pp</td>
|
|
613
|
+
<td>${number(row.repair_attempted)}</td>
|
|
614
|
+
<td>${number(row.repair_successes)}</td>
|
|
615
|
+
</tr>`).join("")}</tbody>
|
|
616
|
+
</table></div>
|
|
617
|
+
</article>`
|
|
618
|
+
: "";
|
|
619
|
+
document.getElementById("robustnessChart").innerHTML = robustnessRows.length
|
|
620
|
+
? `<article class="chart-shell matrix-card">
|
|
621
|
+
<div class="chart-header"><h3 class="chart-title">Choice-order robustness</h3><span class="chart-subtitle">· paired clean comparison</span></div>
|
|
622
|
+
<div class="matrix-wrap"><table>
|
|
623
|
+
<thead><tr><th>Model</th><th>Benchmark</th><th>Clean</th><th>Perturbed</th><th>Delta</th><th>Regressions</th></tr></thead>
|
|
624
|
+
<tbody>${robustnessRows.map(row => `<tr>
|
|
625
|
+
<td>${escapeHtml(row.model)}</td>
|
|
626
|
+
<td>${escapeHtml(row.baseline_benchmark_label || row.benchmark)}</td>
|
|
627
|
+
<td>${number(row.baseline_score).toFixed(1)}%</td>
|
|
628
|
+
<td>${number(row.perturbed_score).toFixed(1)}%</td>
|
|
629
|
+
<td>${number(row.score_delta_pp) >= 0 ? "+" : ""}${number(row.score_delta_pp).toFixed(1)}pp</td>
|
|
630
|
+
<td>${number(row.regressions)}/${number(row.paired_total)}</td>
|
|
631
|
+
</tr>`).join("")}</tbody>
|
|
632
|
+
</table></div>
|
|
633
|
+
</article>`
|
|
634
|
+
: "";
|
|
635
|
+
const maxLoopRate = Math.max(1, ...models.map(model => model.loopRate));
|
|
636
|
+
document.getElementById("loopChart").innerHTML = `<article class="chart-shell">
|
|
637
|
+
<div class="chart-header"><h3 class="chart-title">Generation loops</h3><span class="chart-subtitle">· lower is better</span></div>
|
|
638
|
+
<div class="horizontal-list">${[...models].sort((a, b) => a.loopRate - b.loopRate).slice(0, 7).map(model => `<div class="horizontal-row ${model.loopRate === 0 ? "clean" : ""}">
|
|
639
|
+
<div class="horizontal-name" title="${escapeHtml(model.model)}">${escapeHtml(model.model)}</div>
|
|
640
|
+
<div class="horizontal-track"><div class="horizontal-fill loop-fill" style="--value:${model.loopRate / maxLoopRate * 100}"></div></div>
|
|
641
|
+
<div class="horizontal-value">${model.loopRate.toFixed(1)}%</div>
|
|
642
|
+
</div>`).join("")}</div>
|
|
643
|
+
</article>`;
|
|
644
|
+
document.getElementById("benchmarkCharts").innerHTML = benchmarks.map((_, index) => chart(chartConfigs.get(`benchmark-${index}`))).join("");
|
|
645
|
+
|
|
646
|
+
const visibleModels = models.slice(0, 5);
|
|
647
|
+
const seriesShades = [1, 2, 3, 4, 5].map(step => `var(--series-${step})`);
|
|
648
|
+
const rowLookupForBenchmark = benchmark => new Map(
|
|
649
|
+
rows.filter(row => benchmarkLabel(row) === benchmark).map(row => [row.model, row])
|
|
650
|
+
);
|
|
651
|
+
document.getElementById("suiteChart").innerHTML = `<article class="chart-shell">
|
|
652
|
+
<div class="chart-header"><h3 class="chart-title">Capability overview</h3><span class="chart-subtitle">· accuracy</span></div>
|
|
653
|
+
<div class="suite-plot">${benchmarks.map(benchmark => {
|
|
654
|
+
const lookup = rowLookupForBenchmark(benchmark);
|
|
655
|
+
return `<div class="suite-group">
|
|
656
|
+
<div class="suite-bars">${visibleModels.map((model, index) => {
|
|
657
|
+
const row = lookup.get(model.model);
|
|
658
|
+
return `<div class="suite-bar" style="--value:${clamp(row?.score)};--shade:${seriesShades[index]}" title="${escapeHtml(model.model)} · ${row ? number(row.score).toFixed(1) : "0.0"}%"></div>`;
|
|
659
|
+
}).join("")}</div>
|
|
660
|
+
<div class="suite-name">${escapeHtml(benchmark)}</div>
|
|
661
|
+
</div>`;
|
|
662
|
+
}).join("")}</div>
|
|
663
|
+
<div class="legend">${visibleModels.map((model, index) => `<div class="legend-item"><span class="legend-swatch" style="--shade:${seriesShades[index]}"></span>${escapeHtml(model.model)}</div>`).join("")}</div>
|
|
664
|
+
</article>`;
|
|
665
|
+
|
|
666
|
+
const scatterSvg = () => {
|
|
667
|
+
const width = 1000;
|
|
668
|
+
const height = 480;
|
|
669
|
+
const pad = { left: 62, right: 170, top: 22, bottom: 52 };
|
|
670
|
+
const plotWidth = width - pad.left - pad.right;
|
|
671
|
+
const plotHeight = height - pad.top - pad.bottom;
|
|
672
|
+
const maxSpeed = Math.max(1, ...models.map(model => model.speed));
|
|
673
|
+
const x = value => pad.left + Math.log10(number(value) + 1) / Math.log10(maxSpeed + 1) * plotWidth;
|
|
674
|
+
const y = value => pad.top + (1 - clamp(value) / 100) * plotHeight;
|
|
675
|
+
return `<svg class="scatter" viewBox="0 0 ${width} ${height}" role="img" aria-label="Accuracy versus aggregate throughput">
|
|
676
|
+
${[0, 25, 50, 75, 100].map(tick => `<line class="grid-line" x1="${pad.left}" x2="${pad.left + plotWidth}" y1="${y(tick)}" y2="${y(tick)}"></line><text class="axis-label" x="${pad.left - 10}" y="${y(tick) + 3}" text-anchor="end">${tick}%</text>`).join("")}
|
|
677
|
+
<text class="axis-label" x="${pad.left + plotWidth / 2}" y="${height - 8}" text-anchor="middle">AGGREGATE OUTPUT TOKENS / SECOND →</text>
|
|
678
|
+
${scoredModels.slice(0, 8).map((model, index) => {
|
|
679
|
+
const cx = x(model.speed);
|
|
680
|
+
const cy = y(model.score);
|
|
681
|
+
return `<circle class="scatter-dot ${index === 0 ? "winner" : ""}" cx="${cx}" cy="${cy}" r="9"></circle><text class="scatter-label" x="${cx + 14}" y="${cy + (index % 2 ? 18 : -12)}">${escapeHtml(model.model)} · ${model.speed.toFixed(1)}</text>`;
|
|
682
|
+
}).join("")}
|
|
683
|
+
</svg>`;
|
|
684
|
+
};
|
|
685
|
+
document.getElementById("efficiencyChart").innerHTML = `<article class="chart-shell">
|
|
686
|
+
<div class="chart-header"><h3 class="chart-title">Accuracy × throughput</h3><span class="chart-subtitle">· aggregate tok/s · upper right wins</span></div>
|
|
687
|
+
${scatterSvg()}
|
|
688
|
+
</article>`;
|
|
689
|
+
document.getElementById("speedChart").innerHTML = chart(chartConfigs.get("speed"));
|
|
690
|
+
|
|
691
|
+
document.getElementById("compositionChart").innerHTML = `<article class="chart-shell">
|
|
692
|
+
<div class="chart-header"><h3 class="chart-title">Passed tasks</h3><span class="chart-subtitle">· full suite</span></div>
|
|
693
|
+
<div class="stack-list">${scoredModels.slice(0, 7).map(model => `<div class="stack-row">
|
|
694
|
+
<div class="stack-name" title="${escapeHtml(model.model)}">${escapeHtml(model.model)}</div>
|
|
695
|
+
<div class="stack-track"><div class="stack-pass" style="--value:${clamp(model.score)}"></div></div>
|
|
696
|
+
<div class="stack-score">${number(model.score).toFixed(1)}%</div>
|
|
697
|
+
</div>`).join("")}</div>
|
|
698
|
+
<div class="stack-key"><span>Passed</span><span>Failed</span></div>
|
|
699
|
+
</article>`;
|
|
700
|
+
|
|
701
|
+
const maxRuntime = Math.max(1, ...models.map(model => model.runtime));
|
|
702
|
+
document.getElementById("runtimeChart").innerHTML = `<article class="chart-shell">
|
|
703
|
+
<div class="chart-header"><h3 class="chart-title">Total runtime</h3><span class="chart-subtitle">· lower is better</span></div>
|
|
704
|
+
<div class="horizontal-list">${[...models].sort((a, b) => a.runtime - b.runtime).slice(0, 7).map((model, index) => `<div class="horizontal-row ${index === 0 ? "winner" : ""}">
|
|
705
|
+
<div class="horizontal-name" title="${escapeHtml(model.model)}">${escapeHtml(model.model)}</div>
|
|
706
|
+
<div class="horizontal-track"><div class="horizontal-fill" style="--value:${model.runtime / maxRuntime * 100}"></div></div>
|
|
707
|
+
<div class="horizontal-value">${formatTime(model.runtime)}</div>
|
|
708
|
+
</div>`).join("")}</div>
|
|
709
|
+
</article>`;
|
|
710
|
+
|
|
711
|
+
const heat = value => `color-mix(in srgb, var(--green) ${(clamp(value) / 5).toFixed(2)}%, transparent)`;
|
|
712
|
+
const rowLookup = new Map();
|
|
713
|
+
rows.forEach(row => {
|
|
714
|
+
if (!rowLookup.has(row.model)) rowLookup.set(row.model, new Map());
|
|
715
|
+
rowLookup.get(row.model).set(benchmarkLabel(row), row);
|
|
716
|
+
});
|
|
717
|
+
document.getElementById("matrix").innerHTML = `<article class="chart-shell matrix-card">
|
|
718
|
+
<div class="chart-header"><h3 class="chart-title">Score matrix</h3><span class="chart-subtitle">· full suite</span></div>
|
|
719
|
+
<div class="matrix-wrap"><table>
|
|
720
|
+
<thead><tr><th>Model</th><th>Overall</th>${benchmarks.map(item => `<th>${escapeHtml(item)}</th>`).join("")}<th>Loops</th><th>Agg tok/s</th><th>Stream</th><th>Eff.</th></tr></thead>
|
|
721
|
+
<tbody>${models.slice(0, 8).map(model => `<tr>
|
|
722
|
+
<td>${escapeHtml(model.model)}</td><td class="score-cell" style="background:${model.score == null ? "transparent" : heat(model.score)}">${model.score == null ? "—" : `${number(model.score).toFixed(1)}%`}</td>
|
|
723
|
+
${benchmarks.map(benchmark => {
|
|
724
|
+
const row = rowLookup.get(model.model)?.get(benchmark);
|
|
725
|
+
return row
|
|
726
|
+
? `<td style="background:${heat(row.score)}">${number(row.score).toFixed(1)}%</td>`
|
|
727
|
+
: "<td>—</td>";
|
|
728
|
+
}).join("")}<td>${model.loopRate.toFixed(1)}%</td><td>${model.speed.toFixed(1)}</td><td>${model.streamSpeed.toFixed(1)}</td><td>${model.effective.toFixed(2)}x</td>
|
|
729
|
+
</tr>`).join("")}</tbody>
|
|
730
|
+
</table></div>
|
|
731
|
+
</article>`;
|
|
732
|
+
|
|
733
|
+
const arenaRows = rows.filter(row => (row.tasks || []).some(task => task.workspace?.render_status));
|
|
734
|
+
if (arenaRows.length) {
|
|
735
|
+
const arenaTasks = arenaRows.flatMap(row => (row.tasks || []).filter(task => task.workspace?.render_status));
|
|
736
|
+
const renderedTasks = arenaTasks.filter(task => task.workspace.render_status === "rendered");
|
|
737
|
+
document.getElementById("arenaLink").innerHTML = `<a class="arena-link" href="arena.html">Open the arena gallery ↗</a>
|
|
738
|
+
<span class="arena-note">${renderedTasks.length}/${arenaTasks.length} pages rendered · big previews that open each generated page</span>`;
|
|
739
|
+
}
|
|
740
|
+
|
|
741
|
+
document.getElementById("details").innerHTML = rows
|
|
742
|
+
.sort((a, b) => number(b.score) - number(a.score))
|
|
743
|
+
.map(row => `<details>
|
|
744
|
+
<summary><span class="detail-title">${escapeHtml(row.model)}${row.harness_version ? ` ${escapeHtml(row.harness_version)}` : ""} / ${escapeHtml(benchmarkLabel(row))}</span><span class="detail-score">${row.passed}/${number(row.scored_total ?? row.total)} scored · ${number(row.total)} attempted · ${number(row.score).toFixed(1)}% · failures ${number(row.failures)}/${number(row.loop_kills)}/${number(row.timeouts)}/${number(row.length_exceeded)}/${number(row.harness_errors)} (fail/loop/timeout/length/harness)${row.harness_score_delta_pp == null ? "" : ` · harness score ${number(row.harness_score_delta_pp) >= 0 ? "+" : ""}${number(row.harness_score_delta_pp).toFixed(1)}pp · loop-kill ${number(row.loop_kill_delta_pp) >= 0 ? "+" : ""}${number(row.loop_kill_delta_pp).toFixed(1)}pp`}${number(row.repair_attempts) ? ` · repair ${number(row.repair_delta_pp) >= 0 ? "+" : ""}${number(row.repair_delta_pp).toFixed(1)}pp · ${number(row.repair_successes)}/${number(row.repair_attempted)} fixed` : ""}${robustnessMeta(row)}${row.macro_f1 == null ? "" : ` · macro-F1 ${number(row.macro_f1).toFixed(1)}%`}${row.verifier_pass_accuracy == null ? "" : ` · on passing ${number(row.verifier_pass_accuracy).toFixed(1)}% · on failing ${number(row.verifier_fail_accuracy).toFixed(1)}%`} · ${row.harness === "decision" ? `${number(row.decisions_per_s).toFixed(1)} dec/s · p50 ${number(row.decision_latency_p50_ms).toFixed(0)} ms · p95 ${number(row.decision_latency_p95_ms).toFixed(0)} ms · Brier ${number(row.decision_brier).toFixed(3)} · ECE ${number(row.decision_ece).toFixed(3)}` : `${aggregateSpeed(row).toFixed(1)} agg tok/s · ${streamSpeed(row).toFixed(1)} stream · ${effectiveConcurrency(row).toFixed(2)}x eff · ${number(row.throughput_coverage ?? 100).toFixed(1)}% coverage`}</span></summary>
|
|
745
|
+
${row.pi_system_prompt ? `<div class="code thinking-code"><div class="code-label">Pi scaffold · ${number(row.pi_system_prompt_tokens)} tokens · sha256 ${escapeHtml(row.pi_system_prompt_sha256 || "unavailable")} · tools available: ${escapeHtml((row.pi_tools_available || []).join(", ") || "none")}${row.pi_max_output_tokens ? ` · output limit ${number(row.pi_max_output_tokens)} via ${escapeHtml(row.pi_max_output_tokens_field || "unknown")}` : ""}</div><pre>${escapeHtml(row.pi_system_prompt)}</pre></div>` : ""}
|
|
746
|
+
${(row.task_statistics || []).length ? `<div class="code"><div class="code-label">RULER per-task 95% confidence intervals</div><pre>${escapeHtml(row.task_statistics.map(item => `${item.task} · n=${item.samples} · ${number(item.score).toFixed(1)}% · [${number(item.ci95_low).toFixed(1)}%, ${number(item.ci95_high).toFixed(1)}%]`).join("\n"))}</pre></div>` : ""}
|
|
747
|
+
${row.tool_schema_validity_rate == null ? "" : `<div class="code thinking-code"><div class="code-label">Agentic metrics</div><pre>schema valid ${number(row.tool_schema_validity_rate).toFixed(1)}% · avg turns to solve ${row.avg_turns_to_solve == null ? "—" : number(row.avg_turns_to_solve).toFixed(1)} · avg tokens to solve ${row.avg_tokens_to_solve == null ? "—" : number(row.avg_tokens_to_solve).toFixed(0)} · post-error recovery ${row.post_error_recovery_rate == null ? "—" : `${number(row.post_error_recovery_rate).toFixed(1)}%`} · redundant ${number(row.redundant_action_rate).toFixed(1)}% · destructive ${number(row.destructive_action_count)}</pre></div>`}
|
|
748
|
+
<div class="tasks">${(row.tasks || []).map(task => `<article class="task">
|
|
749
|
+
<div class="task-head"><span>${escapeHtml(task.task_id)} · ${number(task.response_time_s).toFixed(1)}s · ${number(task.output_tokens)} output · ${number(task.input_tokens)} input · ${number(task.model_turns || 1)} turns · ${number(task.tool_calls)} tools</span><span><span class="status ${task.passed ? "pass" : number(task.score) > 0 ? "partial" : ""}">${task.contaminated ? "CONTAMINATED" : task.loop_killed ? "LOOP KILLED" : task.timed_out ? "TIMEOUT" : task.length_exceeded ? "LENGTH EXCEEDED" : task.harness_error ? "HARNESS ERROR" : task.passed ? "PASS" : number(task.score) > 0 ? `PARTIAL ${number(task.score).toFixed(0)}%` : "FAIL"}</span><span class="loop-status ${escapeHtml(task.loop_state)}">${escapeHtml(loopLabel(task))}</span></span></div>
|
|
750
|
+
<div class="exchange">
|
|
751
|
+
${task.error ? `<div class="code"><div class="code-label">Error</div><pre>${escapeHtml(task.error)}</pre></div>` : ""}
|
|
752
|
+
${(task.workspace?.checkpoints || []).length ? `<div class="code"><div class="code-label">Checkpoints · ${number(task.workspace.positive_points)}/${number(task.workspace.max_points)} positive · −${number(task.workspace.penalty_points)} penalty</div><pre>${escapeHtml(task.workspace.checkpoints.map(checkpoint => `${checkpoint.awarded >= 0 ? "+" : ""}${number(checkpoint.awarded)} / ${checkpoint.weight >= 0 ? "+" : ""}${number(checkpoint.weight)} ${checkpoint.id}\n${checkpoint.evidence || ""}`).join("\n\n"))}</pre></div>` : ""}
|
|
753
|
+
${task.workspace?.test_output ? `<div class="code"><div class="code-label">Workspace verifier · exit ${number(task.workspace.test_exit_code)} · ${escapeHtml((task.workspace.test_command || []).join(" "))}</div><pre>${escapeHtml(task.workspace.test_output)}</pre></div>` : ""}
|
|
754
|
+
${task.workspace?.patch ? `<div class="code"><div class="code-label">Workspace patch${task.workspace.patch_truncated ? " · truncated" : ""}</div><pre>${escapeHtml(task.workspace.patch)}</pre></div>` : ""}
|
|
755
|
+
${task.workspace?.render_status ? `<div class="code thinking-code"><div class="code-label">Headless render · ${escapeHtml(task.workspace.render_status)} · ${number(task.workspace.animation_frames)} frames · ${number(task.repair_attempts_used)} repair(s)${task.workspace.page_html ? ` · <a href="${escapeHtml(task.workspace.page_html)}" target="_blank" rel="noopener noreferrer">page</a>` : ""}${task.workspace.screenshot ? ` · <a href="${escapeHtml(task.workspace.screenshot)}" target="_blank" rel="noopener noreferrer">screenshot</a>` : ""}</div>${renderDiagnostics(task) ? `<pre>${escapeHtml(renderDiagnostics(task))}</pre>` : ""}</div>` : ""}
|
|
756
|
+
${task.perturbation ? `<div class="code"><div class="code-label">Choice order · seed ${number(task.perturbation.seed)}</div><pre>${escapeHtml(task.perturbation.choice_order.join(", "))}\nanswer ${escapeHtml(task.perturbation.original_answer)} → ${escapeHtml(task.perturbation.perturbed_answer)}</pre></div>` : ""}
|
|
757
|
+
${(task.attempts || []).length ? `<div class="code thinking-code"><div class="code-label">Verifier-feedback attempts</div><pre>${escapeHtml(task.attempts.map(attempt => `Attempt ${number(attempt.attempt)} · ${attempt.passed ? "pass" : "fail"} · ${number(attempt.score).toFixed(1)}%${attempt.feedback ? `\nfeedback: ${attempt.feedback}` : ""}\n\n${attempt.response || ""}`).join("\n\n────────────────────────────────\n\n"))}</pre></div>` : ""}
|
|
758
|
+
${(task.tool_trace || []).length ? `<div class="code thinking-code"><div class="code-label">Pi native tool trace</div><pre>${escapeHtml(task.tool_trace.map(call => `${call.name || "tool"} · attempt ${number(call.attempt || 1)} · ${call.is_error ? "error" : "ok"} · ${number(call.duration_s).toFixed(3)}s\n${JSON.stringify(call.arguments || {}, null, 2)}${call.output ? `\noutput${call.output_truncated ? " (last 8,000 chars)" : ""}:\n${call.output}` : ""}`).join("\n\n"))}</pre></div>` : ""}
|
|
759
|
+
${task.thinking ? `<div class="code thinking-code"><div class="code-label">Thinking · ${number(task.thinking_time_s).toFixed(1)}s · loop score ${(number(task.loop_score) * 100).toFixed(0)}%</div><pre>${escapeHtml(task.thinking)}</pre></div>` : ""}
|
|
760
|
+
<div class="code"><div class="code-label">Prompt</div><pre>${escapeHtml(task.prompt)}</pre></div>
|
|
761
|
+
<div class="code"><div class="code-label">Response</div><pre>${escapeHtml(task.response)}</pre></div>
|
|
762
|
+
</div>
|
|
763
|
+
</article>`).join("")}</div>
|
|
764
|
+
</details>`).join("");
|
|
765
|
+
document.getElementById("footerMeta").textContent = [date, environment].filter(Boolean).join(" · ");
|
|
766
|
+
|
|
767
|
+
</script>
|
|
768
|
+
</body>
|
|
769
|
+
</html>
|