benchkit-cli 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (149) hide show
  1. benchkit/__init__.py +1 -0
  2. benchkit/_pi_proxy.py +219 -0
  3. benchkit/answer_key_guard.ts +49 -0
  4. benchkit/artifacts.py +87 -0
  5. benchkit/benchmarks/__init__.py +105 -0
  6. benchkit/benchmarks/aider_polyglot.py +405 -0
  7. benchkit/benchmarks/arc.py +52 -0
  8. benchkit/benchmarks/banking77.py +149 -0
  9. benchkit/benchmarks/base.py +19 -0
  10. benchkit/benchmarks/boolq.py +49 -0
  11. benchkit/benchmarks/chartqa.py +133 -0
  12. benchkit/benchmarks/code_verifier.py +77 -0
  13. benchkit/benchmarks/downloads.py +166 -0
  14. benchkit/benchmarks/evalplus.py +288 -0
  15. benchkit/benchmarks/git_surgery.py +485 -0
  16. benchkit/benchmarks/gpqa.py +56 -0
  17. benchkit/benchmarks/gsm8k.py +97 -0
  18. benchkit/benchmarks/hellaswag.py +49 -0
  19. benchkit/benchmarks/humaneval.py +103 -0
  20. benchkit/benchmarks/ifeval.py +56 -0
  21. benchkit/benchmarks/ifeval_instructions.py +477 -0
  22. benchkit/benchmarks/mbpp.py +60 -0
  23. benchkit/benchmarks/mc_arena.py +479 -0
  24. benchkit/benchmarks/mcq.py +94 -0
  25. benchkit/benchmarks/medfailbench.py +537 -0
  26. benchkit/benchmarks/mmlu.py +54 -0
  27. benchkit/benchmarks/mmlu_pro.py +62 -0
  28. benchkit/benchmarks/mmvp.py +115 -0
  29. benchkit/benchmarks/ocrbench.py +96 -0
  30. benchkit/benchmarks/openbookqa.py +52 -0
  31. benchkit/benchmarks/patcheval.py +929 -0
  32. benchkit/benchmarks/piqa.py +52 -0
  33. benchkit/benchmarks/ruler.py +172 -0
  34. benchkit/benchmarks/ruler_generator.py +535 -0
  35. benchkit/benchmarks/sanity.py +174 -0
  36. benchkit/benchmarks/treejs_arena.py +268 -0
  37. benchkit/benchmarks/truthfulqa.py +52 -0
  38. benchkit/benchmarks/utils.py +25 -0
  39. benchkit/benchmarks/vision.py +230 -0
  40. benchkit/benchmarks/vstar.py +108 -0
  41. benchkit/benchmarks/winogrande.py +52 -0
  42. benchkit/benchmarks/xstest.py +116 -0
  43. benchkit/browser.py +429 -0
  44. benchkit/catalog.py +121 -0
  45. benchkit/cli.py +1000 -0
  46. benchkit/client.py +1413 -0
  47. benchkit/controls.py +87 -0
  48. benchkit/datasets/MMLU_LICENSE.txt +21 -0
  49. benchkit/datasets/README.md +142 -0
  50. benchkit/datasets/arc_challenge.jsonl +1172 -0
  51. benchkit/datasets/banking77.jsonl +3080 -0
  52. benchkit/datasets/banking77_intents.json +79 -0
  53. benchkit/datasets/boolq.jsonl +3270 -0
  54. benchkit/datasets/code_verifier.jsonl +500 -0
  55. benchkit/datasets/gpqa_diamond.jsonl +198 -0
  56. benchkit/datasets/gsm8k.jsonl +1319 -0
  57. benchkit/datasets/hellaswag.jsonl +1000 -0
  58. benchkit/datasets/humaneval.jsonl +164 -0
  59. benchkit/datasets/ifeval.jsonl +541 -0
  60. benchkit/datasets/mbpp.jsonl +500 -0
  61. benchkit/datasets/mc_blocks_1_20_1.jsonl +1003 -0
  62. benchkit/datasets/medfailbench_hard30.jsonl +30 -0
  63. benchkit/datasets/medfailbench_hard30.md +13 -0
  64. benchkit/datasets/openbookqa.jsonl +500 -0
  65. benchkit/datasets/piqa.jsonl +1838 -0
  66. benchkit/datasets/sanity_code.jsonl +6 -0
  67. benchkit/datasets/truthfulqa.jsonl +817 -0
  68. benchkit/datasets/winogrande.jsonl +1267 -0
  69. benchkit/datasets/xstest.csv +451 -0
  70. benchkit/decision.py +194 -0
  71. benchkit/demo.py +451 -0
  72. benchkit/dockerfiles.py +247 -0
  73. benchkit/engine.py +1270 -0
  74. benchkit/evaluation.py +195 -0
  75. benchkit/events.py +141 -0
  76. benchkit/executor.py +96 -0
  77. benchkit/git_surgery/backport-release-stack/setup.sh +328 -0
  78. benchkit/git_surgery/backport-release-stack/verify.sh +119 -0
  79. benchkit/git_surgery/bisect-the-regression/setup.sh +49 -0
  80. benchkit/git_surgery/bisect-the-regression/verify.sh +34 -0
  81. benchkit/git_surgery/rebase-conflict-chain/setup.sh +65 -0
  82. benchkit/git_surgery/rebase-conflict-chain/verify.sh +43 -0
  83. benchkit/git_surgery/recover-complex-stash/setup.sh +152 -0
  84. benchkit/git_surgery/recover-complex-stash/verify.sh +95 -0
  85. benchkit/git_surgery/recover-lost-work/setup.sh +40 -0
  86. benchkit/git_surgery/recover-lost-work/verify.sh +36 -0
  87. benchkit/git_surgery/repair-force-pushed-remote/setup.sh +171 -0
  88. benchkit/git_surgery/repair-force-pushed-remote/verify.sh +83 -0
  89. benchkit/git_surgery/revert-merge-with-followups/setup.sh +198 -0
  90. benchkit/git_surgery/revert-merge-with-followups/verify.sh +98 -0
  91. benchkit/git_surgery/secret-in-history/README.md +27 -0
  92. benchkit/git_surgery/secret-in-history/setup.sh +107 -0
  93. benchkit/git_surgery/secret-in-history/verify.sh +106 -0
  94. benchkit/git_surgery/split-the-mega-commit/setup.sh +50 -0
  95. benchkit/git_surgery/split-the-mega-commit/verify.sh +48 -0
  96. benchkit/git_surgery/untangle-nested-submodules/setup.sh +193 -0
  97. benchkit/git_surgery/untangle-nested-submodules/verify.sh +90 -0
  98. benchkit/history.py +474 -0
  99. benchkit/jobs.py +332 -0
  100. benchkit/leakage.py +51 -0
  101. benchkit/looping.py +873 -0
  102. benchkit/mc_render.py +274 -0
  103. benchkit/mc_viewer/dist/atlas.png +0 -0
  104. benchkit/mc_viewer/dist/blockStates.json.gz +0 -0
  105. benchkit/mc_viewer/dist/version.json +4 -0
  106. benchkit/mc_viewer/dist/viewer.html +23 -0
  107. benchkit/mc_viewer/dist/viewer.js +2 -0
  108. benchkit/mc_viewer/dist/viewer.js.LICENSE.txt +25 -0
  109. benchkit/mc_viewer/dist/worker.js +2 -0
  110. benchkit/mc_viewer/dist/worker.js.LICENSE.txt +19 -0
  111. benchkit/metrics.py +100 -0
  112. benchkit/ollama.py +60 -0
  113. benchkit/perf.py +353 -0
  114. benchkit/perf_report.py +207 -0
  115. benchkit/perturbations.py +208 -0
  116. benchkit/pi_agent.py +476 -0
  117. benchkit/pi_package/package-lock.json +1827 -0
  118. benchkit/pi_package/package.json +8 -0
  119. benchkit/report.py +142 -0
  120. benchkit/report_markdown.py +645 -0
  121. benchkit/results.py +688 -0
  122. benchkit/retry.py +163 -0
  123. benchkit/runner.py +769 -0
  124. benchkit/sandbox.py +1167 -0
  125. benchkit/template_check.py +291 -0
  126. benchkit/templates/arena.html +504 -0
  127. benchkit/templates/history.html +58 -0
  128. benchkit/templates/perf.html +509 -0
  129. benchkit/templates/report.html +769 -0
  130. benchkit/tui/__init__.py +5 -0
  131. benchkit/tui/app.py +87 -0
  132. benchkit/tui/app.tcss +512 -0
  133. benchkit/tui/formatting.py +84 -0
  134. benchkit/tui/screens/__init__.py +1 -0
  135. benchkit/tui/screens/connect.py +191 -0
  136. benchkit/tui/screens/detail.py +288 -0
  137. benchkit/tui/screens/help.py +84 -0
  138. benchkit/tui/screens/modals.py +544 -0
  139. benchkit/tui/screens/results.py +361 -0
  140. benchkit/tui/screens/run.py +988 -0
  141. benchkit/tui/screens/setup.py +733 -0
  142. benchkit/tui/theme.py +135 -0
  143. benchkit/tui/widgets.py +120 -0
  144. benchkit_cli-0.1.0.dist-info/METADATA +294 -0
  145. benchkit_cli-0.1.0.dist-info/RECORD +149 -0
  146. benchkit_cli-0.1.0.dist-info/WHEEL +5 -0
  147. benchkit_cli-0.1.0.dist-info/entry_points.txt +3 -0
  148. benchkit_cli-0.1.0.dist-info/licenses/LICENSE +201 -0
  149. benchkit_cli-0.1.0.dist-info/top_level.txt +1 -0
benchkit/templates/arena.html ADDED
@@ -0,0 +1,504 @@
1
+ <!doctype html>
2
+ <html lang="en">
3
+ <head>
4
+ <meta charset="utf-8">
5
+ <meta name="viewport" content="width=device-width, initial-scale=1">
6
+ <meta name="color-scheme" content="light dark">
7
+ <title>BenchKit Arena</title>
8
+ <script>
9
+ // Applied before paint so the gallery never flashes the wrong theme, and
10
+ // shares the report's stored choice.
11
+ (() => {
12
+ let stored = null;
13
+ try { stored = localStorage.getItem("benchkit-theme"); } catch (error) { stored = null; }
14
+ document.documentElement.dataset.theme = stored === "dark" ? "dark" : "light";
15
+ })();
16
+ </script>
17
+ <style>
18
+ /* Dogi (https://github.com/DogukanUrker/DogiZed): flat black or white,
19
+ colour only where it carries meaning. Kept in step with report.html. */
20
+ :root,
21
+ :root[data-theme="light"] {
22
+ color-scheme: light;
23
+ --page: #ffffff;
24
+ --card: #ffffff;
25
+ --ink: #0b0b0b;
26
+ --muted: #6b7280;
27
+ --line: rgba(11, 11, 11, .12);
28
+ --raise: rgba(11, 11, 11, .04);
29
+ --track: rgba(11, 11, 11, .09);
30
+ --green: #059669;
31
+ --red: #dc2626;
32
+ --blue: #2563eb;
33
+ --amber: #d97706;
34
+ }
35
+
36
+ :root[data-theme="dark"] {
37
+ color-scheme: dark;
38
+ --page: #000000;
39
+ --card: #000000;
40
+ --ink: #ffffff;
41
+ --muted: #6b7280;
42
+ --line: rgba(255, 255, 255, .14);
43
+ --raise: rgba(255, 255, 255, .06);
44
+ --track: rgba(255, 255, 255, .12);
45
+ --green: #34d399;
46
+ --red: #ef4444;
47
+ --blue: #60a5fa;
48
+ --amber: #facc15;
49
+ }
50
+
51
+ :root {
52
+ --sans: "Avenir Next", Avenir, "Century Gothic", sans-serif;
53
+ --mono: "SFMono-Regular", "Cascadia Mono", Menlo, monospace;
54
+ }
55
+
56
+ * { box-sizing: border-box; }
57
+ html { background: var(--page); }
58
+ body {
59
+ margin: 0;
60
+ min-height: 100vh;
61
+ color: var(--ink);
62
+ background: var(--page);
63
+ font-family: var(--sans);
64
+ -webkit-font-smoothing: antialiased;
65
+ }
66
+ button { font: inherit; }
67
+ a { color: inherit; }
68
+ .page { width: min(1360px, calc(100% - 40px)); margin: 0 auto; padding: 54px 0 70px; }
69
+
70
+ .head {
71
+ display: flex;
72
+ justify-content: space-between;
73
+ gap: 32px;
74
+ align-items: end;
75
+ padding-bottom: 30px;
76
+ border-bottom: 1px solid var(--line);
77
+ }
78
+ h1, h2, h3, p { margin-top: 0; }
79
+ h1 { margin-bottom: 7px; font-size: 42px; line-height: 1; letter-spacing: -.045em; }
80
+ .meta { margin: 0; color: var(--muted); font: 11px/1.55 var(--mono); }
81
+ .head-right { display: flex; flex-direction: column; align-items: flex-end; gap: 18px; }
82
+ .rate { color: var(--green); font-size: 42px; font-weight: 800; line-height: 1; letter-spacing: -.045em; }
83
+ .rate-label { margin-top: 6px; font-size: 12px; font-weight: 800; text-align: right; }
84
+
85
+ .theme-toggle {
86
+ display: inline-flex;
87
+ align-items: center;
88
+ gap: 8px;
89
+ padding: 7px 14px;
90
+ border: 1px solid var(--line);
91
+ border-radius: 999px;
92
+ background: var(--raise);
93
+ color: var(--ink);
94
+ cursor: pointer;
95
+ font: 700 9px/1 var(--mono);
96
+ letter-spacing: .09em;
97
+ text-transform: uppercase;
98
+ }
99
+ .theme-toggle:hover { border-color: var(--blue); color: var(--blue); }
100
+ .theme-toggle[aria-pressed="false"] { color: var(--muted); }
101
+ .theme-toggle[aria-pressed="false"] .theme-dot { background: var(--track); }
102
+ .toggles { display: flex; gap: 10px; flex-wrap: wrap; justify-content: flex-end; }
103
+ .theme-dot { width: 9px; height: 9px; border-radius: 50%; background: var(--blue); }
104
+
105
+ .summary {
106
+ display: grid;
107
+ grid-template-columns: repeat(4, 1fr);
108
+ margin: 28px 0 42px;
109
+ border: 1px solid var(--line);
110
+ border-radius: 16px;
111
+ overflow: hidden;
112
+ }
113
+ .stat { padding: 20px 22px; }
114
+ .stat + .stat { border-left: 1px solid var(--line); }
115
+ .stat-value { font-size: 27px; font-weight: 800; letter-spacing: -.03em; }
116
+ .stat-label { margin-top: 6px; color: var(--muted); font: 8px/1.3 var(--mono); letter-spacing: .09em; text-transform: uppercase; }
117
+
118
+ .models { margin-bottom: 54px; border: 1px solid var(--line); border-radius: 16px; overflow: hidden; }
119
+ .model-row { display: grid; grid-template-columns: minmax(140px, 240px) 1fr 132px; gap: 16px; align-items: center; padding: 15px 22px; }
120
+ .model-row + .model-row { border-top: 1px solid var(--line); }
121
+ .model-name { overflow-wrap: anywhere; font: 800 11px/1.3 var(--mono); }
122
+ .track { height: 9px; border-radius: 999px; background: var(--track); overflow: hidden; }
123
+ .fill { height: 100%; border-radius: 999px; background: var(--green); }
124
+ .model-value { color: var(--muted); text-align: right; font: 9px/1.4 var(--mono); }
125
+
126
+ .scene { margin-bottom: 46px; }
127
+ .scene-head { display: flex; justify-content: space-between; align-items: baseline; gap: 20px; margin-bottom: 16px; }
128
+ .scene-title { font-size: 15px; font-weight: 800; letter-spacing: -.01em; }
129
+ .scene-id { color: var(--muted); font: 9px/1 var(--mono); }
130
+ .grid { display: grid; grid-template-columns: repeat(auto-fit, minmax(400px, 540px)); justify-content: start; gap: 22px; }
131
+
132
+ .card { border: 1px solid var(--line); border-radius: 14px; background: var(--card); overflow: hidden; }
133
+ /* The stage holds the screenshot the run captured and, when live previews
134
+ are on, the page itself running in this browser on top of it. */
135
+ .stage { position: relative; width: 100%; aspect-ratio: 16 / 10; background: #000; overflow: hidden; }
136
+ .stage img { position: absolute; inset: 0; width: 100%; height: 100%; object-fit: cover; }
137
+ .stage iframe { position: absolute; inset: 0; width: 100%; height: 100%; border: 0; z-index: 1; background: #000; }
138
+ .stage-empty { position: absolute; inset: 0; display: grid; place-items: center; background: var(--card); box-shadow: inset 0 0 0 1px var(--line); color: var(--muted); font: 9px/1 var(--mono); letter-spacing: .09em; text-transform: uppercase; }
139
+ .stage-link { position: absolute; inset: 0; z-index: 2; text-decoration: none; }
140
+ .is-live .stage-link { display: none; }
141
+ .open-hint {
142
+ position: absolute;
143
+ inset: auto 0 0 0;
144
+ padding: 10px 14px;
145
+ background: linear-gradient(transparent, rgba(0, 0, 0, .78));
146
+ color: #fff;
147
+ opacity: 0;
148
+ transition: opacity .16s ease;
149
+ font: 700 9px/1 var(--mono);
150
+ letter-spacing: .09em;
151
+ text-transform: uppercase;
152
+ }
153
+ .stage-link:hover .open-hint, .stage-link:focus-visible .open-hint { opacity: 1; }
154
+ .stage-link:focus-visible { outline: 2px solid var(--blue); outline-offset: -2px; }
155
+ /* Sits above a live frame, which swallows clicks meant for the card. */
156
+ .open-badge {
157
+ position: absolute;
158
+ top: 9px;
159
+ right: 9px;
160
+ z-index: 3;
161
+ padding: 6px 9px;
162
+ border-radius: 7px;
163
+ background: rgba(0, 0, 0, .62);
164
+ color: #fff;
165
+ opacity: 0;
166
+ transition: opacity .16s ease;
167
+ text-decoration: none;
168
+ font: 700 8px/1 var(--mono);
169
+ letter-spacing: .09em;
170
+ text-transform: uppercase;
171
+ }
172
+ .card:hover .open-badge, .open-badge:focus-visible { opacity: 1; }
173
+ .live-badge { position: absolute; top: 9px; left: 9px; z-index: 3; padding: 6px 9px; border-radius: 7px; background: rgba(0, 0, 0, .62); color: #fff; font: 700 8px/1 var(--mono); letter-spacing: .09em; text-transform: uppercase; }
174
+
175
+ .card-foot { display: flex; justify-content: space-between; align-items: center; gap: 14px; padding: 12px 15px; border-top: 1px solid var(--line); font: 9px/1.4 var(--mono); }
176
+ .card-model { font-weight: 800; overflow-wrap: anywhere; }
177
+ .card-sub { margin-top: 4px; color: var(--muted); }
178
+ .status { font-weight: 900; color: var(--red); white-space: nowrap; }
179
+ .status.pass { color: var(--green); }
180
+ .status.warn { color: var(--amber); }
181
+ .status.skip { color: var(--muted); }
182
+ .links { display: flex; gap: 12px; padding: 0 15px 13px; font: 9px/1 var(--mono); }
183
+ .links a { color: var(--blue); text-decoration: none; }
184
+ .links a:hover { text-decoration: underline; }
185
+ details.diag { border-top: 1px solid var(--line); }
186
+ details.diag summary { padding: 10px 15px; color: var(--muted); cursor: pointer; font: 8px/1 var(--mono); letter-spacing: .09em; text-transform: uppercase; }
187
+ details.diag pre { max-height: 220px; margin: 0; padding: 0 15px 14px; overflow: auto; white-space: pre-wrap; word-break: break-word; font: 9px/1.55 var(--mono); }
188
+
189
+ .empty { padding: 60px 0; color: var(--muted); font: 11px/1.6 var(--mono); }
190
+ footer { display: flex; justify-content: space-between; gap: 20px; margin-top: 70px; padding-top: 24px; border-top: 1px solid var(--line); color: var(--muted); font: 8px/1.4 var(--mono); }
191
+
192
+ @media (max-width: 720px) {
193
+ .page { width: min(100% - 24px, 600px); padding-top: 34px; }
194
+ .head { display: block; }
195
+ .head-right { align-items: flex-start; margin-top: 24px; }
196
+ .rate-label { text-align: left; }
197
+ .summary { grid-template-columns: repeat(2, 1fr); }
198
+ .stat:nth-child(3) { border-top: 1px solid var(--line); border-left: 0; }
199
+ .stat:nth-child(4) { border-top: 1px solid var(--line); }
200
+ .grid { grid-template-columns: 1fr; }
201
+ .model-row { grid-template-columns: 1fr; gap: 8px; }
202
+ .model-value { text-align: left; }
203
+ }
204
+ </style>
205
+ </head>
206
+ <body>
207
+ <main class="page">
208
+ <header class="head">
209
+ <div>
210
+ <h1>Arena</h1>
211
+ <p class="meta" id="headMeta"></p>
212
+ </div>
213
+ <div class="head-right">
214
+ <div class="toggles">
215
+ <button class="theme-toggle" id="liveToggle" type="button" aria-pressed="true">
216
+ <span class="theme-dot"></span><span id="liveLabel">Live previews on</span>
217
+ </button>
218
+ <button class="theme-toggle" id="themeToggle" type="button" aria-label="Switch colour theme">
219
+ <span class="theme-dot"></span><span id="themeLabel">Dark</span>
220
+ </button>
221
+ </div>
222
+ <div>
223
+ <div class="rate" id="rate"></div>
224
+ <div class="rate-label" id="rateLabel"></div>
225
+ </div>
226
+ </div>
227
+ </header>
228
+
229
+ <section class="summary" id="summary"></section>
230
+ <section class="models" id="models"></section>
231
+ <section id="scenes"></section>
232
+
233
+ <footer><span id="footerMeta"></span><span>Generated by BenchKit</span></footer>
234
+ </main>
235
+
236
+ <script id="arenaData" type="application/json">__BENCHKIT_ARENA_DATA__</script>
237
+ <script>
238
+ const root = document.documentElement;
239
+ const themeLabel = document.getElementById("themeLabel");
240
+ const setTheme = mode => {
241
+ root.dataset.theme = mode;
242
+ themeLabel.textContent = mode === "dark" ? "Light" : "Dark";
243
+ try { localStorage.setItem("benchkit-theme", mode); } catch (error) { /* private mode */ }
244
+ };
245
+ setTheme(root.dataset.theme === "dark" ? "dark" : "light");
246
+ document.getElementById("themeToggle").addEventListener("click", () => {
247
+ setTheme(root.dataset.theme === "dark" ? "light" : "dark");
248
+ });
249
+
250
+ const arena = JSON.parse(document.getElementById("arenaData").textContent);
251
+ const escapeHtml = value => String(value ?? "").replace(/[&<>"']/g, char => ({"&":"&amp;","<":"&lt;",">":"&gt;",'"':"&quot;","'":"&#39;"}[char]));
252
+ const number = value => Number(value) || 0;
253
+ const percent = (part, total) => total > 0 ? `${(part / total * 100).toFixed(0)}%` : "—";
254
+
255
+ // A task that never reached the browser - a timeout, a killed loop, an
256
+ // output-limit stop - still belongs here: leaving it out would quietly
257
+ // shrink the denominator and overstate the render rate.
258
+ const status = task => {
259
+ // mc-arena is scored on the block list its script printed, so its own
260
+ // verdict wins over whether a picture came back.
261
+ const built = task.workspace?.build_status;
262
+ if (built) return built;
263
+ const rendered = task.workspace?.render_status;
264
+ if (rendered) return rendered;
265
+ if (task.loop_killed) return "loop_killed";
266
+ if (task.timed_out) return "timed_out";
267
+ if (task.length_exceeded) return "length_exceeded";
268
+ if (task.harness_error) return "harness_error";
269
+ return "not_rendered";
270
+ };
271
+ const sceneOf = task => task.workspace?.scene
272
+ || String(task.prompt || "").trim().split("\n").filter(line => line.trim()).pop()
273
+ || task.task_id;
274
+
275
+ const entries = arena.results.flatMap(row => (row.tasks || [])
276
+ .map(task => ({ model: row.model, harness: row.harness_label || "", row, task, status: status(task) })));
277
+
278
+ const PASSED = new Set(["rendered", "valid"]);
279
+ const rendered = entries.filter(entry => PASSED.has(entry.status));
280
+ // Machines without a browser, a GPU stack or a route to the CDN are
281
+ // ordinary: those tasks keep their credit and are not counted against
282
+ // the render rate.
283
+ const skipped = entries.filter(entry => entry.status === "skipped");
284
+ const scored = entries.filter(entry => entry.status !== "skipped");
285
+ const harnessErrors = entries.filter(entry => entry.status === "harness_error");
286
+ const models = [...new Set(entries.map(entry => entry.model))];
287
+ const promptSets = [...new Set(arena.results.map(row => row.prompt_set_version).filter(Boolean))];
288
+ const date = String(arena.generated_at || "").replace("_", " · ").replaceAll("-", ".");
289
+ const environment = [arena.provider, arena.hardware].filter(Boolean).join(" · ") || "Local inference";
290
+
291
+ document.getElementById("headMeta").textContent = [
292
+ date,
293
+ environment,
294
+ promptSets.length ? `prompt set ${promptSets.join(", ")}` : "",
295
+ "previews run live in this browser · drag to orbit · Open ↗ for a full tab",
296
+ ].filter(Boolean).join(" · ");
297
+ document.getElementById("footerMeta").textContent = [date, environment].filter(Boolean).join(" · ");
298
+ document.getElementById("rate").textContent = percent(rendered.length, scored.length);
299
+ // treejs-arena passes on rendering, mc-arena on producing a usable build.
300
+ const passLabel = entries.some(entry => entry.task.workspace?.build_status)
301
+ ? (entries.every(entry => entry.task.workspace?.build_status) ? "builds valid" : "passed")
302
+ : "pages rendered";
303
+ document.getElementById("rateLabel").textContent = scored.length
304
+ ? `${rendered.length}/${scored.length} ${passLabel}${skipped.length ? ` · ${skipped.length} skipped` : ""}`
305
+ : `${skipped.length} skipped · this machine cannot render`;
306
+
307
+ const scenes = new Map();
308
+ entries.forEach(entry => {
309
+ const id = entry.task.task_id;
310
+ if (!scenes.has(id)) scenes.set(id, { id, scene: sceneOf(entry.task), entries: [] });
311
+ scenes.get(id).entries.push(entry);
312
+ });
313
+
314
+ const repairs = entries.reduce((sum, entry) => sum + number(entry.task.repair_attempts_used), 0);
315
+ document.getElementById("summary").innerHTML = [
316
+ { value: models.length, label: "models" },
317
+ { value: scenes.size, label: "scenes" },
318
+ { value: `${rendered.length}/${scored.length}`, label: skipped.length ? `${passLabel} · ${skipped.length} skipped` : passLabel },
319
+ { value: harnessErrors.length ? `${repairs} · ${harnessErrors.length}` : repairs, label: harnessErrors.length ? "repairs · harness errors" : "repair turns" },
320
+ ].map(stat => `<div class="stat">
321
+ <div class="stat-value">${escapeHtml(stat.value)}</div>
322
+ <div class="stat-label">${escapeHtml(stat.label)}</div>
323
+ </div>`).join("");
324
+
325
+ document.getElementById("models").innerHTML = models.map(model => {
326
+ const own = entries.filter(entry => entry.model === model);
327
+ const judged = own.filter(entry => entry.status !== "skipped");
328
+ const ok = own.filter(entry => PASSED.has(entry.status)).length;
329
+ const share = judged.length ? ok / judged.length * 100 : 0;
330
+ const modelRepairs = own.reduce((sum, entry) => sum + number(entry.task.repair_attempts_used), 0);
331
+ const skips = own.length - judged.length;
332
+ return `<div class="model-row">
333
+ <div class="model-name" title="${escapeHtml(model)}">${escapeHtml(model)}</div>
334
+ <div class="track"><div class="fill" style="width:${share.toFixed(1)}%"></div></div>
335
+ <div class="model-value">${ok}/${judged.length} · ${judged.length ? `${share.toFixed(0)}%` : "—"}${skips ? ` · ${skips} skipped` : ""}${modelRepairs ? ` · ${modelRepairs} repair${modelRepairs === 1 ? "" : "s"}` : ""}</div>
336
+ </div>`;
337
+ }).join("");
338
+
339
+ const STATUS_TEXT = {
340
+ rendered: "RENDERED",
341
+ valid: "VALID BUILD",
342
+ invalid: "INVALID BUILD",
343
+ no_script: "NO SCRIPT",
344
+ script_error: "SCRIPT ERROR",
345
+ script_timeout: "SCRIPT TIMEOUT",
346
+ bad_output: "BAD OUTPUT",
347
+ output_limit: "OUTPUT LIMIT",
348
+ failed: "FAILED",
349
+ no_html: "NO HTML",
350
+ skipped: "SKIPPED",
351
+ harness_error: "HARNESS ERROR",
352
+ timed_out: "TIMED OUT",
353
+ loop_killed: "LOOP KILLED",
354
+ length_exceeded: "LENGTH LIMIT",
355
+ not_rendered: "NOT RENDERED",
356
+ };
357
+ const WARN = new Set(["harness_error", "timed_out", "loop_killed", "length_exceeded", "script_timeout"]);
358
+ const NEUTRAL = new Set(["skipped"]);
359
+ const diagnostics = task => [
360
+ ...(task.workspace?.page_errors || []).map(item => `[uncaught] ${item}`),
361
+ ...(task.workspace?.console_errors || []).map(item => `[console] ${item}`),
362
+ ...(task.workspace?.blocked_requests || []).map(item => `[blocked] ${item}`),
363
+ ...(task.workspace?.failed_requests || []).map(item => `[network] ${item}`),
364
+ ...(task.workspace?.render_diagnostics ? [task.workspace.render_diagnostics] : []),
365
+ ...(task.workspace?.skip_reason ? [`[skipped] ${task.workspace.skip_reason}`] : []),
366
+ ...(task.error ? [`[harness] ${task.error}`] : []),
367
+ ].join("\n");
368
+
369
+ const cards = entry => {
370
+ const { model, harness, task, status } = entry;
371
+ const workspace = task.workspace || {};
372
+ const page = workspace.page_html;
373
+ const answer = workspace.response_text;
374
+ // For a build the three views are one wide strip; the isometric view on
375
+ // its own is what fits a card.
376
+ const shot = workspace.screenshot_iso || workspace.screenshot;
377
+ const sheet = workspace.screenshot;
378
+ const label = `${model} · ${task.task_id}`;
379
+ // Whatever the model actually produced, in the form it survives as.
380
+ const target = page || answer;
381
+ const poster = shot
382
+ ? `<img loading="lazy" src="${escapeHtml(shot)}" alt="${escapeHtml(label)}">`
383
+ : `<div class="stage-empty">${escapeHtml(STATUS_TEXT[status] || "no screenshot")}</div>`;
384
+ // A build-scored task carries numbers a screenshot cannot show.
385
+ const builtStats = workspace.build_status && workspace.blocks_total !== undefined
386
+ ? [
387
+ `${number(workspace.blocks_total)} blocks`,
388
+ `${number(workspace.palette_size)} block types`,
389
+ `${(number(workspace.floating_fraction) * 100).toFixed(0)}% floating`,
390
+ number(workspace.hallucinated_blocks) ? `${number(workspace.hallucinated_blocks)} fake ids` : "",
391
+ ].filter(Boolean).join(" · ")
392
+ : "";
393
+ const views = ["iso", "side", "top"]
394
+ .map(view => [view, workspace[`screenshot_${view}`]])
395
+ .filter(([, path]) => path);
396
+ // treejs runs the model's own page; mc-arena has no page, so its live
397
+ // preview is BenchKit's viewer pointed at the block list. The viewer is
398
+ // served by `benchkit history`, so this resolves there and nowhere else
399
+ // - opened straight off disk the card keeps the screenshot instead.
400
+ const explore = !page && workspace.blocks_json
401
+ ? `/viewer/viewer.html?interactive=1&build=${encodeURIComponent(new URL(workspace.blocks_json, location.href).href)}`
402
+ : "";
403
+ const trace = diagnostics(task);
404
+ return `<article class="card">
405
+ <div class="stage"${page || explore ? ` data-page="${escapeHtml(page || explore)}" data-label="${escapeHtml(label)}"${explore ? ' data-trusted="1"' : ""}` : ""}>
406
+ ${poster}
407
+ ${target ? `<a class="stage-link" href="${escapeHtml(target)}" target="_blank" rel="noopener noreferrer" title="${page ? "Open the generated page in a new tab" : "Open the model's answer in a new tab"}"><span class="open-hint">${page ? "Open page ↗" : "Open answer ↗"}</span></a>` : ""}
408
+ ${page || explore ? `<a class="open-badge" href="${escapeHtml(page || explore)}" target="_blank" rel="noopener noreferrer">Open ↗</a>` : ""}
409
+ </div>
410
+ <div class="card-foot">
411
+ <div>
412
+ <div class="card-model" title="${escapeHtml(model)}">${escapeHtml(model)}</div>
413
+ <div class="card-sub">${escapeHtml(harness)}${number(task.repair_attempts_used) ? ` · ${number(task.repair_attempts_used)} repair${number(task.repair_attempts_used) === 1 ? "" : "s"}` : ""}${builtStats ? ` · ${escapeHtml(builtStats)}` : workspace.render_status && status !== "skipped" ? ` · ${number(workspace.animation_frames)} frames` : ""}</div>
414
+ </div>
415
+ <span class="status ${PASSED.has(status) ? "pass" : NEUTRAL.has(status) ? "skip" : WARN.has(status) ? "warn" : ""}">${escapeHtml(STATUS_TEXT[status] || "FAILED")}</span>
416
+ </div>
417
+ <div class="links">
418
+ ${page ? `<a href="${escapeHtml(page)}" target="_blank" rel="noopener noreferrer">Open page ↗</a>` : ""}
419
+ ${answer ? `<a href="${escapeHtml(answer)}" target="_blank" rel="noopener noreferrer">Open answer ↗</a>` : ""}
420
+ ${sheet ? `<a href="${escapeHtml(sheet)}" target="_blank" rel="noopener noreferrer">Full screenshot ↗</a>` : ""}
421
+ ${views.map(([view, path]) => `<a href="${escapeHtml(path)}" target="_blank" rel="noopener noreferrer">${escapeHtml(view)} ↗</a>`).join("")}
422
+ ${workspace.script_py ? `<a href="${escapeHtml(workspace.script_py)}" target="_blank" rel="noopener noreferrer">Script ↗</a>` : ""}
423
+ ${workspace.blocks_json ? `<a href="${escapeHtml(workspace.blocks_json)}" target="_blank" rel="noopener noreferrer">Blocks ↗</a>` : ""}
424
+ </div>
425
+ ${trace ? `<details class="diag"><summary>${status === "skipped" ? "Why this was skipped" : "Browser diagnostics"}</summary><pre>${escapeHtml(trace)}</pre></details>` : ""}
426
+ </article>`;
427
+ };
428
+
429
+ document.getElementById("scenes").innerHTML = scenes.size
430
+ ? [...scenes.values()].map(scene => `<section class="scene">
431
+ <div class="scene-head">
432
+ <h2 class="scene-title">${escapeHtml(scene.scene || scene.id)}</h2>
433
+ <span class="scene-id">${escapeHtml(scene.id)} · ${scene.entries.filter(entry => PASSED.has(entry.status)).length}/${scene.entries.length} ${passLabel}</span>
434
+ </div>
435
+ <div class="grid">${scene.entries.map(cards).join("")}</div>
436
+ </section>`).join("")
437
+ : `<p class="empty">This run captured no arena scenes.</p>`;
438
+
439
+ // The screenshots come from the machine that ran the benchmark, which may
440
+ // have had no browser at all. The pages themselves are here, so this
441
+ // browser can just run them: each stage loads its page while on screen and
442
+ // drops it again after, since a browser only keeps a handful of WebGL
443
+ // contexts alive.
444
+ const MAX_LIVE = 8;
445
+ const live = [];
446
+ let liveEnabled = true;
447
+ try { liveEnabled = localStorage.getItem("benchkit-arena-live") !== "off"; } catch (error) { /* private mode */ }
448
+
449
+ const unload = stage => {
450
+ const frame = stage.querySelector("iframe");
451
+ if (frame) frame.remove();
452
+ stage.classList.remove("is-live");
453
+ const index = live.indexOf(stage);
454
+ if (index !== -1) live.splice(index, 1);
455
+ };
456
+ const load = stage => {
457
+ if (!liveEnabled || stage.classList.contains("is-live") || !stage.dataset.page) return;
458
+ while (live.length >= MAX_LIVE) unload(live[0]);
459
+ const frame = document.createElement("iframe");
460
+ // Untrusted model output: scripts only, no access to this page.
461
+ // A treejs scene is a page the model wrote, so it is sandboxed into an
462
+ // opaque origin and can touch nothing here. The mc-arena viewer is
463
+ // BenchKit's own page - the model contributed block coordinates, not
464
+ // code - and it needs a real origin: prismarine-viewer meshes chunks in
465
+ // a web worker, and an opaque origin cannot start one.
466
+ frame.setAttribute(
467
+ "sandbox",
468
+ stage.dataset.trusted
469
+ ? "allow-scripts allow-same-origin"
470
+ : "allow-scripts allow-pointer-lock"
471
+ );
472
+ frame.setAttribute("loading", "lazy");
473
+ frame.setAttribute("title", `${stage.dataset.label || "scene"} (live)`);
474
+ frame.src = stage.dataset.page;
475
+ stage.appendChild(frame);
476
+ stage.classList.add("is-live");
477
+ live.push(stage);
478
+ };
479
+
480
+ const stages = [...document.querySelectorAll(".stage[data-page]")];
481
+ const observer = new IntersectionObserver(entries => {
482
+ entries.forEach(entry => entry.isIntersecting ? load(entry.target) : unload(entry.target));
483
+ }, { rootMargin: "150px 0px" });
484
+ const observe = () => stages.forEach(stage => observer.observe(stage));
485
+
486
+ const liveToggle = document.getElementById("liveToggle");
487
+ const liveLabel = document.getElementById("liveLabel");
488
+ const setLive = enabled => {
489
+ liveEnabled = enabled;
490
+ liveToggle.setAttribute("aria-pressed", String(enabled));
491
+ liveLabel.textContent = enabled ? "Live previews on" : "Live previews off";
492
+ try { localStorage.setItem("benchkit-arena-live", enabled ? "on" : "off"); } catch (error) { /* private mode */ }
493
+ if (enabled) {
494
+ observe();
495
+ } else {
496
+ observer.disconnect();
497
+ [...live].forEach(unload);
498
+ }
499
+ };
500
+ liveToggle.addEventListener("click", () => setLive(!liveEnabled));
501
+ setLive(liveEnabled);
502
+ </script>
503
+ </body>
504
+ </html>
benchkit/templates/history.html ADDED
@@ -0,0 +1,58 @@
1
+ <!doctype html>
2
+ <html lang="en">
3
+ <head>
4
+ <meta charset="utf-8">
5
+ <meta name="viewport" content="width=device-width, initial-scale=1">
6
+ <meta name="color-scheme" content="light dark">
7
+ <title>BenchKit History</title>
8
+ <script>
9
+ (() => {
10
+ let stored = null;
11
+ try { stored = localStorage.getItem("benchkit-history-theme"); } catch (_) {}
12
+ document.documentElement.dataset.theme = stored === "dark" ? "dark" : "light";
13
+ })();
14
+ </script>
15
+ <style>
16
+ :root,:root[data-theme="light"]{color-scheme:light;--page:#f7f7f5;--card:#fff;--ink:#111;--muted:#68707d;--line:#dedfdb;--soft:#f1f2ef;--green:#087f5b;--green-bg:#e8f7f0;--blue:#1d5fd1;--blue-bg:#eaf1ff;--amber:#a85d00;--amber-bg:#fff3df;--red:#c92a2a;--red-bg:#fff0f0;--shadow:0 10px 30px rgba(20,25,30,.07)}
17
+ :root[data-theme="dark"]{color-scheme:dark;--page:#090a0d;--card:#121419;--ink:#f7f5ef;--muted:#9aa1ad;--line:#2d313b;--soft:#191c23;--green:#5ee0a7;--green-bg:#10291f;--blue:#7fb0ff;--blue-bg:#14233d;--amber:#ffc078;--amber-bg:#302314;--red:#ff8787;--red-bg:#321b1b;--shadow:none}
18
+ :root{--sans:Inter,-apple-system,BlinkMacSystemFont,"Segoe UI",sans-serif;--mono:"SFMono-Regular",Consolas,monospace}*{box-sizing:border-box}body{margin:0;min-height:100vh;background:var(--page);color:var(--ink);font:14px/1.45 var(--sans)}button,input,select{font:inherit;color:inherit}.page{width:min(1380px,calc(100% - 36px));margin:auto;padding:34px 0 60px}.top{display:flex;justify-content:space-between;align-items:flex-start;gap:24px;margin-bottom:26px}.brand{display:flex;align-items:center;gap:12px}.logo{display:grid;place-items:center;width:42px;height:42px;border-radius:11px;background:#111;color:#fff;font:800 13px/1 var(--mono)}h1{font-size:27px;line-height:1;margin:0;letter-spacing:-.04em}.subtitle{color:var(--muted);margin:7px 0 0}.theme{border:1px solid var(--line);background:var(--card);padding:8px 12px;border-radius:9px;cursor:pointer;font:700 10px/1 var(--mono);text-transform:uppercase;letter-spacing:.06em}
19
+ .steps{display:grid;grid-template-columns:repeat(3,1fr);border:1px solid var(--line);border-radius:14px;background:var(--card);box-shadow:var(--shadow);overflow:hidden;margin-bottom:18px}.step{padding:18px 20px}.step+.step{border-left:1px solid var(--line)}.step-label{color:var(--blue);font:800 10px/1 var(--mono);text-transform:uppercase;letter-spacing:.08em}.step strong{display:block;margin-top:7px;font-size:15px}.step span{display:block;color:var(--muted);font-size:12px;margin-top:3px}
20
+ .tabs{display:flex;gap:5px;margin-bottom:13px}.tab{border:0;background:transparent;color:var(--muted);padding:9px 13px;border-radius:8px;cursor:pointer;font-weight:700}.tab.active{background:var(--ink);color:var(--card)}.view{display:none}.view.active{display:block}.panel{background:var(--card);border:1px solid var(--line);border-radius:14px;box-shadow:var(--shadow);overflow:hidden}.filters{display:grid;grid-template-columns:minmax(230px,1fr) minmax(220px,1fr) 170px 170px;gap:12px;padding:17px;border-bottom:1px solid var(--line)}.field label{display:block;color:var(--muted);font:800 9px/1 var(--mono);text-transform:uppercase;letter-spacing:.07em;margin:0 0 7px}.control{width:100%;padding:10px 11px;border:1px solid var(--line);border-radius:8px;background:var(--card);outline:none}.control:focus{border-color:var(--blue);box-shadow:0 0 0 3px var(--blue-bg)}
21
+ .summary{display:grid;grid-template-columns:repeat(5,1fr);border-bottom:1px solid var(--line)}.stat{padding:17px}.stat+.stat{border-left:1px solid var(--line)}.stat label{color:var(--muted);font:800 9px/1 var(--mono);text-transform:uppercase}.stat b{display:block;font:800 23px/1.1 var(--mono);margin-top:8px;letter-spacing:-.04em}.stat small{display:block;color:var(--muted);font-size:10px;margin-top:5px}.content{display:grid;grid-template-columns:minmax(0,1.5fr) minmax(350px,.8fr);min-height:560px}.runs{border-right:1px solid var(--line);min-width:0}.section-head{display:flex;justify-content:space-between;align-items:center;gap:16px;padding:15px 17px;border-bottom:1px solid var(--line)}.section-title{font-weight:800}.section-note{color:var(--muted);font-size:11px;margin-top:3px}.count{color:var(--blue);font:700 11px/1 var(--mono)}.table-wrap{overflow:auto;max-height:660px}table{width:100%;border-collapse:collapse}th{position:sticky;top:0;z-index:2;padding:10px 12px;background:var(--soft);color:var(--muted);border-bottom:1px solid var(--line);font:800 9px/1 var(--mono);text-transform:uppercase;letter-spacing:.06em;text-align:left;white-space:nowrap;cursor:pointer}th.num,td.num{text-align:right}td{padding:12px;border-bottom:1px solid var(--line);vertical-align:middle}tbody tr{cursor:pointer}tbody tr:hover,tbody tr.selected{background:var(--blue-bg)}.run-id{font:700 11px/1.25 var(--mono)}.subline{color:var(--muted);font-size:10px;margin-top:4px}.score{font:800 17px/1 var(--mono)}.pill{display:inline-flex;padding:5px 7px;border-radius:999px;font:800 9px/1 var(--mono);white-space:nowrap}.pill.clean{color:var(--green);background:var(--green-bg)}.pill.loop{color:var(--amber);background:var(--amber-bg)}.pill.unknown{color:var(--muted);background:var(--soft)}
22
+ .detail{min-width:0}.empty{min-height:480px;display:grid;place-items:center;text-align:center;color:var(--muted);padding:44px}.detail-body{padding:19px}.detail-score{font:850 52px/.9 var(--mono);letter-spacing:-.07em}.detail-score small{font-size:14px;color:var(--muted);letter-spacing:0}.detail-run{color:var(--blue);font:700 11px/1.4 var(--mono)}.metrics{display:grid;grid-template-columns:repeat(2,1fr);gap:8px;margin-top:18px}.metric{padding:11px;border:1px solid var(--line);border-radius:8px;background:var(--soft)}.metric label{display:block;color:var(--muted);font:800 8px/1 var(--mono);text-transform:uppercase;letter-spacing:.06em}.metric b{display:block;font:700 12px/1.3 var(--mono);margin-top:7px}.explain{padding:11px 12px;border-left:3px solid var(--blue);background:var(--blue-bg);color:var(--muted);font-size:11px;margin-top:14px}.task-section{margin-top:20px}.task-title{font:800 9px/1 var(--mono);text-transform:uppercase;letter-spacing:.07em;margin-bottom:9px}.tasks{max-height:280px;overflow:auto;border:1px solid var(--line);border-radius:9px}.task{display:grid;grid-template-columns:minmax(0,1fr) 55px 88px;gap:8px;padding:9px 10px;border-bottom:1px solid var(--line);align-items:center;font:10px/1.2 var(--mono)}.task:last-child{border:0}.task-name{overflow:hidden;text-overflow:ellipsis;white-space:nowrap}.pass{color:var(--green)}.fail{color:var(--red)}.task-state{text-align:right}.empty-row{text-align:center;color:var(--muted);padding:60px}
23
+ .perf-layout{display:grid;grid-template-columns:280px minmax(0,1fr);min-height:580px}.model-list{border-right:1px solid var(--line);padding:8px;max-height:690px;overflow:auto}.model-button{display:flex;justify-content:space-between;width:100%;gap:10px;padding:10px;border:0;border-radius:7px;background:transparent;text-align:left;cursor:pointer;font:11px/1.25 var(--mono)}.model-button:hover,.model-button.active{background:var(--blue-bg);color:var(--blue)}.perf-main{padding:18px}.bar-chart{display:flex;align-items:flex-end;gap:12px;height:250px;padding:30px 15px 0;border-left:1px solid var(--line);border-bottom:1px solid var(--line);margin:18px 0 24px}.bar-group{display:flex;align-items:flex-end;justify-content:center;gap:3px;position:relative;flex:1;height:100%}.bar{width:min(28px,40%);min-height:2px;border-radius:4px 4px 0 0}.bar.pp{background:var(--blue)}.bar.tg{background:var(--green)}.bar-label{position:absolute;top:calc(100% + 7px);font:9px/1 var(--mono);color:var(--muted);white-space:nowrap}.legend{display:flex;gap:14px;color:var(--muted);font-size:11px}.legend i{display:inline-block;width:9px;height:9px;border-radius:2px;margin-right:5px}.health{padding:18px}.health-summary{display:grid;grid-template-columns:repeat(4,1fr);gap:10px;margin-bottom:18px}.health-card{border:1px solid var(--line);border-radius:9px;padding:14px}.health-card b{display:block;font:800 22px/1 var(--mono)}.health-card span{display:block;color:var(--muted);font-size:10px;margin-top:6px}.warning{border:1px solid var(--line);border-radius:9px;padding:13px;margin-top:8px}.warning strong{display:block;font:700 11px/1.3 var(--mono)}.warning p{color:var(--muted);font-size:11px;margin:6px 0 0;overflow-wrap:anywhere}.page-links{display:flex;gap:8px;margin-bottom:14px}.page-link{flex:1;text-align:center;padding:9px 12px;border:1px solid var(--line);border-radius:9px;background:var(--soft);color:var(--blue);text-decoration:none;font:700 11px/1 var(--mono);text-transform:uppercase;letter-spacing:.06em}.page-link:hover{border-color:var(--blue);background:var(--blue-bg)}.footer{color:var(--muted);font:10px/1.6 var(--mono);margin-top:18px;display:flex;justify-content:space-between;gap:20px}
24
+ @media(max-width:1000px){.content{grid-template-columns:1fr}.runs{border-right:0;border-bottom:1px solid var(--line)}.filters{grid-template-columns:1fr 1fr}.summary{grid-template-columns:repeat(3,1fr)}.perf-layout{grid-template-columns:1fr}.model-list{border-right:0;border-bottom:1px solid var(--line);max-height:220px}}@media(max-width:650px){.page{width:min(100% - 20px,1380px);padding-top:20px}.top{display:block}.theme{margin-top:16px}.steps,.filters,.summary,.health-summary{grid-template-columns:1fr}.step+.step,.stat+.stat{border-left:0;border-top:1px solid var(--line)}.content{display:block}.metrics{grid-template-columns:1fr}.footer{display:block}.tabs{overflow:auto}.tab{white-space:nowrap}}
25
+ </style>
26
+ </head>
27
+ <body>
28
+ <main class="page">
29
+ <header class="top"><div class="brand"><div class="logo">BK</div><div><h1>Benchmark history</h1><p class="subtitle">Compare the scores BenchKit actually recorded—run by run.</p></div></div><button class="theme" id="theme">Dark theme</button></header>
30
+ <section class="steps"><div class="step"><div class="step-label">Step 1</div><strong>Choose a model</strong><span>Only models found in your results folders appear.</span></div><div class="step"><div class="step-label">Step 2</div><strong>Choose a benchmark</strong><span>Different slices and harnesses remain separate runs.</span></div><div class="step"><div class="step-label">Step 3</div><strong>Inspect what happened</strong><span>Click a score for loop, repair and task diagnostics.</span></div></section>
31
+ <nav class="tabs"><button class="tab active" data-view="history">Run history</button><button class="tab" data-view="performance">Performance</button><button class="tab" data-view="health">Data health</button></nav>
32
+
33
+ <section class="view active" id="view-history"><div class="panel">
34
+ <div class="filters"><div class="field"><label>Model</label><select class="control" id="model"></select></div><div class="field"><label>Benchmark</label><select class="control" id="benchmark"></select></div><div class="field"><label>Harness</label><select class="control" id="harness"><option value="all">All harnesses</option></select></div><div class="field"><label>Loop status</label><select class="control" id="loop"><option value="all">All runs</option><option value="loop">Loop evidence</option><option value="clean">No loop evidence</option><option value="unknown">Not captured</option></select></div></div>
35
+ <div class="summary" id="summary"></div>
36
+ <div class="content"><section class="runs"><div class="section-head"><div><div class="section-title" id="run-title">Runs</div><div class="section-note">Each row is one recorded benchmark result. No averages or estimates.</div></div><div class="count" id="run-count"></div></div><div class="table-wrap"><table><thead><tr><th data-sort="run_id">Run</th><th class="num" data-sort="score">Score</th><th class="num" data-sort="passed">Passed</th><th data-sort="loops">Loop status</th><th class="num" data-sort="repair_attempted">Repair</th><th class="num" data-sort="tok_s">tok/s</th><th data-sort="harness">Harness</th></tr></thead><tbody id="run-rows"></tbody></table></div></section><aside class="detail" id="detail"><div class="empty">Choose a run to see its diagnostics.</div></aside></div>
37
+ </div></section>
38
+
39
+ <section class="view" id="view-performance"><div class="panel"><div class="perf-layout"><aside class="model-list" id="perf-models"></aside><section class="perf-main" id="perf-main"><div class="empty">Choose a model to inspect its measured prompt and decode throughput.</div></section></div></div></section>
40
+ <section class="view" id="view-health"><div class="panel health"><div class="health-summary" id="health-summary"></div><div id="warnings"></div></div></section>
41
+ <footer class="footer"><span>Reads results.json and perf.json from every selected results directory.</span><span>Reload the page to rescan files.</span></footer>
42
+ </main>
43
+ <script id="history-data" type="application/json">__BENCHKIT_HISTORY_DATA__</script>
44
+ <script>
45
+ const DATA=JSON.parse(document.getElementById("history-data").textContent);const rows=DATA.benchmark_rows;const state={model:"",benchmark:"",harness:"all",loop:"all",sort:"run_id",direction:-1,selected:null};
46
+ const escapeHtml=value=>String(value??"").replace(/[&<>'"]/g,char=>({"&":"&amp;","<":"&lt;",">":"&gt;","'":"&#39;",'"':"&quot;"}[char]));const number=value=>value===null||value===undefined?"Not captured":new Intl.NumberFormat("en-US",{maximumFractionDigits:1}).format(value);const models=[...new Set(rows.map(row=>row.model))].sort();
47
+ function loopKind(row){if(row.loops===null&&row.suspected_loops===null&&row.loop_kills===null)return"unknown";return Number(row.loops||0)+Number(row.suspected_loops||0)+Number(row.loop_kills||0)>0?"loop":"clean"}function filtered(){return rows.filter(row=>row.model===state.model&&row.benchmark===state.benchmark&&(state.harness==="all"||row.harness===state.harness)&&(state.loop==="all"||loopKind(row)===state.loop)).sort((a,b)=>{let left=a[state.sort],right=b[state.sort];if(typeof left==="string")return left.localeCompare(right)*state.direction;return((Number(left)||0)-(Number(right)||0))*state.direction})}
48
+ function resetBenchmarks(){const benchmarks=[...new Set(rows.filter(row=>row.model===state.model).map(row=>row.benchmark))].sort();benchmarkSelect.innerHTML=benchmarks.map(value=>`<option>${escapeHtml(value)}</option>`).join("");state.benchmark=benchmarks[0]||"";benchmarkSelect.value=state.benchmark;resetHarnesses()}function resetHarnesses(){const harnesses=[...new Set(rows.filter(row=>row.model===state.model&&row.benchmark===state.benchmark).map(row=>row.harness))].sort();harnessSelect.innerHTML='<option value="all">All harnesses</option>'+harnesses.map(value=>`<option>${escapeHtml(value)}</option>`).join("");state.harness="all"}
49
+ const modelSelect=document.getElementById("model"),benchmarkSelect=document.getElementById("benchmark"),harnessSelect=document.getElementById("harness");modelSelect.innerHTML=models.map(value=>`<option>${escapeHtml(value)}</option>`).join("");state.model=models[0]||"";resetBenchmarks();
50
+ function render(){const all=rows.filter(row=>row.model===state.model&&row.benchmark===state.benchmark),visible=filtered(),scores=all.map(row=>Number(row.score)),known=all.filter(row=>loopKind(row)!=="unknown"),loopRuns=known.filter(row=>loopKind(row)==="loop"),repairs=all.reduce((sum,row)=>sum+Number(row.repair_attempted||0),0),latest=[...all].sort((a,b)=>b.run_id.localeCompare(a.run_id))[0];document.getElementById("run-title").textContent=`${state.model} · ${state.benchmark}`;document.getElementById("run-count").textContent=`${visible.length} of ${all.length} runs`;document.getElementById("summary").innerHTML=[["Actual runs",all.length,"No estimates"],["Observed range",scores.length?`${number(Math.min(...scores))}–${number(Math.max(...scores))}`:"—","Lowest–highest score"],["Latest score",latest?number(latest.score):"—",latest?.run_id||"No result"],["Loop evidence",loopRuns.length,`${known.length} runs captured loop metrics`],["Repair attempts",repairs,"Across selected runs"]].map(item=>`<div class="stat"><label>${item[0]}</label><b>${item[1]}</b><small>${item[2]}</small></div>`).join("");document.getElementById("run-rows").innerHTML=visible.length?visible.map(row=>{const kind=loopKind(row),loopText=kind==="unknown"?"Not captured":kind==="clean"?"No loop evidence":`${row.loops||0} loop · ${row.suspected_loops||0} suspected · ${row.loop_kills||0} killed`;return`<tr data-id="${row.id}" class="${state.selected===row.id?"selected":""}"><td><div class="run-id">${escapeHtml(row.run_id)}</div><div class="subline">${escapeHtml(row.schema_generation)} · ${row.sources.length} source${row.sources.length===1?"":"s"}</div></td><td class="num"><span class="score">${number(row.score)}</span></td><td class="num">${number(row.passed)} / ${number(row.scored_total)}</td><td><span class="pill ${kind}">${escapeHtml(loopText)}</span></td><td class="num">${row.repair_attempted===null?"Not captured":`${number(row.repair_successes||0)} / ${number(row.repair_attempted)}`}</td><td class="num">${number(row.tok_s)}</td><td><div class="run-id">${escapeHtml(row.harness_label)}</div><div class="subline">${escapeHtml(row.slice??"full")}${row.variant?` · ${escapeHtml(row.variant)}`:""}</div></td></tr>`}).join(""):'<tr><td colspan="7" class="empty-row">No runs match these filters.</td></tr>';document.querySelectorAll("tr[data-id]").forEach(element=>element.onclick=()=>selectRun(element.dataset.id))}
51
+ function metric(label,value){return`<div class="metric"><label>${escapeHtml(label)}</label><b>${escapeHtml(value===null||value===undefined?"Not captured":value)}</b></div>`}function selectRun(id){state.selected=id;const row=rows.find(item=>item.id===id);if(!row)return;render();const tasks=row.task_diagnostics||[],flagged=tasks.filter(task=>(task.loop_state&&task.loop_state!=="clean")||task.loop_killed||task.timed_out||task.length_exceeded||task.harness_error),shown=flagged.length?flagged:tasks,kind=loopKind(row);const pages=row.pages||{},links=[pages.gallery?`<a class="page-link" href="${escapeHtml(pages.gallery)}" target="_blank" rel="noopener noreferrer">Gallery &#8599;</a>`:"",pages.report?`<a class="page-link" href="${escapeHtml(pages.report)}" target="_blank" rel="noopener noreferrer">Report &#8599;</a>`:""].filter(Boolean).join("");document.getElementById("detail").innerHTML=`<div class="section-head"><div><div class="detail-run">${escapeHtml(row.run_id)}</div><div class="section-title">Run details</div></div><span class="pill ${kind}">${kind==="unknown"?"Loop data not captured":kind==="clean"?"No loop evidence":"Loop evidence"}</span></div><div class="detail-body">${links?`<div class="page-links">${links}</div>`:""}<div class="detail-score">${number(row.score)}<small> / 100</small></div><div class="subline">${number(row.passed)} passed out of ${number(row.scored_total)} scored tasks</div><div class="metrics">${metric("Harness",row.harness_label)}${metric("Slice / variant",`${row.slice??"full"}${row.variant?` · ${row.variant}`:""}`)}${metric("Loops",row.loops)}${metric("Suspected loops",row.suspected_loops)}${metric("Loop kills",row.loop_kills)}${metric("Recovered loops",row.recovered_loops)}${metric("Repair success",row.repair_attempted===null?null:`${row.repair_successes||0} / ${row.repair_attempted}`)}${metric("Timeouts",row.timeouts)}${metric("Length exceeded",row.length_exceeded)}${metric("Harness errors",row.harness_errors)}${metric("Token speed",row.tok_s===null?null:`${number(row.tok_s)} tok/s`)}${metric("Average response",row.avg_response_time===null?null:`${number(row.avg_response_time)}s`)}</div>${kind==="unknown"?'<div class="explain">This report predates BenchKit loop metrics. “Not captured” means unknown—not zero.</div>':kind==="loop"?'<div class="explain"><strong>Loop:</strong> confirmed repetitive generation. <strong>Suspected:</strong> repetition evidence that did not necessarily trigger a kill. <strong>Killed:</strong> BenchKit stopped the generation.</div>':'<div class="explain">This run captured loop metrics and recorded no loop evidence.</div>'}${tasks.length?`<div class="task-section"><div class="task-title">${flagged.length?"Flagged tasks":"Task diagnostics"} · ${shown.length} shown</div><div class="tasks">${shown.map(task=>`<div class="task"><div class="task-name" title="${escapeHtml(task.task_id)}">${escapeHtml(task.task_id)}</div><div class="task-state ${task.passed?"pass":"fail"}">${number(task.score)}</div><div class="task-state">${escapeHtml(task.loop_killed?"loop killed":task.timed_out?"timeout":task.length_exceeded?"length":task.harness_error?"harness error":task.loop_state||task.outcome||"—")}</div></div>`).join("")}</div></div>`:'<div class="explain">Task-level diagnostics were not stored by this report version.</div>'}</div>`}
52
+ modelSelect.onchange=event=>{state.model=event.target.value;state.selected=null;resetBenchmarks();render()};benchmarkSelect.onchange=event=>{state.benchmark=event.target.value;state.selected=null;resetHarnesses();render()};harnessSelect.onchange=event=>{state.harness=event.target.value;render()};document.getElementById("loop").onchange=event=>{state.loop=event.target.value;render()};document.querySelectorAll("th[data-sort]").forEach(header=>header.onclick=()=>{const key=header.dataset.sort;if(state.sort===key)state.direction*=-1;else{state.sort=key;state.direction=key==="run_id"?-1:1}render()});
53
+ function renderPerformance(){const byModel={};DATA.performance_rows.forEach(row=>(byModel[row.model]??=[]).push(row));const names=Object.keys(byModel).sort(),list=document.getElementById("perf-models");list.innerHTML=names.length?names.map(name=>`<button class="model-button" data-model="${escapeHtml(name)}"><span>${escapeHtml(name)}</span><span>${byModel[name].length}</span></button>`).join(""):'<div class="empty">No perf.json reports found.</div>';list.querySelectorAll("button").forEach(button=>button.onclick=()=>{list.querySelectorAll("button").forEach(item=>item.classList.toggle("active",item===button));const points=[...byModel[button.dataset.model]].sort((a,b)=>(a.depth||0)-(b.depth||0)),max=Math.max(...points.flatMap(point=>[point.pp_tps||0,point.tg_tps||0]),1);document.getElementById("perf-main").innerHTML=`<div class="section-title">${escapeHtml(button.dataset.model)}</div><div class="section-note">Measured throughput by actual input depth.</div><div class="legend"><span><i style="background:var(--blue)"></i>Prompt processing</span><span><i style="background:var(--green)"></i>Token generation</span></div><div class="bar-chart">${points.map(point=>`<div class="bar-group"><div class="bar pp" title="PP ${number(point.pp_tps)} tok/s" style="height:${Math.max(1,(point.pp_tps||0)/max*100)}%"></div><div class="bar tg" title="TG ${number(point.tg_tps)} tok/s" style="height:${Math.max(1,(point.tg_tps||0)/max*100)}%"></div><div class="bar-label">${escapeHtml(point.depth_label)}</div></div>`).join("")}</div><div class="table-wrap"><table><thead><tr><th>Run</th><th>Depth</th><th class="num">PP tok/s</th><th class="num">TG tok/s</th><th class="num">TTFT</th><th class="num">Reps</th></tr></thead><tbody>${points.map(point=>`<tr><td class="run-id">${escapeHtml(point.run_id)}</td><td>${number(point.depth)} <span class="subline">${escapeHtml(point.depth_label)}</span></td><td class="num">${number(point.pp_tps)}</td><td class="num">${number(point.tg_tps)}</td><td class="num">${point.ttft_s===null?"Not captured":`${number(point.ttft_s)}s`}</td><td class="num">${number(point.repetitions)}</td></tr>`).join("")}</tbody></table></div>`})}
54
+ function renderHealth(){const counts=DATA.counts;document.getElementById("health-summary").innerHTML=[[counts.discovered_reports,"Reports found"],[counts.unique_reports,"Unique reports"],[counts.benchmark_results,"Benchmark results"],[DATA.warnings.length,"Warnings"]].map(item=>`<div class="health-card"><b>${item[0]}</b><span>${item[1]}</span></div>`).join("");document.getElementById("warnings").innerHTML=DATA.warnings.length?DATA.warnings.map(item=>`<div class="warning"><strong>${escapeHtml(item.kind.replaceAll("_"," "))}</strong><p>${escapeHtml(item.path)}<br>${escapeHtml(item.message)}</p></div>`).join(""):'<div class="explain">All discovered reports were readable.</div>'}
55
+ document.querySelectorAll(".tab").forEach(tab=>tab.onclick=()=>{document.querySelectorAll(".tab").forEach(item=>item.classList.toggle("active",item===tab));document.querySelectorAll(".view").forEach(view=>view.classList.toggle("active",view.id===`view-${tab.dataset.view}`))});document.getElementById("theme").onclick=()=>{const dark=document.documentElement.dataset.theme!=="dark";document.documentElement.dataset.theme=dark?"dark":"light";document.getElementById("theme").textContent=dark?"Light theme":"Dark theme";try{localStorage.setItem("benchkit-history-theme",dark?"dark":"light")}catch(_){}};if(document.documentElement.dataset.theme==="dark")document.getElementById("theme").textContent="Light theme";render();renderPerformance();renderHealth();const first=filtered()[0];if(first)selectRun(first.id);
56
+ </script>
57
+ </body>
58
+ </html>