benchkit-cli 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (149) hide show
  1. benchkit/__init__.py +1 -0
  2. benchkit/_pi_proxy.py +219 -0
  3. benchkit/answer_key_guard.ts +49 -0
  4. benchkit/artifacts.py +87 -0
  5. benchkit/benchmarks/__init__.py +105 -0
  6. benchkit/benchmarks/aider_polyglot.py +405 -0
  7. benchkit/benchmarks/arc.py +52 -0
  8. benchkit/benchmarks/banking77.py +149 -0
  9. benchkit/benchmarks/base.py +19 -0
  10. benchkit/benchmarks/boolq.py +49 -0
  11. benchkit/benchmarks/chartqa.py +133 -0
  12. benchkit/benchmarks/code_verifier.py +77 -0
  13. benchkit/benchmarks/downloads.py +166 -0
  14. benchkit/benchmarks/evalplus.py +288 -0
  15. benchkit/benchmarks/git_surgery.py +485 -0
  16. benchkit/benchmarks/gpqa.py +56 -0
  17. benchkit/benchmarks/gsm8k.py +97 -0
  18. benchkit/benchmarks/hellaswag.py +49 -0
  19. benchkit/benchmarks/humaneval.py +103 -0
  20. benchkit/benchmarks/ifeval.py +56 -0
  21. benchkit/benchmarks/ifeval_instructions.py +477 -0
  22. benchkit/benchmarks/mbpp.py +60 -0
  23. benchkit/benchmarks/mc_arena.py +479 -0
  24. benchkit/benchmarks/mcq.py +94 -0
  25. benchkit/benchmarks/medfailbench.py +537 -0
  26. benchkit/benchmarks/mmlu.py +54 -0
  27. benchkit/benchmarks/mmlu_pro.py +62 -0
  28. benchkit/benchmarks/mmvp.py +115 -0
  29. benchkit/benchmarks/ocrbench.py +96 -0
  30. benchkit/benchmarks/openbookqa.py +52 -0
  31. benchkit/benchmarks/patcheval.py +929 -0
  32. benchkit/benchmarks/piqa.py +52 -0
  33. benchkit/benchmarks/ruler.py +172 -0
  34. benchkit/benchmarks/ruler_generator.py +535 -0
  35. benchkit/benchmarks/sanity.py +174 -0
  36. benchkit/benchmarks/treejs_arena.py +268 -0
  37. benchkit/benchmarks/truthfulqa.py +52 -0
  38. benchkit/benchmarks/utils.py +25 -0
  39. benchkit/benchmarks/vision.py +230 -0
  40. benchkit/benchmarks/vstar.py +108 -0
  41. benchkit/benchmarks/winogrande.py +52 -0
  42. benchkit/benchmarks/xstest.py +116 -0
  43. benchkit/browser.py +429 -0
  44. benchkit/catalog.py +121 -0
  45. benchkit/cli.py +1000 -0
  46. benchkit/client.py +1413 -0
  47. benchkit/controls.py +87 -0
  48. benchkit/datasets/MMLU_LICENSE.txt +21 -0
  49. benchkit/datasets/README.md +142 -0
  50. benchkit/datasets/arc_challenge.jsonl +1172 -0
  51. benchkit/datasets/banking77.jsonl +3080 -0
  52. benchkit/datasets/banking77_intents.json +79 -0
  53. benchkit/datasets/boolq.jsonl +3270 -0
  54. benchkit/datasets/code_verifier.jsonl +500 -0
  55. benchkit/datasets/gpqa_diamond.jsonl +198 -0
  56. benchkit/datasets/gsm8k.jsonl +1319 -0
  57. benchkit/datasets/hellaswag.jsonl +1000 -0
  58. benchkit/datasets/humaneval.jsonl +164 -0
  59. benchkit/datasets/ifeval.jsonl +541 -0
  60. benchkit/datasets/mbpp.jsonl +500 -0
  61. benchkit/datasets/mc_blocks_1_20_1.jsonl +1003 -0
  62. benchkit/datasets/medfailbench_hard30.jsonl +30 -0
  63. benchkit/datasets/medfailbench_hard30.md +13 -0
  64. benchkit/datasets/openbookqa.jsonl +500 -0
  65. benchkit/datasets/piqa.jsonl +1838 -0
  66. benchkit/datasets/sanity_code.jsonl +6 -0
  67. benchkit/datasets/truthfulqa.jsonl +817 -0
  68. benchkit/datasets/winogrande.jsonl +1267 -0
  69. benchkit/datasets/xstest.csv +451 -0
  70. benchkit/decision.py +194 -0
  71. benchkit/demo.py +451 -0
  72. benchkit/dockerfiles.py +247 -0
  73. benchkit/engine.py +1270 -0
  74. benchkit/evaluation.py +195 -0
  75. benchkit/events.py +141 -0
  76. benchkit/executor.py +96 -0
  77. benchkit/git_surgery/backport-release-stack/setup.sh +328 -0
  78. benchkit/git_surgery/backport-release-stack/verify.sh +119 -0
  79. benchkit/git_surgery/bisect-the-regression/setup.sh +49 -0
  80. benchkit/git_surgery/bisect-the-regression/verify.sh +34 -0
  81. benchkit/git_surgery/rebase-conflict-chain/setup.sh +65 -0
  82. benchkit/git_surgery/rebase-conflict-chain/verify.sh +43 -0
  83. benchkit/git_surgery/recover-complex-stash/setup.sh +152 -0
  84. benchkit/git_surgery/recover-complex-stash/verify.sh +95 -0
  85. benchkit/git_surgery/recover-lost-work/setup.sh +40 -0
  86. benchkit/git_surgery/recover-lost-work/verify.sh +36 -0
  87. benchkit/git_surgery/repair-force-pushed-remote/setup.sh +171 -0
  88. benchkit/git_surgery/repair-force-pushed-remote/verify.sh +83 -0
  89. benchkit/git_surgery/revert-merge-with-followups/setup.sh +198 -0
  90. benchkit/git_surgery/revert-merge-with-followups/verify.sh +98 -0
  91. benchkit/git_surgery/secret-in-history/README.md +27 -0
  92. benchkit/git_surgery/secret-in-history/setup.sh +107 -0
  93. benchkit/git_surgery/secret-in-history/verify.sh +106 -0
  94. benchkit/git_surgery/split-the-mega-commit/setup.sh +50 -0
  95. benchkit/git_surgery/split-the-mega-commit/verify.sh +48 -0
  96. benchkit/git_surgery/untangle-nested-submodules/setup.sh +193 -0
  97. benchkit/git_surgery/untangle-nested-submodules/verify.sh +90 -0
  98. benchkit/history.py +474 -0
  99. benchkit/jobs.py +332 -0
  100. benchkit/leakage.py +51 -0
  101. benchkit/looping.py +873 -0
  102. benchkit/mc_render.py +274 -0
  103. benchkit/mc_viewer/dist/atlas.png +0 -0
  104. benchkit/mc_viewer/dist/blockStates.json.gz +0 -0
  105. benchkit/mc_viewer/dist/version.json +4 -0
  106. benchkit/mc_viewer/dist/viewer.html +23 -0
  107. benchkit/mc_viewer/dist/viewer.js +2 -0
  108. benchkit/mc_viewer/dist/viewer.js.LICENSE.txt +25 -0
  109. benchkit/mc_viewer/dist/worker.js +2 -0
  110. benchkit/mc_viewer/dist/worker.js.LICENSE.txt +19 -0
  111. benchkit/metrics.py +100 -0
  112. benchkit/ollama.py +60 -0
  113. benchkit/perf.py +353 -0
  114. benchkit/perf_report.py +207 -0
  115. benchkit/perturbations.py +208 -0
  116. benchkit/pi_agent.py +476 -0
  117. benchkit/pi_package/package-lock.json +1827 -0
  118. benchkit/pi_package/package.json +8 -0
  119. benchkit/report.py +142 -0
  120. benchkit/report_markdown.py +645 -0
  121. benchkit/results.py +688 -0
  122. benchkit/retry.py +163 -0
  123. benchkit/runner.py +769 -0
  124. benchkit/sandbox.py +1167 -0
  125. benchkit/template_check.py +291 -0
  126. benchkit/templates/arena.html +504 -0
  127. benchkit/templates/history.html +58 -0
  128. benchkit/templates/perf.html +509 -0
  129. benchkit/templates/report.html +769 -0
  130. benchkit/tui/__init__.py +5 -0
  131. benchkit/tui/app.py +87 -0
  132. benchkit/tui/app.tcss +512 -0
  133. benchkit/tui/formatting.py +84 -0
  134. benchkit/tui/screens/__init__.py +1 -0
  135. benchkit/tui/screens/connect.py +191 -0
  136. benchkit/tui/screens/detail.py +288 -0
  137. benchkit/tui/screens/help.py +84 -0
  138. benchkit/tui/screens/modals.py +544 -0
  139. benchkit/tui/screens/results.py +361 -0
  140. benchkit/tui/screens/run.py +988 -0
  141. benchkit/tui/screens/setup.py +733 -0
  142. benchkit/tui/theme.py +135 -0
  143. benchkit/tui/widgets.py +120 -0
  144. benchkit_cli-0.1.0.dist-info/METADATA +294 -0
  145. benchkit_cli-0.1.0.dist-info/RECORD +149 -0
  146. benchkit_cli-0.1.0.dist-info/WHEEL +5 -0
  147. benchkit_cli-0.1.0.dist-info/entry_points.txt +3 -0
  148. benchkit_cli-0.1.0.dist-info/licenses/LICENSE +201 -0
  149. benchkit_cli-0.1.0.dist-info/top_level.txt +1 -0
benchkit/templates/perf.html ADDED
@@ -0,0 +1,509 @@
1
+ <!doctype html>
2
+ <html lang="en">
3
+ <head>
4
+ <meta charset="utf-8">
5
+ <meta name="viewport" content="width=device-width, initial-scale=1">
6
+ <meta name="color-scheme" content="light dark">
7
+ <title>BenchKit Performance Profile</title>
8
+ <script>
9
+ (() => {
10
+ let stored = null;
11
+ try { stored = localStorage.getItem("benchkit-theme"); } catch (error) { stored = null; }
12
+ document.documentElement.dataset.theme = stored === "dark" ? "dark" : "light";
13
+ })();
14
+ </script>
15
+ <style>
16
+ :root,
17
+ :root[data-theme="light"] {
18
+ color-scheme: light;
19
+ --page: #ffffff;
20
+ --card: #ffffff;
21
+ --ink: #0b0b0b;
22
+ --muted: #6b7280;
23
+ --line: rgba(11, 11, 11, .12);
24
+ --raise: rgba(11, 11, 11, .04);
25
+ --track: rgba(11, 11, 11, .09);
26
+ --green: #059669;
27
+ --green-soft: #10b981;
28
+ --red: #dc2626;
29
+ --blue: #2563eb;
30
+ --amber: #d97706;
31
+ --bar: #6b7280;
32
+ --bar-light: rgba(11, 11, 11, .09);
33
+ --shade-1: #dbeafe;
34
+ --shade-2: #bfdbfe;
35
+ --shade-3: #93c5fd;
36
+ --shade-4: #60a5fa;
37
+ --shade-5: #3b82f6;
38
+ --shade-6: #2563eb;
39
+ --shade-7: #1d4ed8;
40
+ }
41
+
42
+ :root[data-theme="dark"] {
43
+ color-scheme: dark;
44
+ --page: #000000;
45
+ --card: #000000;
46
+ --ink: #ffffff;
47
+ --muted: #6b7280;
48
+ --line: rgba(255, 255, 255, .14);
49
+ --raise: rgba(255, 255, 255, .06);
50
+ --track: rgba(255, 255, 255, .12);
51
+ --green: #34d399;
52
+ --green-soft: #6ee7b7;
53
+ --red: #ef4444;
54
+ --blue: #60a5fa;
55
+ --amber: #facc15;
56
+ --bar: #9aa1ad;
57
+ --bar-light: rgba(255, 255, 255, .12);
58
+ --shade-1: #14304f;
59
+ --shade-2: #1b4470;
60
+ --shade-3: #235891;
61
+ --shade-4: #2c6cb2;
62
+ --shade-5: #3f86d4;
63
+ --shade-6: #4f95e8;
64
+ --shade-7: #60a5fa;
65
+ }
66
+
67
+ :root {
68
+ --sans: "Avenir Next", Avenir, "Century Gothic", sans-serif;
69
+ --mono: "SFMono-Regular", "Cascadia Mono", Menlo, monospace;
70
+ }
71
+
72
+ * { box-sizing: border-box; }
73
+ html { background: var(--page); }
74
+ body {
75
+ margin: 0;
76
+ min-height: 100vh;
77
+ color: var(--ink);
78
+ background: var(--page);
79
+ font-family: var(--sans);
80
+ -webkit-font-smoothing: antialiased;
81
+ }
82
+ button { font: inherit; }
83
+ .page { width: min(1180px, calc(100% - 40px)); margin: 0 auto; padding: 54px 0 70px; }
84
+
85
+ .report-head {
86
+ display: flex;
87
+ justify-content: space-between;
88
+ gap: 32px;
89
+ align-items: end;
90
+ padding-bottom: 30px;
91
+ border-bottom: 1px solid var(--line);
92
+ }
93
+ h1, h2, h3, p { margin-top: 0; }
94
+ h1 { margin-bottom: 7px; font-size: 42px; line-height: 1; letter-spacing: -.045em; }
95
+ .meta { margin: 0; color: var(--muted); font: 11px/1.55 var(--mono); }
96
+ .head-right { display: flex; flex-direction: column; align-items: flex-end; gap: 18px; }
97
+ .best { text-align: right; }
98
+ .best-score { color: var(--green); font-size: 42px; font-weight: 800; line-height: 1; letter-spacing: -.045em; }
99
+ .best-model { margin-top: 6px; font-size: 12px; font-weight: 800; }
100
+
101
+ .theme-toggle {
102
+ display: inline-flex;
103
+ align-items: center;
104
+ gap: 8px;
105
+ padding: 7px 14px;
106
+ border: 1px solid var(--line);
107
+ border-radius: 999px;
108
+ background: var(--raise);
109
+ color: var(--ink);
110
+ cursor: pointer;
111
+ font: 700 9px/1 var(--mono);
112
+ letter-spacing: .09em;
113
+ text-transform: uppercase;
114
+ }
115
+ .theme-toggle:hover { border-color: var(--blue); color: var(--blue); }
116
+ .theme-dot { width: 9px; height: 9px; border-radius: 50%; background: var(--blue); }
117
+
118
+ .summary {
119
+ display: grid;
120
+ grid-template-columns: repeat(4, 1fr);
121
+ margin: 28px 0 0;
122
+ background: var(--card);
123
+ border: 1px solid var(--line);
124
+ }
125
+ .stat { padding: 21px 24px; }
126
+ .stat + .stat { border-left: 1px solid var(--line); }
127
+ .stat-label { color: var(--muted); font: 9px/1 var(--mono); letter-spacing: .07em; text-transform: uppercase; }
128
+ .stat-value { margin-top: 8px; font-size: 25px; font-weight: 800; letter-spacing: -.035em; }
129
+ .stat-detail { margin-top: 5px; color: var(--muted); font-size: 9px; }
130
+
131
+ .notice {
132
+ display: none;
133
+ margin-top: 14px;
134
+ padding: 13px 16px;
135
+ border: 1px solid var(--line);
136
+ border-left: 3px solid var(--blue);
137
+ border-radius: 7px;
138
+ background: var(--raise);
139
+ color: var(--muted);
140
+ font: 9px/1.65 var(--mono);
141
+ }
142
+ .notice.show { display: block; }
143
+ .notice.warn { border-left-color: var(--amber); color: var(--ink); }
144
+
145
+ .section { margin-top: 64px; }
146
+ .section-head {
147
+ display: flex;
148
+ align-items: center;
149
+ justify-content: space-between;
150
+ gap: 20px;
151
+ margin-bottom: 18px;
152
+ }
153
+ h2 { margin: 0; font-size: 24px; letter-spacing: -.035em; }
154
+ .section-note { margin: 0; color: var(--muted); font-size: 10px; }
155
+ .charts { display: grid; gap: 30px; }
156
+
157
+ .chart-shell {
158
+ position: relative;
159
+ width: 100%;
160
+ min-width: 0;
161
+ padding: 42px 52px 36px;
162
+ overflow: hidden;
163
+ border: 1px solid var(--line);
164
+ border-radius: 22px;
165
+ background: var(--card);
166
+ }
167
+ .chart-header { display: flex; align-items: baseline; gap: 8px; }
168
+ .chart-title { margin: 0; font-size: 24px; font-weight: 800; letter-spacing: -.035em; }
169
+ .chart-subtitle { color: var(--muted); font-size: 22px; font-weight: 700; }
170
+
171
+ .bar-plot {
172
+ display: flex;
173
+ align-items: stretch;
174
+ gap: clamp(18px, 4vw, 54px);
175
+ height: 440px;
176
+ margin-top: 32px;
177
+ padding: 0 24px;
178
+ border-bottom: 2px solid var(--line);
179
+ }
180
+ .bar-item {
181
+ display: grid;
182
+ grid-template-rows: 54px 1fr;
183
+ flex: 1 1 0;
184
+ min-width: 0;
185
+ text-align: center;
186
+ }
187
+ .bar-value {
188
+ align-self: end;
189
+ padding-bottom: 12px;
190
+ color: var(--bar);
191
+ font-size: clamp(22px, 3vw, 38px);
192
+ font-weight: 800;
193
+ line-height: 1;
194
+ letter-spacing: -.05em;
195
+ }
196
+ .bar-item.winner .bar-value { color: var(--green); }
197
+ .bar-slot { display: flex; align-items: end; min-height: 0; }
198
+ .bar {
199
+ width: 100%;
200
+ height: calc(var(--value) * 1%);
201
+ min-height: 3px;
202
+ border-radius: 11px 11px 0 0;
203
+ background: var(--shade, var(--bar));
204
+ transform-origin: bottom;
205
+ animation: rise .55s cubic-bezier(.25,.7,.2,1) both;
206
+ }
207
+ .bar-item.winner .bar { background: linear-gradient(var(--green-soft), var(--green)); }
208
+ @keyframes rise { from { transform: scaleY(0); } }
209
+ .bar-labels {
210
+ display: flex;
211
+ gap: clamp(18px, 4vw, 54px);
212
+ padding: 17px 24px 0;
213
+ }
214
+ .bar-label { flex: 1 1 0; min-width: 0; text-align: center; }
215
+ .model-name { overflow: hidden; font-size: clamp(10px, 1.5vw, 14px); font-weight: 800; text-overflow: ellipsis; white-space: nowrap; }
216
+ .model-meta { margin-top: 5px; color: var(--muted); font-size: clamp(8px, 1vw, 10px); font-weight: 700; }
217
+
218
+ .horizontal-list { display: grid; gap: 20px; margin-top: 46px; }
219
+ .horizontal-row { display: grid; grid-template-columns: 110px 1fr 94px; gap: 18px; align-items: center; }
220
+ .horizontal-name { overflow: hidden; font-size: 12px; font-weight: 800; text-overflow: ellipsis; white-space: nowrap; }
221
+ .horizontal-track { height: 18px; border-radius: 5px; background: var(--track); overflow: hidden; }
222
+ .horizontal-fill { width: calc(var(--value) * 1%); height: 100%; border-radius: inherit; background: var(--bar); }
223
+ .horizontal-row.winner .horizontal-fill { background: var(--green); }
224
+ .horizontal-value { color: var(--bar); font-size: 15px; font-weight: 800; text-align: right; }
225
+ .horizontal-row.winner .horizontal-value { color: var(--green); }
226
+
227
+ .matrix-card { display: flex; flex-direction: column; }
228
+ .matrix-wrap { margin-top: 34px; overflow-x: auto; border: 1px solid var(--line); border-radius: 10px; background: var(--card); }
229
+ table { width: 100%; border-collapse: collapse; font-variant-numeric: tabular-nums; }
230
+ th, td { padding: 13px 14px; border-bottom: 1px solid var(--line); font-size: 10px; text-align: right; white-space: nowrap; }
231
+ tr:last-child td { border-bottom: 0; }
232
+ th { color: var(--muted); background: var(--raise); font: 8px/1 var(--mono); letter-spacing: .07em; text-transform: uppercase; }
233
+ th:first-child, td:first-child { position: sticky; left: 0; text-align: left; }
234
+ th:first-child { z-index: 2; background: var(--raise); }
235
+ td:first-child { z-index: 1; background: var(--card); font-weight: 800; }
236
+ .winner-cell { color: var(--green); font-size: 13px; font-weight: 800; }
237
+ .warning { color: var(--amber); font-weight: 800; }
238
+ .range { display: block; margin-top: 4px; color: var(--muted); font-size: 8px; }
239
+
240
+ .details { display: grid; gap: 8px; }
241
+ details { border: 1px solid var(--line); border-radius: 10px; background: var(--card); overflow: hidden; }
242
+ summary {
243
+ display: grid;
244
+ grid-template-columns: 1fr auto auto;
245
+ gap: 18px;
246
+ align-items: center;
247
+ padding: 17px 19px;
248
+ list-style: none;
249
+ cursor: pointer;
250
+ font-size: 10px;
251
+ }
252
+ summary::-webkit-details-marker { display: none; }
253
+ summary::after { content: "+"; color: var(--muted); font: 15px/1 var(--mono); }
254
+ details[open] summary::after { content: "−"; }
255
+ .detail-title { font-weight: 800; }
256
+ .detail-score { color: var(--muted); }
257
+ .raw-wrap { padding: 0 10px 10px; overflow-x: auto; }
258
+ .raw-wrap table { border-top: 1px solid var(--line); }
259
+
260
+ footer {
261
+ display: flex;
262
+ justify-content: space-between;
263
+ gap: 20px;
264
+ margin-top: 70px;
265
+ padding-top: 24px;
266
+ border-top: 1px solid var(--line);
267
+ color: var(--muted);
268
+ font: 8px/1.4 var(--mono);
269
+ }
270
+
271
+ @media (max-width: 720px) {
272
+ .page { width: min(100% - 24px, 600px); padding-top: 34px; }
273
+ .report-head { display: block; }
274
+ .head-right { align-items: flex-start; margin-top: 24px; }
275
+ .best { text-align: left; }
276
+ .summary { grid-template-columns: repeat(2, 1fr); }
277
+ .stat:nth-child(3) { border-top: 1px solid var(--line); border-left: 0; }
278
+ .stat:nth-child(4) { border-top: 1px solid var(--line); }
279
+ .chart-shell { padding: 25px 18px 24px; border-radius: 14px; }
280
+ .chart-title { font-size: 18px; }
281
+ .chart-subtitle { font-size: 17px; }
282
+ .bar-plot { height: 340px; gap: 12px; padding: 0 4px; }
283
+ .bar-labels { gap: 12px; padding: 14px 4px 0; }
284
+ .horizontal-row { grid-template-columns: 70px 1fr 74px; gap: 10px; }
285
+ }
286
+ </style>
287
+ </head>
288
+ <body>
289
+ <main class="page">
290
+ <header class="report-head">
291
+ <div>
292
+ <h1>BenchKit performance</h1>
293
+ <p class="meta" id="reportMeta"></p>
294
+ </div>
295
+ <div class="head-right">
296
+ <button class="theme-toggle" id="themeToggle" type="button" aria-label="Switch colour theme">
297
+ <span class="theme-dot"></span><span id="themeLabel">Dark</span>
298
+ </button>
299
+ <div class="best" id="best"></div>
300
+ </div>
301
+ </header>
302
+
303
+ <section class="summary" id="summary"></section>
304
+ <div class="notice" id="notice"></div>
305
+
306
+ <section class="section">
307
+ <div class="section-head"><h2>Chart gallery</h2><p class="section-note">Performance across context depth</p></div>
308
+ <div class="charts">
309
+ <div id="generationChart"></div>
310
+ <div id="promptChart"></div>
311
+ <div id="latencyChart"></div>
312
+ </div>
313
+ </section>
314
+
315
+ <section class="section">
316
+ <div class="section-head"><h2>Measurement matrix</h2><p class="section-note">Median with min—max range</p></div>
317
+ <div id="matrix"></div>
318
+ </section>
319
+
320
+ <section class="section">
321
+ <div class="section-head"><h2>Raw repetitions</h2><p class="section-note">Open a depth to inspect every run</p></div>
322
+ <div class="details" id="details"></div>
323
+ </section>
324
+
325
+ <footer><span id="footerMeta"></span><span>Generated by BenchKit</span></footer>
326
+ </main>
327
+
328
+ <script id="perfData" type="application/json">__BENCHKIT_PERF_DATA__</script>
329
+ <script>
330
+ const root = document.documentElement;
331
+ const themeLabel = document.getElementById("themeLabel");
332
+ const setTheme = mode => {
333
+ root.dataset.theme = mode;
334
+ themeLabel.textContent = mode === "dark" ? "Light" : "Dark";
335
+ };
336
+ setTheme(root.dataset.theme === "dark" ? "dark" : "light");
337
+ document.getElementById("themeToggle").addEventListener("click", () => {
338
+ const next = root.dataset.theme === "dark" ? "light" : "dark";
339
+ setTheme(next);
340
+ try { localStorage.setItem("benchkit-theme", next); } catch (error) { /* file:// */ }
341
+ });
342
+
343
+ const profile = JSON.parse(document.getElementById("perfData").textContent);
344
+ const cases = profile.cases || [];
345
+ const settings = profile.settings || {};
346
+ const environment = profile.environment || {};
347
+ const startup = profile.startup || {};
348
+ const drift = profile.drift_check || {};
349
+ const sweep = profile.sweep_summary || {};
350
+ const escapeHtml = value => String(value ?? "").replace(/[&<>"']/g, char => ({"&":"&amp;","<":"&lt;",">":"&gt;",'"':"&quot;","'":"&#39;"}[char]));
351
+ const number = value => Number(value) || 0;
352
+ const median = (item, key) => number(item?.[key]?.median);
353
+ const metricRange = (item, key, digits = 1) =>
354
+ `${number(item?.[key]?.min).toFixed(digits)}—${number(item?.[key]?.max).toFixed(digits)}`;
355
+ const formatTime = (value, digits = 2) => `${number(value).toFixed(digits)}s`;
356
+ const date = String(profile.generated_at || "").replace("T", " · ");
357
+ const config = environment.config_note || environment.hardware || "";
358
+ const meta = [
359
+ date,
360
+ profile.provider,
361
+ profile.host,
362
+ `${number(settings.gen_tokens)} output tokens`,
363
+ `${number(settings.reps)} measured reps`,
364
+ config,
365
+ ].filter(Boolean);
366
+ document.getElementById("reportMeta").textContent = meta.join(" · ");
367
+ document.getElementById("footerMeta").textContent = [profile.model, profile.provider, config].filter(Boolean).join(" · ");
368
+
369
+ const peakTg = Math.max(0, ...cases.map(item => median(item, "tg_tps")));
370
+ document.getElementById("best").innerHTML =
371
+ `<div class="best-score">${peakTg.toFixed(1)}</div><div class="best-model">tok/s · peak TG · ${escapeHtml(profile.model)}</div>`;
372
+
373
+ const summaryItems = [
374
+ ["Depths", cases.length, cases.map(item => item.depth_label).join(" · ")],
375
+ ["Median TG", `${number(sweep.tg_tps_median).toFixed(1)}`, `tok/s · ${number(sweep.tg_depths)} depths`],
376
+ ["Median PP", `${number(sweep.pp_tps_median).toFixed(1)}`, `tok/s · ${number(sweep.pp_depths)} reliable depths`],
377
+ ["First request", startup.ttft_s != null ? formatTime(startup.ttft_s, 3) : "—", startup.label || "not captured"],
378
+ ];
379
+ document.getElementById("summary").innerHTML = summaryItems.map(item => `
380
+ <article class="stat"><div class="stat-label">${escapeHtml(item[0])}</div>
381
+ <div class="stat-value">${escapeHtml(item[1])}</div><div class="stat-detail">${escapeHtml(item[2])}</div></article>
382
+ `).join("");
383
+
384
+ const calibrated = cases.filter(item => item.depth_method === "calibrated");
385
+ const inaccurate = cases.filter(item => !item.depth_within_tolerance);
386
+ const estimated = cases.filter(item => item.depth_method === "estimated");
387
+ const notices = ["Tip: Run the ruler benchmark to verify how much of the configured context window is actually usable."];
388
+ if (calibrated.length) notices.push("Depth method: calibrated · the server does not expose its native tokenizer route, so server-reported input tokens were used.");
389
+ if (inaccurate.length) notices.push(`⚠ Outside ±1% depth tolerance: ${inaccurate.map(item => item.depth_label).join(", ")}. Use Actual input.`);
390
+ if (estimated.length) notices.push(`⚠ Token-depth calibration unavailable: ${estimated.map(item => item.depth_label).join(", ")}. Use Actual input.`);
391
+ if (drift.error) {
392
+ notices.push(`⚠ End-of-sweep drift check failed: ${drift.error}`);
393
+ } else if (drift.depth_label) {
394
+ notices.push(
395
+ `Drift check · ${drift.depth_label} TG ${number(drift.initial_tg_tps).toFixed(1)} → ${number(drift.final_tg_tps).toFixed(1)} tok/s (${number(drift.change * 100).toFixed(1)}%)`
396
+ + (drift.stable ? "" : " · possible warmup/drift anomaly")
397
+ );
398
+ }
399
+ const notice = document.getElementById("notice");
400
+ if (notices.length) {
401
+ notice.textContent = notices.join(" ");
402
+ notice.classList.add("show");
403
+ if (inaccurate.length || estimated.length || drift.error || drift.stable === false) notice.classList.add("warn");
404
+ }
405
+
406
+ const shades = [1, 2, 3, 4, 5, 6, 7].map(step => `var(--shade-${step})`);
407
+ const barChart = ({title, subtitle, items, value, meta}) => {
408
+ const maximum = Math.max(1, ...items.map(value));
409
+ const bestValue = Math.max(0, ...items.map(value));
410
+ return `<article class="chart-shell">
411
+ <div class="chart-header"><h3 class="chart-title">${escapeHtml(title)}</h3><span class="chart-subtitle">· ${escapeHtml(subtitle)}</span></div>
412
+ <div class="bar-plot">${items.map((item, index) => {
413
+ const current = value(item);
414
+ const winner = current === bestValue;
415
+ const height = Math.max(0, Math.min(100, current / maximum * 100));
416
+ return `<div class="bar-item ${winner ? "winner" : ""}">
417
+ <div class="bar-value">${current.toFixed(1)}</div>
418
+ <div class="bar-slot"><div class="bar" style="--value:${height};--shade:${shades[Math.min(index, shades.length - 1)]};animation-delay:${index * 45}ms"></div></div>
419
+ </div>`;
420
+ }).join("")}</div>
421
+ <div class="bar-labels">${items.map(item => `<div class="bar-label">
422
+ <div class="model-name">${escapeHtml(item.depth_label)}</div>
423
+ <div class="model-meta">${escapeHtml(meta(item))}</div>
424
+ </div>`).join("")}</div>
425
+ </article>`;
426
+ };
427
+
428
+ document.getElementById("generationChart").innerHTML = barChart({
429
+ title: "Generation speed", subtitle: "TG tok/s", items: cases,
430
+ value: item => median(item, "tg_tps"),
431
+ meta: item => `${number(item.actual_input_tokens).toLocaleString()} input tokens`,
432
+ });
433
+
434
+ const promptCases = cases.filter(item => item.pp_reliable);
435
+ document.getElementById("promptChart").innerHTML = barChart({
436
+ title: "Prompt processing", subtitle: "PP tok/s", items: promptCases,
437
+ value: item => median(item, "pp_tps"),
438
+ meta: item => `${number(item.actual_input_tokens).toLocaleString()} input tokens`,
439
+ });
440
+
441
+ const maxTtft = Math.max(1, ...cases.map(item => median(item, "ttft_s")));
442
+ const fastestTtft = Math.min(...cases.map(item => median(item, "ttft_s")).filter(Boolean));
443
+ document.getElementById("latencyChart").innerHTML = `<article class="chart-shell">
444
+ <div class="chart-header"><h3 class="chart-title">Time to first token</h3><span class="chart-subtitle">· lower is better</span></div>
445
+ <div class="horizontal-list">${cases.map(item => {
446
+ const value = median(item, "ttft_s");
447
+ return `<div class="horizontal-row ${value === fastestTtft ? "winner" : ""}">
448
+ <div class="horizontal-name">${escapeHtml(item.depth_label)}</div>
449
+ <div class="horizontal-track"><div class="horizontal-fill" style="--value:${value / maxTtft * 100}"></div></div>
450
+ <div class="horizontal-value">${formatTime(value, 3)}</div>
451
+ </div>`;
452
+ }).join("")}</div>
453
+ </article>`;
454
+
455
+ const bestTg = Math.max(0, ...cases.map(item => median(item, "tg_tps")));
456
+ document.getElementById("matrix").innerHTML = `<article class="chart-shell matrix-card">
457
+ <div class="chart-header"><h3 class="chart-title">Performance matrix</h3><span class="chart-subtitle">· measured medians</span></div>
458
+ <div class="matrix-wrap"><table>
459
+ <thead><tr><th>Context</th><th>Actual input</th><th>PP tok/s</th><th>TG tok/s</th><th>TTFT</th><th>Wall</th><th>Overhead</th><th>Output</th><th>Runs</th></tr></thead>
460
+ <tbody>${cases.map(item => {
461
+ const warning = item.depth_within_tolerance ? "" : "!";
462
+ const failed = number(item.successful_reps) < number(settings.reps);
463
+ const pp = item.pp_reliable
464
+ ? `${median(item, "pp_tps").toFixed(1)}<span class="range">${metricRange(item, "pp_tps")}</span>`
465
+ : `n/a<span class="range">tiny prompt</span>`;
466
+ const overhead = item.overhead_available
467
+ ? `${formatTime(median(item, "overhead_s"), 3)}<span class="range">${metricRange(item, "overhead_s", 3)}s</span>`
468
+ : "n/a";
469
+ return `<tr>
470
+ <td class="${warning ? "warning" : ""}">${escapeHtml(item.depth_label)}${warning}</td>
471
+ <td>${number(item.actual_input_tokens).toLocaleString()}</td>
472
+ <td>${pp}</td>
473
+ <td class="${median(item, "tg_tps") === bestTg ? "winner-cell" : ""}">${median(item, "tg_tps").toFixed(1)}<span class="range">${metricRange(item, "tg_tps")}</span></td>
474
+ <td>${formatTime(median(item, "ttft_s"), 3)}<span class="range">${metricRange(item, "ttft_s", 3)}s</span></td>
475
+ <td>${formatTime(median(item, "wall_time_s"), 2)}<span class="range">${metricRange(item, "wall_time_s", 2)}s</span></td>
476
+ <td>${overhead}</td>
477
+ <td>${median(item, "output_tokens").toFixed(0)}</td>
478
+ <td class="${failed ? "warning" : ""}">${number(item.successful_reps)}/${number(settings.reps)}</td>
479
+ </tr>`;
480
+ }).join("")}</tbody>
481
+ </table></div>
482
+ </article>`;
483
+
484
+ document.getElementById("details").innerHTML = cases.map(item => `<details>
485
+ <summary>
486
+ <span class="detail-title">${escapeHtml(item.depth_label)} context · ${number(item.actual_input_tokens).toLocaleString()} actual tokens</span>
487
+ <span class="detail-score">${median(item, "tg_tps").toFixed(1)} TG tok/s · ${formatTime(median(item, "ttft_s"), 3)} TTFT</span>
488
+ </summary>
489
+ <div class="raw-wrap"><table>
490
+ <thead><tr><th>Run</th><th>Input</th><th>Output</th><th>PP tok/s</th><th>TG tok/s</th><th>Wall TG</th><th>TTFT</th><th>Wall</th><th>Overhead</th></tr></thead>
491
+ <tbody>${(item.runs || []).map(run => run.error
492
+ ? `<tr><td>${number(run.repetition)}</td><td colspan="8" class="warning">${escapeHtml(run.error)}</td></tr>`
493
+ : `<tr>
494
+ <td>${number(run.repetition)}</td>
495
+ <td>${number(run.input_tokens).toLocaleString()}</td>
496
+ <td>${number(run.output_tokens)}</td>
497
+ <td>${number(run.pp_tps).toFixed(1)}</td>
498
+ <td>${number(run.tg_tps).toFixed(1)}</td>
499
+ <td>${number(run.wall_tg_tps).toFixed(1)}</td>
500
+ <td>${formatTime(run.ttft_s, 3)}</td>
501
+ <td>${formatTime(run.wall_time_s, 2)}</td>
502
+ <td>${run.server_timings ? formatTime(run.overhead_s, 3) : "n/a"}</td>
503
+ </tr>`
504
+ ).join("")}</tbody>
505
+ </table></div>
506
+ </details>`).join("");
507
+ </script>
508
+ </body>
509
+ </html>