pluginbench 0.5.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pluginbench/__init__.py +3 -0
- pluginbench/__main__.py +3 -0
- pluginbench/assessment-charts.html +790 -0
- pluginbench/assessment.py +316 -0
- pluginbench/assessment_charts.py +1225 -0
- pluginbench/assessment_metrics.py +1956 -0
- pluginbench/assessment_models.py +127 -0
- pluginbench/baselines.py +792 -0
- pluginbench/claude.py +374 -0
- pluginbench/cli.py +672 -0
- pluginbench/code_diff.py +207 -0
- pluginbench/codex.py +451 -0
- pluginbench/codex_events.py +585 -0
- pluginbench/collectors.py +591 -0
- pluginbench/compare.py +927 -0
- pluginbench/components.py +129 -0
- pluginbench/coverage_report.py +610 -0
- pluginbench/datasets.py +207 -0
- pluginbench/evaluation.example.yaml +45 -0
- pluginbench/evaluation.py +1651 -0
- pluginbench/evaluation_report.py +1004 -0
- pluginbench/fallow_parse.mjs +101 -0
- pluginbench/harbor_backend.py +2946 -0
- pluginbench/isolation.py +314 -0
- pluginbench/judge.py +683 -0
- pluginbench/language_analysis.py +915 -0
- pluginbench/ledger.py +217 -0
- pluginbench/measurement_profiles.py +616 -0
- pluginbench/native_treatment_runtime.js +221 -0
- pluginbench/native_treatments.py +155 -0
- pluginbench/observer_data/go-analysis-v1/go.mod +10 -0
- pluginbench/observer_data/go-analysis-v1/go.sum +10 -0
- pluginbench/observer_data/go-analysis-v1/main.go +64 -0
- pluginbench/observer_data/oxc-parser-0.126.0/package-lock.json +449 -0
- pluginbench/observer_data/oxc-parser-0.126.0/package.json +5 -0
- pluginbench/parallel.py +101 -0
- pluginbench/processes.py +100 -0
- pluginbench/remeasure.py +992 -0
- pluginbench/run_lock.py +227 -0
- pluginbench/run_status.py +518 -0
- pluginbench/study.py +1858 -0
- pluginbench/study_review.py +924 -0
- pluginbench/submissions.py +486 -0
- pluginbench/task_sources.py +403 -0
- pluginbench/treatments.py +388 -0
- pluginbench/trial_containers.py +94 -0
- pluginbench/vendor/ECHARTS-LICENSE +222 -0
- pluginbench/vendor/ECHARTS-NOTICE +5 -0
- pluginbench/vendor/ZRENDER-LICENSE +29 -0
- pluginbench/vendor/echarts.min.js +48 -0
- pluginbench-0.5.0.dist-info/METADATA +294 -0
- pluginbench-0.5.0.dist-info/RECORD +58 -0
- pluginbench-0.5.0.dist-info/WHEEL +4 -0
- pluginbench-0.5.0.dist-info/entry_points.txt +2 -0
- pluginbench-0.5.0.dist-info/licenses/LICENSE +201 -0
- pluginbench-0.5.0.dist-info/licenses/src/pluginbench/vendor/ECHARTS-LICENSE +222 -0
- pluginbench-0.5.0.dist-info/licenses/src/pluginbench/vendor/ECHARTS-NOTICE +5 -0
- pluginbench-0.5.0.dist-info/licenses/src/pluginbench/vendor/ZRENDER-LICENSE +29 -0
|
@@ -0,0 +1,790 @@
|
|
|
1
|
+
<!doctype html>
|
|
2
|
+
<html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1"><title>Assessment charts</title><style>
|
|
3
|
+
/*
|
|
4
|
+
* Design language tokens. Generated from themes/graphite-indigo.json in
|
|
5
|
+
* @filipgutica/ui; edit that file and run `pnpm themes:build`. Values live there.
|
|
6
|
+
* DESIGN.md names these tokens; it never repeats their values.
|
|
7
|
+
*
|
|
8
|
+
* Standalone HTML: import or inline this file.
|
|
9
|
+
* Inline visuals: inherit the host font and theme variables; see DESIGN.md.
|
|
10
|
+
* Repos: map these names onto the repository's own token system.
|
|
11
|
+
*
|
|
12
|
+
* Theme states: bare :root is light. prefers-color-scheme picks dark unless
|
|
13
|
+
* data-theme="light" is set. data-theme="dark" forces dark.
|
|
14
|
+
*/
|
|
15
|
+
|
|
16
|
+
:root {
|
|
17
|
+
color-scheme: light;
|
|
18
|
+
|
|
19
|
+
/* Cool graphite neutrals. */
|
|
20
|
+
--bg: #f4f5f8;
|
|
21
|
+
--surface: #e7e8ec;
|
|
22
|
+
--surface-raised: #ffffff;
|
|
23
|
+
--surface-hover: #e4e6eb;
|
|
24
|
+
--surface-selected: #dde0e6;
|
|
25
|
+
--row-hover: #dfe1e7;
|
|
26
|
+
--row-selected: #ffffff;
|
|
27
|
+
--border-subtle: #d8dbe2;
|
|
28
|
+
--border: #777d88;
|
|
29
|
+
--text: #222326;
|
|
30
|
+
--text-muted: #5d626c;
|
|
31
|
+
--text-faint: #626772;
|
|
32
|
+
|
|
33
|
+
/* Primary actions, links, and focus have separate roles. */
|
|
34
|
+
--accent: #6359a7;
|
|
35
|
+
--accent-text: #ffffff;
|
|
36
|
+
--accent-hover: #554b94;
|
|
37
|
+
--link: #075fc7;
|
|
38
|
+
--focus: #075fc7;
|
|
39
|
+
|
|
40
|
+
/* Status: only for state, never for decoration. */
|
|
41
|
+
--success: #1f8a5b;
|
|
42
|
+
--warning: #86532e;
|
|
43
|
+
--danger: #c7372d;
|
|
44
|
+
--info: #175cd3;
|
|
45
|
+
--success-soft: color-mix(in srgb, var(--success) 11%, var(--bg));
|
|
46
|
+
--warning-soft: color-mix(in srgb, var(--warning) 12%, var(--bg));
|
|
47
|
+
--danger-soft: color-mix(in srgb, var(--danger) 10%, var(--bg));
|
|
48
|
+
--info-soft: color-mix(in srgb, var(--info) 10%, var(--bg));
|
|
49
|
+
|
|
50
|
+
/* Code */
|
|
51
|
+
--code-bg: var(--surface);
|
|
52
|
+
--code-text: var(--text);
|
|
53
|
+
--code-key: #715b42;
|
|
54
|
+
--code-string: #346347;
|
|
55
|
+
--code-number: #86532e;
|
|
56
|
+
--code-comment: var(--text-faint);
|
|
57
|
+
|
|
58
|
+
/* Type: system faces, no font loading. */
|
|
59
|
+
--font-sans: -apple-system, BlinkMacSystemFont, "SF Pro Text", "Segoe UI", system-ui, sans-serif;
|
|
60
|
+
--font-mono: ui-monospace, "SF Mono", SFMono-Regular, Menlo, Consolas, monospace;
|
|
61
|
+
--text-xs: 12px;
|
|
62
|
+
--text-sm: 13px;
|
|
63
|
+
--text-md: 14px;
|
|
64
|
+
--text-lg: 16px;
|
|
65
|
+
--text-xl: 20px;
|
|
66
|
+
--text-2xl: 24px;
|
|
67
|
+
--text-3xl: 32px;
|
|
68
|
+
--text-reading: 16px;
|
|
69
|
+
--text-display: 56px;
|
|
70
|
+
--text-display-sm: 38px;
|
|
71
|
+
--leading-tight: 1.25;
|
|
72
|
+
--leading: 1.5;
|
|
73
|
+
--weight-regular: 400;
|
|
74
|
+
--weight-medium: 500;
|
|
75
|
+
--weight-semibold: 600;
|
|
76
|
+
--tracking-tight: -0.01em;
|
|
77
|
+
|
|
78
|
+
/* Space: 4px rhythm. */
|
|
79
|
+
--space-1: 4px;
|
|
80
|
+
--space-2: 8px;
|
|
81
|
+
--space-3: 12px;
|
|
82
|
+
--space-4: 16px;
|
|
83
|
+
--space-5: 20px;
|
|
84
|
+
--space-6: 24px;
|
|
85
|
+
--space-8: 32px;
|
|
86
|
+
--space-12: 48px;
|
|
87
|
+
--space-16: 64px;
|
|
88
|
+
--space-20: 80px;
|
|
89
|
+
--space-24: 96px;
|
|
90
|
+
|
|
91
|
+
/* Size */
|
|
92
|
+
--control-sm: 32px;
|
|
93
|
+
--control-md: 36px;
|
|
94
|
+
--control-lg: 44px;
|
|
95
|
+
--row: 36px;
|
|
96
|
+
--measure: 68ch;
|
|
97
|
+
--page-max: 960px;
|
|
98
|
+
--page-wide: 1120px;
|
|
99
|
+
--sidebar: 224px;
|
|
100
|
+
|
|
101
|
+
/* Shape */
|
|
102
|
+
--radius-sm: 4px;
|
|
103
|
+
--radius-md: 6px;
|
|
104
|
+
--radius-lg: 8px;
|
|
105
|
+
--radius-xl: 12px;
|
|
106
|
+
--radius-pill: 999px;
|
|
107
|
+
|
|
108
|
+
/* Elevation: floating layers only (menus, popovers, dialogs). */
|
|
109
|
+
--shadow-popover: 0 8px 24px rgb(16 16 24 / 12%), 0 0 0 1px var(--border-subtle);
|
|
110
|
+
|
|
111
|
+
/* Motion */
|
|
112
|
+
--duration-fast: 120ms;
|
|
113
|
+
--duration: 180ms;
|
|
114
|
+
--ease-out: cubic-bezier(0.2, 0.8, 0.2, 1);
|
|
115
|
+
|
|
116
|
+
/* Focus */
|
|
117
|
+
--focus-ring: 0 0 0 2px var(--bg), 0 0 0 4px var(--focus);
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
@media (prefers-color-scheme: dark) {
|
|
121
|
+
:root:not([data-theme="light"]) {
|
|
122
|
+
color-scheme: dark;
|
|
123
|
+
--bg: #151619;
|
|
124
|
+
--surface: #191a1d;
|
|
125
|
+
--surface-raised: #212226;
|
|
126
|
+
--surface-hover: #303238;
|
|
127
|
+
--surface-selected: #34373e;
|
|
128
|
+
--row-hover: #25272d;
|
|
129
|
+
--row-selected: #38334f;
|
|
130
|
+
--border-subtle: #3b3e45;
|
|
131
|
+
--border: #818894;
|
|
132
|
+
--text: #f4f5f8;
|
|
133
|
+
--text-muted: #b0b5bf;
|
|
134
|
+
--text-faint: #a0a6b1;
|
|
135
|
+
--accent: #a99ce2;
|
|
136
|
+
--accent-text: #151619;
|
|
137
|
+
--accent-hover: #b7ace9;
|
|
138
|
+
--link: #58a6ff;
|
|
139
|
+
--focus: #58a6ff;
|
|
140
|
+
--success: #4cc38a;
|
|
141
|
+
--warning: #e2a54b;
|
|
142
|
+
--danger: #f0776c;
|
|
143
|
+
--info: #84adff;
|
|
144
|
+
--code-key: #d8c2a5;
|
|
145
|
+
--code-string: #a8bd9c;
|
|
146
|
+
--code-number: #d5b98c;
|
|
147
|
+
--shadow-popover: 0 8px 24px rgb(0 0 0 / 45%), 0 0 0 1px var(--border);
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
:root[data-theme="dark"] {
|
|
152
|
+
color-scheme: dark;
|
|
153
|
+
--bg: #151619;
|
|
154
|
+
--surface: #191a1d;
|
|
155
|
+
--surface-raised: #212226;
|
|
156
|
+
--surface-hover: #303238;
|
|
157
|
+
--surface-selected: #34373e;
|
|
158
|
+
--row-hover: #25272d;
|
|
159
|
+
--row-selected: #38334f;
|
|
160
|
+
--border-subtle: #3b3e45;
|
|
161
|
+
--border: #818894;
|
|
162
|
+
--text: #f4f5f8;
|
|
163
|
+
--text-muted: #b0b5bf;
|
|
164
|
+
--text-faint: #a0a6b1;
|
|
165
|
+
--accent: #a99ce2;
|
|
166
|
+
--accent-text: #151619;
|
|
167
|
+
--accent-hover: #b7ace9;
|
|
168
|
+
--link: #58a6ff;
|
|
169
|
+
--focus: #58a6ff;
|
|
170
|
+
--success: #4cc38a;
|
|
171
|
+
--warning: #e2a54b;
|
|
172
|
+
--danger: #f0776c;
|
|
173
|
+
--info: #84adff;
|
|
174
|
+
--code-key: #d8c2a5;
|
|
175
|
+
--code-string: #a8bd9c;
|
|
176
|
+
--code-number: #d5b98c;
|
|
177
|
+
--shadow-popover: 0 8px 24px rgb(0 0 0 / 45%), 0 0 0 1px var(--border);
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
/* Minimal base. Component styles live in reference.html. */
|
|
181
|
+
*, *::before, *::after { box-sizing: border-box; }
|
|
182
|
+
[hidden] { display: none !important; }
|
|
183
|
+
body {
|
|
184
|
+
margin: 0;
|
|
185
|
+
background: var(--bg);
|
|
186
|
+
color: var(--text);
|
|
187
|
+
font: var(--weight-regular) var(--text-md) / var(--leading) var(--font-sans);
|
|
188
|
+
-webkit-font-smoothing: antialiased;
|
|
189
|
+
}
|
|
190
|
+
h1, h2, h3, h4, p { margin: 0; }
|
|
191
|
+
h1, h2, h3, h4 { font-weight: var(--weight-semibold); line-height: var(--leading-tight); letter-spacing: var(--tracking-tight); text-wrap: balance; }
|
|
192
|
+
code, kbd, pre { font-family: var(--font-mono); }
|
|
193
|
+
table, .num { font-variant-numeric: tabular-nums; }
|
|
194
|
+
a { color: var(--link); text-decoration: none; }
|
|
195
|
+
a:hover { text-decoration: underline; }
|
|
196
|
+
p a, dd a { text-decoration: underline; text-underline-offset: 0.18em; }
|
|
197
|
+
:focus-visible { outline: none; box-shadow: var(--focus-ring); border-radius: var(--radius-sm); }
|
|
198
|
+
@media (prefers-reduced-motion: reduce) {
|
|
199
|
+
*, *::before, *::after { transition-duration: 0ms !important; animation-duration: 0ms !important; scroll-behavior: auto !important; }
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
:root { --pb-bg:var(--background,var(--bg)); --pb-surface:var(--card,var(--surface-raised)); --pb-text:var(--foreground,var(--text)); --pb-muted:var(--muted-foreground,var(--text-muted)); --pb-border:var(--input,var(--border-subtle)); --pb-accent:var(--primary,var(--accent)); }
|
|
203
|
+
/* Categorical package colors in fixed order, validated against --bg in each mode. Vanilla stays neutral. */
|
|
204
|
+
:root { --series-1:#2a78d6; --series-2:#eb6834; --series-3:#1baf7a; --series-4:#eda100; --series-5:#e87ba4; --series-6:#008300; --series-7:#6250d6; --series-8:#e34948; }
|
|
205
|
+
@media (prefers-color-scheme: dark) { :root:not([data-theme="light"]) { --series-1:#3987e5; --series-2:#d95926; --series-3:#199e70; --series-4:#c98500; --series-5:#d55181; --series-6:#008300; --series-7:#9085e9; --series-8:#e66767; } }
|
|
206
|
+
:root[data-theme="dark"] { --series-1:#3987e5; --series-2:#d95926; --series-3:#199e70; --series-4:#c98500; --series-5:#d55181; --series-6:#008300; --series-7:#9085e9; --series-8:#e66767; }
|
|
207
|
+
body { background:var(--pb-bg); color:var(--pb-text); }
|
|
208
|
+
main { max-width:var(--page-wide); margin:auto; padding:var(--space-6); }
|
|
209
|
+
h1 { font-size:var(--text-2xl); } h2 { font-size:var(--text-xl); }
|
|
210
|
+
header > * { min-width:0; } header { display:grid; gap:var(--space-3); margin-bottom:var(--space-4); } .chart-options { display:grid; gap:var(--space-3); margin-top:var(--space-6); }
|
|
211
|
+
p { max-width:var(--measure); } .muted,.counts { color:var(--pb-muted); font-size:var(--text-sm); }
|
|
212
|
+
.controls { min-width:0; max-width:100%; display:flex; flex-wrap:wrap; gap:var(--space-4); align-items:end; }
|
|
213
|
+
label { display:grid; gap:var(--space-1); min-width:0; } select { min-width:0; min-height:var(--control-lg); font:inherit; color:inherit; background:var(--pb-surface); border:1px solid var(--pb-border); border-radius:var(--radius-md); padding:var(--space-2); max-width:100%; }
|
|
214
|
+
.overview-button { min-height:var(--control-lg); padding:var(--space-2) var(--space-3); font:inherit; color:var(--pb-text); background:var(--pb-surface); border:1px solid var(--pb-border); border-radius:var(--radius-md); cursor:pointer; }
|
|
215
|
+
.overview-button:hover { background:var(--surface-hover); } .overview-button[aria-pressed=true] { border-color:var(--pb-accent); font-weight:var(--weight-medium); }
|
|
216
|
+
.scope-context { display:grid; gap:var(--space-2); margin-top:var(--space-3); } .scope-context h2 { overflow-wrap:anywhere; }
|
|
217
|
+
.runtime { display:flex; flex-wrap:wrap; gap:var(--space-2) var(--space-4); font-size:var(--text-sm); } .runtime div { display:flex; gap:6px; } .runtime dt { color:var(--pb-muted); } .runtime dd { margin:0; overflow-wrap:anywhere; }
|
|
218
|
+
.grid { display:grid; gap:var(--space-8); }
|
|
219
|
+
section { border-top:1px solid var(--pb-border); padding-top:var(--space-6); min-width:0; }
|
|
220
|
+
section > * + * { margin-top:var(--space-3); } .task { border-top:1px solid var(--pb-border); padding-top:var(--space-3); }
|
|
221
|
+
h3 { font-size:var(--text-md); overflow-wrap:anywhere; } .chart { width:100%; height:320px; }
|
|
222
|
+
.legend { display:flex; gap:var(--space-4); flex-wrap:wrap; font-size:var(--text-sm); }
|
|
223
|
+
.legend span::before, .swatch::before { content:''; display:inline-block; width:10px; height:10px; border-radius:50%; margin-right:var(--space-2); background:var(--series-color, var(--pb-accent)); }
|
|
224
|
+
.run-toggle { display:flex; align-items:center; gap:8px; font-size:var(--text-sm); cursor:pointer; min-height:32px; }
|
|
225
|
+
summary { cursor:pointer; min-height:32px; padding-top:var(--space-2); } pre { font-size:var(--text-xs); white-space:pre-wrap; overflow-wrap:anywhere; } details { color:var(--pb-muted); } footer { margin-top:var(--space-6); font-size:var(--text-sm); }
|
|
226
|
+
.review-findings { overflow-wrap:anywhere; }
|
|
227
|
+
.table-wrap { overflow:auto; } table { width:100%; border-collapse:collapse; font-size:var(--text-sm); } th,td { text-align:left; padding:var(--space-2); border-bottom:1px solid var(--pb-border); } .tooltip { max-width:260px; white-space:pre-wrap; overflow-wrap:anywhere; }
|
|
228
|
+
.package-summary { margin:var(--space-5) 0; }
|
|
229
|
+
.package-summary h2 { margin-bottom:var(--space-2); }
|
|
230
|
+
.package-summary th { font-weight:var(--weight-medium); }
|
|
231
|
+
.package-summary td { vertical-align:top; }
|
|
232
|
+
.package-summary small { display:block; color:var(--pb-muted); }
|
|
233
|
+
.package-summary thead th { white-space:nowrap; }
|
|
234
|
+
.package-summary h3 { margin-top:var(--space-5); margin-bottom:var(--space-2); }
|
|
235
|
+
.scope-context h2 { font-size:var(--text-lg); }
|
|
236
|
+
.metric-link { padding:0; font:inherit; color:var(--pb-text); background:none; border:0; cursor:pointer; text-align:left; text-decoration:underline; text-underline-offset:3px; }
|
|
237
|
+
.metric-link[aria-pressed=true] { color:var(--pb-accent); }
|
|
238
|
+
.chart-controls { display:flex; flex-wrap:wrap; align-items:end; gap:var(--space-4); margin:var(--space-5) 0 var(--space-3); }
|
|
239
|
+
.chart-controls label { flex:1; min-width:180px; }
|
|
240
|
+
body.evaluation header { gap:var(--space-2); }
|
|
241
|
+
body.evaluation .scope-context { margin-top:0; }
|
|
242
|
+
body.evaluation section { padding-top:var(--space-4); }
|
|
243
|
+
body.evaluation .package-summary { margin:var(--space-4) 0; }
|
|
244
|
+
body.evaluation .chart-options { margin-top:var(--space-2); }
|
|
245
|
+
@media(max-width:700px) { main { padding:var(--space-4); } .grid { grid-template-columns:1fr; } .controls label { max-width:100%; } }
|
|
246
|
+
</style></head><body><main>
|
|
247
|
+
<header><h1>Guidance versus vanilla</h1><p id="study" class="muted"></p>
|
|
248
|
+
<div class="controls"><label>Individual task<select id="task"></select></label></div>
|
|
249
|
+
<dl id="run-metadata" class="runtime"></dl>
|
|
250
|
+
<section id="collection-status" hidden></section>
|
|
251
|
+
</header>
|
|
252
|
+
<details id="task-results"><summary>Task checks</summary><section id="outcomes" aria-label="Task results"></section></details>
|
|
253
|
+
<details id="measurement-completeness"><summary></summary><div></div></details>
|
|
254
|
+
<div class="chart-options"><label class="run-toggle"><input id="show-runs" type="checkbox" checked>Show individual runs</label><p class="muted">Grey is vanilla; the colored series uses the selected guidance. Each row is one task. Bars show the median (middle result); each dot is one run. A zero has no bar length. Squares mark failed checks. Exact values are in the details.</p></div>
|
|
255
|
+
<details id="limitations"><summary>Study details and limitations</summary><p id="metadata-source" class="muted"></p><ul></ul></details>
|
|
256
|
+
<div id="chart-controls" class="chart-controls"></div>
|
|
257
|
+
<div id="panels" class="grid"></div><footer></footer>
|
|
258
|
+
</main><script>__ECHARTS_SCRIPT__</script><script id="assessment-data" type="application/json">__ASSESSMENT_DATA__</script><script>
|
|
259
|
+
"use strict";
|
|
260
|
+
const data = JSON.parse(document.getElementById("assessment-data").textContent);
|
|
261
|
+
// Diagnostic panels stay in results JSON/CSV only.
|
|
262
|
+
const observations = data.observations.filter(row => !(data.diagnostic_panels || []).includes(row.panel));
|
|
263
|
+
const create = (tag, text, parent) => {
|
|
264
|
+
const node = document.createElement(tag);
|
|
265
|
+
if (text !== void 0) node.textContent = text;
|
|
266
|
+
if (parent) parent.append(node);
|
|
267
|
+
return node;
|
|
268
|
+
};
|
|
269
|
+
const unique = (items) => [...new Set(items)];
|
|
270
|
+
const format = (value) => value === null ? "Unknown" : new Intl.NumberFormat(void 0, { maximumFractionDigits: 2 }).format(value);
|
|
271
|
+
const formatTick = value => new Intl.NumberFormat(undefined, { notation: Math.abs(value) >= 10000 ? "compact" : "standard", maximumFractionDigits: 1 }).format(value);
|
|
272
|
+
// Summaries and bar labels; exact values stay in the detail tables, tooltips and CSV.
|
|
273
|
+
// Values below one keep two significant digits so a small real difference never reads as zero.
|
|
274
|
+
const formatCompact = value => value === null || value === undefined ? "Unknown" : new Intl.NumberFormat(undefined, Math.abs(value) >= 10000 ? { notation: "compact", maximumFractionDigits: 2 } : value !== 0 && Math.abs(value) < 1 ? { maximumSignificantDigits: 2 } : { maximumFractionDigits: 2 }).format(value);
|
|
275
|
+
const formatSigned = value => (value > 0 ? "+" : "") + formatCompact(value);
|
|
276
|
+
const armLabel = (arm) => arm === "baseline" ? "Vanilla" : arm === "unknown" ? "Unlabelled skill set" : arm;
|
|
277
|
+
const choices = (select, values) => {
|
|
278
|
+
for (const value of values) create("option", value, select).value = value;
|
|
279
|
+
};
|
|
280
|
+
const taskSelect = document.getElementById("task");
|
|
281
|
+
create("option", "Choose a task…", taskSelect).value = "";
|
|
282
|
+
const knownTasks = unique(observations.map(row => row.task).filter(task => task !== "unknown"));
|
|
283
|
+
choices(taskSelect, knownTasks);
|
|
284
|
+
taskSelect.value = "";
|
|
285
|
+
// One package order and color per package everywhere: Vanilla first, then packages by name.
|
|
286
|
+
const packageNames = unique([...observations.map(row => row.arm), ...data.attempts.map(row => row.guidance)]).filter(arm => arm !== "baseline").sort();
|
|
287
|
+
const armRank = arm => arm === "baseline" ? -1 : packageNames.indexOf(arm);
|
|
288
|
+
const byArm = (a, b) => armRank(a) - armRank(b);
|
|
289
|
+
const seriesVariable = arm => arm !== "baseline" && armRank(arm) >= 0 ? "--series-" + (armRank(arm) % 8 + 1) : "--pb-muted";
|
|
290
|
+
const viewLabel = create("label", "Chart values", document.querySelector(".controls"));
|
|
291
|
+
const comparisonView = create("select", undefined, viewLabel);
|
|
292
|
+
for (const [value, label] of [["delta", "Delta vs vanilla"], ["raw", "Raw values"]]) create("option", label, comparisonView).value = value;
|
|
293
|
+
comparisonView.value = "raw";
|
|
294
|
+
let overviewButton, scopeHeading, scopeDescription;
|
|
295
|
+
document.body.classList.add("evaluation");
|
|
296
|
+
const taskLabel = taskSelect.parentElement;
|
|
297
|
+
overviewButton = create("button", "Overview");
|
|
298
|
+
overviewButton.type = "button";
|
|
299
|
+
overviewButton.className = "overview-button";
|
|
300
|
+
overviewButton.setAttribute("aria-controls", "panels outcomes");
|
|
301
|
+
taskLabel.before(overviewButton);
|
|
302
|
+
overviewButton.addEventListener("click", () => { taskSelect.value = ""; update(); });
|
|
303
|
+
const scopeContext = create("div", undefined, document.querySelector("header"));
|
|
304
|
+
scopeContext.className = "scope-context";
|
|
305
|
+
scopeContext.setAttribute("aria-live", "polite");
|
|
306
|
+
scopeHeading = create("h2", undefined, scopeContext);
|
|
307
|
+
scopeDescription = create("p", undefined, scopeContext);
|
|
308
|
+
scopeDescription.className = "muted";
|
|
309
|
+
const pageTitle = document.querySelector("header > h1");
|
|
310
|
+
if (data.name) { pageTitle.textContent = data.name; document.title = data.name; }
|
|
311
|
+
pageTitle.after(scopeContext);
|
|
312
|
+
const footer = document.querySelector("footer"); footer.replaceChildren();
|
|
313
|
+
create("span", "Each set's delta is relative to this evaluation's vanilla runs. Negative is not automatically better. ", footer);
|
|
314
|
+
const download = create("a", "Download observations CSV", footer); download.href = data.download_csv; download.download = "";
|
|
315
|
+
|
|
316
|
+
const showRuns = document.getElementById("show-runs");
|
|
317
|
+
document.getElementById("study").textContent = data.assigned + " assigned attempts · " + knownTasks.length + (knownTasks.length === 1 ? " task" : " tasks") + (data.study_progress ? " · " + data.study_progress.status + " · " + data.study_progress.completed_attempts + " finished" : "");
|
|
318
|
+
const limitations = document.getElementById("limitations");
|
|
319
|
+
for (const limitation of data.limitations) create("li", limitation, limitations.querySelector("ul"));
|
|
320
|
+
const panels = document.getElementById("panels");
|
|
321
|
+
let metricSelect, chartBreakdown, packageSummary;
|
|
322
|
+
packageSummary = create("section"); packageSummary.className = "package-summary";
|
|
323
|
+
document.getElementById("task-results").before(packageSummary);
|
|
324
|
+
const chartControls = document.getElementById("chart-controls");
|
|
325
|
+
chartControls.after(document.querySelector(".chart-options"));
|
|
326
|
+
panels.after(document.getElementById("task-results"), document.getElementById("measurement-completeness"), document.getElementById("limitations"));
|
|
327
|
+
metricSelect = create("select", undefined, create("label", "Measurement", chartControls));
|
|
328
|
+
metricSelect.id = "measurement";
|
|
329
|
+
for (const panel of unique(observations.map(row => row.panel))) {
|
|
330
|
+
const group = create("optgroup", undefined, metricSelect); group.label = panel;
|
|
331
|
+
for (const metric of unique(observations.filter(row => row.panel === panel).map(row => row.metric))) create("option", data.metric_guide[metric].title, group).value = metric;
|
|
332
|
+
}
|
|
333
|
+
chartBreakdown = create("select", undefined, create("label", "Chart", chartControls));
|
|
334
|
+
chartBreakdown.id = "chart-breakdown";
|
|
335
|
+
for (const [value, label] of [["average", "Package averages"], ["tasks", "Values by task"]]) create("option", label, chartBreakdown).value = value;
|
|
336
|
+
const methodology = create("details", undefined, document.querySelector("header"));
|
|
337
|
+
create("summary", "Evaluation setup and runtime evidence", methodology);
|
|
338
|
+
create("p", "Vanilla receives no supplied guidance. Native harness instructions still apply. All packages use the same tasks and runner; reviews use fixed criteria. Requested settings and observed runtime receipts are separate.", methodology).className = "counts";
|
|
339
|
+
create("pre", JSON.stringify({ runner: data.runner, reviewer: data.reviewer, comparison_identity: data.comparison_identity, guidance_packages: data.guidance_packages, attempts: data.attempts }, null, 2), methodology);
|
|
340
|
+
document.getElementById("limitations").after(methodology);
|
|
341
|
+
document.getElementById("limitations").querySelector("summary").textContent = "Measurement limits";
|
|
342
|
+
|
|
343
|
+
for (const option of metricSelect.querySelectorAll("option")) {
|
|
344
|
+
const rows = observations.filter(row => row.metric === option.value);
|
|
345
|
+
if (rows.length && rows.every(row => row.value === null)) option.textContent += " — no values";
|
|
346
|
+
}
|
|
347
|
+
const KEY_METRICS = ["Uncached input tokens", "Output tokens", "Coding time", "Source lines added", "Net cognitive complexity added", "Net cyclomatic complexity added", "Functions added", "Complexity hotspots added", "Introduced dead findings", "Test lines added", "Changed-line coverage %", "Redundant added tests", "Duplicated added test lines", "Total review findings"];
|
|
348
|
+
function taskCheckSummary(attempts) {
|
|
349
|
+
const executions = attempts.filter(row => ["failed", "timed_out"].includes(row.execution?.status));
|
|
350
|
+
const completed = attempts.filter(row => !executions.includes(row));
|
|
351
|
+
const passed = completed.filter(row => row.correctness === "pass").length;
|
|
352
|
+
const failed = completed.filter(row => row.correctness === "fail" && row.execution?.status === "complete").length;
|
|
353
|
+
// Overlap passes: the verifier passed every test, but trusted tests overwrote files the agent changed.
|
|
354
|
+
const overlap = completed.filter(row => row.correctness === "overlap_pass").length;
|
|
355
|
+
const unknown = attempts.length - passed - failed - overlap - executions.length;
|
|
356
|
+
return {
|
|
357
|
+
passed: passed + "/" + attempts.length + " attempts passed",
|
|
358
|
+
remainder: failed + " failed task check" + (failed === 1 ? "" : "s") + (overlap ? " · " + overlap + " overlap pass" + (overlap === 1 ? "" : "es") : "") + " · " + executions.length + " execution failure" + (executions.length === 1 ? "" : "s") + " · " + unknown + " unknown",
|
|
359
|
+
};
|
|
360
|
+
}
|
|
361
|
+
// Overview: equal-weight mean of per-task ratios. Task view: this task's ratio, or a difference when the reference is zero.
|
|
362
|
+
function referenceText(arm, metric) {
|
|
363
|
+
if (!taskSelect.value) {
|
|
364
|
+
const row = (data.reference_aggregates || []).find(item => item.guidance === arm && item.metric === metric);
|
|
365
|
+
return row && row.ratio_to_reference !== null ? formatCompact(row.ratio_to_reference) + "× reference (" + row.measured_tasks + "/" + row.assigned_tasks + " tasks)" : null;
|
|
366
|
+
}
|
|
367
|
+
const rows = (data.reference_comparisons || []).filter(item => item.task === taskSelect.value && item.guidance === arm && item.metric === metric);
|
|
368
|
+
if (rows.length !== 1) return null;
|
|
369
|
+
const row = rows[0];
|
|
370
|
+
return row.ratio_to_reference !== null ? formatCompact(row.ratio_to_reference) + "× reference" : row.difference_vs_reference !== null ? formatSigned(row.difference_vs_reference) + " vs reference" : null;
|
|
371
|
+
}
|
|
372
|
+
// Passing-only summaries are labelled separately and never replace all-attempt values.
|
|
373
|
+
function passingText(arm, metric, group) {
|
|
374
|
+
if (!taskSelect.value) {
|
|
375
|
+
const row = (data.passing_aggregates || []).find(item => item.arm === arm && item.metric === metric && (item.analysis_group ?? null) === (group ?? null));
|
|
376
|
+
return row && row.value !== null ? "Passing only: " + formatCompact(row.value) + " (" + row.measured_tasks + "/" + row.assigned_tasks + " tasks)" : null;
|
|
377
|
+
}
|
|
378
|
+
const row = (data.passing_medians || []).find(item => item.task === taskSelect.value && item.arm === arm && item.metric === metric);
|
|
379
|
+
return row ? "Passing only: " + formatCompact(row.value) + " (" + row.passing + " runs)" : null;
|
|
380
|
+
}
|
|
381
|
+
function showPackageSummary() {
|
|
382
|
+
const metricsOpen = packageSummary.querySelector("details")?.open || false;
|
|
383
|
+
packageSummary.replaceChildren();
|
|
384
|
+
create("h2", taskSelect.value ? "Task comparison" : "Package comparison", packageSummary);
|
|
385
|
+
const arms = unique(data.attempts.map(row => row.guidance)).sort(byArm);
|
|
386
|
+
const headerRow = parent => {
|
|
387
|
+
const head = create("tr", undefined, create("thead", undefined, parent));
|
|
388
|
+
create("th", "Measure", head).scope = "col";
|
|
389
|
+
for (const arm of arms) {
|
|
390
|
+
const cell = create("th", armLabel(arm), head); cell.scope = "col";
|
|
391
|
+
const swatch = create("span", undefined); swatch.className = "swatch"; swatch.setAttribute("aria-hidden", "true");
|
|
392
|
+
swatch.style.setProperty("--series-color", "var(" + seriesVariable(arm) + ")"); cell.prepend(swatch);
|
|
393
|
+
}
|
|
394
|
+
};
|
|
395
|
+
const metricTable = (parent, label, metrics) => {
|
|
396
|
+
const metricsWrap = create("div", undefined, parent); metricsWrap.className = "table-wrap";
|
|
397
|
+
const metricsTable = create("table", undefined, metricsWrap);
|
|
398
|
+
metricsTable.setAttribute("aria-label", label);
|
|
399
|
+
headerRow(metricsTable);
|
|
400
|
+
const metricsBody = create("tbody", undefined, metricsTable);
|
|
401
|
+
for (const metric of metrics) {
|
|
402
|
+
const row = create("tr", undefined, metricsBody); create("th", "", row).scope = "row";
|
|
403
|
+
const button = create("button", data.metric_guide[metric].title, row.firstChild); button.className = "metric-link"; button.type = "button";
|
|
404
|
+
const referenceValue = taskSelect.value ? (data.reference_values || []).find(item => item.task === taskSelect.value && item.metric === metric) : null;
|
|
405
|
+
if (referenceValue) create("small", "Reference: " + formatCompact(referenceValue.value), row.firstChild);
|
|
406
|
+
button.setAttribute("aria-pressed", String(metricSelect.value === metric));
|
|
407
|
+
button.addEventListener("click", () => { metricSelect.value = metric; update(); metricSelect.focus(); });
|
|
408
|
+
for (const arm of arms) {
|
|
409
|
+
const valueRows = taskSelect.value ? (data.raw_medians || []).filter(row => row.task === taskSelect.value && row.metric === metric && row.arm === arm) : (data.aggregates || []).filter(row => row.metric === metric && row.guidance === arm);
|
|
410
|
+
// Separate analyzer groups have unlike scores: show each group on its own line, never pooled.
|
|
411
|
+
const groups = valueRows.length > 1 && !taskSelect.value ? valueRows : [valueRows.length === 1 ? valueRows[0] : null];
|
|
412
|
+
const cell = create("td", undefined, row);
|
|
413
|
+
for (const result of groups) {
|
|
414
|
+
const text = result?.value !== undefined && result.value !== null ? formatCompact(result.value) : "Unknown";
|
|
415
|
+
create("div", text + (groups.length > 1 && result?.analysis_group ? " · " + result.analysis_group : ""), cell);
|
|
416
|
+
if (!taskSelect.value && result) create("small", result.measured_tasks + "/" + result.assigned_tasks + " tasks", cell);
|
|
417
|
+
if (arm !== "baseline") {
|
|
418
|
+
const deltaRows = taskSelect.value ? (data.comparisons || []).filter(row => row.task === taskSelect.value && row.metric === metric && row.guidance === arm) : [result];
|
|
419
|
+
const delta = deltaRows.length === 1 ? deltaRows[0] : null;
|
|
420
|
+
const value = delta?.absolute_delta_vs_vanilla ?? null;
|
|
421
|
+
create("small", value === null ? "Delta unavailable" : formatSigned(value) + " " + delta.absolute_delta_unit + " vs vanilla", cell);
|
|
422
|
+
}
|
|
423
|
+
// Each analyzer group keeps its own passing-only value.
|
|
424
|
+
const passing = passingText(arm, metric, result?.analysis_group);
|
|
425
|
+
if (passing) create("small", passing, cell);
|
|
426
|
+
}
|
|
427
|
+
const reference = referenceText(arm, metric);
|
|
428
|
+
if (reference) create("small", reference, cell).className = "reference";
|
|
429
|
+
}
|
|
430
|
+
}
|
|
431
|
+
};
|
|
432
|
+
const wrap = create("div", undefined, packageSummary); wrap.className = "table-wrap";
|
|
433
|
+
const table = create("table", undefined, wrap); table.setAttribute("aria-label", "Package comparison");
|
|
434
|
+
headerRow(table);
|
|
435
|
+
const body = create("tbody", undefined, table);
|
|
436
|
+
const rowFor = label => { const row = create("tr", undefined, body); create("th", label, row).scope = "row"; return row; };
|
|
437
|
+
const checks = rowFor("Task checks");
|
|
438
|
+
const counts = rowFor("Measurement values");
|
|
439
|
+
for (const arm of arms) {
|
|
440
|
+
const attempts = data.attempts.filter(row => row.guidance === arm && (!taskSelect.value || row.task === taskSelect.value));
|
|
441
|
+
const summary = taskCheckSummary(attempts);
|
|
442
|
+
const cell = create("td", summary.passed, checks);
|
|
443
|
+
create("small", summary.remainder, cell);
|
|
444
|
+
const rows = observations.filter(row => row.arm === arm && (!taskSelect.value || row.task === taskSelect.value));
|
|
445
|
+
const numeric = rows.filter(row => row.value !== null).length;
|
|
446
|
+
const countCell = create("td", numeric + "/" + rows.length + " numeric", counts);
|
|
447
|
+
create("small", rows.length - numeric + " unknown", countCell);
|
|
448
|
+
}
|
|
449
|
+
const priced = (data.cost || []).filter(row => row.dollars !== null && (!taskSelect.value || row.task === taskSelect.value));
|
|
450
|
+
if (priced.length) {
|
|
451
|
+
const cost = rowFor("Coding cost");
|
|
452
|
+
for (const arm of arms) {
|
|
453
|
+
const values = priced.filter(row => row.arm === arm).map(row => row.dollars);
|
|
454
|
+
const cell = create("td", values.length ? "$" + (values.reduce((a, b) => a + b, 0) / values.length).toFixed(2) + " per attempt" : "Unknown", cost);
|
|
455
|
+
create("small", values.length + " priced attempts · prices as of " + unique(priced.map(row => row.price_as_of)).join(", "), cell);
|
|
456
|
+
}
|
|
457
|
+
}
|
|
458
|
+
const mainMetrics = unique(observations.map(row => row.metric));
|
|
459
|
+
const keyMetrics = KEY_METRICS.filter(metric => mainMetrics.includes(metric));
|
|
460
|
+
if (keyMetrics.length) {
|
|
461
|
+
create("h3", "Key results", packageSummary);
|
|
462
|
+
create("p", (taskSelect.value ? "Medians of this task's runs. " : "Averages of task medians, each task weighted equally. ") + "Deltas compare each package with vanilla. \"× reference\" compares code measures with the task's reference solution: 1× means the same amount. Lower is not automatically better. Choose a metric name to chart it.", packageSummary).className = "counts";
|
|
463
|
+
metricTable(packageSummary, "Key results", keyMetrics);
|
|
464
|
+
}
|
|
465
|
+
const metricsDetails = create("details", undefined, packageSummary);
|
|
466
|
+
metricsDetails.open = metricsOpen;
|
|
467
|
+
create("summary", "Compare all metrics", metricsDetails);
|
|
468
|
+
metricTable(metricsDetails, "All metrics", mainMetrics);
|
|
469
|
+
}
|
|
470
|
+
function showMetadata() {
|
|
471
|
+
const target = document.getElementById("run-metadata");
|
|
472
|
+
target.replaceChildren();
|
|
473
|
+
const requested = data.runner || {};
|
|
474
|
+
const versions = unique(data.attempts.map(attempt => attempt.runtime.harness_version).filter(Boolean));
|
|
475
|
+
for (const [label, value] of [["Requested model", requested.model || "unknown"], ["Requested reasoning", requested.reasoning || "unknown"], ["Harness", (requested.harness || "unknown") + (versions.length ? " · " + versions.join(", ") : "")], ["Parallel jobs", String(data.jobs || 1)]]) {
|
|
476
|
+
const group = create("div", undefined, target); create("dt", label, group); create("dd", value, group);
|
|
477
|
+
}
|
|
478
|
+
if (data.reviewer) { const group = create("div", undefined, target); create("dt", "Reviewer", group); create("dd", data.reviewer.model + " · " + data.reviewer.reasoning + " · " + data.reviewer.harness, group); }
|
|
479
|
+
document.getElementById("metadata-source").textContent = String(data.name) + ". Actual runtime evidence, including unreported effective reasoning, is in each attempt's details. Separate evaluations use their own vanilla samples.";
|
|
480
|
+
}
|
|
481
|
+
|
|
482
|
+
function showCompleteness() {
|
|
483
|
+
const target = document.getElementById("measurement-completeness");
|
|
484
|
+
const rows = (data.measurement_completeness || []).filter(row => !taskSelect.value || row.task === taskSelect.value);
|
|
485
|
+
target.hidden = !rows.length;
|
|
486
|
+
if (!rows.length) return;
|
|
487
|
+
target.querySelector("summary").textContent = rows.filter(row => row.complete).length + "/" + rows.length + " coding submissions have complete metric counts";
|
|
488
|
+
const body = target.querySelector("div"); body.replaceChildren();
|
|
489
|
+
create("p", "Counts are separate from correctness. Zero findings are measured; a submission that adds no source lines has no changed-line coverage percentage.", body).className = "counts";
|
|
490
|
+
const table = create("table", undefined, body);
|
|
491
|
+
for (const metric of Object.keys(rows[0].counts)) {
|
|
492
|
+
const tr = create("tr", undefined, table);
|
|
493
|
+
create("th", metric, tr);
|
|
494
|
+
create("td", rows.filter(row => row.counts[metric]).length + "/" + rows.length + " complete", tr);
|
|
495
|
+
}
|
|
496
|
+
// Task × measure × package: measured, unknown with its reason, or not configured for the task.
|
|
497
|
+
const matrix = (data.completeness_matrix || []).filter(row => !taskSelect.value || row.task === taskSelect.value);
|
|
498
|
+
if (!matrix.length) return;
|
|
499
|
+
const arms = unique(matrix.flatMap(row => Object.keys(row.cells)));
|
|
500
|
+
const grid = create("table", undefined, body);
|
|
501
|
+
const head = create("tr", undefined, grid);
|
|
502
|
+
for (const label of ["Task", "Measure", ...arms.map(armLabel)]) create("th", label, head);
|
|
503
|
+
for (const row of matrix) {
|
|
504
|
+
const tr = create("tr", undefined, grid);
|
|
505
|
+
create("td", row.task, tr);
|
|
506
|
+
create("td", row.metric, tr);
|
|
507
|
+
for (const arm of arms) create("td", row.cells[arm] ? row.cells[arm].text : "", tr);
|
|
508
|
+
}
|
|
509
|
+
}
|
|
510
|
+
function showCollectionStatus() {
|
|
511
|
+
const target = document.getElementById("collection-status"), collection = data.measurement_collection;
|
|
512
|
+
target.replaceChildren();
|
|
513
|
+
const attempts = (data.attempts || []).filter(row => !taskSelect.value || row.task === taskSelect.value);
|
|
514
|
+
const byId = new Map(attempts.map(row => [row.id, row]));
|
|
515
|
+
const failures = (collection?.failures || []).filter(row => byId.has(row.attempt_id));
|
|
516
|
+
target.hidden = !failures.length;
|
|
517
|
+
if (target.hidden) return;
|
|
518
|
+
create("h2", "Some measurements are unavailable", target);
|
|
519
|
+
create("p", unique(failures.map(row => row.attempt_id)).length + "/" + attempts.length + " submissions have missing code measurements. Available metrics and task checks are still shown; missing values are not zero.", target).className = "counts";
|
|
520
|
+
const details = create("details", undefined, target);
|
|
521
|
+
create("summary", "Why measurements are missing", details);
|
|
522
|
+
const wrap = create("div", undefined, details); wrap.className = "table-wrap";
|
|
523
|
+
const table = create("table", undefined, wrap), head = create("tr", undefined, create("thead", undefined, table));
|
|
524
|
+
for (const label of ["Task", "Package", "Run", "Measurement", "Reason"]) create("th", label, head).scope = "col";
|
|
525
|
+
const body = create("tbody", undefined, table);
|
|
526
|
+
for (const failure of failures) {
|
|
527
|
+
const attempt = byId.get(failure.attempt_id), row = create("tr", undefined, body);
|
|
528
|
+
const family = failure.family === "coverage" ? "Coverage" : failure.family === "fallow" ? "Complexity and unused-code warnings" : failure.family;
|
|
529
|
+
const diagnostic = (failure.diagnostics || []).find(value => typeof value === "string" && value);
|
|
530
|
+
const reason = [failure.reason || failure.status, diagnostic?.split(/\r?\n/)[0]].filter(Boolean).join(": ");
|
|
531
|
+
for (const value of [attempt.task, armLabel(attempt.guidance), failure.stage === "before" ? "Starting code" : attempt.repeat, family, reason.slice(0, 450)]) create("td", value, row);
|
|
532
|
+
}
|
|
533
|
+
const evidence = create("details", undefined, details);
|
|
534
|
+
create("summary", "Collection receipts", evidence);
|
|
535
|
+
create("pre", JSON.stringify({ revision: collection.revision, status: collection.status, revision_receipt: collection.revision_receipt, history: collection.history, failures }, null, 2), evidence);
|
|
536
|
+
}
|
|
537
|
+
function showOutcomes() {
|
|
538
|
+
const target = document.getElementById("outcomes");
|
|
539
|
+
target.replaceChildren();
|
|
540
|
+
create("h2", "Task checks", target);
|
|
541
|
+
create("p", "Completed task-check failures, native execution failures, and unknown outcomes are counted separately. All attempts remain assigned; these counts do not establish why an execution failed.", target).className = "counts";
|
|
542
|
+
const attempts = (data.attempts || []).filter(row => !taskSelect.value || row.task === taskSelect.value);
|
|
543
|
+
const table = create("table", undefined, target);
|
|
544
|
+
const header = create("tr", undefined, table);
|
|
545
|
+
for (const text of ["Task", "Guidance", "Checks passed / assigned"]) create("th", text, header).scope = "col";
|
|
546
|
+
const keys = unique(attempts.map(row => JSON.stringify([row.task, row.guidance]))).map(key => JSON.parse(key)).sort((a, b) => a[0].localeCompare(b[0]) || byArm(a[1], b[1]));
|
|
547
|
+
for (const [task, arm] of keys) {
|
|
548
|
+
const members = attempts.filter(row => row.task === task && row.guidance === arm);
|
|
549
|
+
const tr = create("tr", undefined, table);
|
|
550
|
+
for (const value of [task, armLabel(arm)]) create("td", value, tr);
|
|
551
|
+
const summary = taskCheckSummary(members), cell = create("td", summary.passed, tr);
|
|
552
|
+
create("small", summary.remainder, cell);
|
|
553
|
+
}
|
|
554
|
+
const failures = attempts.filter(row => ["failed", "timed_out"].includes(row.execution?.status));
|
|
555
|
+
if (!failures.length) return;
|
|
556
|
+
create("h3", "Native execution failures", target);
|
|
557
|
+
create("p", "Execution failures are separate from task checks. Available metrics remain shown; partial token receipts are lower bounds, not final totals.", target).className = "counts";
|
|
558
|
+
const wrap = create("div", undefined, target); wrap.className = "table-wrap";
|
|
559
|
+
const failureTable = create("table", undefined, wrap), head = create("tr", undefined, create("thead", undefined, failureTable));
|
|
560
|
+
for (const label of ["Task", "Guidance", "Run", "Execution reason"]) create("th", label, head).scope = "col";
|
|
561
|
+
const body = create("tbody", undefined, failureTable);
|
|
562
|
+
for (const failure of failures) {
|
|
563
|
+
const row = create("tr", undefined, body);
|
|
564
|
+
for (const value of [failure.task, armLabel(failure.guidance), failure.repeat ?? failure.id, failure.execution.reason]) create("td", value, row);
|
|
565
|
+
}
|
|
566
|
+
}
|
|
567
|
+
|
|
568
|
+
function resolvedColor(variable) {
|
|
569
|
+
const probe = create("span", void 0, document.body);
|
|
570
|
+
probe.style.color = "var(" + variable + ")";
|
|
571
|
+
const color = getComputedStyle(probe).color;
|
|
572
|
+
probe.remove();
|
|
573
|
+
return color;
|
|
574
|
+
}
|
|
575
|
+
function observationValue(row) {
|
|
576
|
+
const counts = row.provenance.covered !== undefined && row.provenance.covered !== null && row.provenance.total !== undefined && row.provenance.total !== null ? " (" + format(row.provenance.covered) + "/" + format(row.provenance.total) + " covered)" : row.per_test_count !== null ? " / " + format(row.per_test_count) + " supplied tests" : "";
|
|
577
|
+
return format(row.value) + " " + data.metric_guide[row.metric].unit + counts;
|
|
578
|
+
}
|
|
579
|
+
function observationText(row) {
|
|
580
|
+
const identity = row.panel.startsWith("Review ") ? "\nJudge: " + (row.provenance.judge_model || ["unknown"]).join(", ") + " \xB7 reasoning: " + (row.provenance.judge_reasoning_effort?.length ? row.provenance.judge_reasoning_effort : ["unknown"]).join(", ") + "\nImplementation model: " + row.model + " \xB7 reasoning: " + row.reasoning_effort : "\nModel: " + row.model + " \xB7 reasoning: " + row.reasoning_effort;
|
|
581
|
+
return row.task + " · " + armLabel(row.arm) + " \xB7 " + row.attempt_id + "\n" + data.metric_guide[row.metric].title + ": " + observationValue(row) + "\nChecks: " + row.correctness + " \xB7 " + row.status + "\n" + row.reason + identity + "\nHarness: " + row.harness + " \xB7 " + row.harness_version;
|
|
582
|
+
}
|
|
583
|
+
function addDetails(parent, rows) {
|
|
584
|
+
const details = create("details", void 0, parent);
|
|
585
|
+
create("summary", "Attempt values and provenance (" + rows.length + ")", details);
|
|
586
|
+
const wrap = create("div", void 0, details);
|
|
587
|
+
wrap.className = "table-wrap";
|
|
588
|
+
const table = create("table", void 0, wrap);
|
|
589
|
+
const head = create("tr", void 0, create("thead", void 0, table));
|
|
590
|
+
for (const label of ["Task", "Skill set", "Attempt", "Value", "Checks", "Measurement"]) create("th", label, head).scope = "col";
|
|
591
|
+
const body = create("tbody", void 0, table);
|
|
592
|
+
for (const row of rows) {
|
|
593
|
+
const tr = create("tr", void 0, body);
|
|
594
|
+
for (const value of [row.task, armLabel(row.arm), row.attempt_id.slice(0, 8), observationValue(row), row.correctness, row.status]) create("td", value, tr);
|
|
595
|
+
}
|
|
596
|
+
const source = create("details", void 0, details);
|
|
597
|
+
create("summary", "Full source and measurement details", source);
|
|
598
|
+
create("pre", JSON.stringify(rows, null, 2), source);
|
|
599
|
+
}
|
|
600
|
+
function showReviewFindings(parent, rows) {
|
|
601
|
+
const ids = new Set(rows.map(row => row.attempt_id));
|
|
602
|
+
const records = (data.review_findings || []).filter(row => ids.has(row.attempt_id));
|
|
603
|
+
const details = create("details", undefined, parent);
|
|
604
|
+
details.className = "review-findings";
|
|
605
|
+
create("summary", "Read the findings for each attempt (" + records.length + ")", details);
|
|
606
|
+
create("p", "AI review findings are judgments. Each issue has one severity and category; repeated symptoms of one cause count as one finding. These counts do not measure human review effort.", details).className = "counts";
|
|
607
|
+
for (const record of records) {
|
|
608
|
+
const attempt = create("details", undefined, details);
|
|
609
|
+
create("summary", record.task + " · " + armLabel(record.arm) + " · " + record.attempt_id + " · " + record.status, attempt);
|
|
610
|
+
create("p", "Reviewed stage: " + (record.stage || "unknown") + ". Criteria: " + (record.criteria_sha256 || "not recorded") + ".", attempt).className = "counts";
|
|
611
|
+
if (record.status !== "reviewed" || !Array.isArray(record.issues)) {
|
|
612
|
+
create("p", "Findings unavailable: " + (record.reason || "a complete review was not captured"), attempt);
|
|
613
|
+
continue;
|
|
614
|
+
}
|
|
615
|
+
if (!record.issues.length) create("p", "No actionable issues reported. This does not prove the submission is defect-free.", attempt);
|
|
616
|
+
for (const issue of record.issues) {
|
|
617
|
+
create("h3", (issue.severity || "Severity unknown") + " · " + issue.category + " · " + issue.description, attempt);
|
|
618
|
+
create("p", issue.path + ":" + issue.line, attempt).className = "counts";
|
|
619
|
+
create("pre", issue.evidence, attempt);
|
|
620
|
+
create("p", "Why it matters: " + issue.rationale, attempt);
|
|
621
|
+
create("p", "Suggested fix: " + issue.smallest_fix, attempt);
|
|
622
|
+
}
|
|
623
|
+
}
|
|
624
|
+
}
|
|
625
|
+
function evaluationPlotRows({ rows, metric }) {
|
|
626
|
+
const raw = comparisonView.value === "raw", overview = !taskSelect.value;
|
|
627
|
+
if (overview && chartBreakdown.value === "tasks") {
|
|
628
|
+
const medians = raw ? (data.raw_medians || []) : (data.comparisons || []);
|
|
629
|
+
return medians.filter(row => row.metric === metric).map(row => ({ ...row, guidance: row.guidance || row.arm, label: row.task.replaceAll("-", " "), plotted: raw ? row.value : row.absolute_delta_vs_vanilla, status: raw ? "measured" : row.status, reason: raw ? "Raw task median for one measurement profile. Comparison eligibility is separate." : row.reason, population: raw ? row.measured + " runs" : "Vanilla " + row.baseline_measured + "/" + row.baseline_assigned + " · guidance " + row.guidance_measured + "/" + row.guidance_assigned + " runs", rawRuns: [] })).sort((a, b) => a.task.localeCompare(b.task) || Number(b.guidance === "baseline") - Number(a.guidance === "baseline") || a.guidance.localeCompare(b.guidance));
|
|
630
|
+
}
|
|
631
|
+
if (overview) return (data.aggregates || []).filter(row => row.metric === metric).map(row => ({ ...row, status: raw ? row.status : row.comparison_status, label: armLabel(row.guidance) + (row.analysis_group ? " · " + row.analysis_group : ""), plotted: raw ? row.value : row.absolute_delta_vs_vanilla, population: row.measured_tasks + "/" + row.assigned_tasks + " tasks", rawRuns: [] }));
|
|
632
|
+
if (!raw) return (data.comparisons || []).filter(row => row.metric === metric && row.task === taskSelect.value).map(row => ({ ...row, label: row.guidance, plotted: row.absolute_delta_vs_vanilla, population: "Vanilla " + row.baseline_measured + "/" + row.baseline_assigned + " · guidance " + row.guidance_measured + "/" + row.guidance_assigned + " runs", rawRuns: [] }));
|
|
633
|
+
const medians = (data.raw_medians || []).filter(row => row.role === "handoff" && row.metric === metric && row.task === taskSelect.value);
|
|
634
|
+
const arms = unique(rows.map(row => row.arm));
|
|
635
|
+
return arms.flatMap(arm => {
|
|
636
|
+
const values = medians.filter(row => row.arm === arm);
|
|
637
|
+
if (!values.length) return [{ guidance: arm, label: armLabel(arm), plotted: null, value: null, population: "No defined value", status: "unknown", reason: "See scan counts and attempt details.", rawRuns: [] }];
|
|
638
|
+
return values.map(median => ({ ...median, guidance: arm, label: armLabel(arm), plotted: median.value, population: median.measured + " runs", status: "measured", reason: "Raw task median. Comparison eligibility is separate.", rawRuns: rows.filter(row => row.arm === arm && row.value !== null && (median.compatibility !== null ? row.compatibility === median.compatibility : median.attempt_ids.includes(row.attempt_id))) }));
|
|
639
|
+
});
|
|
640
|
+
}
|
|
641
|
+
function drawEvaluationPlot({ parent, rows, metric }) {
|
|
642
|
+
const raw = comparisonView.value === "raw", overview = !taskSelect.value;
|
|
643
|
+
const byTask = overview && chartBreakdown.value === "tasks";
|
|
644
|
+
const groupOf = row => byTask ? row.label : overview && row.analysis_group ? row.analysis_group : null;
|
|
645
|
+
const plotRows = evaluationPlotRows({ rows, metric }).map(row => ({ ...row, group: groupOf(row) })).sort((a, b) => (a.group ?? "").localeCompare(b.group ?? "") || byArm(a.guidance, b.guidance));
|
|
646
|
+
// Group bars only when each group has one value per package and colors are unique (eight packages at most);
|
|
647
|
+
// otherwise keep one labelled bar per row.
|
|
648
|
+
const grouped = packageNames.length <= 8 && plotRows.some(row => row.group !== null) && unique(plotRows.map(row => row.group + "\u0000" + row.guidance)).length === plotRows.length;
|
|
649
|
+
const details = create("details", undefined, parent); create("summary", overview && !byTask ? (raw ? "Package averages" : "Package averages and deltas vs vanilla") : (raw ? "Task medians" : "Task medians and deltas vs vanilla"), details);
|
|
650
|
+
const wrap = create("div", undefined, details); wrap.className = "table-wrap";
|
|
651
|
+
const table = create("table", undefined, wrap), head = create("tr", undefined, table);
|
|
652
|
+
for (const label of ["Set", raw ? "Value" : "Delta vs vanilla", ...(!raw ? ["Vanilla reference", "% vs vanilla"] : []), "Status", "Measured / assigned"]) create("th", label, head);
|
|
653
|
+
for (const row of plotRows) {
|
|
654
|
+
const tr = create("tr", undefined, table);
|
|
655
|
+
for (const value of [byTask ? row.label + " · " + armLabel(row.guidance) : row.label, format(row.plotted), ...(!raw ? [format(row.baseline_value ?? row.baseline_median ?? null), format(row.percentage_delta_vs_vanilla)] : []), row.status, row.population]) create("td", value, tr);
|
|
656
|
+
}
|
|
657
|
+
if (!plotRows.some(row => row.plotted !== null || row.rawRuns.length)) { create("p", raw ? "No defined values to plot. Complete empty scans and missing measurements are listed separately." : "No matching baseline values for a delta. Switch to Raw values to see the available measurements.", parent); return 0; }
|
|
658
|
+
const container = create("div", undefined, parent); container.className = "chart";
|
|
659
|
+
const arms = unique(plotRows.map(row => row.guidance));
|
|
660
|
+
const categories = grouped ? unique(plotRows.map(row => row.group)) : plotRows.map(row => byTask ? row.label + " · " + armLabel(row.guidance) : row.label);
|
|
661
|
+
container.style.height = Math.max(260, (grouped ? categories.length * (arms.length * 20 + 28) : plotRows.length * 64) + 70) + "px";
|
|
662
|
+
const text = resolvedColor("--pb-text"), border = resolvedColor("--pb-border");
|
|
663
|
+
const chart = echarts.init(container, null, { renderer: "svg" });
|
|
664
|
+
const unit = raw ? data.metric_guide[metric].unit : plotRows[0]?.absolute_delta_unit;
|
|
665
|
+
const colors = new Map(arms.map(arm => [arm, resolvedColor(seriesVariable(arm))]));
|
|
666
|
+
const color = row => colors.get(row.guidance);
|
|
667
|
+
if (grouped) {
|
|
668
|
+
const legend = create("div", undefined, parent); legend.className = "legend";
|
|
669
|
+
container.before(legend);
|
|
670
|
+
for (const arm of arms) create("span", armLabel(arm), legend).style.setProperty("--series-color", color({ guidance: arm }));
|
|
671
|
+
}
|
|
672
|
+
// Run dots carry the values when shown; bar labels would collide with them.
|
|
673
|
+
const label = { show: !(raw && !overview && showRuns.checked), position: "right", color: text, formatter: params => formatCompact(params.value) };
|
|
674
|
+
const barItem = row => !row || row.plotted === null ? null : ({ value: row.plotted, comparison: row, itemStyle: { color: color(row), borderRadius: row.plotted < 0 ? [4, 0, 0, 4] : [0, 4, 4, 0] } });
|
|
675
|
+
const series = grouped
|
|
676
|
+
? arms.map(arm => ({ type: "bar", name: armLabel(arm), barMaxWidth: 14, barGap: "15%", label, data: categories.map(group => barItem(plotRows.find(row => row.group === group && row.guidance === arm))) }))
|
|
677
|
+
: [{ type: "bar", name: raw ? "Raw values" : "Delta vs vanilla", barMaxWidth: 16, label, data: plotRows.map(barItem) }];
|
|
678
|
+
if (!raw) series[0].markLine = { silent: true, symbol: "none", data: [{ xAxis: 0 }], lineStyle: { color: text }, label: { show: false } };
|
|
679
|
+
if (raw && !overview && showRuns.checked) series.push({ type: "scatter", symbolSize: 7, data: plotRows.flatMap((row, index) => row.rawRuns.map((observation, repeat) => ({ value: [observation.value, index], observation, symbol: observation.correctness === "fail" ? "rect" : observation.correctness === "unknown" ? "emptyCircle" : "circle", symbolOffset: [0, (repeat % 3 - 1) * 4], itemStyle: { color: color(row) } }))) });
|
|
680
|
+
chart.setOption({ animation: false,
|
|
681
|
+
aria: { enabled: true, label: { description: overview && chartBreakdown.value === "tasks" ? "One bar per task and package; select a task to inspect repeated runs." : overview ? "Equal-weight average of each task's median. Each package is compared with vanilla; negative deltas are not automatically better." : "Task-level medians or deltas vs vanilla, with individual runs available in raw mode." } },
|
|
682
|
+
grid: { left: container.clientWidth < 500 ? 120 : 180, right: 100, top: 16, bottom: 48 }, textStyle: { color: text, fontFamily: getComputedStyle(document.body).fontFamily },
|
|
683
|
+
xAxis: { type: "value", min: raw && plotRows.every(row => row.plotted === null || row.plotted >= 0) ? 0 : undefined, name: unit + (raw ? "" : " vs vanilla"), nameLocation: "middle", nameGap: 32, splitNumber: container.clientWidth < 500 ? 2 : 5, axisLabel: { color: text, hideOverlap: true, formatter: formatTick }, splitLine: { lineStyle: { color: border, type: "dashed" } } },
|
|
684
|
+
yAxis: { type: "category", inverse: true, data: categories, axisLabel: { color: text, width: container.clientWidth < 500 ? 110 : 168, overflow: "break" }, axisLine: { show: false }, axisTick: { show: false } },
|
|
685
|
+
tooltip: { trigger: "item", confine: true, backgroundColor: resolvedColor("--pb-surface"), borderColor: border, textStyle: { color: text }, formatter: params => {
|
|
686
|
+
const node = create("div"); node.className = "tooltip";
|
|
687
|
+
if (params.data.observation) node.textContent = observationText(params.data.observation);
|
|
688
|
+
else {
|
|
689
|
+
const row = params.data.comparison;
|
|
690
|
+
node.textContent = row.label + (byTask ? " · " + armLabel(row.guidance) : "") + "\n" + (raw ? "Value: " : "Delta vs vanilla: ") + format(row.plotted) + " " + unit + "\n" + row.population + " · " + row.status + "\n" + row.reason;
|
|
691
|
+
}
|
|
692
|
+
return node;
|
|
693
|
+
} }, series
|
|
694
|
+
});
|
|
695
|
+
if (overview && chartBreakdown.value === "tasks") chart.on("click", params => { const task = params.data.comparison?.task; if (task) { taskSelect.value = task; update(); taskSelect.focus(); } });
|
|
696
|
+
parent.plots.push(chart); return 0;
|
|
697
|
+
}
|
|
698
|
+
function complexityNote(parent, rows) {
|
|
699
|
+
for (const arm of unique(rows.map((row) => row.arm)).sort(byArm)) {
|
|
700
|
+
const values = rows.filter((row) => row.arm === arm);
|
|
701
|
+
const counts = unique(values.map((row) => row.provenance.reported_functions ?? "unknown"));
|
|
702
|
+
const completeness = unique(values.map((row) => row.provenance.complete_for_declared_scope));
|
|
703
|
+
const scan = completeness.every((value) => value === true) ? "complete declared scan" : completeness.every((value) => value === true || value === false) ? "partial declared scan" : "scan completeness unknown";
|
|
704
|
+
create("p", armLabel(arm) + ": " + counts.join(" / ") + " functions reported \xB7 " + scan, parent).className = "counts";
|
|
705
|
+
}
|
|
706
|
+
}
|
|
707
|
+
function renderPanel({ section, panel, metric }) {
|
|
708
|
+
const rows = observations.filter(row => row.panel === panel && row.metric === metric && (!taskSelect.value || row.task === taskSelect.value));
|
|
709
|
+
const help = data.metric_guide[metric];
|
|
710
|
+
create("h2", help.title, section);
|
|
711
|
+
create("p", help.explanation, section).className = "metric-explanation";
|
|
712
|
+
if (panel.startsWith("Review ")) create("p", "Independent review of " + (data.independent_review?.stage || "handoff") + " submissions. Judge costs are separate from implementation.", section).className = "counts";
|
|
713
|
+
const counting = {
|
|
714
|
+
"Code added": "Source code only; tests are excluded. Complexity added sums the scores of changed functions after the edit and subtracts the same functions before, so editing an inherited complex function counts only its growth. Functions are matched on path and name; renames count as added. Each task's reference solution is measured the same way.",
|
|
715
|
+
"Tests added": "Changed-line coverage uses the submission's own tests and only the source lines it added. Redundant added tests come from per-test coverage in one run: an added test is a candidate when other tests cover every added line it covers. Equal coverage does not prove equal assertions.",
|
|
716
|
+
"LOC": "Totals use the declared source/test scope when available, otherwise file-name and extension classification. Line changes count additions and removals separately; a moved or rewritten block can appear in both. Exact scope and exclusions are in each run's details.",
|
|
717
|
+
"Tests with zero unique contribution": "Uses recorded per-test coverage results among the supplied tests. This is not a claim about the complete suite. Raw counts are not comparison bars."
|
|
718
|
+
}[panel];
|
|
719
|
+
if (counting) {
|
|
720
|
+
const details = create("details", undefined, section);
|
|
721
|
+
create("summary", "How this is counted", details);
|
|
722
|
+
create("p", counting, details);
|
|
723
|
+
}
|
|
724
|
+
if (panel === "Tests added") {
|
|
725
|
+
const controlled = unique(rows.filter(row => row.provenance.clock).map(row => row.task));
|
|
726
|
+
if (controlled.length) create("p", "Controlled-clock coverage: " + controlled.join(", ") + ". Ambient Date reads advance by a configured step. Native correctness checks use their original clock; the clock policy is in each run's details.", section).className = "counts";
|
|
727
|
+
}
|
|
728
|
+
if (!rows.length) { create("p", "No observations for these settings.", section); return; }
|
|
729
|
+
if (!taskSelect.value) create("p", (data.aggregates || []).filter(row => row.metric === metric).sort((a, b) => byArm(a.guidance, b.guidance)).map(row => {
|
|
730
|
+
const tasks = new Set(row.task_medians.map(item => item.task));
|
|
731
|
+
const observed = rows.filter(attempt => attempt.arm === row.guidance && tasks.has(attempt.task));
|
|
732
|
+
const unknown = observed.filter(attempt => attempt.value === null).length;
|
|
733
|
+
return armLabel(row.guidance) + (row.analysis_group ? " · " + row.analysis_group : "") + ": " + row.measured_tasks + "/" + row.assigned_tasks + (row.assigned_tasks === 1 ? " task" : " tasks") + " with defined values" + (unknown ? " · " + unknown + (unknown === 1 ? " unknown attempt" : " unknown attempts") : "");
|
|
734
|
+
}).join(" · "), section).className = "counts";
|
|
735
|
+
else create("p", unique(rows.map(row => row.arm)).sort(byArm).map(arm => {
|
|
736
|
+
const observed = rows.filter(row => row.arm === arm);
|
|
737
|
+
const denominator = observed.length;
|
|
738
|
+
const defined = observed.filter(row => row.value !== null).length;
|
|
739
|
+
return armLabel(arm) + ": " + defined + "/" + denominator + (denominator === 1 ? " measured attempt" : " measured attempts");
|
|
740
|
+
}).join(" · "), section).className = "counts";
|
|
741
|
+
if (panel === "Code added") {
|
|
742
|
+
const scope = create("details", undefined, section);
|
|
743
|
+
create("summary", "Scan completeness by task", scope);
|
|
744
|
+
for (const task of unique(rows.map(row => row.task))) {
|
|
745
|
+
create("h3", task, scope);
|
|
746
|
+
complexityNote(scope, rows.filter(row => row.task === task));
|
|
747
|
+
}
|
|
748
|
+
}
|
|
749
|
+
if (rows.every(row => row.value === null)) {
|
|
750
|
+
for (const reason of unique(rows.map(row => row.reason || "Measurement unavailable."))) create("p", reason, section);
|
|
751
|
+
}
|
|
752
|
+
else {
|
|
753
|
+
drawEvaluationPlot({ parent: section, rows, metric });
|
|
754
|
+
}
|
|
755
|
+
addDetails(section, rows);
|
|
756
|
+
if (["Total review findings", "P0 review findings", "Correctness review findings"].includes(metric)) showReviewFindings(section, rows);
|
|
757
|
+
}
|
|
758
|
+
function update() {
|
|
759
|
+
const overview = !taskSelect.value;
|
|
760
|
+
overviewButton.setAttribute("aria-pressed", String(overview));
|
|
761
|
+
scopeHeading.textContent = overview ? "Overview · all tasks" : "Task · " + taskSelect.value;
|
|
762
|
+
scopeDescription.textContent = overview ? "Each task has equal weight. Select a task to inspect repeats. All attempts stay included, even failed checks." : "Medians of this task's repeated runs. Use Overview to return to all tasks.";
|
|
763
|
+
const scopedAttempts = data.attempts.filter(attempt => overview || attempt.task === taskSelect.value);
|
|
764
|
+
document.getElementById("study").textContent = scopedAttempts.length + " assigned attempts · " + (overview ? knownTasks.length + " tasks" : "1 task · from a " + data.assigned + "-attempt evaluation");
|
|
765
|
+
document.querySelector(".run-toggle").hidden = comparisonView.value === "delta" || !taskSelect.value;
|
|
766
|
+
chartBreakdown.parentElement.hidden = !overview;
|
|
767
|
+
document.querySelector(".chart-options > p").textContent = comparisonView.value === "delta" ? (overview && chartBreakdown.value === "average" ? "Each bar averages the task deltas versus vanilla, using the same matched tasks. " : "Each bar is the guidance median minus the vanilla median. ") + "Zero means they match; negative is not automatically better." : overview ? (chartBreakdown.value === "tasks" ? "One bar per task and package shows its median. Click a bar or choose a task to inspect repeats. " : "Bars show the average of task medians. Counts show how many tasks have defined values. ") + "Grey is vanilla; colors identify packages." : "Bars show medians. Each dot is one run; squares mark failed task checks. Exact values are below.";
|
|
768
|
+
showPackageSummary();
|
|
769
|
+
showMetadata();
|
|
770
|
+
showOutcomes();
|
|
771
|
+
showCompleteness();
|
|
772
|
+
showCollectionStatus();
|
|
773
|
+
for (const section of panels.children) for (const chart of section.plots) chart.dispose();
|
|
774
|
+
panels.replaceChildren();
|
|
775
|
+
const selectedMetrics = [metricSelect.value];
|
|
776
|
+
for (const metric of selectedMetrics) {
|
|
777
|
+
const panel = observations.find(row => row.metric === metric).panel;
|
|
778
|
+
const section = create("section", undefined, panels); section.plots = [];
|
|
779
|
+
renderPanel({ section, panel, metric });
|
|
780
|
+
}
|
|
781
|
+
}
|
|
782
|
+
for (const control of [taskSelect, showRuns, comparisonView, metricSelect, chartBreakdown]) control.addEventListener("change", update);
|
|
783
|
+
window.matchMedia("(prefers-color-scheme: dark)").addEventListener("change", update);
|
|
784
|
+
new MutationObserver(update).observe(document.documentElement, { attributes: true, attributeFilter: ["style", "class", "data-theme"] });
|
|
785
|
+
new MutationObserver(update).observe(document.head, { subtree: true, childList: true, characterData: true });
|
|
786
|
+
new ResizeObserver(() => {
|
|
787
|
+
for (const section of panels.children) for (const chart of section.plots) chart.resize();
|
|
788
|
+
}).observe(panels);
|
|
789
|
+
update();
|
|
790
|
+
</script></body></html>
|