skillfid 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +303 -0
- package/package.json +39 -0
- package/src/baseline.js +85 -0
- package/src/calibration.js +32 -0
- package/src/cli.js +329 -0
- package/src/concurrency.js +77 -0
- package/src/copilot-sdk.js +137 -0
- package/src/corpus.js +112 -0
- package/src/dataset.js +486 -0
- package/src/diagnosis.js +40 -0
- package/src/evaluation.js +486 -0
- package/src/files.js +56 -0
- package/src/inventory.js +83 -0
- package/src/journal.js +193 -0
- package/src/json.js +26 -0
- package/src/judge.js +39 -0
- package/src/progress.js +223 -0
- package/src/prompts.js +42 -0
- package/src/questions.js +80 -0
- package/src/report.js +233 -0
- package/src/structured.js +35 -0
- package/src/verification.js +33 -0
- package/templates/evaluation-report.html +537 -0
|
@@ -0,0 +1,537 @@
|
|
|
1
|
+
<!doctype html>
|
|
2
|
+
<html lang="en">
|
|
3
|
+
<head>
|
|
4
|
+
<meta charset="utf-8">
|
|
5
|
+
<meta name="viewport" content="width=device-width, initial-scale=1">
|
|
6
|
+
<script>
|
|
7
|
+
(() => {
|
|
8
|
+
const param = new URLSearchParams(window.location.search).get("clawpilotTheme");
|
|
9
|
+
const theme =
|
|
10
|
+
param || (window.matchMedia("(prefers-color-scheme: dark)").matches ? "dark" : "light");
|
|
11
|
+
document.documentElement.setAttribute("data-theme", theme);
|
|
12
|
+
})();
|
|
13
|
+
</script>
|
|
14
|
+
<title>Skill evaluation report</title>
|
|
15
|
+
<style>
|
|
16
|
+
:root {
|
|
17
|
+
color-scheme: light;
|
|
18
|
+
--cp-bg: #f7f4ef;
|
|
19
|
+
--cp-bg-elevated: #fcfbf8;
|
|
20
|
+
--cp-surface: #ffffff;
|
|
21
|
+
--cp-surface-soft: #f5f5f5;
|
|
22
|
+
--cp-border: #dedede;
|
|
23
|
+
--cp-border-strong: #919191;
|
|
24
|
+
--cp-text: #242424;
|
|
25
|
+
--cp-text-muted: #5c5c5c;
|
|
26
|
+
--cp-text-soft: #6f6f6f;
|
|
27
|
+
--cp-accent: #b11f4b;
|
|
28
|
+
--cp-accent-hover: #9a1a41;
|
|
29
|
+
--cp-accent-soft: rgba(177, 31, 75, 0.08);
|
|
30
|
+
--cp-accent-fg: #ffffff;
|
|
31
|
+
--cp-success: #16a34a;
|
|
32
|
+
--cp-danger: #dc2626;
|
|
33
|
+
--cp-warning: #f59e0b;
|
|
34
|
+
--cp-link: #0078d4;
|
|
35
|
+
--cp-shadow: 0 18px 48px rgba(0, 0, 0, 0.12);
|
|
36
|
+
--cp-overlay: rgba(255, 255, 255, 0.8);
|
|
37
|
+
--cp-panel: rgba(255, 255, 255, 0.86);
|
|
38
|
+
--cp-panel-strong: rgba(255, 255, 255, 0.96);
|
|
39
|
+
--cp-sheen: rgba(255, 255, 255, 0.55);
|
|
40
|
+
--cp-highlight: rgba(177, 31, 75, 0.12);
|
|
41
|
+
}
|
|
42
|
+
html[data-theme="dark"] {
|
|
43
|
+
color-scheme: dark;
|
|
44
|
+
--cp-bg: #3d3b3a;
|
|
45
|
+
--cp-bg-elevated: #343231;
|
|
46
|
+
--cp-surface: #292929;
|
|
47
|
+
--cp-surface-soft: #2e2e2e;
|
|
48
|
+
--cp-border: #474747;
|
|
49
|
+
--cp-border-strong: #5f5f5f;
|
|
50
|
+
--cp-text: #dedede;
|
|
51
|
+
--cp-text-muted: #919191;
|
|
52
|
+
--cp-text-soft: #b0b0b0;
|
|
53
|
+
--cp-accent: #fd8ea1;
|
|
54
|
+
--cp-accent-hover: #fb7b91;
|
|
55
|
+
--cp-accent-soft: rgba(253, 142, 161, 0.14);
|
|
56
|
+
--cp-accent-fg: #1a1a1a;
|
|
57
|
+
--cp-success: #4ade80;
|
|
58
|
+
--cp-danger: #f87171;
|
|
59
|
+
--cp-warning: #fbbf24;
|
|
60
|
+
--cp-link: #4da6ff;
|
|
61
|
+
--cp-shadow: 0 18px 48px rgba(0, 0, 0, 0.32);
|
|
62
|
+
--cp-overlay: rgba(41, 41, 41, 0.88);
|
|
63
|
+
--cp-panel: rgba(41, 41, 41, 0.72);
|
|
64
|
+
--cp-panel-strong: rgba(41, 41, 41, 0.96);
|
|
65
|
+
--cp-sheen: rgba(255, 255, 255, 0.04);
|
|
66
|
+
--cp-highlight: rgba(253, 142, 161, 0.12);
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
* { box-sizing: border-box; }
|
|
70
|
+
html { scroll-behavior: smooth; }
|
|
71
|
+
body {
|
|
72
|
+
margin: 0;
|
|
73
|
+
background: var(--cp-bg);
|
|
74
|
+
color: var(--cp-text);
|
|
75
|
+
font-family: "Segoe UI", Aptos, Calibri, -apple-system, BlinkMacSystemFont, sans-serif;
|
|
76
|
+
letter-spacing: 0;
|
|
77
|
+
}
|
|
78
|
+
button, input, select { font: inherit; }
|
|
79
|
+
button, select { cursor: pointer; }
|
|
80
|
+
a { color: var(--cp-link); }
|
|
81
|
+
.shell { width: min(1180px, calc(100% - 40px)); margin: 0 auto; }
|
|
82
|
+
.topbar {
|
|
83
|
+
border-bottom: 1px solid var(--cp-border);
|
|
84
|
+
background: var(--cp-bg-elevated);
|
|
85
|
+
}
|
|
86
|
+
.topbar-inner {
|
|
87
|
+
min-height: 52px;
|
|
88
|
+
display: flex;
|
|
89
|
+
align-items: center;
|
|
90
|
+
justify-content: space-between;
|
|
91
|
+
gap: 20px;
|
|
92
|
+
}
|
|
93
|
+
.product { font-size: 14px; font-weight: 700; }
|
|
94
|
+
.nav { display: flex; gap: 20px; }
|
|
95
|
+
.nav a { color: var(--cp-text-muted); font-size: 13px; text-decoration: none; }
|
|
96
|
+
.nav a:hover { color: var(--cp-text); }
|
|
97
|
+
.masthead { padding: 48px 0 36px; border-bottom: 1px solid var(--cp-border); }
|
|
98
|
+
.eyebrow {
|
|
99
|
+
margin: 0 0 12px;
|
|
100
|
+
color: var(--cp-accent);
|
|
101
|
+
font-size: 12px;
|
|
102
|
+
font-weight: 700;
|
|
103
|
+
text-transform: uppercase;
|
|
104
|
+
letter-spacing: 0;
|
|
105
|
+
}
|
|
106
|
+
h1 { margin: 0; max-width: 760px; font-size: clamp(34px, 6vw, 62px); line-height: 1; font-weight: 650; }
|
|
107
|
+
.lede { max-width: 720px; margin: 18px 0 0; color: var(--cp-text-muted); font-size: 18px; line-height: 1.55; }
|
|
108
|
+
.meta { display: flex; align-items: center; flex-wrap: wrap; gap: 8px 20px; margin-top: 24px; color: var(--cp-text-soft); font-size: 13px; }
|
|
109
|
+
.run-id {
|
|
110
|
+
display: inline-flex;
|
|
111
|
+
align-items: center;
|
|
112
|
+
min-height: 28px;
|
|
113
|
+
padding: 4px 10px;
|
|
114
|
+
border: 1px solid var(--cp-border-strong);
|
|
115
|
+
border-radius: 0.625rem;
|
|
116
|
+
color: var(--cp-text);
|
|
117
|
+
background: var(--cp-surface);
|
|
118
|
+
font-weight: 600;
|
|
119
|
+
}
|
|
120
|
+
main section { padding: 40px 0; border-bottom: 1px solid var(--cp-border); }
|
|
121
|
+
.section-heading { display: flex; justify-content: space-between; align-items: end; gap: 24px; margin-bottom: 24px; }
|
|
122
|
+
h2 { margin: 0; font-size: 24px; font-weight: 650; }
|
|
123
|
+
h3 { margin: 0; font-size: 16px; }
|
|
124
|
+
.section-note { max-width: 580px; margin: 6px 0 0; color: var(--cp-text-muted); line-height: 1.5; }
|
|
125
|
+
.verdict {
|
|
126
|
+
display: grid;
|
|
127
|
+
grid-template-columns: minmax(0, 1.5fr) minmax(240px, 0.7fr);
|
|
128
|
+
gap: 32px;
|
|
129
|
+
align-items: center;
|
|
130
|
+
padding: 28px;
|
|
131
|
+
border: 1px solid var(--cp-border-strong);
|
|
132
|
+
border-radius: 16px;
|
|
133
|
+
background: var(--cp-surface);
|
|
134
|
+
}
|
|
135
|
+
.verdict .verdict-label { margin: 0 0 8px; color: var(--cp-accent); font-size: 13px; font-weight: 700; text-transform: uppercase; }
|
|
136
|
+
.verdict h2 { font-size: 30px; }
|
|
137
|
+
.verdict p { margin: 10px 0 0; color: var(--cp-text-muted); line-height: 1.5; }
|
|
138
|
+
.verdict-score { text-align: right; }
|
|
139
|
+
.score-value { display: block; font-size: 52px; line-height: 1; font-weight: 700; font-variant-numeric: tabular-nums; }
|
|
140
|
+
.score-caption { display: block; margin-top: 8px; color: var(--cp-text-muted); }
|
|
141
|
+
.metrics { display: grid; grid-template-columns: repeat(4, 1fr); gap: 12px; margin-top: 16px; }
|
|
142
|
+
.metric { min-width: 0; padding: 20px; border: 1px solid var(--cp-border); border-radius: 16px; background: var(--cp-surface); }
|
|
143
|
+
.metric-value { display: block; font-size: 25px; font-weight: 700; font-variant-numeric: tabular-nums; overflow-wrap: anywhere; }
|
|
144
|
+
.metric-value.success { color: var(--cp-success); }
|
|
145
|
+
.metric-label { display: block; margin-top: 8px; color: var(--cp-text-muted); font-size: 13px; }
|
|
146
|
+
.counts { display: grid; grid-template-columns: repeat(4, 1fr); gap: 20px; }
|
|
147
|
+
.distribution-counts { grid-template-columns: repeat(5, 1fr); }
|
|
148
|
+
.count { padding: 18px 0; border-top: 2px solid var(--cp-border); }
|
|
149
|
+
.count strong { display: block; font-size: 26px; font-variant-numeric: tabular-nums; }
|
|
150
|
+
.count span { color: var(--cp-text-muted); font-size: 13px; }
|
|
151
|
+
.action-list { display: grid; gap: 10px; }
|
|
152
|
+
.action {
|
|
153
|
+
display: grid;
|
|
154
|
+
grid-template-columns: 42px minmax(0, 1fr) auto;
|
|
155
|
+
gap: 14px;
|
|
156
|
+
align-items: start;
|
|
157
|
+
padding: 16px 0;
|
|
158
|
+
border-top: 1px solid var(--cp-border);
|
|
159
|
+
}
|
|
160
|
+
.priority {
|
|
161
|
+
display: grid;
|
|
162
|
+
place-items: center;
|
|
163
|
+
width: 36px;
|
|
164
|
+
height: 36px;
|
|
165
|
+
border: 1px solid var(--cp-border-strong);
|
|
166
|
+
border-radius: 50%;
|
|
167
|
+
font-weight: 700;
|
|
168
|
+
}
|
|
169
|
+
.action-file { color: var(--cp-link); font-family: Consolas, "Courier New", Courier, monospace; font-size: 13px; }
|
|
170
|
+
.action-changes { display: grid; gap: 8px; margin: 10px 0 0; padding-left: 20px; color: var(--cp-text-muted); font-size: 14px; line-height: 1.45; }
|
|
171
|
+
.change-impact { margin-left: 6px; color: var(--cp-text-soft); font-size: 12px; white-space: nowrap; }
|
|
172
|
+
.affected { color: var(--cp-text-muted); font-size: 12px; white-space: nowrap; }
|
|
173
|
+
.filters { display: grid; grid-template-columns: minmax(220px, 1fr) 180px 180px; gap: 10px; margin-bottom: 14px; }
|
|
174
|
+
.control {
|
|
175
|
+
width: 100%;
|
|
176
|
+
min-height: 40px;
|
|
177
|
+
padding: 8px 12px;
|
|
178
|
+
border: 1px solid var(--cp-border-strong);
|
|
179
|
+
border-radius: 0.625rem;
|
|
180
|
+
color: var(--cp-text);
|
|
181
|
+
background: var(--cp-surface);
|
|
182
|
+
}
|
|
183
|
+
.table-wrap { overflow-x: auto; border: 1px solid var(--cp-border); border-radius: 16px; background: var(--cp-surface); }
|
|
184
|
+
table { width: 100%; min-width: 820px; border-collapse: collapse; }
|
|
185
|
+
th, td { padding: 13px 14px; border-bottom: 1px solid var(--cp-border); text-align: left; vertical-align: middle; }
|
|
186
|
+
th { color: var(--cp-text-muted); background: var(--cp-surface-soft); font-size: 12px; font-weight: 650; }
|
|
187
|
+
td { font-size: 13px; }
|
|
188
|
+
tbody tr { cursor: pointer; }
|
|
189
|
+
tbody tr:hover, tbody tr:focus { background: var(--cp-accent-soft); outline: none; }
|
|
190
|
+
tbody tr:last-child td { border-bottom: 0; }
|
|
191
|
+
.test-name { max-width: 400px; font-weight: 600; }
|
|
192
|
+
.numeric { text-align: right; font-variant-numeric: tabular-nums; }
|
|
193
|
+
.status { display: inline-flex; align-items: center; gap: 7px; white-space: nowrap; }
|
|
194
|
+
.status::before { content: ""; width: 8px; height: 8px; border-radius: 50%; background: var(--cp-border-strong); }
|
|
195
|
+
.status.perfect::before, .status.strong::before { background: var(--cp-success); }
|
|
196
|
+
.status.mixed::before { background: var(--cp-warning); }
|
|
197
|
+
.status.weak::before, .status.zero::before { background: var(--cp-danger); }
|
|
198
|
+
.empty { padding: 32px; text-align: center; color: var(--cp-text-muted); }
|
|
199
|
+
.detail {
|
|
200
|
+
margin-top: 14px;
|
|
201
|
+
padding: 24px;
|
|
202
|
+
border: 1px solid var(--cp-border);
|
|
203
|
+
border-radius: 16px;
|
|
204
|
+
background: var(--cp-surface);
|
|
205
|
+
}
|
|
206
|
+
.detail[hidden] { display: none; }
|
|
207
|
+
.detail-head { display: flex; justify-content: space-between; gap: 24px; align-items: start; }
|
|
208
|
+
.detail-grid { display: grid; grid-template-columns: 1fr 1fr; gap: 28px; margin-top: 22px; }
|
|
209
|
+
.detail-label { margin: 0 0 6px; color: var(--cp-text-muted); font-size: 12px; font-weight: 650; text-transform: uppercase; }
|
|
210
|
+
.detail p { line-height: 1.55; }
|
|
211
|
+
.trial-evidence { margin-top: 24px; }
|
|
212
|
+
.trial-list { display: grid; gap: 8px; margin-top: 10px; }
|
|
213
|
+
.trial {
|
|
214
|
+
border: 1px solid var(--cp-border);
|
|
215
|
+
border-radius: 0.625rem;
|
|
216
|
+
background: var(--cp-surface-soft);
|
|
217
|
+
}
|
|
218
|
+
.trial summary { padding: 12px 14px; cursor: pointer; font-weight: 650; }
|
|
219
|
+
.trial-body { padding: 0 14px 14px; }
|
|
220
|
+
.trial-answer { margin: 0; white-space: pre-wrap; }
|
|
221
|
+
.trial-findings { margin: 12px 0 0; padding-left: 20px; color: var(--cp-text-muted); }
|
|
222
|
+
.close {
|
|
223
|
+
width: 36px;
|
|
224
|
+
height: 36px;
|
|
225
|
+
border: 1px solid var(--cp-border);
|
|
226
|
+
border-radius: 0.625rem;
|
|
227
|
+
color: var(--cp-text);
|
|
228
|
+
background: var(--cp-surface);
|
|
229
|
+
font-size: 20px;
|
|
230
|
+
}
|
|
231
|
+
.close:hover { border-color: var(--cp-border-strong); }
|
|
232
|
+
footer { padding: 28px 0 48px; color: var(--cp-text-muted); font-size: 12px; }
|
|
233
|
+
@media (max-width: 800px) {
|
|
234
|
+
.shell { width: min(100% - 24px, 1180px); }
|
|
235
|
+
.nav { display: none; }
|
|
236
|
+
.masthead { padding-top: 32px; }
|
|
237
|
+
.verdict, .detail-grid { grid-template-columns: 1fr; }
|
|
238
|
+
.verdict-score { text-align: left; }
|
|
239
|
+
.metrics { grid-template-columns: repeat(2, 1fr); }
|
|
240
|
+
.counts { grid-template-columns: repeat(2, 1fr); }
|
|
241
|
+
.filters { grid-template-columns: 1fr; }
|
|
242
|
+
}
|
|
243
|
+
@media (max-width: 480px) {
|
|
244
|
+
h1 { font-size: 36px; }
|
|
245
|
+
.metrics, .counts { grid-template-columns: 1fr; }
|
|
246
|
+
.metric { padding: 16px; }
|
|
247
|
+
.verdict { padding: 20px; }
|
|
248
|
+
.action { grid-template-columns: 38px minmax(0, 1fr); }
|
|
249
|
+
.affected { grid-column: 2; }
|
|
250
|
+
}
|
|
251
|
+
</style>
|
|
252
|
+
</head>
|
|
253
|
+
<body>
|
|
254
|
+
<header>
|
|
255
|
+
<div class="topbar">
|
|
256
|
+
<div class="shell topbar-inner">
|
|
257
|
+
<div class="product">skillfid</div>
|
|
258
|
+
<nav class="nav" aria-label="Report sections">
|
|
259
|
+
<a href="#outcome">Outcome</a>
|
|
260
|
+
<a href="#findings">Improvements</a>
|
|
261
|
+
<a href="#analyzed">Scope</a>
|
|
262
|
+
<a href="#tests">Evidence</a>
|
|
263
|
+
</nav>
|
|
264
|
+
</div>
|
|
265
|
+
</div>
|
|
266
|
+
<div class="masthead">
|
|
267
|
+
<div class="shell">
|
|
268
|
+
<p class="eyebrow">Skill evaluation report</p>
|
|
269
|
+
<h1 id="report-title"></h1>
|
|
270
|
+
<p class="lede">Does the skill help an agent use the source guidance correctly? This report compares unaided answers with skill-assisted answers, then traces remaining failures to changes you can make.</p>
|
|
271
|
+
<div class="meta">
|
|
272
|
+
<span class="run-id" id="run-id"></span>
|
|
273
|
+
<span id="evaluated-at"></span>
|
|
274
|
+
<span id="question-total"></span>
|
|
275
|
+
<span id="trial-total"></span>
|
|
276
|
+
</div>
|
|
277
|
+
</div>
|
|
278
|
+
</div>
|
|
279
|
+
</header>
|
|
280
|
+
|
|
281
|
+
<main>
|
|
282
|
+
<section id="outcome">
|
|
283
|
+
<div class="shell">
|
|
284
|
+
<div class="verdict">
|
|
285
|
+
<div>
|
|
286
|
+
<p class="verdict-label" id="verdict-label"></p>
|
|
287
|
+
<h2 id="verdict-headline"></h2>
|
|
288
|
+
<p id="verdict-detail"></p>
|
|
289
|
+
</div>
|
|
290
|
+
<div class="verdict-score">
|
|
291
|
+
<span class="score-value" id="skill-score"></span>
|
|
292
|
+
<span class="score-caption">skill-assisted score</span>
|
|
293
|
+
</div>
|
|
294
|
+
</div>
|
|
295
|
+
<div class="metrics" aria-label="Evaluation metrics">
|
|
296
|
+
<div class="metric"><span class="metric-value" id="closed-book-score"></span><span class="metric-label">Without the skill</span></div>
|
|
297
|
+
<div class="metric"><span class="metric-value success" id="uplift-score"></span><span class="metric-label">Percentage-point uplift</span></div>
|
|
298
|
+
<div class="metric"><span class="metric-value" id="fully-correct"></span><span class="metric-label">Questions perfect in every trial</span></div>
|
|
299
|
+
</div>
|
|
300
|
+
</div>
|
|
301
|
+
</section>
|
|
302
|
+
|
|
303
|
+
<section id="findings">
|
|
304
|
+
<div class="shell">
|
|
305
|
+
<div class="section-heading">
|
|
306
|
+
<div><h2 id="findings-title">Concrete fix targets</h2><p class="section-note" id="findings-note">Only diagnoses with a specific file target appear here. They remain proposals for review, not confirmed changes.</p></div>
|
|
307
|
+
</div>
|
|
308
|
+
<h3 id="recommendations-title">Recommended changes</h3>
|
|
309
|
+
<div class="action-list" id="recommendations" style="margin-top: 12px"></div>
|
|
310
|
+
</div>
|
|
311
|
+
</section>
|
|
312
|
+
|
|
313
|
+
<section id="distribution">
|
|
314
|
+
<div class="shell">
|
|
315
|
+
<div class="section-heading">
|
|
316
|
+
<div><h2>Score distribution</h2><p class="section-note">Question scores average all skill-assisted trials. Bands distinguish marginal misses from consequential failures.</p></div>
|
|
317
|
+
</div>
|
|
318
|
+
<div class="counts distribution-counts">
|
|
319
|
+
<div class="count"><strong id="perfect-count"></strong><span>Perfect: 100%</span></div>
|
|
320
|
+
<div class="count"><strong id="strong-count"></strong><span>Strong: 90%–<100%</span></div>
|
|
321
|
+
<div class="count"><strong id="mixed-count"></strong><span>Mixed: 70%–<90%</span></div>
|
|
322
|
+
<div class="count"><strong id="weak-count"></strong><span>Weak: >0%–<70%</span></div>
|
|
323
|
+
<div class="count"><strong id="zero-count"></strong><span>Zero: 0%</span></div>
|
|
324
|
+
</div>
|
|
325
|
+
</div>
|
|
326
|
+
</section>
|
|
327
|
+
|
|
328
|
+
<section id="analyzed">
|
|
329
|
+
<div class="shell">
|
|
330
|
+
<div class="section-heading">
|
|
331
|
+
<div><h2>What was analyzed</h2><p class="section-note">The evaluation covers the published source set and the skill generated from it. Every question is grounded in evidence from those documents.</p></div>
|
|
332
|
+
</div>
|
|
333
|
+
<div class="counts">
|
|
334
|
+
<div class="count"><strong id="document-count"></strong><span>source documents</span></div>
|
|
335
|
+
<div class="count"><strong id="knowledge-count"></strong><span>knowledge items</span></div>
|
|
336
|
+
<div class="count"><strong id="question-count"></strong><span>grounded questions</span></div>
|
|
337
|
+
<div class="count"><strong id="judgment-count"></strong><span>answers judged</span></div>
|
|
338
|
+
</div>
|
|
339
|
+
</div>
|
|
340
|
+
</section>
|
|
341
|
+
|
|
342
|
+
<section id="tests">
|
|
343
|
+
<div class="shell">
|
|
344
|
+
<div class="section-heading">
|
|
345
|
+
<div><h2>Test evidence</h2><p class="section-note"><span id="result-count"></span> questions connect aggregate scores to source evidence, each recorded answer, and criterion-level judge findings.</p></div>
|
|
346
|
+
</div>
|
|
347
|
+
<div class="filters">
|
|
348
|
+
<input class="control" id="search" type="search" placeholder="Search questions, evidence, or diagnoses" aria-label="Search questions, evidence, or diagnoses">
|
|
349
|
+
<select class="control" id="status-filter" aria-label="Filter by result">
|
|
350
|
+
<option value="all">All results</option>
|
|
351
|
+
<option value="perfect">Perfect</option>
|
|
352
|
+
<option value="strong">Strong</option>
|
|
353
|
+
<option value="mixed">Mixed</option>
|
|
354
|
+
<option value="weak">Weak</option>
|
|
355
|
+
<option value="zero">Zero</option>
|
|
356
|
+
</select>
|
|
357
|
+
<select class="control" id="cause-filter" aria-label="Filter by diagnosed cause">
|
|
358
|
+
<option value="all">All causes</option>
|
|
359
|
+
<option value="none">No issue</option>
|
|
360
|
+
<option value="skill_activation">Activation</option>
|
|
361
|
+
<option value="skill_knowledge_gap">Knowledge gap</option>
|
|
362
|
+
<option value="skill_retrieval">Retrieval</option>
|
|
363
|
+
<option value="grounding">Grounding</option>
|
|
364
|
+
<option value="answer_variability">Answer variability</option>
|
|
365
|
+
<option value="test_or_model_limitation">Test or model limitation</option>
|
|
366
|
+
<option value="unknown">Unknown</option>
|
|
367
|
+
</select>
|
|
368
|
+
</div>
|
|
369
|
+
<div class="table-wrap">
|
|
370
|
+
<table>
|
|
371
|
+
<thead><tr><th>Question</th><th>Result</th><th class="numeric">Without skill</th><th class="numeric">With skill</th><th>Diagnosis</th></tr></thead>
|
|
372
|
+
<tbody id="test-rows"></tbody>
|
|
373
|
+
</table>
|
|
374
|
+
<div class="empty" id="empty" hidden>No tests match these filters.</div>
|
|
375
|
+
</div>
|
|
376
|
+
<article class="detail" id="detail" hidden aria-live="polite">
|
|
377
|
+
<div class="detail-head"><div><p class="detail-label">Selected test</p><h3 id="detail-question"></h3></div><button class="close" id="close-detail" type="button" aria-label="Close test details" title="Close">×</button></div>
|
|
378
|
+
<div class="detail-grid">
|
|
379
|
+
<div><p class="detail-label">What the answer needed</p><p id="detail-criterion"></p><p class="detail-label">Judge finding</p><p id="detail-rationale"></p></div>
|
|
380
|
+
<div><p class="detail-label">Source evidence</p><p id="detail-source"></p><p class="detail-label">Diagnosis evidence</p><p id="detail-diagnosis"></p><p class="detail-label">Recommended change</p><p id="detail-fix"></p></div>
|
|
381
|
+
</div>
|
|
382
|
+
<div class="trial-evidence"><p class="detail-label">Recorded trial answers</p><div class="trial-list" id="detail-trials"></div></div>
|
|
383
|
+
</article>
|
|
384
|
+
</div>
|
|
385
|
+
</section>
|
|
386
|
+
</main>
|
|
387
|
+
|
|
388
|
+
<footer><div class="shell" id="report-provenance"></div></footer>
|
|
389
|
+
|
|
390
|
+
<script>
|
|
391
|
+
const report = __SKILLFID_REPORT_DATA__;
|
|
392
|
+
const tests = report.tests;
|
|
393
|
+
const labels = { perfect: "Perfect", strong: "Strong", mixed: "Mixed", weak: "Weak", zero: "Zero", none: "No issue", multiple: "Multiple diagnoses", skill_activation: "Activation", skill_knowledge_gap: "Knowledge gap", skill_retrieval: "Retrieval", grounding: "Grounding", answer_variability: "Answer variability", interpretation: "Interpretation", application: "Application", test_or_model_limitation: "Test or model limitation", unknown: "Unresolved" };
|
|
394
|
+
const rows = document.getElementById("test-rows");
|
|
395
|
+
const empty = document.getElementById("empty");
|
|
396
|
+
const resultCount = document.getElementById("result-count");
|
|
397
|
+
const search = document.getElementById("search");
|
|
398
|
+
const statusFilter = document.getElementById("status-filter");
|
|
399
|
+
const causeFilter = document.getElementById("cause-filter");
|
|
400
|
+
const detail = document.getElementById("detail");
|
|
401
|
+
|
|
402
|
+
document.title = `${report.title} evaluation report`;
|
|
403
|
+
document.getElementById("report-title").textContent = report.title;
|
|
404
|
+
document.getElementById("run-id").textContent = report.runId;
|
|
405
|
+
document.getElementById("evaluated-at").textContent = `Evaluated ${report.createdAt}`;
|
|
406
|
+
document.getElementById("question-total").textContent = `${report.counts.questions} questions`;
|
|
407
|
+
document.getElementById("trial-total").textContent = `${report.trialsPerQuestion} trial${report.trialsPerQuestion === 1 ? "" : "s"} per condition`;
|
|
408
|
+
document.getElementById("verdict-label").textContent = report.verdict.label;
|
|
409
|
+
document.getElementById("verdict-headline").textContent = report.verdict.headline;
|
|
410
|
+
document.getElementById("verdict-detail").textContent = report.verdict.detail;
|
|
411
|
+
document.getElementById("skill-score").textContent = report.scores.skill;
|
|
412
|
+
document.getElementById("closed-book-score").textContent = report.scores.closedBook;
|
|
413
|
+
document.getElementById("uplift-score").textContent = report.scores.uplift;
|
|
414
|
+
document.getElementById("fully-correct").textContent = `${report.counts.fullyCorrect} / ${report.counts.questions}`;
|
|
415
|
+
document.getElementById("document-count").textContent = report.counts.documents;
|
|
416
|
+
document.getElementById("knowledge-count").textContent = report.counts.knowledge;
|
|
417
|
+
document.getElementById("question-count").textContent = report.counts.questions;
|
|
418
|
+
document.getElementById("judgment-count").textContent = report.counts.judgments;
|
|
419
|
+
document.getElementById("perfect-count").textContent = report.distribution.perfect;
|
|
420
|
+
document.getElementById("strong-count").textContent = report.distribution.strong;
|
|
421
|
+
document.getElementById("mixed-count").textContent = report.distribution.mixed;
|
|
422
|
+
document.getElementById("weak-count").textContent = report.distribution.weak;
|
|
423
|
+
document.getElementById("zero-count").textContent = report.distribution.zero;
|
|
424
|
+
document.getElementById("report-provenance").textContent = [`skillfid ${report.evaluatorVersion}`, report.runId, report.datasetId, report.baselineId, `${report.model} subject`, `${report.judgeModel} judge`].filter(Boolean).join(" · ");
|
|
425
|
+
|
|
426
|
+
const recommendations = document.getElementById("recommendations");
|
|
427
|
+
if (!report.recommendations.length) {
|
|
428
|
+
document.getElementById("findings-title").textContent = "No changes recommended";
|
|
429
|
+
document.getElementById("findings-note").textContent = "Every tested skill-assisted answer received a perfect score.";
|
|
430
|
+
document.getElementById("recommendations-title").hidden = true;
|
|
431
|
+
} else {
|
|
432
|
+
report.recommendations.forEach((recommendation, index) => {
|
|
433
|
+
const action = document.createElement("div");
|
|
434
|
+
action.className = "action";
|
|
435
|
+
const priority = document.createElement("span");
|
|
436
|
+
priority.className = "priority";
|
|
437
|
+
priority.textContent = String(index + 1);
|
|
438
|
+
const content = document.createElement("div");
|
|
439
|
+
const heading = document.createElement("strong");
|
|
440
|
+
heading.className = "action-file";
|
|
441
|
+
heading.textContent = recommendation.file;
|
|
442
|
+
const changes = document.createElement("ul");
|
|
443
|
+
changes.className = "action-changes";
|
|
444
|
+
recommendation.changes.forEach((change) => {
|
|
445
|
+
const item = document.createElement("li");
|
|
446
|
+
const text = document.createElement("span");
|
|
447
|
+
text.textContent = change.recommendation;
|
|
448
|
+
item.appendChild(text);
|
|
449
|
+
if (change.affectedTests > 1) {
|
|
450
|
+
const impact = document.createElement("span");
|
|
451
|
+
impact.className = "change-impact";
|
|
452
|
+
impact.textContent = `${change.affectedTests} tests`;
|
|
453
|
+
item.appendChild(impact);
|
|
454
|
+
}
|
|
455
|
+
changes.appendChild(item);
|
|
456
|
+
});
|
|
457
|
+
content.append(heading, changes);
|
|
458
|
+
const affected = document.createElement("span");
|
|
459
|
+
affected.className = "affected";
|
|
460
|
+
affected.textContent = `${recommendation.affectedTests} test${recommendation.affectedTests === 1 ? "" : "s"}`;
|
|
461
|
+
action.append(priority, content, affected);
|
|
462
|
+
recommendations.appendChild(action);
|
|
463
|
+
});
|
|
464
|
+
}
|
|
465
|
+
|
|
466
|
+
function showDetail(test) {
|
|
467
|
+
document.getElementById("detail-question").textContent = test.question;
|
|
468
|
+
document.getElementById("detail-criterion").textContent = test.criterion;
|
|
469
|
+
document.getElementById("detail-rationale").textContent = test.rationale;
|
|
470
|
+
document.getElementById("detail-source").textContent = test.source;
|
|
471
|
+
document.getElementById("detail-diagnosis").textContent = test.diagnosis?.evidence?.join(" ") || "No diagnosis recorded.";
|
|
472
|
+
document.getElementById("detail-fix").textContent = test.fix;
|
|
473
|
+
const trials = document.getElementById("detail-trials");
|
|
474
|
+
trials.replaceChildren();
|
|
475
|
+
[["Skill-assisted", test.trials.skill], ["Without skill", test.trials.closedBook]].forEach(([condition, entries]) => {
|
|
476
|
+
entries.forEach((entry) => {
|
|
477
|
+
const item = document.createElement("details");
|
|
478
|
+
item.className = "trial";
|
|
479
|
+
const summary = document.createElement("summary");
|
|
480
|
+
summary.textContent = `${condition} · trial ${entry.trial + 1} · ${entry.score}`;
|
|
481
|
+
const body = document.createElement("div");
|
|
482
|
+
body.className = "trial-body";
|
|
483
|
+
const answer = document.createElement("p");
|
|
484
|
+
answer.className = "trial-answer";
|
|
485
|
+
answer.textContent = entry.answer || "No answer text recorded.";
|
|
486
|
+
body.appendChild(answer);
|
|
487
|
+
if (entry.failedCriteria.length) {
|
|
488
|
+
const findings = document.createElement("ul");
|
|
489
|
+
findings.className = "trial-findings";
|
|
490
|
+
entry.failedCriteria.forEach((finding) => {
|
|
491
|
+
const row = document.createElement("li");
|
|
492
|
+
row.textContent = finding;
|
|
493
|
+
findings.appendChild(row);
|
|
494
|
+
});
|
|
495
|
+
body.appendChild(findings);
|
|
496
|
+
}
|
|
497
|
+
item.append(summary, body);
|
|
498
|
+
trials.appendChild(item);
|
|
499
|
+
});
|
|
500
|
+
});
|
|
501
|
+
detail.hidden = false;
|
|
502
|
+
detail.scrollIntoView({ behavior: "smooth", block: "nearest" });
|
|
503
|
+
}
|
|
504
|
+
|
|
505
|
+
function render() {
|
|
506
|
+
const query = search.value.trim().toLowerCase();
|
|
507
|
+
const filtered = tests.filter((test) => {
|
|
508
|
+
const matchesText = !query || `${test.question} ${test.source} ${labels[test.cause] ?? test.cause}`.toLowerCase().includes(query);
|
|
509
|
+
const matchesStatus = statusFilter.value === "all" || test.status === statusFilter.value;
|
|
510
|
+
const matchesCause = causeFilter.value === "all" || test.cause === causeFilter.value;
|
|
511
|
+
return matchesText && matchesStatus && matchesCause;
|
|
512
|
+
});
|
|
513
|
+
rows.replaceChildren();
|
|
514
|
+
filtered.forEach((test) => {
|
|
515
|
+
const row = document.createElement("tr");
|
|
516
|
+
row.tabIndex = 0;
|
|
517
|
+
row.innerHTML = `<td class="test-name"></td><td><span class="status ${test.status}">${labels[test.status]}</span></td><td class="numeric">${test.baseline}</td><td class="numeric">${test.skill}</td><td></td>`;
|
|
518
|
+
row.children[0].textContent = test.question;
|
|
519
|
+
row.children[4].textContent = labels[test.cause] ?? test.cause;
|
|
520
|
+
row.addEventListener("click", () => showDetail(test));
|
|
521
|
+
row.addEventListener("keydown", (event) => { if (event.key === "Enter" || event.key === " ") { event.preventDefault(); showDetail(test); } });
|
|
522
|
+
rows.appendChild(row);
|
|
523
|
+
});
|
|
524
|
+
resultCount.textContent = String(filtered.length);
|
|
525
|
+
empty.hidden = filtered.length !== 0;
|
|
526
|
+
document.querySelector("table").hidden = filtered.length === 0;
|
|
527
|
+
detail.hidden = true;
|
|
528
|
+
}
|
|
529
|
+
|
|
530
|
+
search.addEventListener("input", render);
|
|
531
|
+
statusFilter.addEventListener("change", render);
|
|
532
|
+
causeFilter.addEventListener("change", render);
|
|
533
|
+
document.getElementById("close-detail").addEventListener("click", () => { detail.hidden = true; });
|
|
534
|
+
render();
|
|
535
|
+
</script>
|
|
536
|
+
</body>
|
|
537
|
+
</html>
|