static-fire 0.44.0 → 0.54.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +40 -34
- package/package.json +2 -1
- package/src/adapters/index.ts +3 -3
- package/src/api.ts +38 -6
- package/src/cli.ts +106 -20
- package/src/core/batch.ts +16 -1
- package/src/core/commands.ts +7 -6
- package/src/core/compare.ts +4 -1
- package/src/core/coverage.ts +84 -0
- package/src/core/db.ts +36 -0
- package/src/core/facets.ts +23 -8
- package/src/core/init.ts +99 -9
- package/src/core/plan.ts +84 -0
- package/src/core/project.ts +95 -9
- package/src/core/regrade.ts +10 -1
- package/src/core/score.ts +1 -1
- package/src/core/task-page.ts +54 -0
- package/src/core/trial-detail.ts +3 -3
- package/src/core/types.ts +1 -1
- package/src/core/view-query.ts +30 -13
- package/src/core/view.ts +131 -53
- package/src/guide.md +128 -51
- package/src/server.ts +7 -3
- package/web/app.tsx +276 -133
- package/web/filter-menu.ts +35 -4
- package/web/how-it-works.tsx +92 -0
- package/web/index-switches.ts +29 -0
- package/web/levels.tsx +15 -0
- package/web/notices.ts +74 -0
- package/web/output-view.ts +61 -0
- package/web/pages.ts +26 -0
- package/web/styles.css +57 -22
- package/web/task-card.ts +6 -0
- package/web/tasks.tsx +122 -0
- package/web/suite-notes.ts +0 -20
package/web/app.tsx
CHANGED
|
@@ -5,17 +5,24 @@ import type { AgentConfig, Criterion, JudgeConfig, Suite, Trial } from "../src/c
|
|
|
5
5
|
import type { TrialDetail } from "../src/core/trial-detail";
|
|
6
6
|
import type { View, ViewConfigSummary, ViewTrial } from "../src/core/view";
|
|
7
7
|
import { parseViewQuery, viewQueryParams, type Scope, type ViewQuery } from "../src/core/view-query";
|
|
8
|
-
import {
|
|
8
|
+
import { coverageLine } from "../src/core/coverage";
|
|
9
|
+
import { titleHref, toggleSuite, viewTitle } from "./index-switches";
|
|
10
|
+
import { shownTask } from "./task-card";
|
|
9
11
|
import type { Facets } from "../src/core/facets";
|
|
10
12
|
import { configLabel } from "../src/core/label";
|
|
11
13
|
import type { RegradeEnded, RegradeRun } from "../src/api";
|
|
12
14
|
import { BarList, ConfigTable, dateRange, FrontierChart, money, secs, type HumanLine, type Point, type Row } from "./charts";
|
|
13
15
|
import { familyName, familySlot, HARNESS_NAME, HARNESS_ORDER, Marker } from "./encoding";
|
|
14
|
-
import { PAGES, pageOf, pathOf, type PageId } from "./pages";
|
|
16
|
+
import { howItWorksHref, PAGES, pageOf, pathOf, type PageId } from "./pages";
|
|
15
17
|
import { api } from "./api";
|
|
16
18
|
import { ComparePage } from "./compare";
|
|
17
|
-
import {
|
|
19
|
+
import { HowItWorksPage } from "./how-it-works";
|
|
20
|
+
import { TasksPage } from "./tasks";
|
|
21
|
+
import { clearAll, filterTags, menuSections, pick, searchMenu, SECTION_NAMES, withGradingPick, type MenuValue, type SectionId } from "./filter-menu";
|
|
18
22
|
import { runStatus, STATUS_REFRESH_MS, statusText, type RunStatus } from "./run-status";
|
|
23
|
+
import { LevelLegend, LevelTag } from "./levels";
|
|
24
|
+
import { alertText, gradingItem, notPlottedGroups, regradeNote, ungradedItem, type NotPlotted } from "./notices";
|
|
25
|
+
import { HTML_SANDBOX, outputKind, PROSE_SANDBOX, PROSE_TOKENS, proseDoc, sourceLabel, type OutputKind } from "./output-view";
|
|
19
26
|
|
|
20
27
|
/** The authored definitions the page lists: every suite and task (for the scope selector) and the config files. */
|
|
21
28
|
interface Catalog { suites: Suite[]; titles: Record<string, string>; configs: AgentConfig[] }
|
|
@@ -57,7 +64,12 @@ function App() {
|
|
|
57
64
|
}, []);
|
|
58
65
|
useEffect(() => {
|
|
59
66
|
Promise.all([api<Suite[]>("/api/suites"), api<{ id: string; title: string }[]>("/api/tasks"), api<AgentConfig[]>("/api/configs")])
|
|
60
|
-
.then(([suites, tasks, configs]) =>
|
|
67
|
+
.then(([suites, tasks, configs]) => {
|
|
68
|
+
// The titles in the project's order: each suite's tasks in turn, then the tasks in no suite (the Filter menu's Task section).
|
|
69
|
+
const order = [...new Set([...suites.flatMap((s) => s.tasks), ...tasks.map((t) => t.id)])];
|
|
70
|
+
const title = new Map(tasks.map((t) => [t.id, t.title]));
|
|
71
|
+
setCatalog({ suites, titles: Object.fromEntries(order.filter((id) => title.has(id)).map((id) => [id, title.get(id)!])), configs });
|
|
72
|
+
})
|
|
61
73
|
.catch((e: Error) => setCatalogError(e.message));
|
|
62
74
|
}, []);
|
|
63
75
|
// Every change to the query is a history entry; Back and Forward set the query from the URL again.
|
|
@@ -133,6 +145,12 @@ function App() {
|
|
|
133
145
|
setPage(to);
|
|
134
146
|
window.scrollTo(0, 0);
|
|
135
147
|
};
|
|
148
|
+
/** Opens an address inside the dashboard, such as an entry of the Tasks page, as a history entry. */
|
|
149
|
+
const goHref = (href: string) => {
|
|
150
|
+
history.pushState(null, "", href);
|
|
151
|
+
setPage(pageOf(new URL(href, location.href).pathname));
|
|
152
|
+
window.scrollTo(0, 0);
|
|
153
|
+
};
|
|
136
154
|
const hrefOf = (to: PageId) => (to === "results" ? `${pathOf(to)}${search ? `?${search}` : ""}` : pathOf(to));
|
|
137
155
|
|
|
138
156
|
const view = loaded?.view;
|
|
@@ -153,30 +171,44 @@ function App() {
|
|
|
153
171
|
|
|
154
172
|
{page === "runs" ? <RunsPage batches={batches} tick={tick} onNew={() => setNewOpen(true)} />
|
|
155
173
|
: page === "compare" ? <ComparePage tick={tick} titles={catalog?.titles ?? {}} />
|
|
174
|
+
: page === "tasks" ? <TasksPage />
|
|
175
|
+
: page === "how" ? <HowItWorksPage />
|
|
156
176
|
: urlError ? <div className="empty card"><div className="kicker">This link has a filter the dashboard cannot read</div><p className="error" style={{ whiteSpace: "pre-wrap" }}>{urlError}</p><a className="btn" href={location.pathname}>Reset filters</a></div>
|
|
157
177
|
: catalogError ? <div className="empty card"><div className="kicker">Could not load the suites, tasks and configs</div><p className="error" style={{ whiteSpace: "pre-wrap" }}>{catalogError}</p><a className="btn" href={location.href}>Reload</a></div>
|
|
158
178
|
: loadError ? <div className="empty card"><div className="kicker">Could not load the view</div><p className="error" style={{ whiteSpace: "pre-wrap" }}>{loadError}</p></div>
|
|
159
179
|
: !loaded || !catalog ? <div className="empty card"><div className="kicker">Static Fire</div><h1>Loading…</h1></div>
|
|
160
|
-
: <Dashboard loaded={loaded} catalog={catalog} onScope={(scope) => setQuery((q) => withScope(q, scope, catalog, facets))} onTrial={setTrialId} onNew={() => go("runs")}
|
|
161
|
-
bar={<FilterBar query={query} view={loaded.view} facets={facets} titles={catalog.titles}
|
|
162
|
-
|
|
163
|
-
|
|
180
|
+
: <Dashboard loaded={loaded} catalog={catalog} onScope={(scope, suites) => setQuery((q) => withScope(q, scope, suites, catalog, facets))} onTrial={setTrialId} onNew={() => go("runs")} onLink={goHref}
|
|
181
|
+
bar={<FilterBar query={query} view={loaded.view} facets={facets} titles={catalog.titles}
|
|
182
|
+
onChange={(patch) => setQuery((q) => ("scope" in patch ? withScope({ ...q, ...patch }, patch.scope ?? { kind: "index" }, patch.suites, catalog, facets)
|
|
183
|
+
: patch.grading ? gradingPicked({ ...q, ...patch }, loaded.view, catalog, facets) : { ...q, ...patch }))} />}
|
|
184
|
+
notice={(notPlotted) => <NoticeAlert view={loaded.view} scope={loaded.scope} titles={catalog.titles} search={search} regrade={regrade} error={regradeError}
|
|
185
|
+
notPlotted={notPlotted} onRegrade={startRegrade} onVersion={(grading) => setQuery((q) => gradingPicked({ ...q, grading, judge: undefined }, loaded.view, catalog, facets))} onLink={goHref} />} />}
|
|
164
186
|
{page === "results" && trial && view && <TrialDrawer view={view} trial={trial} tick={tick} onClose={() => setTrialId(null)} />}
|
|
165
187
|
{page === "runs" && newOpen && <NewRun onClose={() => setNewOpen(false)} onStarted={() => { setNewOpen(false); setTick((t) => t + 1); }} />}
|
|
166
188
|
</div>
|
|
167
189
|
);
|
|
168
190
|
}
|
|
169
191
|
|
|
192
|
+
/** The query after the page sets a grading pick: on the index, the suites become an explicit pick (web/filter-menu.ts withGradingPick). */
|
|
193
|
+
function gradingPicked(q: ViewQuery, view: View, catalog: Catalog, facets: Facets | null): ViewQuery {
|
|
194
|
+
const taskId = facets?.grading.find((g) => g.setups.some((s) => s.hash.startsWith(q.grading!)))?.taskId;
|
|
195
|
+
return withGradingPick(q, taskId, view.filters.suiteIds ?? [], catalog.suites);
|
|
196
|
+
}
|
|
197
|
+
|
|
170
198
|
/**
|
|
171
|
-
* The query with a new scope
|
|
172
|
-
* still covers that task, because the view refuses
|
|
199
|
+
* The query with a new scope and suite pick (`suites` undefined: nothing picked, the default set). A picked grading
|
|
200
|
+
* version belongs to one task: it is kept only when the new scope still covers that task, because the view refuses
|
|
201
|
+
* a grading version outside its scope. Back on the index with no suite picked, the kept grading pick picks the
|
|
202
|
+
* suite of its task, since the default set has no exception for it (withGradingPick).
|
|
173
203
|
*/
|
|
174
|
-
function withScope(q: ViewQuery, scope: Scope, catalog: Catalog, facets: Facets | null): ViewQuery {
|
|
175
|
-
|
|
204
|
+
function withScope(q: ViewQuery, scope: Scope, suites: string[] | undefined, catalog: Catalog, facets: Facets | null): ViewQuery {
|
|
205
|
+
const next: ViewQuery = { ...q, scope, suites: scope.kind === "index" && suites?.length ? suites : undefined };
|
|
206
|
+
if (!q.grading) return next;
|
|
176
207
|
const taskId = facets?.grading.find((g) => g.setups.some((s) => s.hash.startsWith(q.grading!)))?.taskId;
|
|
177
208
|
const covered = scope.kind === "task" ? [scope.id]
|
|
178
|
-
: catalog.suites.filter((s) =>
|
|
179
|
-
|
|
209
|
+
: catalog.suites.filter((s) => !next.suites || next.suites.includes(s.id)).flatMap((s) => s.tasks);
|
|
210
|
+
if (!taskId || !covered.includes(taskId)) return { ...next, grading: undefined };
|
|
211
|
+
return scope.kind === "index" && !next.suites ? withGradingPick(next, taskId, [], catalog.suites) : next;
|
|
180
212
|
}
|
|
181
213
|
|
|
182
214
|
/**
|
|
@@ -270,22 +302,21 @@ function FilterBar({ query, view, facets, titles, onChange }: {
|
|
|
270
302
|
<button type="button" aria-label={`Remove filter ${t.text}`} title="Remove this filter" onClick={() => onChange(t.remove)}>×</button>
|
|
271
303
|
</span>
|
|
272
304
|
))}
|
|
273
|
-
{tags.length > 0 && <button type="button" className="btn clear-all" onClick={() => onChange(
|
|
305
|
+
{tags.length > 0 && <button type="button" className="btn clear-all" onClick={() => onChange(clearAll(query))}>Clear all</button>}
|
|
274
306
|
</div>
|
|
275
307
|
);
|
|
276
308
|
}
|
|
277
309
|
|
|
278
310
|
/** The task ids a scope covers in this view. */
|
|
279
|
-
const tasksOf = (view: View, sel: Scope
|
|
280
|
-
sel.kind === "task" ? [sel.id] : sel.kind === "suite" ? suites.get(sel.id)?.tasks ?? [] : Object.keys(view.tasks);
|
|
311
|
+
const tasksOf = (view: View, sel: Scope) => (sel.kind === "task" ? [sel.id] : Object.keys(view.tasks));
|
|
281
312
|
|
|
282
313
|
/**
|
|
283
314
|
* What the selected scope says about one config: a table row. Score and cost are null when the scope has
|
|
284
315
|
* none. The chart point is this same row (pointFor), so the chart and the table never disagree.
|
|
285
316
|
*/
|
|
286
|
-
function rowFor(view: View, s: ViewConfigSummary, sel: Scope
|
|
287
|
-
const tasks = tasksOf(view, sel
|
|
288
|
-
const stat = sel.kind === "index" ? s.index :
|
|
317
|
+
function rowFor(view: View, s: ViewConfigSummary, sel: Scope): Row {
|
|
318
|
+
const tasks = tasksOf(view, sel);
|
|
319
|
+
const stat = sel.kind === "index" ? s.index : s.tasks[sel.id]?.score;
|
|
289
320
|
const per = tasks.map((t) => s.tasks[t]).filter((t) => t && t.valid > 0);
|
|
290
321
|
const costs = per.map((t) => t.costUsd).filter((c): c is number => c != null);
|
|
291
322
|
const times = per.map((t) => t.durationMs).filter((x): x is number => x != null);
|
|
@@ -302,38 +333,52 @@ function rowFor(view: View, s: ViewConfigSummary, sel: Scope, suites: Map<string
|
|
|
302
333
|
}
|
|
303
334
|
|
|
304
335
|
/** The chart point of one config: its row, when it has both a score and a cost. */
|
|
305
|
-
function pointFor(view: View, s: ViewConfigSummary, sel: Scope
|
|
306
|
-
const r = rowFor(view, s, sel
|
|
336
|
+
function pointFor(view: View, s: ViewConfigSummary, sel: Scope): Point | null {
|
|
337
|
+
const r = rowFor(view, s, sel);
|
|
307
338
|
return r.score == null || r.cost == null ? null : { ...r, score: r.score, cost: r.cost };
|
|
308
339
|
}
|
|
309
340
|
|
|
310
|
-
/** Why a config with trials in view has no point: a suite it does not cover in full. */
|
|
341
|
+
/** Why a config with trials in view has no point: a suite in view it does not cover in full. */
|
|
311
342
|
function partialText(s: ViewConfigSummary, sel: Scope, suites: Map<string, Suite>): string | null {
|
|
312
|
-
const ids = sel.kind === "
|
|
343
|
+
const ids = sel.kind === "index" ? Object.keys(s.partial) : [];
|
|
313
344
|
const parts = ids.flatMap((id) => {
|
|
314
345
|
const p = s.partial[id];
|
|
315
|
-
return p ? [`${
|
|
346
|
+
return p ? [`${ids.length > 1 ? `${suites.get(id)?.title ?? id} ` : ""}partial (${p.have} of ${p.of} tasks)`] : [];
|
|
316
347
|
});
|
|
317
348
|
return parts.length ? parts.join(", ") : null;
|
|
318
349
|
}
|
|
319
350
|
|
|
320
|
-
|
|
351
|
+
/** A config the chart cannot plot, for the alert's details: the suites in view it has not completed, and its score. */
|
|
352
|
+
function notPlotted(view: View, s: ViewConfigSummary, sel: Scope, suites: Map<string, Suite>): NotPlotted {
|
|
353
|
+
const partial = sel.kind === "index"
|
|
354
|
+
? Object.entries(s.partial).flatMap(([id, p]) => p ? [{ suite: suites.get(id)?.title ?? id, have: p.have, of: p.of }] : [])
|
|
355
|
+
: [];
|
|
356
|
+
return { label: s.label, partial, score: rowFor(view, s, sel).score };
|
|
357
|
+
}
|
|
321
358
|
|
|
322
|
-
function Dashboard({ loaded, catalog, onScope, onTrial, onNew, bar, notice }: {
|
|
359
|
+
function Dashboard({ loaded, catalog, onScope, onTrial, onNew, onLink, bar, notice }: {
|
|
360
|
+
loaded: Loaded; catalog: Catalog; onScope: (s: Scope, suites?: string[]) => void; onTrial: (id: string) => void; onNew: () => void;
|
|
361
|
+
/** Opens an address inside the dashboard: the title's entry on the Tasks page. */
|
|
362
|
+
onLink: (href: string) => void; bar?: ReactNode;
|
|
363
|
+
/** The one alert (#82), given the configs the chart cannot plot (none in the Table, which lists them with a reason). */
|
|
364
|
+
notice?: (notPlotted: NotPlotted[]) => ReactNode;
|
|
365
|
+
}) {
|
|
323
366
|
const { view, scope: sel } = loaded;
|
|
324
367
|
const { trials, summary } = view;
|
|
325
368
|
const suites = new Map(catalog.suites.map((s) => [s.id, s]));
|
|
326
369
|
const taskIds = Object.keys(view.tasks);
|
|
327
|
-
const allTasks = [...new Set(catalog.suites.flatMap((s) => s.tasks))];
|
|
328
370
|
const taskTitle = (id: string) => view.tasks[id]?.def.title ?? catalog.titles[id] ?? id;
|
|
329
371
|
const valid = trials.filter((t) => t.status === "done" && t.grades);
|
|
330
372
|
const starts = valid.map((t) => t.startedAt).filter((x): x is string => !!x).sort();
|
|
331
373
|
|
|
332
|
-
const
|
|
374
|
+
const inView = sel.kind === "index" ? view.filters.suiteIds : [];
|
|
375
|
+
const title = viewTitle(sel, inView, suites, taskTitle);
|
|
376
|
+
const titleLink = titleHref(sel, inView);
|
|
377
|
+
const coverage = sel.kind === "index" ? coverageLine(view.coverage, view.filters.judge.label) : null;
|
|
333
378
|
const agents = summary.configs.filter((s) => s.harness !== "human");
|
|
334
|
-
const points = agents.map((s) => pointFor(view, s, sel
|
|
379
|
+
const points = agents.map((s) => pointFor(view, s, sel)).filter((p): p is Point => p != null);
|
|
335
380
|
const humans: HumanLine[] = summary.configs.filter((s) => s.harness === "human").flatMap((s) => {
|
|
336
|
-
const stat = sel.kind === "index" ? s.index :
|
|
381
|
+
const stat = sel.kind === "index" ? s.index : s.tasks[sel.id]?.score;
|
|
337
382
|
return stat?.mean != null ? [{ s, score: stat.mean }] : [];
|
|
338
383
|
});
|
|
339
384
|
// Every config in view the chart draws neither as a point nor as a human line, agents and humans alike.
|
|
@@ -350,30 +395,35 @@ function Dashboard({ loaded, catalog, onScope, onTrial, onNew, bar, notice }: {
|
|
|
350
395
|
const [shown, setShown] = useState<"chart" | "table">("chart");
|
|
351
396
|
// The table: every config the chart draws (points and human lines, by score), then the ones it cannot draw.
|
|
352
397
|
const tableRows: Row[] = [
|
|
353
|
-
...[...points, ...summary.configs.filter((s) => humans.some((h) => h.s.configId === s.configId)).map((s) => rowFor(view, s, sel
|
|
398
|
+
...[...points, ...summary.configs.filter((s) => humans.some((h) => h.s.configId === s.configId)).map((s) => rowFor(view, s, sel))]
|
|
354
399
|
.sort((a, b) => b.score! - a.score!),
|
|
355
|
-
...unplotted.map((s) => { const r = rowFor(view, s, sel
|
|
400
|
+
...unplotted.map((s) => { const r = rowFor(view, s, sel); return { ...r, reason: partialText(s, sel, suites) ?? (r.score == null ? "no valid score" : "no cost data") }; }),
|
|
356
401
|
];
|
|
357
402
|
|
|
358
403
|
return (
|
|
359
404
|
<>
|
|
360
405
|
<section className="hero">
|
|
361
406
|
<div className="kicker">View</div>
|
|
362
|
-
<h1>{
|
|
407
|
+
<h1>{titleLink ? <a className="title-link" href={titleLink} title={`Read about ${title} on the Tasks page`}
|
|
408
|
+
onClick={(e) => { if (e.button !== 0 || e.metaKey || e.ctrlKey || e.shiftKey || e.altKey) return; e.preventDefault(); onLink(titleLink); }}>{title}</a> : title}</h1>
|
|
363
409
|
<div className="meta">
|
|
364
410
|
<span><b>{taskIds.length}</b> task{taskIds.length === 1 ? "" : "s"} × <b>{summary.configs.length}</b> config{summary.configs.length === 1 ? "" : "s"} · <b>{valid.length}</b> valid trial{valid.length === 1 ? "" : "s"}</span>
|
|
365
411
|
<span>Judge <b>{view.filters.judge.label}</b></span>
|
|
366
412
|
{starts.length > 0 && <span>Ran {dateRange(starts[0]!, starts.at(-1)!)}</span>}
|
|
367
413
|
{one && <span><VersionLabel label={one.label} hash={one.taskHash} /></span>}
|
|
368
414
|
</div>
|
|
369
|
-
{suiteNotes(sel, view.filters.suiteIds, suites).map((n) => <p key={n.title} className="note">{sel.kind !== "suite" && <b>{n.title}: </b>}{n.description}</p>)}
|
|
370
415
|
</section>
|
|
371
416
|
|
|
372
|
-
<div className="chips" role="
|
|
373
|
-
<button className={`chip ${sel.kind === "index" ? "on" : ""}`}
|
|
374
|
-
|
|
375
|
-
{
|
|
417
|
+
<div className="chips" role="group" aria-label="Score shown">
|
|
418
|
+
<button className={`chip ${sel.kind === "index" && !view.coverage?.picked ? "on" : ""}`} aria-pressed={sel.kind === "index" && !view.coverage?.picked}
|
|
419
|
+
title="The best set of suites the data supports" onClick={() => onScope({ kind: "index" })}>Agent Performance Index</button>
|
|
420
|
+
{catalog.suites.map((s) => {
|
|
421
|
+
const on = inView.includes(s.id), next = toggleSuite(sel, inView, s.id, catalog.suites);
|
|
422
|
+
return <button key={s.id} className={`chip ${on ? "on" : ""}`} aria-pressed={on} disabled={!next}
|
|
423
|
+
title={next ? `${on ? "Leave out" : "Add"} ${s.title}` : "At least one suite stays in view"} onClick={() => next && onScope({ kind: "index" }, next)}>{s.title}</button>;
|
|
424
|
+
})}
|
|
376
425
|
</div>
|
|
426
|
+
{coverage && <p className="note coverage">{coverage}</p>}
|
|
377
427
|
{bar}
|
|
378
428
|
|
|
379
429
|
{trials.length === 0 ? (
|
|
@@ -382,7 +432,7 @@ function Dashboard({ loaded, catalog, onScope, onTrial, onNew, bar, notice }: {
|
|
|
382
432
|
<h1>No trials in this view</h1>
|
|
383
433
|
<p>Run agent configs on these tasks and each trial shows here as soon as it is graded.</p>
|
|
384
434
|
<button className="btn btn-primary" onClick={onNew}>New run</button>
|
|
385
|
-
{notice}
|
|
435
|
+
{notice?.([])}
|
|
386
436
|
</div>
|
|
387
437
|
) : <>
|
|
388
438
|
<section className="card">
|
|
@@ -409,8 +459,7 @@ function Dashboard({ loaded, catalog, onScope, onTrial, onNew, bar, notice }: {
|
|
|
409
459
|
</div>
|
|
410
460
|
<FrontierChart points={points} yTitle={title} humans={humans} onPick={pickConfig} />
|
|
411
461
|
</> : <ConfigTable rows={tableRows} onPick={pickConfig} />}
|
|
412
|
-
{notice}
|
|
413
|
-
{shown === "chart" && unplotted.length > 0 && <p className="note">Not plotted (no valid score for every task in this view, or no cost data): {unplotted.map((s) => { const why = partialText(s, sel, suites); return why ? `${s.label}: ${why}` : s.label; }).join("; ")}.</p>}
|
|
462
|
+
{notice?.(shown === "chart" ? unplotted.map((s) => notPlotted(view, s, sel, suites)) : [])}
|
|
414
463
|
<p className="note">Costs are API-equivalent. Claude Code and Pi report their own; Codex and Antigravity are estimated from tokens × <code>pricing.yaml</code>. On a flat subscription you pay in usage limits, not dollars.</p>
|
|
415
464
|
</section>
|
|
416
465
|
|
|
@@ -418,22 +467,22 @@ function Dashboard({ loaded, catalog, onScope, onTrial, onNew, bar, notice }: {
|
|
|
418
467
|
<section className="card">
|
|
419
468
|
<h3>{title}</h3>
|
|
420
469
|
<p className="sub">Mean of valid trials · higher is better</p>
|
|
421
|
-
<BarList rows={summary.configs.map((s) => ({ s, v: s.harness === "human" ? humans.find((h) => h.s.configId === s.configId)?.score ?? null : pointFor(view, s, sel
|
|
470
|
+
<BarList rows={summary.configs.map((s) => ({ s, v: s.harness === "human" ? humans.find((h) => h.s.configId === s.configId)?.score ?? null : pointFor(view, s, sel)?.score ?? null }))} fmt={(v) => v.toFixed(1)} />
|
|
422
471
|
</section>
|
|
423
472
|
<section className="card">
|
|
424
473
|
<h3>Cost per task</h3>
|
|
425
474
|
<p className="sub">API-equivalent USD · lower is better</p>
|
|
426
|
-
<BarList rows={agents.map((s) => ({ s, v: pointFor(view, s, sel
|
|
475
|
+
<BarList rows={agents.map((s) => ({ s, v: pointFor(view, s, sel)?.cost ?? null }))} fmt={money} lowerIsBetter />
|
|
427
476
|
</section>
|
|
428
477
|
<section className="card">
|
|
429
478
|
<h3>Time per task</h3>
|
|
430
479
|
<p className="sub">Agent wall-clock · lower is better</p>
|
|
431
|
-
<BarList rows={agents.map((s) => ({ s, v: pointFor(view, s, sel
|
|
480
|
+
<BarList rows={agents.map((s) => ({ s, v: pointFor(view, s, sel)?.durationMs ?? null }))} fmt={secs} lowerIsBetter />
|
|
432
481
|
</section>
|
|
433
482
|
</div>
|
|
434
483
|
|
|
435
484
|
{taskIds.length > 1 && <Heatmap view={view} suites={suites} />}
|
|
436
|
-
|
|
485
|
+
<TaskBreakdown view={view} configs={catalog.configs} />
|
|
437
486
|
<TrialsCard view={view} onTrial={onTrial} />
|
|
438
487
|
</>}
|
|
439
488
|
</>
|
|
@@ -447,17 +496,33 @@ const trialsFrom = (c: { trials: number; configs: number }) =>
|
|
|
447
496
|
`${c.trials} trial${c.trials === 1 ? "" : "s"} from ${c.configs} config${c.configs === 1 ? "" : "s"}`;
|
|
448
497
|
|
|
449
498
|
/**
|
|
450
|
-
*
|
|
451
|
-
*
|
|
452
|
-
*
|
|
453
|
-
*
|
|
499
|
+
* The one alert in the chart card (#82, DEC-023): a button with a short count of what the view does not show, and
|
|
500
|
+
* a click that opens the details under it. The details float over the page (like the Filter menu), so the card
|
|
501
|
+
* is one line high whatever the number of notices. Each item says what is not shown and why, then what to do:
|
|
502
|
+
* - trials graded under another grading hash, or never graded: they come back with a regrade (judge calls only;
|
|
503
|
+
* POST /api/regrade with the view's query). One Regrade button covers both.
|
|
504
|
+
* - trials on another task hash did another job: the link switches the view to their version by setting the
|
|
505
|
+
* grading version filter, and reads by that grading version's name.
|
|
506
|
+
* - configs the chart cannot plot, grouped by reason.
|
|
507
|
+
* Escape or a click outside closes the details. Renders nothing when there is nothing to report.
|
|
454
508
|
*/
|
|
455
|
-
function
|
|
509
|
+
function NoticeAlert({ view, scope, titles, search, regrade, error, notPlotted, onRegrade, onVersion, onLink }: {
|
|
456
510
|
view: View; scope: Scope; titles: Record<string, string>; search: string; regrade: RegradeStatus; error: string;
|
|
457
|
-
onRegrade: () => void; onVersion: (gradingHash: string) => void;
|
|
511
|
+
notPlotted: NotPlotted[]; onRegrade: () => void; onVersion: (gradingHash: string) => void; onLink: (href: string) => void;
|
|
458
512
|
}) {
|
|
459
|
-
const
|
|
460
|
-
|
|
513
|
+
const [open, setOpen] = useState(false);
|
|
514
|
+
const root = useRef<HTMLDivElement>(null), button = useRef<HTMLButtonElement>(null);
|
|
515
|
+
useEffect(() => {
|
|
516
|
+
if (!open) return;
|
|
517
|
+
const away = (e: MouseEvent) => { if (!root.current?.contains(e.target as Node)) setOpen(false); };
|
|
518
|
+
const esc = (e: KeyboardEvent) => { if (e.key === "Escape") { setOpen(false); button.current?.focus(); } };
|
|
519
|
+
document.addEventListener("mousedown", away);
|
|
520
|
+
document.addEventListener("keydown", esc);
|
|
521
|
+
return () => { document.removeEventListener("mousedown", away); document.removeEventListener("keydown", esc); };
|
|
522
|
+
}, [open]);
|
|
523
|
+
const { otherGrading, ungraded, otherTask, gradable } = view.leftOut;
|
|
524
|
+
const text = alertText(otherGrading.trials + ungraded.trials + otherTask.trials, notPlotted.length);
|
|
525
|
+
if (!text) return null;
|
|
461
526
|
const current = Object.values(view.tasks).every((t) => t.current);
|
|
462
527
|
const mine = regrade.running?.query === search;
|
|
463
528
|
const lastError = !regrade.running && regrade.last?.error && regrade.last.query === search ? regrade.last.error : "";
|
|
@@ -465,31 +530,58 @@ function LeftOutNotice({ view, scope, titles, search, regrade, error, onRegrade,
|
|
|
465
530
|
const setupFor = (v: OtherVersion) => v.gradingSetups.find((g) => g.hash === v.link);
|
|
466
531
|
const byTask = new Map<string, typeof otherTask.versions>();
|
|
467
532
|
for (const v of otherTask.versions) byTask.set(v.taskId, [...(byTask.get(v.taskId) ?? []), v]);
|
|
468
|
-
const
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
533
|
+
const groups = notPlottedGroups(notPlotted);
|
|
534
|
+
const grading = otherGrading.trials > 0 && gradingItem(otherGrading, { current, ungraded: ungraded.trials });
|
|
535
|
+
const notGraded = ungraded.trials > 0 && ungradedItem(ungraded, { current, otherGrading: otherGrading.trials });
|
|
536
|
+
const regradeBlock = current && (grading || notGraded) && (
|
|
537
|
+
<li className="notice-action">
|
|
538
|
+
<button className="btn btn-quiet" disabled={!!regrade.running} onClick={onRegrade}>
|
|
539
|
+
{mine ? `Regrading… ${regrade.running!.graded} of ${regrade.running!.total}` : regrade.running ? "Another regrade is running" : "Regrade"}
|
|
540
|
+
</button>
|
|
541
|
+
<p className="note">{regradeNote(gradable.length, !!view.coverage?.leftOut.length)}</p>
|
|
542
|
+
{(error || lastError) && <p className="error">The regrade did not finish: {error || lastError}</p>}
|
|
543
|
+
</li>
|
|
472
544
|
);
|
|
545
|
+
const how = howItWorksHref("left-out");
|
|
473
546
|
return (
|
|
474
|
-
<div className="
|
|
475
|
-
{
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
{
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
547
|
+
<div className="notice pop" ref={root}>
|
|
548
|
+
{/* A live region holds only the count, so a change is announced and opening the details is not. */}
|
|
549
|
+
<div role="status"><button ref={button} type="button" className={`notice-btn${open ? " on" : ""}`} aria-expanded={open} aria-controls="notice-details" onClick={() => setOpen(!open)}>
|
|
550
|
+
<svg width="14" height="14" viewBox="0 0 16 16" aria-hidden="true"><path d="M8 1.5 15 14H1z" fill="none" stroke="currentColor" strokeWidth="1.8" strokeLinejoin="round" /><path d="M8 6v4M8 11.8v.4" stroke="currentColor" strokeWidth="1.8" strokeLinecap="round" /></svg>
|
|
551
|
+
<span>{text}</span>
|
|
552
|
+
<span className="notice-more">{open ? "Hide details" : "Details"}</span>
|
|
553
|
+
</button></div>
|
|
554
|
+
{open && (
|
|
555
|
+
<div className="notice-details" id="notice-details" role="region" aria-label="What is not shown">
|
|
556
|
+
<ul>
|
|
557
|
+
{grading && <li><b>{grading.what}</b> {grading.todo}</li>}
|
|
558
|
+
{notGraded && <li><b>{notGraded.what}</b> {notGraded.todo}</li>}
|
|
559
|
+
{regradeBlock}
|
|
560
|
+
{[...byTask].map(([taskId, versions]) => {
|
|
561
|
+
const n = versions.reduce((a, v) => a + v.trials, 0);
|
|
562
|
+
const which = view.tasks[taskId]?.current ? "an earlier" : "another";
|
|
563
|
+
const what = scope.kind === "task" ? "this task" : titles[taskId] ?? taskId;
|
|
564
|
+
const links = versions.flatMap((v) => { const g = setupFor(v); return g ? [{ v, g }] : []; });
|
|
565
|
+
const tip = (v: OtherVersion, hash: string) => `Task hash ${v.taskHash} · grading hash ${hash}`;
|
|
566
|
+
return (
|
|
567
|
+
<li key={taskId}>
|
|
568
|
+
<b>{n} trial{n === 1 ? "" : "s"} ran on {which} version of {what}.</b> {n === 1 ? "It" : "They"} did a different job, so a regrade cannot bring {n === 1 ? "it" : "them"} in.{" "}
|
|
569
|
+
{links.length === 1 ? <button className="linkish" title={tip(links[0]!.v, links[0]!.g.hash)} onClick={() => onVersion(links[0]!.g.hash)}>Switch to that version: {links[0]!.g.name}</button>
|
|
570
|
+
: links.map(({ v, g }, i) => <span key={v.taskHash}>{i > 0 ? " · " : "Switch to: "}<button className="linkish" title={tip(v, g.hash)} onClick={() => onVersion(g.hash)}>{g.name} · {v.label != null ? `v${v.label}` : `task ${short(v.taskHash)}`} ({v.trials} trial{v.trials === 1 ? "" : "s"})</button></span>)}
|
|
571
|
+
</li>
|
|
572
|
+
);
|
|
573
|
+
})}
|
|
574
|
+
{groups.length > 0 && (
|
|
575
|
+
<li>
|
|
576
|
+
<b>{notPlotted.length} config{notPlotted.length === 1 ? " is" : "s are"} not on the chart.</b> A point needs a valid score on every task in view and a cost.
|
|
577
|
+
<ul className="notice-groups">{groups.map((g) => <li key={g.why}>{g.why}: {g.configs.join(", ")}. <span className="muted">{g.todo}</span></li>)}</ul>
|
|
578
|
+
The Table lists {notPlotted.length === 1 ? "it" : "them"} with the reason.
|
|
579
|
+
</li>
|
|
580
|
+
)}
|
|
581
|
+
</ul>
|
|
582
|
+
<a className="notice-how" href={how} onClick={(e) => { if (e.button !== 0 || e.metaKey || e.ctrlKey || e.shiftKey || e.altKey) return; e.preventDefault(); onLink(how); }}>Why trials are left out (How it works)</a>
|
|
583
|
+
</div>
|
|
584
|
+
)}
|
|
493
585
|
</div>
|
|
494
586
|
);
|
|
495
587
|
}
|
|
@@ -510,11 +602,11 @@ function Heatmap({ view, suites }: { view: View; suites: Map<string, Suite> }) {
|
|
|
510
602
|
return (
|
|
511
603
|
<section className="card">
|
|
512
604
|
<h2>Where each config is strong</h2>
|
|
513
|
-
<p className="sub">Score per task, Suite Index and
|
|
605
|
+
<p className="sub">Score per task, Suite Index and Agent Performance Index (0–100). A blank cell means no valid trials; a config without a score on every task of a suite is partial there.</p>
|
|
514
606
|
<div className="table-wrap"><table>
|
|
515
607
|
<thead><tr><th>Configuration</th>
|
|
516
608
|
{suiteIds.flatMap((sid) => [...suites.get(sid)!.tasks.map((t) => <th key={`${sid}-${t}`}>{title(t)}</th>), <th key={sid} className="strong">{suites.get(sid)!.title} Index</th>])}
|
|
517
|
-
<th className="strong">
|
|
609
|
+
<th className="strong">Agent Performance Index</th></tr></thead>
|
|
518
610
|
<tbody>{sortedConfigs(view, (s) => s.index.mean).map((s) => (
|
|
519
611
|
<tr key={s.configId}>
|
|
520
612
|
<td className="cfg"><Marker s={s} />{s.label}<ConditionsNote c={s.conditions} /></td>
|
|
@@ -533,21 +625,38 @@ function Heatmap({ view, suites }: { view: View; suites: Map<string, Suite> }) {
|
|
|
533
625
|
|
|
534
626
|
type OtherVersion = View["leftOut"]["otherTask"]["versions"][number];
|
|
535
627
|
|
|
536
|
-
|
|
628
|
+
/**
|
|
629
|
+
* A menu of the tasks in view, for a per-task card (#81): the card shows one task at a time. With one task in
|
|
630
|
+
* view it has that one task.
|
|
631
|
+
*/
|
|
632
|
+
function TaskMenu({ view, value, onPick }: { view: View; value: string; onPick: (id: string) => void }) {
|
|
633
|
+
return (
|
|
634
|
+
<select className="task-menu" aria-label="Task" value={value} onChange={(e) => onPick(e.target.value)}>
|
|
635
|
+
{Object.keys(view.tasks).map((id) => <option key={id} value={id}>{view.tasks[id]!.def.title}</option>)}
|
|
636
|
+
</select>
|
|
637
|
+
);
|
|
638
|
+
}
|
|
537
639
|
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
const
|
|
640
|
+
/** "Where the points come from": one card with a menu of the tasks in view, opening on the first (DEC-022). */
|
|
641
|
+
function TaskBreakdown({ view, configs }: { view: View; configs: AgentConfig[] }) {
|
|
642
|
+
const [picked, setPicked] = useState<string | null>(null);
|
|
541
643
|
const [open, setOpen] = useState(false);
|
|
644
|
+
const taskId = shownTask(Object.keys(view.tasks), picked);
|
|
645
|
+
if (!taskId) return null;
|
|
646
|
+
const vt = view.tasks[taskId]!;
|
|
647
|
+
const task = vt.def;
|
|
542
648
|
const inView = new Set(view.summary.configs.map((s) => s.configKey));
|
|
543
649
|
// A human config's note on its file, from the config file (its config key is human:<id>).
|
|
544
650
|
const notes = configs.flatMap((c) => (c.harness === "human" && inView.has(`human:${c.id}`) && c.outputs?.[taskId]?.note ? [{ id: c.id, label: configLabel(c), note: c.outputs[taskId].note! }] : []));
|
|
545
651
|
const rows = sortedConfigs(view, (s) => s.tasks[taskId]?.score.mean ?? null).filter((s) => s.tasks[taskId]?.valid > 0);
|
|
546
652
|
const dead = new Set(view.summary.deadCriteria[taskId] ?? []);
|
|
547
653
|
return (
|
|
548
|
-
<section className="card">
|
|
549
|
-
<div className="
|
|
550
|
-
|
|
654
|
+
<section className="card points-card">
|
|
655
|
+
<div className="card-head">
|
|
656
|
+
<h2>Where the points come from</h2>
|
|
657
|
+
<TaskMenu view={view} value={taskId} onPick={setPicked} />
|
|
658
|
+
</div>
|
|
659
|
+
<div className="kicker"><VersionLabel label={vt.label} hash={vt.taskHash} /></div>
|
|
551
660
|
<p className="sub">Measure score = % of its criteria passed (judge criteria count their share of passing votes). The task score is the mean of the measures.</p>
|
|
552
661
|
<div className="table-wrap"><table>
|
|
553
662
|
<thead><tr><th>Configuration</th>{task.measures.map((m) => <th key={m.id}>{m.label}</th>)}<th className="strong">Task score</th><th>Valid</th></tr></thead>
|
|
@@ -577,7 +686,7 @@ function TaskBreakdown({ view, taskId, configs }: { view: View; taskId: string;
|
|
|
577
686
|
...m.criteria.map((c) => (
|
|
578
687
|
<tr key={c.id}>
|
|
579
688
|
<td className="q" title={c.question}>{c.id}{dead.has(c.id) && <span className="tag warn" title="Every config passed or every config failed: no signal">dead</span>}</td>
|
|
580
|
-
<td><
|
|
689
|
+
<td><LevelTag level={c.level} /></td>
|
|
581
690
|
<td className="muted">{c.grader}</td>
|
|
582
691
|
{rows.map((s) => { const r = s.tasks[taskId].criteria[c.id]; return <td key={s.configId} style={heat(r == null ? null : r * 100)}>{r == null ? "—" : Math.round(r * 100)}</td>; })}
|
|
583
692
|
</tr>
|
|
@@ -586,22 +695,27 @@ function TaskBreakdown({ view, taskId, configs }: { view: View; taskId: string;
|
|
|
586
695
|
</tbody>
|
|
587
696
|
</table></div>
|
|
588
697
|
)}
|
|
698
|
+
{open && <LevelLegend />}
|
|
589
699
|
</section>
|
|
590
700
|
);
|
|
591
701
|
}
|
|
592
702
|
|
|
593
703
|
const STATUS_TEXT: Record<Trial["status"], string> = { queued: "Queued", running: "Working…", grading: "Grading…", done: "", invalid: "Invalid", failed: "Failed" };
|
|
594
704
|
|
|
705
|
+
/** "Trials": one card with a menu of the tasks in view, opening on the first (DEC-022). */
|
|
595
706
|
function TrialsCard({ view, onTrial }: { view: View; onTrial: (id: string) => void }) {
|
|
596
707
|
const { trials } = view;
|
|
597
|
-
const
|
|
708
|
+
const [picked, setPicked] = useState<string | null>(null);
|
|
709
|
+
const tid = shownTask(Object.keys(view.tasks), picked);
|
|
598
710
|
return (
|
|
599
|
-
<section className="card">
|
|
600
|
-
<
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
711
|
+
<section className="card trials-card">
|
|
712
|
+
<div className="card-head">
|
|
713
|
+
<h2>Trials</h2>
|
|
714
|
+
{tid && <TaskMenu view={view} value={tid} onPick={setPicked} />}
|
|
715
|
+
</div>
|
|
716
|
+
<p className="sub">Every trial of this task in the view, from every batch. Click a finished trial to read each criterion's verdict. Invalid = a grader error (re-grade it); failed = the harness could not run.</p>
|
|
717
|
+
{tid && (
|
|
718
|
+
<div className="trial-task">
|
|
605
719
|
<div className="trial-rows">
|
|
606
720
|
{view.summary.configs.filter((s) => trials.some((t) => t.configKey === s.configKey && t.taskId === tid)).map((s) => (
|
|
607
721
|
<div className="trial-row" key={s.configKey}>
|
|
@@ -619,7 +733,7 @@ function TrialsCard({ view, onTrial }: { view: View; onTrial: (id: string) => vo
|
|
|
619
733
|
))}
|
|
620
734
|
</div>
|
|
621
735
|
</div>
|
|
622
|
-
)
|
|
736
|
+
)}
|
|
623
737
|
</section>
|
|
624
738
|
);
|
|
625
739
|
}
|
|
@@ -680,32 +794,52 @@ function GradingList({ gradings, counted }: { gradings: TrialDetail["gradings"];
|
|
|
680
794
|
);
|
|
681
795
|
}
|
|
682
796
|
|
|
797
|
+
type DrawerTab = "output" | "verdicts" | "run";
|
|
798
|
+
const DRAWER_TABS: { id: DrawerTab; label: string }[] = [{ id: "output", label: "Output" }, { id: "verdicts", label: "Verdicts" }, { id: "run", label: "Run details" }];
|
|
799
|
+
|
|
800
|
+
/** The dashboard's token values the Markdown frame's styles use (web/output-view.ts), read from this page. */
|
|
801
|
+
const proseTokens = () => {
|
|
802
|
+
const css = getComputedStyle(document.documentElement);
|
|
803
|
+
return Object.fromEntries(PROSE_TOKENS.map((k) => [k, css.getPropertyValue(k).trim()]));
|
|
804
|
+
};
|
|
805
|
+
|
|
683
806
|
function TrialDrawer({ view, trial, tick, onClose }: { view: View; trial: ViewTrial; tick: number; onClose: () => void }) {
|
|
684
|
-
const
|
|
807
|
+
const s = view.summary.configs.find((x) => x.configId === trial.configId)!;
|
|
808
|
+
const config = s;
|
|
809
|
+
const task = view.tasks[trial.taskId].def;
|
|
810
|
+
const [text, setText] = useState("");
|
|
811
|
+
const [kind, setKind] = useState<OutputKind | null>(null); // null until the output has loaded
|
|
812
|
+
const [missing, setMissing] = useState(""); // why there is no output to show
|
|
685
813
|
const [image, setImage] = useState<string | null>(null); // an object URL when the output is an image
|
|
686
814
|
const [detail, setDetail] = useState<TrialDetail | null>(null);
|
|
687
815
|
const [detailError, setDetailError] = useState("");
|
|
688
|
-
|
|
689
|
-
|
|
816
|
+
const [tab, setTab] = useState<DrawerTab>("output");
|
|
817
|
+
const [view2, setView2] = useState<"rendered" | "source">("rendered");
|
|
818
|
+
// The output never changes once the trial has ended: loaded once per trial. How it shows comes from the server's
|
|
819
|
+
// media type (an image) and the output path it sends (web/output-view.ts). It never goes into this page itself.
|
|
690
820
|
useEffect(() => {
|
|
691
821
|
let live = true, objectUrl: string | null = null;
|
|
692
|
-
|
|
822
|
+
setText("");
|
|
693
823
|
setImage(null);
|
|
824
|
+
setKind(null);
|
|
825
|
+
setMissing("");
|
|
694
826
|
fetch(`/api/trials/${encodeURIComponent(trial.id)}/output`)
|
|
695
827
|
.then(async (r) => {
|
|
696
|
-
if (!r.ok) return;
|
|
697
|
-
|
|
828
|
+
if (!r.ok) { const why = await r.text(); if (live) setMissing(why); return; }
|
|
829
|
+
const k = outputKind(decodeURIComponent(r.headers.get("x-output-path") ?? ""), r.headers.get("content-type"));
|
|
830
|
+
if (k === "image") {
|
|
698
831
|
const blob = await r.blob();
|
|
699
|
-
if (live) { objectUrl = URL.createObjectURL(blob); setImage(objectUrl); }
|
|
832
|
+
if (live) { objectUrl = URL.createObjectURL(blob); setImage(objectUrl); setKind(k); }
|
|
700
833
|
} else {
|
|
701
834
|
const t = await r.text();
|
|
702
|
-
if (live)
|
|
835
|
+
if (live) { setText(t); setKind(k); }
|
|
703
836
|
}
|
|
704
837
|
})
|
|
705
838
|
.catch(() => {});
|
|
706
839
|
return () => { live = false; if (objectUrl) URL.revokeObjectURL(objectUrl); };
|
|
707
840
|
}, [trial.id]);
|
|
708
|
-
|
|
841
|
+
// Another trial opens on its output, rendered.
|
|
842
|
+
useEffect(() => { setDetail(null); setDetailError(""); setTab("output"); setView2("rendered"); }, [trial.id]);
|
|
709
843
|
// Its gradings do change (a regrade adds one): loaded again on each database event.
|
|
710
844
|
useEffect(() => {
|
|
711
845
|
let live = true;
|
|
@@ -714,24 +848,26 @@ function TrialDrawer({ view, trial, tick, onClose }: { view: View; trial: ViewTr
|
|
|
714
848
|
.catch((e: Error) => { if (live) setDetailError(e.message); });
|
|
715
849
|
return () => { live = false; };
|
|
716
850
|
}, [trial.id, tick]);
|
|
717
|
-
const [tab, setTab] = useState<"rendered" | "raw">("rendered");
|
|
718
|
-
const s = view.summary.configs.find((x) => x.configId === trial.configId)!;
|
|
719
|
-
const config = s;
|
|
720
|
-
const task = view.tasks[trial.taskId].def;
|
|
721
851
|
useEffect(() => {
|
|
722
852
|
const k = (e: KeyboardEvent) => e.key === "Escape" && onClose();
|
|
723
853
|
window.addEventListener("keydown", k);
|
|
724
854
|
return () => window.removeEventListener("keydown", k);
|
|
725
855
|
}, []);
|
|
726
856
|
const { fields, body } = useMemo(() => {
|
|
727
|
-
const m =
|
|
857
|
+
const m = text.match(/^---\r?\n([\s\S]*?)\r?\n---\r?\n?/);
|
|
728
858
|
const field = (k: string) => m?.[1].match(new RegExp(`^${k}:\\s*["']?(.*?)["']?\\s*$`, "m"))?.[1];
|
|
729
|
-
return { fields: { title: field("title"), description: field("description") }, body: m ?
|
|
730
|
-
}, [
|
|
731
|
-
// Image placeholders (often ``) are
|
|
732
|
-
|
|
859
|
+
return { fields: { title: field("title"), description: field("description") }, body: m ? text.slice(m[0].length) : text };
|
|
860
|
+
}, [text]);
|
|
861
|
+
// Markdown and other text: rendered into a frame that runs no code. Image placeholders (often ``) are
|
|
862
|
+
// shown as labels, not loaded.
|
|
863
|
+
const doc = useMemo(() => {
|
|
864
|
+
if (kind !== "markdown" && kind !== "text") return "";
|
|
865
|
+
const html = marked.parse(body.replace(/!\[([^\]]*)\]\([^)]*\)/g, (_, alt) => `<span class="img-ph">Image: ${alt || "untitled"}</span>`), { async: false }) as string;
|
|
866
|
+
return proseDoc(html, proseTokens());
|
|
867
|
+
}, [body, kind]);
|
|
733
868
|
const g = trial.grades;
|
|
734
869
|
const groups = [{ id: "gates", label: "Gates", criteria: [{ id: "output-exists", question: "Did the agent write the output file?", level: "floor", grader: "check" } as Criterion, ...task.gates] }, ...task.measures];
|
|
870
|
+
const frameTitle = `Output of ${s.label} for ${task.title}, trial ${trial.index}`;
|
|
735
871
|
|
|
736
872
|
return (
|
|
737
873
|
<>
|
|
@@ -752,27 +888,30 @@ function TrialDrawer({ view, trial, tick, onClose }: { view: View; trial: ViewTr
|
|
|
752
888
|
</div>
|
|
753
889
|
<button className="close" onClick={onClose} aria-label="Close">×</button>
|
|
754
890
|
</div>
|
|
755
|
-
<div className="drawer-
|
|
756
|
-
|
|
757
|
-
{
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
|
|
891
|
+
<div className="drawer-tabs" role="tablist" aria-label="Trial detail">
|
|
892
|
+
{DRAWER_TABS.map((t) => (
|
|
893
|
+
<button key={t.id} role="tab" id={`drawer-tab-${t.id}`} aria-controls="drawer-panel" aria-selected={tab === t.id} className={`drawer-tab ${tab === t.id ? "on" : ""}`} onClick={() => setTab(t.id)}>{t.label}</button>
|
|
894
|
+
))}
|
|
895
|
+
</div>
|
|
896
|
+
<div className={`drawer-panel ${tab}`} id="drawer-panel" role="tabpanel" aria-labelledby={`drawer-tab-${tab}`}>
|
|
897
|
+
{tab === "output" && <>
|
|
898
|
+
{trial.error && <p className="error">{trial.error}</p>}
|
|
899
|
+
{kind === null ? <p className="muted">{missing || "Loading…"}</p>
|
|
900
|
+
: kind === "image" ? (image && <img className="output-image" src={image} alt={frameTitle} />) : <>
|
|
761
901
|
<div className="tabs">
|
|
762
|
-
<button className={`tab ${
|
|
763
|
-
<button className={`tab ${
|
|
902
|
+
<button className={`tab ${view2 === "rendered" ? "on" : ""}`} aria-pressed={view2 === "rendered"} onClick={() => setView2("rendered")}>Rendered</button>
|
|
903
|
+
<button className={`tab ${view2 === "source" ? "on" : ""}`} aria-pressed={view2 === "source"} onClick={() => setView2("source")}>{sourceLabel(kind)}</button>
|
|
764
904
|
</div>
|
|
765
|
-
{
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
|
|
905
|
+
{view2 === "source" ? <pre className="raw">{text}</pre>
|
|
906
|
+
: kind === "html" ? <iframe className="output-frame page" title={frameTitle} sandbox={HTML_SANDBOX} referrerPolicy="no-referrer" srcDoc={text} />
|
|
907
|
+
: <>
|
|
908
|
+
{(fields.title || fields.description) && <div className="envelope"><div className="k">Subject</div><div className="subject">{fields.title ?? "—"}</div><div className="k">Preview</div><div>{fields.description ?? "—"}</div></div>}
|
|
909
|
+
<iframe className="output-frame prose-frame" title={frameTitle} sandbox={PROSE_SANDBOX} referrerPolicy="no-referrer" srcDoc={doc} />
|
|
910
|
+
</>}
|
|
770
911
|
</>}
|
|
771
|
-
|
|
772
|
-
|
|
773
|
-
|
|
774
|
-
{detail && <><RunConditions trial={detail.trial} /><GradingList gradings={detail.gradings} counted={trial.gradingId} /></>}
|
|
775
|
-
<h3>Verdicts</h3>
|
|
912
|
+
</>}
|
|
913
|
+
{tab === "verdicts" && <>
|
|
914
|
+
<LevelLegend />
|
|
776
915
|
{groups.map((m) => (
|
|
777
916
|
<div key={m.id} className="verdict-group">
|
|
778
917
|
<div className="crit-head"><b>{m.label}</b>{m.id !== "gates" && g && <span className="crit-score">{pct(g.measures[m.id])}</span>}</div>
|
|
@@ -785,7 +924,7 @@ function TrialDrawer({ view, trial, tick, onClose }: { view: View; trial: ViewTr
|
|
|
785
924
|
<span className={`badge ${state}`}>{state === "err" ? "Error" : state === "split" ? `${v!.votes.filter((x) => x.pass).length}/${v!.votes.length}` : state === "pass" ? "Pass" : "Fail"}</span>
|
|
786
925
|
<span className="q">{c.question}</span>
|
|
787
926
|
</div>
|
|
788
|
-
<div className="crit-meta"><
|
|
927
|
+
<div className="crit-meta"><LevelTag level={c.level} /><span className="muted">{c.grader}</span></div>
|
|
789
928
|
{v?.error && <p className="error">{v.error}</p>}
|
|
790
929
|
{v?.votes.map((x, i) => <p key={i} className="reason"><b>{x.pass ? "Yes" : "No"}.</b> {x.reason}</p>)}
|
|
791
930
|
</div>
|
|
@@ -793,7 +932,11 @@ function TrialDrawer({ view, trial, tick, onClose }: { view: View; trial: ViewTr
|
|
|
793
932
|
})}
|
|
794
933
|
</div>
|
|
795
934
|
))}
|
|
796
|
-
|
|
935
|
+
</>}
|
|
936
|
+
{tab === "run" && <>
|
|
937
|
+
{detailError && <p className="error">{detailError}</p>}
|
|
938
|
+
{detail ? <><RunConditions trial={detail.trial} /><GradingList gradings={detail.gradings} counted={trial.gradingId} /></> : !detailError && <p className="muted">Loading…</p>}
|
|
939
|
+
</>}
|
|
797
940
|
</div>
|
|
798
941
|
</div>
|
|
799
942
|
</>
|