static-fire 0.44.0 → 0.54.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/web/app.tsx CHANGED
@@ -5,17 +5,24 @@ import type { AgentConfig, Criterion, JudgeConfig, Suite, Trial } from "../src/c
5
5
  import type { TrialDetail } from "../src/core/trial-detail";
6
6
  import type { View, ViewConfigSummary, ViewTrial } from "../src/core/view";
7
7
  import { parseViewQuery, viewQueryParams, type Scope, type ViewQuery } from "../src/core/view-query";
8
- import { suiteNotes } from "./suite-notes";
8
+ import { coverageLine } from "../src/core/coverage";
9
+ import { titleHref, toggleSuite, viewTitle } from "./index-switches";
10
+ import { shownTask } from "./task-card";
9
11
  import type { Facets } from "../src/core/facets";
10
12
  import { configLabel } from "../src/core/label";
11
13
  import type { RegradeEnded, RegradeRun } from "../src/api";
12
14
  import { BarList, ConfigTable, dateRange, FrontierChart, money, secs, type HumanLine, type Point, type Row } from "./charts";
13
15
  import { familyName, familySlot, HARNESS_NAME, HARNESS_ORDER, Marker } from "./encoding";
14
- import { PAGES, pageOf, pathOf, type PageId } from "./pages";
16
+ import { howItWorksHref, PAGES, pageOf, pathOf, type PageId } from "./pages";
15
17
  import { api } from "./api";
16
18
  import { ComparePage } from "./compare";
17
- import { CLEAR_ALL, filterTags, menuSections, pick, searchMenu, SECTION_NAMES, type MenuValue, type SectionId } from "./filter-menu";
19
+ import { HowItWorksPage } from "./how-it-works";
20
+ import { TasksPage } from "./tasks";
21
+ import { clearAll, filterTags, menuSections, pick, searchMenu, SECTION_NAMES, withGradingPick, type MenuValue, type SectionId } from "./filter-menu";
18
22
  import { runStatus, STATUS_REFRESH_MS, statusText, type RunStatus } from "./run-status";
23
+ import { LevelLegend, LevelTag } from "./levels";
24
+ import { alertText, gradingItem, notPlottedGroups, regradeNote, ungradedItem, type NotPlotted } from "./notices";
25
+ import { HTML_SANDBOX, outputKind, PROSE_SANDBOX, PROSE_TOKENS, proseDoc, sourceLabel, type OutputKind } from "./output-view";
19
26
 
20
27
  /** The authored definitions the page lists: every suite and task (for the scope selector) and the config files. */
21
28
  interface Catalog { suites: Suite[]; titles: Record<string, string>; configs: AgentConfig[] }
@@ -57,7 +64,12 @@ function App() {
57
64
  }, []);
58
65
  useEffect(() => {
59
66
  Promise.all([api<Suite[]>("/api/suites"), api<{ id: string; title: string }[]>("/api/tasks"), api<AgentConfig[]>("/api/configs")])
60
- .then(([suites, tasks, configs]) => setCatalog({ suites, titles: Object.fromEntries(tasks.map((t) => [t.id, t.title])), configs }))
67
+ .then(([suites, tasks, configs]) => {
68
+ // The titles in the project's order: each suite's tasks in turn, then the tasks in no suite (the Filter menu's Task section).
69
+ const order = [...new Set([...suites.flatMap((s) => s.tasks), ...tasks.map((t) => t.id)])];
70
+ const title = new Map(tasks.map((t) => [t.id, t.title]));
71
+ setCatalog({ suites, titles: Object.fromEntries(order.filter((id) => title.has(id)).map((id) => [id, title.get(id)!])), configs });
72
+ })
61
73
  .catch((e: Error) => setCatalogError(e.message));
62
74
  }, []);
63
75
  // Every change to the query is a history entry; Back and Forward set the query from the URL again.
@@ -133,6 +145,12 @@ function App() {
133
145
  setPage(to);
134
146
  window.scrollTo(0, 0);
135
147
  };
148
+ /** Opens an address inside the dashboard, such as an entry of the Tasks page, as a history entry. */
149
+ const goHref = (href: string) => {
150
+ history.pushState(null, "", href);
151
+ setPage(pageOf(new URL(href, location.href).pathname));
152
+ window.scrollTo(0, 0);
153
+ };
136
154
  const hrefOf = (to: PageId) => (to === "results" ? `${pathOf(to)}${search ? `?${search}` : ""}` : pathOf(to));
137
155
 
138
156
  const view = loaded?.view;
@@ -153,30 +171,44 @@ function App() {
153
171
 
154
172
  {page === "runs" ? <RunsPage batches={batches} tick={tick} onNew={() => setNewOpen(true)} />
155
173
  : page === "compare" ? <ComparePage tick={tick} titles={catalog?.titles ?? {}} />
174
+ : page === "tasks" ? <TasksPage />
175
+ : page === "how" ? <HowItWorksPage />
156
176
  : urlError ? <div className="empty card"><div className="kicker">This link has a filter the dashboard cannot read</div><p className="error" style={{ whiteSpace: "pre-wrap" }}>{urlError}</p><a className="btn" href={location.pathname}>Reset filters</a></div>
157
177
  : catalogError ? <div className="empty card"><div className="kicker">Could not load the suites, tasks and configs</div><p className="error" style={{ whiteSpace: "pre-wrap" }}>{catalogError}</p><a className="btn" href={location.href}>Reload</a></div>
158
178
  : loadError ? <div className="empty card"><div className="kicker">Could not load the view</div><p className="error" style={{ whiteSpace: "pre-wrap" }}>{loadError}</p></div>
159
179
  : !loaded || !catalog ? <div className="empty card"><div className="kicker">Static Fire</div><h1>Loading…</h1></div>
160
- : <Dashboard loaded={loaded} catalog={catalog} onScope={(scope) => setQuery((q) => withScope(q, scope, catalog, facets))} onTrial={setTrialId} onNew={() => go("runs")}
161
- bar={<FilterBar query={query} view={loaded.view} facets={facets} titles={catalog.titles} onChange={(patch) => setQuery((q) => ({ ...q, ...patch }))} />}
162
- notice={<LeftOutNotice view={loaded.view} scope={loaded.scope} titles={catalog.titles} search={search} regrade={regrade} error={regradeError}
163
- onRegrade={startRegrade} onVersion={(grading) => setQuery((q) => ({ ...q, grading, judge: undefined }))} />} />}
180
+ : <Dashboard loaded={loaded} catalog={catalog} onScope={(scope, suites) => setQuery((q) => withScope(q, scope, suites, catalog, facets))} onTrial={setTrialId} onNew={() => go("runs")} onLink={goHref}
181
+ bar={<FilterBar query={query} view={loaded.view} facets={facets} titles={catalog.titles}
182
+ onChange={(patch) => setQuery((q) => ("scope" in patch ? withScope({ ...q, ...patch }, patch.scope ?? { kind: "index" }, patch.suites, catalog, facets)
183
+ : patch.grading ? gradingPicked({ ...q, ...patch }, loaded.view, catalog, facets) : { ...q, ...patch }))} />}
184
+ notice={(notPlotted) => <NoticeAlert view={loaded.view} scope={loaded.scope} titles={catalog.titles} search={search} regrade={regrade} error={regradeError}
185
+ notPlotted={notPlotted} onRegrade={startRegrade} onVersion={(grading) => setQuery((q) => gradingPicked({ ...q, grading, judge: undefined }, loaded.view, catalog, facets))} onLink={goHref} />} />}
164
186
  {page === "results" && trial && view && <TrialDrawer view={view} trial={trial} tick={tick} onClose={() => setTrialId(null)} />}
165
187
  {page === "runs" && newOpen && <NewRun onClose={() => setNewOpen(false)} onStarted={() => { setNewOpen(false); setTick((t) => t + 1); }} />}
166
188
  </div>
167
189
  );
168
190
  }
169
191
 
192
+ /** The query after the page sets a grading pick: on the index, the suites become an explicit pick (web/filter-menu.ts withGradingPick). */
193
+ function gradingPicked(q: ViewQuery, view: View, catalog: Catalog, facets: Facets | null): ViewQuery {
194
+ const taskId = facets?.grading.find((g) => g.setups.some((s) => s.hash.startsWith(q.grading!)))?.taskId;
195
+ return withGradingPick(q, taskId, view.filters.suiteIds ?? [], catalog.suites);
196
+ }
197
+
170
198
  /**
171
- * The query with a new scope. A picked grading version belongs to one task: it is kept only when the new scope
172
- * still covers that task, because the view refuses a grading version outside its scope.
199
+ * The query with a new scope and suite pick (`suites` undefined: nothing picked, the default set). A picked grading
200
+ * version belongs to one task: it is kept only when the new scope still covers that task, because the view refuses
201
+ * a grading version outside its scope. Back on the index with no suite picked, the kept grading pick picks the
202
+ * suite of its task, since the default set has no exception for it (withGradingPick).
173
203
  */
174
- function withScope(q: ViewQuery, scope: Scope, catalog: Catalog, facets: Facets | null): ViewQuery {
175
- if (!q.grading) return { ...q, scope };
204
+ function withScope(q: ViewQuery, scope: Scope, suites: string[] | undefined, catalog: Catalog, facets: Facets | null): ViewQuery {
205
+ const next: ViewQuery = { ...q, scope, suites: scope.kind === "index" && suites?.length ? suites : undefined };
206
+ if (!q.grading) return next;
176
207
  const taskId = facets?.grading.find((g) => g.setups.some((s) => s.hash.startsWith(q.grading!)))?.taskId;
177
208
  const covered = scope.kind === "task" ? [scope.id]
178
- : catalog.suites.filter((s) => scope.kind === "index" || s.id === scope.id).flatMap((s) => s.tasks);
179
- return taskId && covered.includes(taskId) ? { ...q, scope } : { ...q, scope, grading: undefined };
209
+ : catalog.suites.filter((s) => !next.suites || next.suites.includes(s.id)).flatMap((s) => s.tasks);
210
+ if (!taskId || !covered.includes(taskId)) return { ...next, grading: undefined };
211
+ return scope.kind === "index" && !next.suites ? withGradingPick(next, taskId, [], catalog.suites) : next;
180
212
  }
181
213
 
182
214
  /**
@@ -270,22 +302,21 @@ function FilterBar({ query, view, facets, titles, onChange }: {
270
302
  <button type="button" aria-label={`Remove filter ${t.text}`} title="Remove this filter" onClick={() => onChange(t.remove)}>×</button>
271
303
  </span>
272
304
  ))}
273
- {tags.length > 0 && <button type="button" className="btn clear-all" onClick={() => onChange(CLEAR_ALL)}>Clear all</button>}
305
+ {tags.length > 0 && <button type="button" className="btn clear-all" onClick={() => onChange(clearAll(query))}>Clear all</button>}
274
306
  </div>
275
307
  );
276
308
  }
277
309
 
278
310
  /** The task ids a scope covers in this view. */
279
- const tasksOf = (view: View, sel: Scope, suites: Map<string, Suite>) =>
280
- sel.kind === "task" ? [sel.id] : sel.kind === "suite" ? suites.get(sel.id)?.tasks ?? [] : Object.keys(view.tasks);
311
+ const tasksOf = (view: View, sel: Scope) => (sel.kind === "task" ? [sel.id] : Object.keys(view.tasks));
281
312
 
282
313
  /**
283
314
  * What the selected scope says about one config: a table row. Score and cost are null when the scope has
284
315
  * none. The chart point is this same row (pointFor), so the chart and the table never disagree.
285
316
  */
286
- function rowFor(view: View, s: ViewConfigSummary, sel: Scope, suites: Map<string, Suite>): Row {
287
- const tasks = tasksOf(view, sel, suites);
288
- const stat = sel.kind === "index" ? s.index : sel.kind === "suite" ? s.suites[sel.id] : s.tasks[sel.id]?.score;
317
+ function rowFor(view: View, s: ViewConfigSummary, sel: Scope): Row {
318
+ const tasks = tasksOf(view, sel);
319
+ const stat = sel.kind === "index" ? s.index : s.tasks[sel.id]?.score;
289
320
  const per = tasks.map((t) => s.tasks[t]).filter((t) => t && t.valid > 0);
290
321
  const costs = per.map((t) => t.costUsd).filter((c): c is number => c != null);
291
322
  const times = per.map((t) => t.durationMs).filter((x): x is number => x != null);
@@ -302,38 +333,52 @@ function rowFor(view: View, s: ViewConfigSummary, sel: Scope, suites: Map<string
302
333
  }
303
334
 
304
335
  /** The chart point of one config: its row, when it has both a score and a cost. */
305
- function pointFor(view: View, s: ViewConfigSummary, sel: Scope, suites: Map<string, Suite>): Point | null {
306
- const r = rowFor(view, s, sel, suites);
336
+ function pointFor(view: View, s: ViewConfigSummary, sel: Scope): Point | null {
337
+ const r = rowFor(view, s, sel);
307
338
  return r.score == null || r.cost == null ? null : { ...r, score: r.score, cost: r.cost };
308
339
  }
309
340
 
310
- /** Why a config with trials in view has no point: a suite it does not cover in full. */
341
+ /** Why a config with trials in view has no point: a suite in view it does not cover in full. */
311
342
  function partialText(s: ViewConfigSummary, sel: Scope, suites: Map<string, Suite>): string | null {
312
- const ids = sel.kind === "suite" ? [sel.id] : sel.kind === "index" ? Object.keys(s.partial) : [];
343
+ const ids = sel.kind === "index" ? Object.keys(s.partial) : [];
313
344
  const parts = ids.flatMap((id) => {
314
345
  const p = s.partial[id];
315
- return p ? [`${sel.kind === "index" ? `${suites.get(id)?.title ?? id} ` : ""}partial (${p.have} of ${p.of} tasks)`] : [];
346
+ return p ? [`${ids.length > 1 ? `${suites.get(id)?.title ?? id} ` : ""}partial (${p.have} of ${p.of} tasks)`] : [];
316
347
  });
317
348
  return parts.length ? parts.join(", ") : null;
318
349
  }
319
350
 
320
- const sameScope = (a: Scope, b: Scope) => a.kind === b.kind && (a.kind === "index" || (b.kind !== "index" && a.id === b.id));
351
+ /** A config the chart cannot plot, for the alert's details: the suites in view it has not completed, and its score. */
352
+ function notPlotted(view: View, s: ViewConfigSummary, sel: Scope, suites: Map<string, Suite>): NotPlotted {
353
+ const partial = sel.kind === "index"
354
+ ? Object.entries(s.partial).flatMap(([id, p]) => p ? [{ suite: suites.get(id)?.title ?? id, have: p.have, of: p.of }] : [])
355
+ : [];
356
+ return { label: s.label, partial, score: rowFor(view, s, sel).score };
357
+ }
321
358
 
322
- function Dashboard({ loaded, catalog, onScope, onTrial, onNew, bar, notice }: { loaded: Loaded; catalog: Catalog; onScope: (s: Scope) => void; onTrial: (id: string) => void; onNew: () => void; bar?: ReactNode; notice?: ReactNode }) {
359
+ function Dashboard({ loaded, catalog, onScope, onTrial, onNew, onLink, bar, notice }: {
360
+ loaded: Loaded; catalog: Catalog; onScope: (s: Scope, suites?: string[]) => void; onTrial: (id: string) => void; onNew: () => void;
361
+ /** Opens an address inside the dashboard: the title's entry on the Tasks page. */
362
+ onLink: (href: string) => void; bar?: ReactNode;
363
+ /** The one alert (#82), given the configs the chart cannot plot (none in the Table, which lists them with a reason). */
364
+ notice?: (notPlotted: NotPlotted[]) => ReactNode;
365
+ }) {
323
366
  const { view, scope: sel } = loaded;
324
367
  const { trials, summary } = view;
325
368
  const suites = new Map(catalog.suites.map((s) => [s.id, s]));
326
369
  const taskIds = Object.keys(view.tasks);
327
- const allTasks = [...new Set(catalog.suites.flatMap((s) => s.tasks))];
328
370
  const taskTitle = (id: string) => view.tasks[id]?.def.title ?? catalog.titles[id] ?? id;
329
371
  const valid = trials.filter((t) => t.status === "done" && t.grades);
330
372
  const starts = valid.map((t) => t.startedAt).filter((x): x is string => !!x).sort();
331
373
 
332
- const title = sel.kind === "index" ? "Workflow Index" : sel.kind === "suite" ? `${suites.get(sel.id)?.title ?? sel.id} Index` : taskTitle(sel.id);
374
+ const inView = sel.kind === "index" ? view.filters.suiteIds : [];
375
+ const title = viewTitle(sel, inView, suites, taskTitle);
376
+ const titleLink = titleHref(sel, inView);
377
+ const coverage = sel.kind === "index" ? coverageLine(view.coverage, view.filters.judge.label) : null;
333
378
  const agents = summary.configs.filter((s) => s.harness !== "human");
334
- const points = agents.map((s) => pointFor(view, s, sel, suites)).filter((p): p is Point => p != null);
379
+ const points = agents.map((s) => pointFor(view, s, sel)).filter((p): p is Point => p != null);
335
380
  const humans: HumanLine[] = summary.configs.filter((s) => s.harness === "human").flatMap((s) => {
336
- const stat = sel.kind === "index" ? s.index : sel.kind === "suite" ? s.suites[sel.id] : s.tasks[sel.id]?.score;
381
+ const stat = sel.kind === "index" ? s.index : s.tasks[sel.id]?.score;
337
382
  return stat?.mean != null ? [{ s, score: stat.mean }] : [];
338
383
  });
339
384
  // Every config in view the chart draws neither as a point nor as a human line, agents and humans alike.
@@ -350,30 +395,35 @@ function Dashboard({ loaded, catalog, onScope, onTrial, onNew, bar, notice }: {
350
395
  const [shown, setShown] = useState<"chart" | "table">("chart");
351
396
  // The table: every config the chart draws (points and human lines, by score), then the ones it cannot draw.
352
397
  const tableRows: Row[] = [
353
- ...[...points, ...summary.configs.filter((s) => humans.some((h) => h.s.configId === s.configId)).map((s) => rowFor(view, s, sel, suites))]
398
+ ...[...points, ...summary.configs.filter((s) => humans.some((h) => h.s.configId === s.configId)).map((s) => rowFor(view, s, sel))]
354
399
  .sort((a, b) => b.score! - a.score!),
355
- ...unplotted.map((s) => { const r = rowFor(view, s, sel, suites); return { ...r, reason: partialText(s, sel, suites) ?? (r.score == null ? "no valid score" : "no cost data") }; }),
400
+ ...unplotted.map((s) => { const r = rowFor(view, s, sel); return { ...r, reason: partialText(s, sel, suites) ?? (r.score == null ? "no valid score" : "no cost data") }; }),
356
401
  ];
357
402
 
358
403
  return (
359
404
  <>
360
405
  <section className="hero">
361
406
  <div className="kicker">View</div>
362
- <h1>{sel.kind === "task" ? taskTitle(sel.id) : view.filters.suiteIds.map((s) => suites.get(s)?.title ?? s).join(" + ")}</h1>
407
+ <h1>{titleLink ? <a className="title-link" href={titleLink} title={`Read about ${title} on the Tasks page`}
408
+ onClick={(e) => { if (e.button !== 0 || e.metaKey || e.ctrlKey || e.shiftKey || e.altKey) return; e.preventDefault(); onLink(titleLink); }}>{title}</a> : title}</h1>
363
409
  <div className="meta">
364
410
  <span><b>{taskIds.length}</b> task{taskIds.length === 1 ? "" : "s"} × <b>{summary.configs.length}</b> config{summary.configs.length === 1 ? "" : "s"} · <b>{valid.length}</b> valid trial{valid.length === 1 ? "" : "s"}</span>
365
411
  <span>Judge <b>{view.filters.judge.label}</b></span>
366
412
  {starts.length > 0 && <span>Ran {dateRange(starts[0]!, starts.at(-1)!)}</span>}
367
413
  {one && <span><VersionLabel label={one.label} hash={one.taskHash} /></span>}
368
414
  </div>
369
- {suiteNotes(sel, view.filters.suiteIds, suites).map((n) => <p key={n.title} className="note">{sel.kind !== "suite" && <b>{n.title}: </b>}{n.description}</p>)}
370
415
  </section>
371
416
 
372
- <div className="chips" role="tablist" aria-label="Score shown">
373
- <button className={`chip ${sel.kind === "index" ? "on" : ""}`} onClick={() => onScope({ kind: "index" })}>Workflow Index</button>
374
- {catalog.suites.map((s) => <button key={s.id} className={`chip ${sameScope(sel, { kind: "suite", id: s.id }) ? "on" : ""}`} onClick={() => onScope({ kind: "suite", id: s.id })}>{s.title} Index</button>)}
375
- {allTasks.map((id) => <button key={id} className={`chip ${sameScope(sel, { kind: "task", id }) ? "on" : ""}`} onClick={() => onScope({ kind: "task", id })}>{taskTitle(id)}</button>)}
417
+ <div className="chips" role="group" aria-label="Score shown">
418
+ <button className={`chip ${sel.kind === "index" && !view.coverage?.picked ? "on" : ""}`} aria-pressed={sel.kind === "index" && !view.coverage?.picked}
419
+ title="The best set of suites the data supports" onClick={() => onScope({ kind: "index" })}>Agent Performance Index</button>
420
+ {catalog.suites.map((s) => {
421
+ const on = inView.includes(s.id), next = toggleSuite(sel, inView, s.id, catalog.suites);
422
+ return <button key={s.id} className={`chip ${on ? "on" : ""}`} aria-pressed={on} disabled={!next}
423
+ title={next ? `${on ? "Leave out" : "Add"} ${s.title}` : "At least one suite stays in view"} onClick={() => next && onScope({ kind: "index" }, next)}>{s.title}</button>;
424
+ })}
376
425
  </div>
426
+ {coverage && <p className="note coverage">{coverage}</p>}
377
427
  {bar}
378
428
 
379
429
  {trials.length === 0 ? (
@@ -382,7 +432,7 @@ function Dashboard({ loaded, catalog, onScope, onTrial, onNew, bar, notice }: {
382
432
  <h1>No trials in this view</h1>
383
433
  <p>Run agent configs on these tasks and each trial shows here as soon as it is graded.</p>
384
434
  <button className="btn btn-primary" onClick={onNew}>New run</button>
385
- {notice}
435
+ {notice?.([])}
386
436
  </div>
387
437
  ) : <>
388
438
  <section className="card">
@@ -409,8 +459,7 @@ function Dashboard({ loaded, catalog, onScope, onTrial, onNew, bar, notice }: {
409
459
  </div>
410
460
  <FrontierChart points={points} yTitle={title} humans={humans} onPick={pickConfig} />
411
461
  </> : <ConfigTable rows={tableRows} onPick={pickConfig} />}
412
- {notice}
413
- {shown === "chart" && unplotted.length > 0 && <p className="note">Not plotted (no valid score for every task in this view, or no cost data): {unplotted.map((s) => { const why = partialText(s, sel, suites); return why ? `${s.label}: ${why}` : s.label; }).join("; ")}.</p>}
462
+ {notice?.(shown === "chart" ? unplotted.map((s) => notPlotted(view, s, sel, suites)) : [])}
414
463
  <p className="note">Costs are API-equivalent. Claude Code and Pi report their own; Codex and Antigravity are estimated from tokens × <code>pricing.yaml</code>. On a flat subscription you pay in usage limits, not dollars.</p>
415
464
  </section>
416
465
 
@@ -418,22 +467,22 @@ function Dashboard({ loaded, catalog, onScope, onTrial, onNew, bar, notice }: {
418
467
  <section className="card">
419
468
  <h3>{title}</h3>
420
469
  <p className="sub">Mean of valid trials · higher is better</p>
421
- <BarList rows={summary.configs.map((s) => ({ s, v: s.harness === "human" ? humans.find((h) => h.s.configId === s.configId)?.score ?? null : pointFor(view, s, sel, suites)?.score ?? null }))} fmt={(v) => v.toFixed(1)} />
470
+ <BarList rows={summary.configs.map((s) => ({ s, v: s.harness === "human" ? humans.find((h) => h.s.configId === s.configId)?.score ?? null : pointFor(view, s, sel)?.score ?? null }))} fmt={(v) => v.toFixed(1)} />
422
471
  </section>
423
472
  <section className="card">
424
473
  <h3>Cost per task</h3>
425
474
  <p className="sub">API-equivalent USD · lower is better</p>
426
- <BarList rows={agents.map((s) => ({ s, v: pointFor(view, s, sel, suites)?.cost ?? null }))} fmt={money} lowerIsBetter />
475
+ <BarList rows={agents.map((s) => ({ s, v: pointFor(view, s, sel)?.cost ?? null }))} fmt={money} lowerIsBetter />
427
476
  </section>
428
477
  <section className="card">
429
478
  <h3>Time per task</h3>
430
479
  <p className="sub">Agent wall-clock · lower is better</p>
431
- <BarList rows={agents.map((s) => ({ s, v: pointFor(view, s, sel, suites)?.durationMs ?? null }))} fmt={secs} lowerIsBetter />
480
+ <BarList rows={agents.map((s) => ({ s, v: pointFor(view, s, sel)?.durationMs ?? null }))} fmt={secs} lowerIsBetter />
432
481
  </section>
433
482
  </div>
434
483
 
435
484
  {taskIds.length > 1 && <Heatmap view={view} suites={suites} />}
436
- {(sel.kind === "task" ? [sel.id] : taskIds).map((t) => <TaskBreakdown key={t} view={view} taskId={t} configs={catalog.configs} />)}
485
+ <TaskBreakdown view={view} configs={catalog.configs} />
437
486
  <TrialsCard view={view} onTrial={onTrial} />
438
487
  </>}
439
488
  </>
@@ -447,17 +496,33 @@ const trialsFrom = (c: { trials: number; configs: number }) =>
447
496
  `${c.trials} trial${c.trials === 1 ? "" : "s"} from ${c.configs} config${c.configs === 1 ? "" : "s"}`;
448
497
 
449
498
  /**
450
- * Under the chart: the trials in scope the view left out, and the way to bring them in. Trials graded under
451
- * another grading hash, or never graded, come back with a regrade (judge calls only; POST /api/regrade with the
452
- * view's query). Trials on another task hash did another job: the link switches the view to their version by
453
- * setting the grading version filter, and reads by that grading version's name. Renders nothing when no trial was left out.
499
+ * The one alert in the chart card (#82, DEC-023): a button with a short count of what the view does not show, and
500
+ * a click that opens the details under it. The details float over the page (like the Filter menu), so the card
501
+ * is one line high whatever the number of notices. Each item says what is not shown and why, then what to do:
502
+ * - trials graded under another grading hash, or never graded: they come back with a regrade (judge calls only;
503
+ * POST /api/regrade with the view's query). One Regrade button covers both.
504
+ * - trials on another task hash did another job: the link switches the view to their version by setting the
505
+ * grading version filter, and reads by that grading version's name.
506
+ * - configs the chart cannot plot, grouped by reason.
507
+ * Escape or a click outside closes the details. Renders nothing when there is nothing to report.
454
508
  */
455
- function LeftOutNotice({ view, scope, titles, search, regrade, error, onRegrade, onVersion }: {
509
+ function NoticeAlert({ view, scope, titles, search, regrade, error, notPlotted, onRegrade, onVersion, onLink }: {
456
510
  view: View; scope: Scope; titles: Record<string, string>; search: string; regrade: RegradeStatus; error: string;
457
- onRegrade: () => void; onVersion: (gradingHash: string) => void;
511
+ notPlotted: NotPlotted[]; onRegrade: () => void; onVersion: (gradingHash: string) => void; onLink: (href: string) => void;
458
512
  }) {
459
- const { otherGrading, ungraded, otherTask } = view.leftOut;
460
- if (!otherGrading.trials && !ungraded.trials && !otherTask.trials) return null;
513
+ const [open, setOpen] = useState(false);
514
+ const root = useRef<HTMLDivElement>(null), button = useRef<HTMLButtonElement>(null);
515
+ useEffect(() => {
516
+ if (!open) return;
517
+ const away = (e: MouseEvent) => { if (!root.current?.contains(e.target as Node)) setOpen(false); };
518
+ const esc = (e: KeyboardEvent) => { if (e.key === "Escape") { setOpen(false); button.current?.focus(); } };
519
+ document.addEventListener("mousedown", away);
520
+ document.addEventListener("keydown", esc);
521
+ return () => { document.removeEventListener("mousedown", away); document.removeEventListener("keydown", esc); };
522
+ }, [open]);
523
+ const { otherGrading, ungraded, otherTask, gradable } = view.leftOut;
524
+ const text = alertText(otherGrading.trials + ungraded.trials + otherTask.trials, notPlotted.length);
525
+ if (!text) return null;
461
526
  const current = Object.values(view.tasks).every((t) => t.current);
462
527
  const mine = regrade.running?.query === search;
463
528
  const lastError = !regrade.running && regrade.last?.error && regrade.last.query === search ? regrade.last.error : "";
@@ -465,31 +530,58 @@ function LeftOutNotice({ view, scope, titles, search, regrade, error, onRegrade,
465
530
  const setupFor = (v: OtherVersion) => v.gradingSetups.find((g) => g.hash === v.link);
466
531
  const byTask = new Map<string, typeof otherTask.versions>();
467
532
  for (const v of otherTask.versions) byTask.set(v.taskId, [...(byTask.get(v.taskId) ?? []), v]);
468
- const regradeButton = current && (otherGrading.trials > 0 || ungraded.trials > 0) && (
469
- <button className="btn btn-quiet" disabled={!!regrade.running} onClick={onRegrade}>
470
- {mine ? `Regrading… ${regrade.running!.graded} of ${regrade.running!.total}` : regrade.running ? "Another regrade is running" : "Regrade (judge calls only)"}
471
- </button>
533
+ const groups = notPlottedGroups(notPlotted);
534
+ const grading = otherGrading.trials > 0 && gradingItem(otherGrading, { current, ungraded: ungraded.trials });
535
+ const notGraded = ungraded.trials > 0 && ungradedItem(ungraded, { current, otherGrading: otherGrading.trials });
536
+ const regradeBlock = current && (grading || notGraded) && (
537
+ <li className="notice-action">
538
+ <button className="btn btn-quiet" disabled={!!regrade.running} onClick={onRegrade}>
539
+ {mine ? `Regrading… ${regrade.running!.graded} of ${regrade.running!.total}` : regrade.running ? "Another regrade is running" : "Regrade"}
540
+ </button>
541
+ <p className="note">{regradeNote(gradable.length, !!view.coverage?.leftOut.length)}</p>
542
+ {(error || lastError) && <p className="error">The regrade did not finish: {error || lastError}</p>}
543
+ </li>
472
544
  );
545
+ const how = howItWorksHref("left-out");
473
546
  return (
474
- <div className="left-out" role="status">
475
- {otherGrading.trials > 0 && <p className="note">{trialsFrom(otherGrading)} {otherGrading.trials === 1 ? "was" : "were"} graded under {current ? "earlier" : "other"} criteria.</p>}
476
- {ungraded.trials > 0 && <p className="note">{trialsFrom(ungraded)} {ungraded.trials === 1 ? "has" : "have"} no grading.</p>}
477
- {regradeButton}
478
- {(error || lastError) && <p className="error">The regrade did not finish: {error || lastError}</p>}
479
- {[...byTask].map(([taskId, versions]) => {
480
- const n = versions.reduce((a, v) => a + v.trials, 0);
481
- const which = view.tasks[taskId]?.current ? "an earlier" : "another";
482
- const what = scope.kind === "task" ? "this task" : titles[taskId] ?? taskId;
483
- const links = versions.flatMap((v) => { const g = setupFor(v); return g ? [{ v, g }] : []; });
484
- const tip = (v: OtherVersion, hash: string) => `Task hash ${v.taskHash} · grading hash ${hash}`;
485
- return (
486
- <p className="note" key={taskId}>
487
- {n} trial{n === 1 ? "" : "s"} ran on {which} version of {what}.{" "}
488
- {links.length === 1 ? <button className="linkish" title={tip(links[0]!.v, links[0]!.g.hash)} onClick={() => onVersion(links[0]!.g.hash)}>Switch to that version: {links[0]!.g.name}</button>
489
- : links.map(({ v, g }, i) => <span key={v.taskHash}>{i > 0 && " · "}<button className="linkish" title={tip(v, g.hash)} onClick={() => onVersion(g.hash)}>{g.name} · {v.label != null ? `v${v.label}` : `task ${short(v.taskHash)}`} ({v.trials} trial{v.trials === 1 ? "" : "s"})</button></span>)}
490
- </p>
491
- );
492
- })}
547
+ <div className="notice pop" ref={root}>
548
+ {/* A live region holds only the count, so a change is announced and opening the details is not. */}
549
+ <div role="status"><button ref={button} type="button" className={`notice-btn${open ? " on" : ""}`} aria-expanded={open} aria-controls="notice-details" onClick={() => setOpen(!open)}>
550
+ <svg width="14" height="14" viewBox="0 0 16 16" aria-hidden="true"><path d="M8 1.5 15 14H1z" fill="none" stroke="currentColor" strokeWidth="1.8" strokeLinejoin="round" /><path d="M8 6v4M8 11.8v.4" stroke="currentColor" strokeWidth="1.8" strokeLinecap="round" /></svg>
551
+ <span>{text}</span>
552
+ <span className="notice-more">{open ? "Hide details" : "Details"}</span>
553
+ </button></div>
554
+ {open && (
555
+ <div className="notice-details" id="notice-details" role="region" aria-label="What is not shown">
556
+ <ul>
557
+ {grading && <li><b>{grading.what}</b> {grading.todo}</li>}
558
+ {notGraded && <li><b>{notGraded.what}</b> {notGraded.todo}</li>}
559
+ {regradeBlock}
560
+ {[...byTask].map(([taskId, versions]) => {
561
+ const n = versions.reduce((a, v) => a + v.trials, 0);
562
+ const which = view.tasks[taskId]?.current ? "an earlier" : "another";
563
+ const what = scope.kind === "task" ? "this task" : titles[taskId] ?? taskId;
564
+ const links = versions.flatMap((v) => { const g = setupFor(v); return g ? [{ v, g }] : []; });
565
+ const tip = (v: OtherVersion, hash: string) => `Task hash ${v.taskHash} · grading hash ${hash}`;
566
+ return (
567
+ <li key={taskId}>
568
+ <b>{n} trial{n === 1 ? "" : "s"} ran on {which} version of {what}.</b> {n === 1 ? "It" : "They"} did a different job, so a regrade cannot bring {n === 1 ? "it" : "them"} in.{" "}
569
+ {links.length === 1 ? <button className="linkish" title={tip(links[0]!.v, links[0]!.g.hash)} onClick={() => onVersion(links[0]!.g.hash)}>Switch to that version: {links[0]!.g.name}</button>
570
+ : links.map(({ v, g }, i) => <span key={v.taskHash}>{i > 0 ? " · " : "Switch to: "}<button className="linkish" title={tip(v, g.hash)} onClick={() => onVersion(g.hash)}>{g.name} · {v.label != null ? `v${v.label}` : `task ${short(v.taskHash)}`} ({v.trials} trial{v.trials === 1 ? "" : "s"})</button></span>)}
571
+ </li>
572
+ );
573
+ })}
574
+ {groups.length > 0 && (
575
+ <li>
576
+ <b>{notPlotted.length} config{notPlotted.length === 1 ? " is" : "s are"} not on the chart.</b> A point needs a valid score on every task in view and a cost.
577
+ <ul className="notice-groups">{groups.map((g) => <li key={g.why}>{g.why}: {g.configs.join(", ")}. <span className="muted">{g.todo}</span></li>)}</ul>
578
+ The Table lists {notPlotted.length === 1 ? "it" : "them"} with the reason.
579
+ </li>
580
+ )}
581
+ </ul>
582
+ <a className="notice-how" href={how} onClick={(e) => { if (e.button !== 0 || e.metaKey || e.ctrlKey || e.shiftKey || e.altKey) return; e.preventDefault(); onLink(how); }}>Why trials are left out (How it works)</a>
583
+ </div>
584
+ )}
493
585
  </div>
494
586
  );
495
587
  }
@@ -510,11 +602,11 @@ function Heatmap({ view, suites }: { view: View; suites: Map<string, Suite> }) {
510
602
  return (
511
603
  <section className="card">
512
604
  <h2>Where each config is strong</h2>
513
- <p className="sub">Score per task, Suite Index and Workflow Index (0–100). A blank cell means no valid trials; a config without a score on every task of a suite is partial there.</p>
605
+ <p className="sub">Score per task, Suite Index and Agent Performance Index (0–100). A blank cell means no valid trials; a config without a score on every task of a suite is partial there.</p>
514
606
  <div className="table-wrap"><table>
515
607
  <thead><tr><th>Configuration</th>
516
608
  {suiteIds.flatMap((sid) => [...suites.get(sid)!.tasks.map((t) => <th key={`${sid}-${t}`}>{title(t)}</th>), <th key={sid} className="strong">{suites.get(sid)!.title} Index</th>])}
517
- <th className="strong">Workflow Index</th></tr></thead>
609
+ <th className="strong">Agent Performance Index</th></tr></thead>
518
610
  <tbody>{sortedConfigs(view, (s) => s.index.mean).map((s) => (
519
611
  <tr key={s.configId}>
520
612
  <td className="cfg"><Marker s={s} />{s.label}<ConditionsNote c={s.conditions} /></td>
@@ -533,21 +625,38 @@ function Heatmap({ view, suites }: { view: View; suites: Map<string, Suite> }) {
533
625
 
534
626
  type OtherVersion = View["leftOut"]["otherTask"]["versions"][number];
535
627
 
536
- const LEVEL_TEXT: Record<string, string> = { floor: "Floor", strong: "Strong", ceiling: "Ceiling" };
628
+ /**
629
+ * A menu of the tasks in view, for a per-task card (#81): the card shows one task at a time. With one task in
630
+ * view it has that one task.
631
+ */
632
+ function TaskMenu({ view, value, onPick }: { view: View; value: string; onPick: (id: string) => void }) {
633
+ return (
634
+ <select className="task-menu" aria-label="Task" value={value} onChange={(e) => onPick(e.target.value)}>
635
+ {Object.keys(view.tasks).map((id) => <option key={id} value={id}>{view.tasks[id]!.def.title}</option>)}
636
+ </select>
637
+ );
638
+ }
537
639
 
538
- function TaskBreakdown({ view, taskId, configs }: { view: View; taskId: string; configs: AgentConfig[] }) {
539
- const vt = view.tasks[taskId];
540
- const task = vt.def;
640
+ /** "Where the points come from": one card with a menu of the tasks in view, opening on the first (DEC-022). */
641
+ function TaskBreakdown({ view, configs }: { view: View; configs: AgentConfig[] }) {
642
+ const [picked, setPicked] = useState<string | null>(null);
541
643
  const [open, setOpen] = useState(false);
644
+ const taskId = shownTask(Object.keys(view.tasks), picked);
645
+ if (!taskId) return null;
646
+ const vt = view.tasks[taskId]!;
647
+ const task = vt.def;
542
648
  const inView = new Set(view.summary.configs.map((s) => s.configKey));
543
649
  // A human config's note on its file, from the config file (its config key is human:<id>).
544
650
  const notes = configs.flatMap((c) => (c.harness === "human" && inView.has(`human:${c.id}`) && c.outputs?.[taskId]?.note ? [{ id: c.id, label: configLabel(c), note: c.outputs[taskId].note! }] : []));
545
651
  const rows = sortedConfigs(view, (s) => s.tasks[taskId]?.score.mean ?? null).filter((s) => s.tasks[taskId]?.valid > 0);
546
652
  const dead = new Set(view.summary.deadCriteria[taskId] ?? []);
547
653
  return (
548
- <section className="card">
549
- <div className="kicker">{task.title} · <VersionLabel label={vt.label} hash={vt.taskHash} /></div>
550
- <h2>Where the points come from</h2>
654
+ <section className="card points-card">
655
+ <div className="card-head">
656
+ <h2>Where the points come from</h2>
657
+ <TaskMenu view={view} value={taskId} onPick={setPicked} />
658
+ </div>
659
+ <div className="kicker"><VersionLabel label={vt.label} hash={vt.taskHash} /></div>
551
660
  <p className="sub">Measure score = % of its criteria passed (judge criteria count their share of passing votes). The task score is the mean of the measures.</p>
552
661
  <div className="table-wrap"><table>
553
662
  <thead><tr><th>Configuration</th>{task.measures.map((m) => <th key={m.id}>{m.label}</th>)}<th className="strong">Task score</th><th>Valid</th></tr></thead>
@@ -577,7 +686,7 @@ function TaskBreakdown({ view, taskId, configs }: { view: View; taskId: string;
577
686
  ...m.criteria.map((c) => (
578
687
  <tr key={c.id}>
579
688
  <td className="q" title={c.question}>{c.id}{dead.has(c.id) && <span className="tag warn" title="Every config passed or every config failed: no signal">dead</span>}</td>
580
- <td><span className={`tag ${c.level}`}>{LEVEL_TEXT[c.level]}</span></td>
689
+ <td><LevelTag level={c.level} /></td>
581
690
  <td className="muted">{c.grader}</td>
582
691
  {rows.map((s) => { const r = s.tasks[taskId].criteria[c.id]; return <td key={s.configId} style={heat(r == null ? null : r * 100)}>{r == null ? "—" : Math.round(r * 100)}</td>; })}
583
692
  </tr>
@@ -586,22 +695,27 @@ function TaskBreakdown({ view, taskId, configs }: { view: View; taskId: string;
586
695
  </tbody>
587
696
  </table></div>
588
697
  )}
698
+ {open && <LevelLegend />}
589
699
  </section>
590
700
  );
591
701
  }
592
702
 
593
703
  const STATUS_TEXT: Record<Trial["status"], string> = { queued: "Queued", running: "Working…", grading: "Grading…", done: "", invalid: "Invalid", failed: "Failed" };
594
704
 
705
+ /** "Trials": one card with a menu of the tasks in view, opening on the first (DEC-022). */
595
706
  function TrialsCard({ view, onTrial }: { view: View; onTrial: (id: string) => void }) {
596
707
  const { trials } = view;
597
- const taskIds = Object.keys(view.tasks);
708
+ const [picked, setPicked] = useState<string | null>(null);
709
+ const tid = shownTask(Object.keys(view.tasks), picked);
598
710
  return (
599
- <section className="card">
600
- <h2>Trials</h2>
601
- <p className="sub">Every trial in this view, from every batch. Click a finished trial to read each criterion's verdict. Invalid = a grader error (re-grade it); failed = the harness could not run.</p>
602
- {taskIds.map((tid) => (
603
- <div key={tid} className="trial-task">
604
- {taskIds.length > 1 && <div className="kicker">{view.tasks[tid].def.title}</div>}
711
+ <section className="card trials-card">
712
+ <div className="card-head">
713
+ <h2>Trials</h2>
714
+ {tid && <TaskMenu view={view} value={tid} onPick={setPicked} />}
715
+ </div>
716
+ <p className="sub">Every trial of this task in the view, from every batch. Click a finished trial to read each criterion's verdict. Invalid = a grader error (re-grade it); failed = the harness could not run.</p>
717
+ {tid && (
718
+ <div className="trial-task">
605
719
  <div className="trial-rows">
606
720
  {view.summary.configs.filter((s) => trials.some((t) => t.configKey === s.configKey && t.taskId === tid)).map((s) => (
607
721
  <div className="trial-row" key={s.configKey}>
@@ -619,7 +733,7 @@ function TrialsCard({ view, onTrial }: { view: View; onTrial: (id: string) => vo
619
733
  ))}
620
734
  </div>
621
735
  </div>
622
- ))}
736
+ )}
623
737
  </section>
624
738
  );
625
739
  }
@@ -680,32 +794,52 @@ function GradingList({ gradings, counted }: { gradings: TrialDetail["gradings"];
680
794
  );
681
795
  }
682
796
 
797
+ type DrawerTab = "output" | "verdicts" | "run";
798
+ const DRAWER_TABS: { id: DrawerTab; label: string }[] = [{ id: "output", label: "Output" }, { id: "verdicts", label: "Verdicts" }, { id: "run", label: "Run details" }];
799
+
800
+ /** The dashboard's token values the Markdown frame's styles use (web/output-view.ts), read from this page. */
801
+ const proseTokens = () => {
802
+ const css = getComputedStyle(document.documentElement);
803
+ return Object.fromEntries(PROSE_TOKENS.map((k) => [k, css.getPropertyValue(k).trim()]));
804
+ };
805
+
683
806
  function TrialDrawer({ view, trial, tick, onClose }: { view: View; trial: ViewTrial; tick: number; onClose: () => void }) {
684
- const [md, setMd] = useState("");
807
+ const s = view.summary.configs.find((x) => x.configId === trial.configId)!;
808
+ const config = s;
809
+ const task = view.tasks[trial.taskId].def;
810
+ const [text, setText] = useState("");
811
+ const [kind, setKind] = useState<OutputKind | null>(null); // null until the output has loaded
812
+ const [missing, setMissing] = useState(""); // why there is no output to show
685
813
  const [image, setImage] = useState<string | null>(null); // an object URL when the output is an image
686
814
  const [detail, setDetail] = useState<TrialDetail | null>(null);
687
815
  const [detailError, setDetailError] = useState("");
688
- // The output never changes once the trial has ended: loaded once per trial. An image output (the server sends
689
- // its media type) is shown as an image, any other output as Markdown text.
816
+ const [tab, setTab] = useState<DrawerTab>("output");
817
+ const [view2, setView2] = useState<"rendered" | "source">("rendered");
818
+ // The output never changes once the trial has ended: loaded once per trial. How it shows comes from the server's
819
+ // media type (an image) and the output path it sends (web/output-view.ts). It never goes into this page itself.
690
820
  useEffect(() => {
691
821
  let live = true, objectUrl: string | null = null;
692
- setMd("");
822
+ setText("");
693
823
  setImage(null);
824
+ setKind(null);
825
+ setMissing("");
694
826
  fetch(`/api/trials/${encodeURIComponent(trial.id)}/output`)
695
827
  .then(async (r) => {
696
- if (!r.ok) return;
697
- if (r.headers.get("content-type")?.startsWith("image/")) {
828
+ if (!r.ok) { const why = await r.text(); if (live) setMissing(why); return; }
829
+ const k = outputKind(decodeURIComponent(r.headers.get("x-output-path") ?? ""), r.headers.get("content-type"));
830
+ if (k === "image") {
698
831
  const blob = await r.blob();
699
- if (live) { objectUrl = URL.createObjectURL(blob); setImage(objectUrl); }
832
+ if (live) { objectUrl = URL.createObjectURL(blob); setImage(objectUrl); setKind(k); }
700
833
  } else {
701
834
  const t = await r.text();
702
- if (live) setMd(t);
835
+ if (live) { setText(t); setKind(k); }
703
836
  }
704
837
  })
705
838
  .catch(() => {});
706
839
  return () => { live = false; if (objectUrl) URL.revokeObjectURL(objectUrl); };
707
840
  }, [trial.id]);
708
- useEffect(() => { setDetail(null); setDetailError(""); }, [trial.id]);
841
+ // Another trial opens on its output, rendered.
842
+ useEffect(() => { setDetail(null); setDetailError(""); setTab("output"); setView2("rendered"); }, [trial.id]);
709
843
  // Its gradings do change (a regrade adds one): loaded again on each database event.
710
844
  useEffect(() => {
711
845
  let live = true;
@@ -714,24 +848,26 @@ function TrialDrawer({ view, trial, tick, onClose }: { view: View; trial: ViewTr
714
848
  .catch((e: Error) => { if (live) setDetailError(e.message); });
715
849
  return () => { live = false; };
716
850
  }, [trial.id, tick]);
717
- const [tab, setTab] = useState<"rendered" | "raw">("rendered");
718
- const s = view.summary.configs.find((x) => x.configId === trial.configId)!;
719
- const config = s;
720
- const task = view.tasks[trial.taskId].def;
721
851
  useEffect(() => {
722
852
  const k = (e: KeyboardEvent) => e.key === "Escape" && onClose();
723
853
  window.addEventListener("keydown", k);
724
854
  return () => window.removeEventListener("keydown", k);
725
855
  }, []);
726
856
  const { fields, body } = useMemo(() => {
727
- const m = md.match(/^---\r?\n([\s\S]*?)\r?\n---\r?\n?/);
857
+ const m = text.match(/^---\r?\n([\s\S]*?)\r?\n---\r?\n?/);
728
858
  const field = (k: string) => m?.[1].match(new RegExp(`^${k}:\\s*["']?(.*?)["']?\\s*$`, "m"))?.[1];
729
- return { fields: { title: field("title"), description: field("description") }, body: m ? md.slice(m[0].length) : md };
730
- }, [md]);
731
- // Image placeholders (often `![…](TODO)`) are shown as labels, not loaded.
732
- const html = useMemo(() => marked.parse(body.replace(/!\[([^\]]*)\]\([^)]*\)/g, (_, alt) => `<span class="img-ph">Image: ${alt || "untitled"}</span>`), { async: false }) as string, [body]);
859
+ return { fields: { title: field("title"), description: field("description") }, body: m ? text.slice(m[0].length) : text };
860
+ }, [text]);
861
+ // Markdown and other text: rendered into a frame that runs no code. Image placeholders (often `![…](TODO)`) are
862
+ // shown as labels, not loaded.
863
+ const doc = useMemo(() => {
864
+ if (kind !== "markdown" && kind !== "text") return "";
865
+ const html = marked.parse(body.replace(/!\[([^\]]*)\]\([^)]*\)/g, (_, alt) => `<span class="img-ph">Image: ${alt || "untitled"}</span>`), { async: false }) as string;
866
+ return proseDoc(html, proseTokens());
867
+ }, [body, kind]);
733
868
  const g = trial.grades;
734
869
  const groups = [{ id: "gates", label: "Gates", criteria: [{ id: "output-exists", question: "Did the agent write the output file?", level: "floor", grader: "check" } as Criterion, ...task.gates] }, ...task.measures];
870
+ const frameTitle = `Output of ${s.label} for ${task.title}, trial ${trial.index}`;
735
871
 
736
872
  return (
737
873
  <>
@@ -752,27 +888,30 @@ function TrialDrawer({ view, trial, tick, onClose }: { view: View; trial: ViewTr
752
888
  </div>
753
889
  <button className="close" onClick={onClose} aria-label="Close">×</button>
754
890
  </div>
755
- <div className="drawer-body">
756
- <div>
757
- {image ? <>
758
- {trial.error && <p className="error">{trial.error}</p>}
759
- <img className="output-image" src={image} alt={`Output of ${s.label} for ${task.title}, trial ${trial.index}`} />
760
- </> : <>
891
+ <div className="drawer-tabs" role="tablist" aria-label="Trial detail">
892
+ {DRAWER_TABS.map((t) => (
893
+ <button key={t.id} role="tab" id={`drawer-tab-${t.id}`} aria-controls="drawer-panel" aria-selected={tab === t.id} className={`drawer-tab ${tab === t.id ? "on" : ""}`} onClick={() => setTab(t.id)}>{t.label}</button>
894
+ ))}
895
+ </div>
896
+ <div className={`drawer-panel ${tab}`} id="drawer-panel" role="tabpanel" aria-labelledby={`drawer-tab-${tab}`}>
897
+ {tab === "output" && <>
898
+ {trial.error && <p className="error">{trial.error}</p>}
899
+ {kind === null ? <p className="muted">{missing || "Loading…"}</p>
900
+ : kind === "image" ? (image && <img className="output-image" src={image} alt={frameTitle} />) : <>
761
901
  <div className="tabs">
762
- <button className={`tab ${tab === "rendered" ? "on" : ""}`} onClick={() => setTab("rendered")}>Rendered</button>
763
- <button className={`tab ${tab === "raw" ? "on" : ""}`} onClick={() => setTab("raw")}>Markdown</button>
902
+ <button className={`tab ${view2 === "rendered" ? "on" : ""}`} aria-pressed={view2 === "rendered"} onClick={() => setView2("rendered")}>Rendered</button>
903
+ <button className={`tab ${view2 === "source" ? "on" : ""}`} aria-pressed={view2 === "source"} onClick={() => setView2("source")}>{sourceLabel(kind)}</button>
764
904
  </div>
765
- {trial.error && <p className="error">{trial.error}</p>}
766
- {tab === "rendered" ? <>
767
- {(fields.title || fields.description) && <div className="envelope"><div className="k">Subject</div><div className="subject">{fields.title ?? "—"}</div><div className="k">Preview</div><div>{fields.description ?? "—"}</div></div>}
768
- <div className="prose" dangerouslySetInnerHTML={{ __html: html }} />
769
- </> : <pre className="raw">{md}</pre>}
905
+ {view2 === "source" ? <pre className="raw">{text}</pre>
906
+ : kind === "html" ? <iframe className="output-frame page" title={frameTitle} sandbox={HTML_SANDBOX} referrerPolicy="no-referrer" srcDoc={text} />
907
+ : <>
908
+ {(fields.title || fields.description) && <div className="envelope"><div className="k">Subject</div><div className="subject">{fields.title ?? "—"}</div><div className="k">Preview</div><div>{fields.description ?? "—"}</div></div>}
909
+ <iframe className="output-frame prose-frame" title={frameTitle} sandbox={PROSE_SANDBOX} referrerPolicy="no-referrer" srcDoc={doc} />
910
+ </>}
770
911
  </>}
771
- </div>
772
- <div>
773
- {detailError && <p className="error">{detailError}</p>}
774
- {detail && <><RunConditions trial={detail.trial} /><GradingList gradings={detail.gradings} counted={trial.gradingId} /></>}
775
- <h3>Verdicts</h3>
912
+ </>}
913
+ {tab === "verdicts" && <>
914
+ <LevelLegend />
776
915
  {groups.map((m) => (
777
916
  <div key={m.id} className="verdict-group">
778
917
  <div className="crit-head"><b>{m.label}</b>{m.id !== "gates" && g && <span className="crit-score">{pct(g.measures[m.id])}</span>}</div>
@@ -785,7 +924,7 @@ function TrialDrawer({ view, trial, tick, onClose }: { view: View; trial: ViewTr
785
924
  <span className={`badge ${state}`}>{state === "err" ? "Error" : state === "split" ? `${v!.votes.filter((x) => x.pass).length}/${v!.votes.length}` : state === "pass" ? "Pass" : "Fail"}</span>
786
925
  <span className="q">{c.question}</span>
787
926
  </div>
788
- <div className="crit-meta"><span className={`tag ${c.level}`}>{LEVEL_TEXT[c.level]}</span><span className="muted">{c.grader}</span></div>
927
+ <div className="crit-meta"><LevelTag level={c.level} /><span className="muted">{c.grader}</span></div>
789
928
  {v?.error && <p className="error">{v.error}</p>}
790
929
  {v?.votes.map((x, i) => <p key={i} className="reason"><b>{x.pass ? "Yes" : "No"}.</b> {x.reason}</p>)}
791
930
  </div>
@@ -793,7 +932,11 @@ function TrialDrawer({ view, trial, tick, onClose }: { view: View; trial: ViewTr
793
932
  })}
794
933
  </div>
795
934
  ))}
796
- </div>
935
+ </>}
936
+ {tab === "run" && <>
937
+ {detailError && <p className="error">{detailError}</p>}
938
+ {detail ? <><RunConditions trial={detail.trial} /><GradingList gradings={detail.gradings} counted={trial.gradingId} /></> : !detailError && <p className="muted">Loading…</p>}
939
+ </>}
797
940
  </div>
798
941
  </div>
799
942
  </>