static-fire 0.0.0-stage → 0.54.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (67) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +236 -2
  3. package/bin/static-fire.js +46 -0
  4. package/package.json +30 -4
  5. package/pricing.yaml +20 -0
  6. package/src/adapters/index.ts +153 -0
  7. package/src/api.ts +250 -0
  8. package/src/cli.ts +383 -0
  9. package/src/core/apkit-install.ts +47 -0
  10. package/src/core/authored.ts +47 -0
  11. package/src/core/batch-state.ts +25 -0
  12. package/src/core/batch.ts +450 -0
  13. package/src/core/batches.ts +221 -0
  14. package/src/core/checks.ts +63 -0
  15. package/src/core/commands.ts +43 -0
  16. package/src/core/compare.ts +206 -0
  17. package/src/core/config.ts +25 -0
  18. package/src/core/copying.ts +18 -0
  19. package/src/core/coverage.ts +84 -0
  20. package/src/core/db.ts +608 -0
  21. package/src/core/derived.ts +74 -0
  22. package/src/core/engine.ts +85 -0
  23. package/src/core/facets.ts +143 -0
  24. package/src/core/grade.ts +197 -0
  25. package/src/core/grading-versions.ts +281 -0
  26. package/src/core/identity.ts +263 -0
  27. package/src/core/init.ts +321 -0
  28. package/src/core/judge-schema.ts +37 -0
  29. package/src/core/judge.ts +166 -0
  30. package/src/core/label.ts +43 -0
  31. package/src/core/lock.ts +40 -0
  32. package/src/core/migrate.ts +288 -0
  33. package/src/core/new-task.ts +85 -0
  34. package/src/core/outside-tools.ts +129 -0
  35. package/src/core/plan.ts +84 -0
  36. package/src/core/profiles.ts +42 -0
  37. package/src/core/project.ts +140 -0
  38. package/src/core/regrade.ts +242 -0
  39. package/src/core/score.ts +148 -0
  40. package/src/core/settings.ts +44 -0
  41. package/src/core/task-file.ts +95 -0
  42. package/src/core/task-page.ts +54 -0
  43. package/src/core/trial-detail.ts +73 -0
  44. package/src/core/types.ts +195 -0
  45. package/src/core/view-query.ts +102 -0
  46. package/src/core/view.ts +576 -0
  47. package/src/guide.md +624 -0
  48. package/src/server.ts +28 -0
  49. package/web/api.ts +6 -0
  50. package/web/app.tsx +1162 -0
  51. package/web/brand/offset-tokens.css +126 -0
  52. package/web/charts.tsx +190 -0
  53. package/web/compare.tsx +317 -0
  54. package/web/encoding.tsx +55 -0
  55. package/web/filter-menu.ts +195 -0
  56. package/web/how-it-works.tsx +92 -0
  57. package/web/index-switches.ts +29 -0
  58. package/web/index.html +17 -0
  59. package/web/levels.tsx +15 -0
  60. package/web/notices.ts +74 -0
  61. package/web/output-view.ts +61 -0
  62. package/web/pages.ts +42 -0
  63. package/web/run-status.ts +51 -0
  64. package/web/styles.css +368 -0
  65. package/web/task-card.ts +6 -0
  66. package/web/tasks.tsx +122 -0
  67. package/web/wording.ts +38 -0
@@ -0,0 +1,55 @@
1
+ // One visual encoding for every chart and table (Offset dataviz tokens, web/brand/offset-tokens.css):
2
+ // shape = harness (circle Claude Code, square Codex, diamond Antigravity, triangle Pi, star by hand)
3
+ // colour = model family (fixed slot per family, so a colour never moves between views; ink = a person)
4
+ // fill = apkit Profile (filled = with Profile, hollow = none; a person is always filled)
5
+ // Every mark keeps Offset's 2px ink outline; text is always ink, never the series colour.
6
+ import type { ConfigSummary, Harness } from "../src/core/types";
7
+ export { HARNESS_NAME } from "../src/core/label";
8
+ export const HARNESS_ORDER: Harness[] = ["claude", "codex", "agy", "pi", "human"];
9
+
10
+ const FAMILY_SLOT: Record<string, number> = {
11
+ "claude-opus": 3, "claude-sonnet": 4, "claude-haiku": 5,
12
+ "gpt-sol": 7, "gpt-luna": 6, "gemini-argon": 2, "gemini-flash": 1,
13
+ };
14
+ const FAMILY_NAME: Record<string, string> = {
15
+ "claude-opus": "Opus", "claude-sonnet": "Sonnet", "claude-haiku": "Haiku",
16
+ "gpt-sol": "GPT Sol", "gpt-luna": "GPT Luna", "gemini-argon": "Gemini Argon", "gemini-flash": "Gemini Flash", human: "Person",
17
+ };
18
+ export const familyName = (f: string) => FAMILY_NAME[f] ?? f;
19
+ export const familySlot = (f: string) => FAMILY_SLOT[f] ?? 1;
20
+ export const fill = (f: string) => (f === "human" ? "var(--color-ink)" : `var(--dataviz-${familySlot(f)})`);
21
+ export const strong = (f: string) => (f === "human" ? "var(--color-ink)" : `var(--dataviz-${familySlot(f)}-strong)`);
22
+
23
+ /** SVG path for a marker centred on (0,0) with radius r. */
24
+ export function shapePath(h: Harness, r: number) {
25
+ switch (h) {
26
+ case "codex": { const s = r * 0.9; return `M${-s},${-s}H${s}V${s}H${-s}Z`; }
27
+ case "agy": { const s = r * 1.2; return `M0,${-s}L${s},0L0,${s}L${-s},0Z`; }
28
+ case "pi": { const s = r * 1.15; return `M0,${-s}L${s},${s * 0.8}L${-s},${s * 0.8}Z`; }
29
+ case "human": {
30
+ const R = r * 1.3, ri = R * 0.45;
31
+ const pts = Array.from({ length: 10 }, (_, i) => { const a = -Math.PI / 2 + (i * Math.PI) / 5, d = i % 2 ? ri : R; return `${(d * Math.cos(a)).toFixed(2)},${(d * Math.sin(a)).toFixed(2)}`; });
32
+ return `M${pts.join("L")}Z`;
33
+ }
34
+ default: return `M${r},0A${r},${r} 0 1,1 ${-r},0A${r},${r} 0 1,1 ${r},0Z`;
35
+ }
36
+ }
37
+
38
+ type MarkerLike = Pick<ConfigSummary, "harness" | "profile" | "family">;
39
+
40
+ export function markerProps(s: MarkerLike) {
41
+ const filled = s.profile || s.harness === "human";
42
+ return { fill: filled ? fill(s.family) : "var(--color-white)", stroke: "var(--outline)", strokeWidth: 2 };
43
+ }
44
+
45
+ /** A standalone marker for legends, tables and lists. `neutral` shows the shape (or hollowness) only. */
46
+ export function Marker({ s, size = 14, neutral }: { s: MarkerLike; size?: number; neutral?: boolean }) {
47
+ const r = size / 2 - 2;
48
+ const dot = neutral ? "var(--fg-3)" : fill(s.family);
49
+ return (
50
+ <svg width={size} height={size} viewBox={`${-size / 2} ${-size / 2} ${size} ${size}`} aria-hidden className="marker">
51
+ <path d={shapePath(s.harness, r)} {...markerProps(s)} {...(neutral && s.profile ? { fill: "var(--color-cream)" } : {})} />
52
+ {!s.profile && s.harness !== "human" && <path d={shapePath(s.harness, r * 0.45)} fill={dot} stroke="none" />}
53
+ </svg>
54
+ );
55
+ }
@@ -0,0 +1,195 @@
1
+ // The Results page's Filter menu and filter tags (#26), as plain data: which sections and values the menu lists,
2
+ // what picking a value sets in the view query, what the search lists and which tags show. The page only draws it.
3
+ // A config facet (Harness, Model, Effort, Profile) takes several values (#31): a pick adds or removes one value and
4
+ // each picked value is its own tag. Task, Judge, Grading version and Date range take one value: a pick replaces it.
5
+ // Task (#81) is the task view: a task is picked here, not by a button on the page.
6
+ import type { Facets, FacetValue } from "../src/core/facets";
7
+ import { HARNESS_NAME } from "../src/core/label";
8
+ import { CONFIG_FACETS, valueSet } from "../src/core/view-query";
9
+ import type { ConfigFacet, ViewQuery } from "../src/core/view-query";
10
+
11
+ export type SectionId = "task" | "harness" | "model" | "effort" | "profile" | "judge" | "grading" | "dates";
12
+ export const SECTION_NAMES: Record<SectionId, string> = {
13
+ task: "Task", harness: "Harness", model: "Model", effort: "Effort", profile: "Profile", judge: "Judge", grading: "Grading version", dates: "Date range",
14
+ };
15
+ export const EFFORT_NONE = "No effort setting", PROFILE_NONE = "No Profile (user setup only)";
16
+
17
+ type Patch = Partial<ViewQuery>;
18
+
19
+ /**
20
+ * One value the menu lists. Picking it sets `set`; picking it again when it is picked sets `clear`. In a config
21
+ * section, `set` adds the value to the picked ones and `clear` removes only it.
22
+ */
23
+ export interface MenuValue {
24
+ section: SectionId; sectionName: string; key: string; label: string;
25
+ /** A second line: a grading version's date, judge and current mark. */
26
+ detail?: string;
27
+ /** The task a grading version belongs to, when more than one task is in scope. */
28
+ group?: string;
29
+ /** A title for the value, for example a grading version's full hash. */
30
+ title?: string;
31
+ count: number; picked: boolean; current?: boolean;
32
+ set: Patch; clear: Patch;
33
+ }
34
+
35
+ export interface MenuSection {
36
+ id: SectionId; name: string;
37
+ /** The picked value, "N picked" when there are more, or how many values there are. */
38
+ summary: string;
39
+ picked: boolean;
40
+ values: MenuValue[];
41
+ /** What removing this section's filter sets. */
42
+ remove: Patch;
43
+ }
44
+
45
+ export interface MenuContext {
46
+ facets: Facets | null; query: ViewQuery;
47
+ /** The view's judge: set by the judge filter, or by a picked grading version. */
48
+ judge: { id: string; label: string };
49
+ /** The title of every task in the project, in the project's order: the Task section lists them all. */
50
+ titles: Record<string, string>;
51
+ }
52
+
53
+ export const CLEAR_ALL: Patch = { judge: undefined, grading: undefined, harness: undefined, model: undefined, effort: undefined, profile: undefined, since: undefined, until: undefined };
54
+
55
+ /** What Clear all sets: every filter, and a picked task (back to the index). A suite pick is not a filter: it stays. */
56
+ export const clearAll = (q: ViewQuery): Patch => (q.scope?.kind === "task" ? { ...CLEAR_ALL, scope: undefined } : CLEAR_ALL);
57
+
58
+ const plural = (n: number, one: string) => `${n} ${one}${n === 1 ? "" : "s"}`;
59
+ const short = (hash: string) => hash.slice(0, 8);
60
+ const MONTHS = ["Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"];
61
+ /** "2026-11-06" (a date filter, a calendar day) as "Nov 6, 2026", with no time zone shift. */
62
+ const dayText = (day: string) => {
63
+ const m = /^(\d{4})-(\d{2})-(\d{2})/.exec(day);
64
+ return m ? `${MONTHS[Number(m[2]) - 1]} ${Number(m[3])}, ${m[1]}` : day;
65
+ };
66
+ const shortDate = (iso: string) => new Date(iso).toLocaleDateString(undefined, { month: "short", day: "numeric", year: "numeric" });
67
+
68
+ /** The date range filter in words, or null when it is not set. */
69
+ export function dateRangeText(since?: string, until?: string): string | null {
70
+ if (since && until) return `${dayText(since)} to ${dayText(until)}`;
71
+ if (since) return `from ${dayText(since)}`;
72
+ if (until) return `until ${dayText(until)}`;
73
+ return null;
74
+ }
75
+
76
+ const NAMES: Partial<Record<SectionId, (v: string) => string>> = {
77
+ harness: (v) => HARNESS_NAME[v as keyof typeof HARNESS_NAME] ?? v,
78
+ effort: (v) => (v === "none" ? EFFORT_NONE : v),
79
+ profile: (v) => (v === "none" ? PROFILE_NONE : v),
80
+ };
81
+
82
+ function section(id: SectionId, values: MenuValue[], remove: Patch): MenuSection {
83
+ const on = values.filter((v) => v.picked);
84
+ const summary = on.length > 1 ? `${on.length} picked` : on[0]?.label ?? plural(values.length, "value");
85
+ return { id, name: SECTION_NAMES[id], summary, picked: on.length > 0, values, remove };
86
+ }
87
+
88
+ /** A config facet: its values with counts, and each URL value no trial has, listed with no trials. */
89
+ function facetSection(id: ConfigFacet, listed: FacetValue[], picked: string[]): MenuSection {
90
+ const all: FacetValue[] = [...listed, ...picked.filter((p) => !listed.some((v) => v.value === p)).map((value) => ({ value, count: 0 }))];
91
+ const name = NAMES[id] ?? ((v: string) => v);
92
+ const to = (values: string[]): Patch => { const v = valueSet(values); return { [id]: v.length ? v : undefined }; };
93
+ const values = all.map((v): MenuValue => ({
94
+ section: id, sectionName: SECTION_NAMES[id], key: v.value, label: v.label ?? name(v.value), count: v.count, picked: picked.includes(v.value),
95
+ set: to([...picked, v.value]), clear: to(picked.filter((p) => p !== v.value)),
96
+ }));
97
+ return section(id, values, { [id]: undefined });
98
+ }
99
+
100
+ /** Every section of the menu, in order, for the facets and the query in view. */
101
+ export function menuSections({ facets, query, judge, titles }: MenuContext): MenuSection[] {
102
+ const f = facets;
103
+ // Task: every task in the project. Picking one opens its task view; picking it again goes back to the index.
104
+ const taskPicked = query.scope?.kind === "task" ? query.scope.id : undefined;
105
+ const taskCounts = new Map((f?.scope.tasks ?? []).map((t) => [t.value, t.count]));
106
+ const taskIds = Object.keys(titles);
107
+ if (taskPicked && !taskIds.includes(taskPicked)) taskIds.push(taskPicked); // a URL task the project does not list
108
+ const taskSection = section("task", taskIds.map((id): MenuValue => ({
109
+ section: "task", sectionName: SECTION_NAMES.task, key: id, label: titles[id] ?? id, count: taskCounts.get(id) ?? 0, picked: id === taskPicked,
110
+ set: { scope: { kind: "task", id }, suites: undefined }, clear: { scope: undefined },
111
+ })), { scope: undefined });
112
+ const config = CONFIG_FACETS.map((k) => facetSection(k, f?.[k] ?? [], query[k] ?? []));
113
+
114
+ // Judge: a picked grading version sets the view's judge, so that judge shows as picked.
115
+ const judgePicked = query.judge ?? (query.grading ? judge.id : undefined);
116
+ const judges = f?.judge ?? [];
117
+ const listedJudges = judges.some((j) => j.value === judge.id) ? judges : [...judges, { value: judge.id, label: judge.label, count: 0 }];
118
+ const judgeValues = listedJudges.map((j): MenuValue => ({
119
+ section: "judge", sectionName: SECTION_NAMES.judge, key: j.value, label: j.label ?? j.value, count: j.count, picked: j.value === judgePicked,
120
+ set: { judge: j.value, grading: undefined }, clear: { judge: undefined, grading: undefined },
121
+ }));
122
+ const judgeSection = section("judge", judgeValues, { judge: undefined, grading: undefined });
123
+
124
+ // Grading version: per task, its versions by name. Each judge's current version is marked. A current one clears
125
+ // the filter and sets its own judge: the judge in view (which may come only from the picked hash, so the pick
126
+ // keeps it), or another judge, whose current version that is.
127
+ // Read from the query, not the loaded view, so a tag goes the moment its filter does. A URL may hold a hash prefix.
128
+ const isPicked = (hash: string) => !!query.grading && hash.startsWith(query.grading);
129
+ const groups = f?.grading ?? [];
130
+ const gradingValues = groups.flatMap((g) => {
131
+ const judges = new Set(g.setups.map((s) => s.judgeLabel)).size > 1;
132
+ return g.setups.map((s): MenuValue => ({
133
+ section: "grading", sectionName: SECTION_NAMES.grading, key: s.hash, label: s.name,
134
+ detail: [shortDate(s.firstSeen), judges ? `judge ${s.judgeLabel}` : "", s.current ? "current" : ""].filter(Boolean).join(" · "),
135
+ group: groups.length > 1 ? titles[g.taskId] ?? g.taskId : undefined,
136
+ title: `Grading hash ${s.hash}`,
137
+ count: s.count, picked: isPicked(s.hash), current: s.current,
138
+ set: s.current ? { grading: undefined, judge: s.judgeId === judge.id ? judgePicked : s.judgeId ?? undefined } : { grading: s.hash, judge: undefined },
139
+ clear: { grading: undefined },
140
+ }));
141
+ });
142
+ if (query.grading && !gradingValues.some((v) => v.picked)) // a URL hash no listed version has
143
+ gradingValues.push({ section: "grading", sectionName: SECTION_NAMES.grading, key: query.grading, label: short(query.grading), title: `Grading hash ${query.grading}`, count: 0, picked: true, set: {}, clear: { grading: undefined } });
144
+ const gradingSection = section("grading", gradingValues, { grading: undefined });
145
+
146
+ // Date range: a pair of dates, not a list of values.
147
+ const range = dateRangeText(query.since, query.until);
148
+ const dates: MenuSection = { id: "dates", name: SECTION_NAMES.dates, summary: range ?? "Any date", picked: !!range, values: [], remove: { since: undefined, until: undefined } };
149
+
150
+ return [taskSection, ...config, judgeSection, gradingSection, dates];
151
+ }
152
+
153
+ /** What picking `v` sets: the value, or without it when it is already picked. */
154
+ export const pick = (v: MenuValue): Patch => (v.picked ? v.clear : v.set);
155
+
156
+ /**
157
+ * The query after the page sets a grading pick (#80, DEC-019). The default set of suites has no exception for a
158
+ * picked grading version, and a new grading can change which configs have completed a suite, so on the index the
159
+ * pick also becomes an explicit suite pick: the suites in view (or the picked ones), and the first suite of the
160
+ * version's task when none of them has it. A view of one task needs no suite. `taskId` comes from the facets.
161
+ */
162
+ export function withGradingPick(q: ViewQuery, taskId: string | undefined, suitesInView: string[], suites: { id: string; tasks: string[] }[]): ViewQuery {
163
+ if (!q.grading || q.scope?.kind === "task" || !taskId) return q;
164
+ const ids = new Set(q.suites?.length ? q.suites : suitesInView);
165
+ if (!suites.some((s) => ids.has(s.id) && s.tasks.includes(taskId))) {
166
+ const home = suites.find((s) => s.tasks.includes(taskId));
167
+ if (home) ids.add(home.id);
168
+ }
169
+ const picked = suites.filter((s) => ids.has(s.id)).map((s) => s.id); // catalog order
170
+ return picked.length ? { ...q, scope: { kind: "index" }, suites: picked } : q;
171
+ }
172
+
173
+ /**
174
+ * The search: every value of every section whose section name, name, task or detail line (a grading version's
175
+ * judge) matches the text (case ignored), as one flat list; and whether the date range matches, so it shows as one entry.
176
+ */
177
+ export function searchMenu(sections: MenuSection[], text: string): { values: MenuValue[]; dates: boolean } {
178
+ const q = text.trim().toLowerCase();
179
+ if (!q) return { values: [], dates: false };
180
+ const values = sections.flatMap((s) => s.values).filter((v) => `${v.sectionName} ${v.label} ${v.group ?? ""} ${v.detail ?? ""}`.toLowerCase().includes(q));
181
+ return { values, dates: SECTION_NAMES.dates.toLowerCase().includes(q) };
182
+ }
183
+
184
+ export interface FilterTag { section: SectionId; /** Unique among the tags. */ key: string; name: string; value: string; text: string; remove: Patch }
185
+
186
+ const CONFIG_SECTIONS = new Set<SectionId>(CONFIG_FACETS);
187
+
188
+ /**
189
+ * One tag per active filter, reading "<section>: <value>": one per picked value of a config section, whose
190
+ * remove control removes only that value, and one for each other section with a pick.
191
+ */
192
+ export const filterTags = (sections: MenuSection[]): FilterTag[] =>
193
+ sections.filter((s) => s.picked).flatMap((s): FilterTag[] => CONFIG_SECTIONS.has(s.id)
194
+ ? s.values.filter((v) => v.picked).map((v) => ({ section: s.id, key: `${s.id}:${v.key}`, name: s.name, value: v.label, text: `${s.name}: ${v.label}`, remove: v.clear }))
195
+ : [{ section: s.id, key: s.id, name: s.name, value: s.summary, text: `${s.name}: ${s.summary}`, remove: s.remove }]);
@@ -0,0 +1,92 @@
1
+ // The How it works page (#79): a short explanation of how Static Fire gets from a trial to a number, in the terms
2
+ // of CONTEXT.md. Static text. Its sections and their order are HOW_IT_WORKS_SECTIONS (web/pages.ts), so another
3
+ // page can link to one section, for example /how-it-works#left-out.
4
+ import { useEffect, type ReactNode } from "react";
5
+ import { HOW_IT_WORKS_SECTIONS, pathOf, type HowItWorksSection } from "./pages";
6
+
7
+ const BODY: Record<HowItWorksSection, ReactNode> = {
8
+ trial: <>
9
+ <p>A <b>trial</b> is one run of one agent config on one task. The agent works in a new folder of its own, with
10
+ the task's prompt, its input files and its project files, and with your own setup, as it would in real work.
11
+ It writes its output to the path the task names. Then the output is graded.</p>
12
+ <p>A trial that could not run (the harness did not start, a login expired, it timed out) is <i>failed</i>. It
13
+ shows as a failure and is not in any mean.</p>
14
+ </>,
15
+ score: <>
16
+ <ul>
17
+ <li><b>Gates</b> are yes or no questions that every output must pass, for example "The output file exists". If
18
+ a gate fails, the task score is 0.</li>
19
+ <li>A <b>measure</b> is a group of criteria. Its score is the percent of its criteria that passed.</li>
20
+ <li>The <b>task score</b> is the mean of the measure scores, with each measure's weight (equal by default),
21
+ from 0 to 100.</li>
22
+ <li>The <b>Suite Index</b> is the mean task score over the tasks of a suite. A config gets one only when it
23
+ has a valid score on every task of the suite. Otherwise it is <i>partial</i>.</li>
24
+ <li>The <b>Agent Performance Index</b> is the mean Suite Index over the suites in view. It also needs a valid
25
+ score on every task of every suite in view.</li>
26
+ </ul>
27
+ <p>A point on a chart is the mean over every valid trial of that config in the view, from any batch, with a 95%
28
+ confidence interval.</p>
29
+ </>,
30
+ graders: <>
31
+ <p>Each criterion has one grader:</p>
32
+ <ul>
33
+ <li>A <b>check</b> is code. It gives the same answer every time: a pattern match, a word count, a field in the
34
+ file's front matter, or a command.</li>
35
+ <li>A <b>judge</b> is a model that reads the output and answers the question. The judge configuration in{" "}
36
+ <code>judges.yaml</code> sets which model it is. A judge does not use your own setup.</li>
37
+ </ul>
38
+ <p>A judge answers each question several times (3 by default). Each answer is a <b>vote</b>, with a pass or fail
39
+ and a reason. The criterion's score is the share of votes that passed, so 2 of 3 votes give 0.67.</p>
40
+ <p>When a grader cannot decide, its verdict is an error. That trial is left out of the means until a regrade
41
+ grades it again.</p>
42
+ </>,
43
+ levels: <>
44
+ <p>Each criterion has a level. It tells you how hard the criterion is meant to be:</p>
45
+ <ul>
46
+ <li><span className="tag floor">Floor</span> Most configurations should pass.</li>
47
+ <li><span className="tag strong">Strong</span> Good configurations pass.</li>
48
+ <li><span className="tag ceiling">Ceiling</span> Only the best configurations pass.</li>
49
+ </ul>
50
+ <p>The level does not change the score. A config that fails floor criteria is not ready for the work. Ceiling
51
+ criteria show which configs are the best.</p>
52
+ </>,
53
+ "left-out": <>
54
+ <p>A chart compares only trials that did the same job and were graded the same way. The Results page counts the
55
+ trials it leaves out, by reason:</p>
56
+ <ul>
57
+ <li><b>Another grading version.</b> The criteria, the references or the grading code changed after the trial
58
+ was graded. <i>What to do:</i> regrade, which uses judge calls only and runs no agent. Or pick the earlier
59
+ grading version in the Filter menu to see those trials as they were graded.</li>
60
+ <li><b>Another task version.</b> The prompt, the output path or the files the agent gets changed, so the
61
+ trial did a different job. A regrade cannot bring it in. <i>What to do:</i> switch to that version to see
62
+ its trials, or run new trials on the current one.</li>
63
+ <li><b>Another judge.</b> The trial was graded by a different judge, and the scores of two judges are never
64
+ mixed. <i>What to do:</i> pick that judge in the Filter menu, or regrade with the judge in view.</li>
65
+ <li><b>Not graded yet.</b> The trial has no finished grading. <i>What to do:</i> if its batch is still running,
66
+ wait. If not, regrade.</li>
67
+ </ul>
68
+ </>,
69
+ };
70
+
71
+ export function HowItWorksPage() {
72
+ // A link to one section opens the page there.
73
+ useEffect(() => { if (location.hash) document.getElementById(decodeURIComponent(location.hash.slice(1)))?.scrollIntoView(); }, []);
74
+ return (
75
+ <>
76
+ <section className="hero">
77
+ <div className="kicker">How it works</div>
78
+ <h1>From a trial to a number</h1>
79
+ <p className="page-sub">What each number on the Results page means. Each task's own questions are on the{" "}
80
+ <a href={pathOf("tasks")}>Tasks</a> page.</p>
81
+ </section>
82
+ <section className="card how-it-works">
83
+ {HOW_IT_WORKS_SECTIONS.map((s) => (
84
+ <section key={s.id} id={s.id} className="how-section">
85
+ <h2>{s.title}</h2>
86
+ {BODY[s.id]}
87
+ </section>
88
+ ))}
89
+ </section>
90
+ </>
91
+ );
92
+ }
@@ -0,0 +1,29 @@
1
+ // The index buttons of the Results page (#80): the suites in view are switches, and the title follows them.
2
+ // Kept apart from app.tsx so the rules are tested without a browser.
3
+ import type { Suite } from "../src/core/types";
4
+ import type { Scope } from "../src/core/view-query";
5
+ import { suiteHref, taskHref } from "./pages";
6
+
7
+ /**
8
+ * The title of a view: the task's on a task view, the suite's when one suite is in view, else
9
+ * "Agent Performance Index". It never lists the suites, so it does not grow with them.
10
+ */
11
+ export const viewTitle = (sel: Scope, suiteIds: string[], suites: Map<string, Suite>, taskTitle: (id: string) => string) =>
12
+ sel.kind === "task" ? taskTitle(sel.id) : suiteIds.length === 1 ? suites.get(suiteIds[0]!)?.title ?? suiteIds[0]! : "Agent Performance Index";
13
+
14
+ /**
15
+ * The suites in view after a click on a suite's switch. On a task view it opens the index on that suite alone.
16
+ * Otherwise it turns the suite off when it is in view, else on. Null when the click would leave no suite in view.
17
+ */
18
+ export function toggleSuite(sel: Scope, inView: string[], id: string, catalog: Suite[]): string[] | null {
19
+ if (sel.kind === "task") return [id];
20
+ const next = inView.includes(id) ? inView.filter((x) => x !== id) : [...inView, id];
21
+ return next.length ? catalog.map((s) => s.id).filter((x) => next.includes(x)) : null;
22
+ }
23
+
24
+ /**
25
+ * Where the title links (#81): a task view's title to the task's entry on the Tasks page, one suite's to the
26
+ * suite's entry. The Agent Performance Index over several suites is not a name, so it has no link.
27
+ */
28
+ export const titleHref = (sel: Scope, suiteIds: string[]): string | null =>
29
+ sel.kind === "task" ? taskHref(sel.id) : suiteIds.length === 1 ? suiteHref(suiteIds[0]!) : null;
package/web/index.html ADDED
@@ -0,0 +1,17 @@
1
+ <!doctype html>
2
+ <html lang="en">
3
+ <head>
4
+ <meta charset="utf-8" />
5
+ <meta name="viewport" content="width=device-width, initial-scale=1" />
6
+ <title>Static Fire</title>
7
+ <link rel="icon" href="data:," />
8
+ <link rel="preconnect" href="https://fonts.googleapis.com" />
9
+ <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin />
10
+ <link rel="stylesheet" href="https://fonts.googleapis.com/css2?family=Hanken+Grotesk:wght@400..900&family=IBM+Plex+Mono:wght@400;500;600&display=swap" />
11
+ <link rel="stylesheet" href="./styles.css" />
12
+ </head>
13
+ <body>
14
+ <div id="root"></div>
15
+ <script type="module" src="./app.tsx"></script>
16
+ </body>
17
+ </html>
package/web/levels.tsx ADDED
@@ -0,0 +1,15 @@
1
+ // A criterion's level where the dashboard shows one (#68), with the engine's names (DEC-014 of #71): a tag whose
2
+ // tooltip says what it means, and a one-line legend next to a list of tags.
3
+
4
+ const LEVEL_TEXT: Record<string, string> = { floor: "Floor", strong: "Strong", ceiling: "Ceiling" };
5
+ /** What each level means. */
6
+ export const LEVEL_MEANING: Record<string, string> = { floor: "most configurations should pass", strong: "good ones pass", ceiling: "only the best pass" };
7
+
8
+ export const LevelTag = ({ level }: { level: string }) => (
9
+ <span className={`tag ${level}`} title={`${LEVEL_TEXT[level] ?? level}: ${LEVEL_MEANING[level] ?? ""}`}>{LEVEL_TEXT[level] ?? level}</span>
10
+ );
11
+
12
+ /** One line that says what the levels mean, next to where they are shown. */
13
+ export const LevelLegend = () => (
14
+ <p className="level-legend"><span className="k">Levels</span>{Object.keys(LEVEL_TEXT).map((l) => <span key={l}><LevelTag level={l} /> {LEVEL_MEANING[l]}</span>)}</p>
15
+ );
package/web/notices.ts ADDED
@@ -0,0 +1,74 @@
1
+ // The words of the Results page's one alert and its details (#82, DEC-023): what a view does not show, why, and
2
+ // what to do. Display only: which trials a view leaves out comes from the view (src/core/view.ts LeftOut), and
3
+ // the regrade is the server's. The How it works page's #left-out section says the same in full.
4
+
5
+ /** A count of trials not shown and of the configs they belong to. */
6
+ export interface Counted { trials: number; configs: number }
7
+
8
+ /** One item of the details: what is not shown and why, then what to do. */
9
+ export interface NoticeItem { what: string; todo: string }
10
+
11
+ const plural = (n: number, one: string, many = `${one}s`) => `${n} ${n === 1 ? one : many}`;
12
+
13
+ /** The alert's short count, for example "3 trials and 2 configs not shown". Null when nothing is left out. */
14
+ export function alertText(trials: number, configs: number): string | null {
15
+ const parts = [trials ? plural(trials, "trial") : "", configs ? plural(configs, "config") : ""].filter(Boolean);
16
+ return parts.length ? `${parts.join(" and ")} not shown` : null;
17
+ }
18
+
19
+ const fromConfigs = (c: Counted) => `${plural(c.trials, "trial")} from ${plural(c.configs, "config")}`;
20
+
21
+ /**
22
+ * Trials graded only under another grading version. `current`: every task in view is on its current version, so
23
+ * the view offers Regrade. `ungraded`: how many trials not graded yet the same Regrade button covers.
24
+ */
25
+ export function gradingItem(c: Counted, { current, ungraded }: { current: boolean; ungraded: number }): NoticeItem {
26
+ const them = c.trials === 1 ? "it" : "them", their = c.trials === 1 ? "its" : "their";
27
+ return {
28
+ what: `${fromConfigs(c)} ${c.trials === 1 ? "was" : "were"} graded under ${current ? "earlier" : "other"} criteria or by another judge.`,
29
+ todo: current
30
+ ? `Regrade ${them} with the button below${ungraded ? " (it also grades the trials not graded yet)" : ""}, or pick ${their} grading version in the Filter menu.`
31
+ : `Pick ${their} grading version in the Filter menu.`,
32
+ };
33
+ }
34
+
35
+ /** Trials with no finished grading. `otherGrading`: how many trials of the other item the same button covers. */
36
+ export function ungradedItem(c: Counted, { current, otherGrading }: { current: boolean; otherGrading: number }): NoticeItem {
37
+ const one = c.trials === 1;
38
+ const wait = `If ${one ? "its" : "their"} batch is still running, wait.`;
39
+ return {
40
+ what: `${fromConfigs(c)} ${one ? "is" : "are"} not graded yet.`,
41
+ todo: current
42
+ ? `${wait} If not, regrade ${one ? "it" : "them"} with the ${otherGrading ? "same button below: one regrade covers both lists" : "button below"}.`
43
+ : wait,
44
+ };
45
+ }
46
+
47
+ /** Under the Regrade button: what it does, and no more (#80: with nothing picked, it covers the suites in view only). */
48
+ export function regradeNote(trials: number, suitesLeftOut: boolean): string {
49
+ return `Judge calls only. It grades the ${plural(trials, "trial")} above again and runs no agent.${suitesLeftOut ? " Suites left out of the index are not regraded." : ""}`;
50
+ }
51
+
52
+ /** A config the chart cannot plot: the suites in view it has not completed, and its score (null: no valid score). */
53
+ export interface NotPlotted { label: string; partial: { suite: string; have: number; of: number }[]; score: number | null }
54
+
55
+ /** The configs the chart cannot plot, under one reason. */
56
+ export interface NotPlottedGroup { why: string; configs: string[]; todo: string }
57
+
58
+ /**
59
+ * The configs the chart cannot plot, grouped by reason: one group per suite they have not completed (in the order
60
+ * met), then no valid score, then no cost data (a score but no cost). A config partial in two suites is in both.
61
+ */
62
+ export function notPlottedGroups(list: NotPlotted[]): NotPlottedGroup[] {
63
+ const suites = new Map<string, string[]>();
64
+ const noScore: string[] = [], noCost: string[] = [];
65
+ for (const c of list) {
66
+ if (c.partial.length) for (const p of c.partial) suites.set(p.suite, [...(suites.get(p.suite) ?? []), `${c.label} (${p.have} of ${p.of} tasks)`]);
67
+ else (c.score == null ? noScore : noCost).push(c.label);
68
+ }
69
+ return [
70
+ ...[...suites].map(([suite, configs]) => ({ why: `No score on every task of ${suite}`, configs, todo: "Run the tasks they miss." })),
71
+ ...(noScore.length ? [{ why: "No valid score", configs: noScore, todo: "Open their trials to see why." }] : []),
72
+ ...(noCost.length ? [{ why: "No cost data", configs: noCost, todo: "Check the costs in pricing.yaml." }] : []),
73
+ ];
74
+ }
@@ -0,0 +1,61 @@
1
+ // How the trial drawer shows a trial's output (#68). An output is agent output and is not trusted (DEC-012 of #71):
2
+ // it never goes into the dashboard's own page. An HTML page runs in a sandboxed frame that may run the page's own
3
+ // scripts but has an opaque origin, so it cannot read or change the dashboard or run code on its origin. Markdown and
4
+ // other text are rendered into a frame that runs no code at all. The API refuses changes from such a frame
5
+ // (src/api.ts), since a sandboxed page can still send a request.
6
+
7
+ export type OutputKind = "image" | "html" | "markdown" | "text";
8
+
9
+ /**
10
+ * How to show an output: by the server's media type for an image, else by the extension of the task's output path.
11
+ * Text that is neither HTML nor Markdown is shown as Markdown, as before, but named "text" so its second view is "Source".
12
+ */
13
+ export function outputKind(outputPath: string, contentType: string | null): OutputKind {
14
+ if (contentType?.startsWith("image/")) return "image";
15
+ const ext = /\.([^./\\]+)$/.exec(outputPath)?.[1]?.toLowerCase();
16
+ return ext === "html" || ext === "htm" ? "html" : ext === "md" || ext === "markdown" ? "markdown" : "text";
17
+ }
18
+
19
+ /** The name of the second view, which shows the output's text as it is. */
20
+ export const sourceLabel = (kind: OutputKind) => (kind === "markdown" ? "Markdown" : "Source");
21
+
22
+ /** The HTML frame may run the page's own scripts. No allow-same-origin, forms, popups, modals or top navigation. */
23
+ export const HTML_SANDBOX = "allow-scripts";
24
+ /** The Markdown and text frame runs no code. */
25
+ export const PROSE_SANDBOX = "";
26
+
27
+ /** The rendered Markdown's frame loads only its styles and the dashboard's fonts: no script, image, frame or request. */
28
+ const PROSE_POLICY = "default-src 'none'; style-src 'unsafe-inline' https://fonts.googleapis.com; font-src https://fonts.gstatic.com";
29
+ const FONTS = "https://fonts.googleapis.com/css2?family=Hanken+Grotesk:wght@400..900&family=IBM+Plex+Mono:wght@400;500;600&display=swap";
30
+
31
+ /** The dashboard's tokens the prose styles use; their values are read from the dashboard's page (web/brand). */
32
+ export const PROSE_TOKENS = [
33
+ "--fg", "--fg-2", "--outline", "--color-sky", "--surface", "--line", "--accent", "--outline-width", "--radius-xs", "--spacing-12", "--spacing-16", "--spacing-24",
34
+ "--font-text", "--font-heading", "--font-mono", "--weight-heading-card", "--leading-text", "--text-sm",
35
+ ] as const;
36
+
37
+ const PROSE_CSS = `
38
+ html, body { margin: 0; background: transparent; color: var(--fg); font-family: var(--font-text); font-size: 15px; line-height: var(--leading-text); -webkit-font-smoothing: antialiased; }
39
+ .prose { line-height: 1.65; max-width: 680px; padding: var(--spacing-16) var(--spacing-16) var(--spacing-24); }
40
+ .prose h1, .prose h2, .prose h3 { font-family: var(--font-heading); font-weight: var(--weight-heading-card); letter-spacing: -0.02em; line-height: 1.15; }
41
+ .prose code { font-family: var(--font-mono); font-size: 0.9em; background: var(--surface); padding: 1px 5px; border-radius: 6px; }
42
+ .prose pre { background: var(--surface); border: var(--outline-width) solid var(--line); padding: var(--spacing-12); border-radius: var(--radius-xs); overflow-x: auto; }
43
+ .prose pre code { background: none; padding: 0; }
44
+ .prose blockquote { border-left: 4px solid var(--accent); margin: 0; padding-left: var(--spacing-16); color: var(--fg-2); }
45
+ .prose img { display: none; }
46
+ .prose table { font-size: var(--text-sm); border-collapse: collapse; }
47
+ .prose td, .prose th { border: 1px solid var(--line); padding: 4px 8px; }
48
+ .img-ph { display: inline-block; background: var(--color-sky); border: var(--outline-width) dashed var(--outline); border-radius: var(--radius-xs); padding: 2px 8px; font-size: var(--text-sm); }
49
+ `;
50
+
51
+ /**
52
+ * The document for the Markdown and text frame: its policy first, so the output cannot loosen it, then the prose
53
+ * styles with the dashboard's token values, then the rendered output. A token value that could end the style
54
+ * element is left out.
55
+ */
56
+ export function proseDoc(html: string, tokens: Record<string, string>): string {
57
+ const vars = Object.entries(tokens).filter(([, v]) => !/[<>{};]/.test(v)).map(([k, v]) => `${k}: ${v};`).join(" ");
58
+ return `<!doctype html><html><head><meta charset="utf-8"><meta http-equiv="Content-Security-Policy" content="${PROSE_POLICY}">`
59
+ + `<link rel="stylesheet" href="${FONTS}"><style>:root { ${vars} }${PROSE_CSS}</style></head>`
60
+ + `<body><div class="prose">${html}</div></body></html>`;
61
+ }
package/web/pages.ts ADDED
@@ -0,0 +1,42 @@
1
+ // The dashboard's pages: client-side routes of the one page the server serves (DEC-006 of #21). The server
2
+ // serves index.html on each path; the page picks what to show from the path. A new page is one entry here
3
+ // and its render branch in app.tsx. The URL query belongs to the Results page (its filters), never to another.
4
+ export const PAGES = [
5
+ { id: "results", path: "/", title: "Results" },
6
+ { id: "runs", path: "/runs", title: "Runs" },
7
+ { id: "compare", path: "/compare", title: "Compare" },
8
+ { id: "tasks", path: "/tasks", title: "Tasks" },
9
+ { id: "how", path: "/how-it-works", title: "How it works" },
10
+ ] as const;
11
+
12
+ export type PageId = (typeof PAGES)[number]["id"];
13
+
14
+ /** The page a path shows. A trailing slash is ignored; the server serves only the paths above. */
15
+ export const pageOf = (pathname: string): PageId =>
16
+ PAGES.find((p) => p.path === (pathname.replace(/\/+$/, "") || "/"))?.id ?? "results";
17
+
18
+ export const pathOf = (id: PageId): string => PAGES.find((p) => p.id === id)!.path;
19
+
20
+ // Addresses inside a page, so another page can link to one entry (#79). An entry's element id is its fragment.
21
+ /** The element id of a suite on the Tasks page. */
22
+ export const suiteAnchor = (id: string) => `suite-${id}`;
23
+ /** The element id of a task on the Tasks page. */
24
+ export const taskAnchor = (id: string) => `task-${id}`;
25
+ /** The address of a suite on the Tasks page: `/tasks#suite-<id>`. */
26
+ export const suiteHref = (id: string) => `${pathOf("tasks")}#${encodeURIComponent(suiteAnchor(id))}`;
27
+ /** The address of a task on the Tasks page: `/tasks#task-<id>`. */
28
+ export const taskHref = (id: string) => `${pathOf("tasks")}#${encodeURIComponent(taskAnchor(id))}`;
29
+
30
+ /** The sections of the How it works page, in reading order. A section's address is `/how-it-works#<id>`. */
31
+ export const HOW_IT_WORKS_SECTIONS = [
32
+ { id: "trial", title: "A trial" },
33
+ { id: "score", title: "How a score is made" },
34
+ { id: "graders", title: "Checks and judges" },
35
+ { id: "levels", title: "Levels" },
36
+ { id: "left-out", title: "Why trials are left out of a chart" },
37
+ ] as const;
38
+
39
+ export type HowItWorksSection = (typeof HOW_IT_WORKS_SECTIONS)[number]["id"];
40
+
41
+ /** The address of a section of the How it works page, for example `/how-it-works#left-out`. */
42
+ export const howItWorksHref = (section: HowItWorksSection) => `${pathOf("how")}#${section}`;