static-fire 0.0.0-stage → 0.54.1

package/src/core/task-page.ts

@@ -0,0 +1,54 @@
+// What the dashboard's Tasks page shows of a task (#79): what the agent gets, the output path, the gates (the
+// engine's output gate first) and the criteria by measure. Built from an allowlist, so the graders' own material (references, sources, the held-out
+// file) and grading details (context, check details, evidence, derived evidence scripts) never reach the page.
+import { existsSync, readdirSync, statSync } from "node:fs";
+import { join, relative, sep } from "node:path";
+import { OUTPUT_GATE } from "./grade";
+import type { Criterion, Level, Task } from "./types";
+
+export interface TaskPageCriterion { id: string; question: string; level: Level; grader: Criterion["grader"] }
+
+export interface TaskPage {
+  id: string;
+  title: string;
+  description: string;
+  version?: number;
+  prompt: string;
+  /** The input files as the agent finds them in its workspace (`inputs/brief.md`). */
+  inputFiles: string[];
+  /** The project files, copied to the workspace root. Only when the task has `project`. */
+  projectFiles?: string[];
+  output: string;
+  /** The engine's own output gate (grade.ts OUTPUT_GATE), which every task has, first, then the task's own gates. */
+  gates: TaskPageCriterion[];
+  measures: { id: string; label: string; weight: number; criteria: TaskPageCriterion[] }[];
+}
+
+/** File paths under `dir`, relative to `from`, with "/" separators. Names that start with "." are left out, as the task hash leaves them out. */
+function fileNames(dir: string, from: string): string[] {
+  const out: string[] = [];
+  const walk = (p: string) => {
+    if (statSync(p).isFile()) { out.push(relative(from, p).split(sep).join("/")); return; }
+    for (const f of readdirSync(p).sort()) if (!f.startsWith(".")) walk(join(p, f));
+  };
+  if (existsSync(dir)) walk(dir);
+  return out;
+}
+
+const criterion = (c: Criterion): TaskPageCriterion => ({ id: c.id, question: c.question, level: c.level, grader: c.grader });
+
+export function taskPage(task: Task): TaskPage {
+  const projectDir = task.project ? join(task.dir, task.project) : null;
+  return {
+    id: task.id,
+    title: task.title,
+    description: task.description ?? "",
+    ...(task.version != null ? { version: task.version } : {}),
+    prompt: task.prompt,
+    inputFiles: fileNames(join(task.dir, task.inputs), task.dir),
+    ...(projectDir ? { projectFiles: fileNames(projectDir, projectDir) } : {}),
+    output: task.output,
+    gates: [OUTPUT_GATE, ...(task.gates ?? [])].map(criterion),
+    measures: task.measures.map((m) => ({ id: m.id, label: m.label, weight: m.weight ?? 1, criteria: m.criteria.map(criterion) })),
+  };
+}

package/src/core/trial-detail.ts

@@ -0,0 +1,73 @@
+// One trial as the trial drawer reads it (GET /api/trials/:id and its output): the trial row with its run
+// conditions, and every grading it has, each with its grading setup and verdicts. Read-only.
+import { existsSync, readFileSync, realpathSync } from "node:fs";
+import { extname, isAbsolute, join, relative, sep } from "node:path";
+import { trialDir } from "./db";
+import type { GradingRecord, Store, TrialRecord, UserSetupRecord, VerdictRecord } from "./db";
+import { namedSetup, versionNames } from "./grading-versions";
+import type { NamedGradingSetup } from "./grading-versions";
+import { IMAGE_MEDIA } from "./judge";
+
+export interface TrialGrading extends GradingRecord {
+  /** The grading version of its grading hash, with its name, or null when the store has no row for it. */
+  setup: NamedGradingSetup | null;
+  verdicts: VerdictRecord[];
+}
+
+export interface TrialDetail {
+  /** The trial row as stored, run conditions included (setupHash, profileHash, blockedTools, harnessVersion, startedAt). */
+  trial: TrialRecord;
+  /** The user setup of its setup hash (path → content hash of each file), or null. */
+  userSetup: UserSetupRecord | null;
+  /** Every grading of the trial, oldest first, finished or not. Which one a view counts is the view's ViewTrial.gradingId. */
+  gradings: TrialGrading[];
+}
+
+/** The trial and every grading it has, or null for an unknown id. */
+export function trialDetail(store: Store, trialId: string): TrialDetail | null {
+  const trial = store.getTrial(trialId);
+  if (!trial) return null;
+  const setups = new Map<string, NamedGradingSetup | null>();
+  let names: ReturnType<typeof versionNames> | undefined;   // built once, on the first grading that has a setup
+  const setupOf = (hash: string) => {
+    if (!setups.has(hash)) {
+      const s = store.getGradingSetup(hash);
+      // The task's grading versions include every one its trials were graded under, so each stored setup has a name.
+      setups.set(hash, s && namedSetup(s, names ??= versionNames(store, trial.taskId)));
+    }
+    return setups.get(hash)!;
+  };
+  return {
+    trial,
+    userSetup: trial.setupHash ? store.getUserSetup(trial.setupHash) : null,
+    gradings: store.listGradings(trialId).map((g) => ({ ...g, setup: setupOf(g.gradingHash), verdicts: store.listVerdicts(g.id) })),
+  };
+}
+
+/** The trial's output file as the drawer shows it: an image (by the output path's extension) or text, with the output path it was read from. */
+export type TrialOutputFile = ({ image: Uint8Array<ArrayBuffer>; mediaType: string } | { text: string }) & { path: string };
+
+/**
+ * The trial's output file: the task's `output` path, as recorded with the task version the trial ran on, inside
+ * the trial's saved workspace. An image type the judges take (IMAGE_MEDIA) is read as bytes with its media
+ * type; any other file as UTF-8 text. Otherwise a plain reason: the trial is unknown, its task version has no
+ * recorded output path (never guessed from today's task file), the file is missing, or the path (after
+ * symlinks) leaves the workspace.
+ */
+export function readTrialOutputFile(store: Store, dbPath: string, trialId: string): TrialOutputFile | { missing: string } {
+  const trial = store.getTrial(trialId);
+  if (!trial) return { missing: `no trial ${trialId}` };
+  const snap = store.getTaskVersion(trial.taskHash)?.snapshot as { output?: unknown } | undefined;
+  const output = typeof snap?.output === "string" && snap.output ? snap.output : null;
+  if (!output) return { missing: `trial ${trialId}: its task version ${trial.taskHash.slice(0, 8)} has no recorded output path` };
+  const none = { missing: `trial ${trialId} has no output file at ${output} in its saved workspace` };
+  const workspace = join(trialDir(dbPath, trial.id), "workspace");
+  const file = join(workspace, output);
+  if (!existsSync(file) || !existsSync(workspace)) return none;
+  let real: string;
+  try { real = realpathSync(file); } catch { return none; }
+  const rel = relative(realpathSync(workspace), real);
+  if (!rel || rel === ".." || rel.startsWith(`..${sep}`) || isAbsolute(rel)) return { missing: `trial ${trialId}: its output path ${output} leaves the workspace` };
+  const mediaType = IMAGE_MEDIA[extname(output).toLowerCase()];
+  try { return mediaType ? { image: readFileSync(real), mediaType, path: output } : { text: readFileSync(real, "utf8"), path: output }; } catch { return none; }   // a folder, or unreadable
+}

package/src/core/types.ts

@@ -0,0 +1,195 @@
+// The primitives (see CONTEXT.md): Suite → Task → Measure → Criterion → Grader → Verdict.
+// Trials, gradings and batches are records in the record store (src/core/db.ts).
+
+/** Agent harnesses run a model headless. `human` is a person: its output is a file they wrote. */
+export type AgentHarness = "claude" | "codex" | "pi" | "agy";
+export type Harness = AgentHarness | "human";
+export const HARNESSES: Harness[] = ["claude", "codex", "pi", "agy", "human"];
+
+export interface HumanOutput { path: string; note?: string }
+
+export interface AgentConfig {
+  id: string;            // file name in configs/ (the label is built from the fields: core/label.ts)
+  harness: Harness;
+  model: string;         // "by hand" for a human config
+  effort?: string;
+  provider?: string;     // pi only
+  profile?: string;      // apkit Profile name; omitted = no Profile
+  family?: string;       // model family for chart colour; defaults to a guess from `model`
+  outputs?: Record<string, HumanOutput>;  // human only: task id → the file they wrote (absolute path)
+}
+
+export interface JudgeConfig {
+  id: string;
+  label: string;
+  harness: Exclude<AgentHarness, "pi">;   // judge Hosts must support JSON Schema output
+  model: string;
+  effort?: string;
+}
+
+// ---- Task definition -------------------------------------------------------------------------
+
+export type Level = "floor" | "strong" | "ceiling";
+export type Evidence = "output" | "inputs" | "references" | "project" | "sources";
+
+export type Check =
+  | { type: "regex"; pattern: string; flags?: string; min?: number; max?: number }
+  | { type: "words"; min?: number; max?: number }
+  | { type: "frontmatter"; field: string; minChars?: number; maxChars?: number }
+  | { type: "command"; run: string; timeoutSec?: number };
+
+export interface Criterion {
+  id: string;
+  question: string;           // phrased so that "yes" = pass
+  level: Level;
+  grader: "check" | "judge" | "human";
+  check?: Check;              // grader: check
+  evidence?: Evidence[];      // grader: judge (default: ["output"])
+  isolate?: boolean;          // grader: judge — its own call instead of the measure's batch
+  derived?: string;           // grader: judge — a script under the task's grader/ folder that makes an image from the output; the judge sees it (README, "Derived evidence")
+}
+
+export interface Measure {
+  id: string;
+  label: string;
+  weight?: number;            // default 1
+  criteria: Criterion[];
+}
+
+export interface Task {
+  id: string;
+  version?: number;           // a readable label only, in neither hash; a task file can leave it out
+  title: string;
+  description: string;
+  prompt: string;
+  inputs: string;             // copied into the trial workspace
+  output: string;             // path the agent must write, relative to the workspace
+  references?: string;        // graders only
+  project?: string;           // files the agent would find in the real project (copied into the workspace root, not shown to graders)
+  sources?: string;           // graders only: saved copies of the pages the notes link to, for fact checks
+  heldOut?: string;           // the real original this task was built from; an agent output that copies it is invalid
+  context?: string;           // one paragraph a judge reads before the criteria
+  gates: Criterion[];
+  measures: Measure[];
+  dir: string;                // absolute task directory (set by the loader)
+}
+
+export interface Suite {
+  id: string;
+  title: string;
+  description?: string;
+  tasks: string[];
+  weights?: Record<string, number>;
+}
+
+// ---- Results -----------------------------------------------------------------------------------
+
+export interface Usage {
+  inputTokens: number;
+  outputTokens: number;
+  cacheReadTokens: number;
+  cacheWriteTokens: number;
+}
+
+export interface AgentResult {
+  ok: boolean;
+  error?: string;
+  usage: Usage;
+  costUsd: number | null;
+  costSource: "reported" | "estimated" | "unknown";
+  turns?: number;
+}
+
+export interface Vote { pass: boolean; reason: string }
+
+export interface Verdict {
+  score: number | null;       // 0–1 (judge: share of votes passed); null = error
+  votes: Vote[];
+  error?: string;
+}
+
+export interface Grades {
+  verdicts: Record<string, Verdict>;     // by criterion id, gates included
+  gatesPassed: boolean;
+  measures: Record<string, number>;      // 0–100 per measure
+  score: number;                         // 0–100 task score
+  errors: string[];                      // criterion ids that errored (→ trial invalid)
+  judgeCostUsd: number;
+}
+
+export type TrialStatus = "queued" | "running" | "grading" | "done" | "invalid" | "failed";
+
+export interface Trial {
+  id: string;                 // the trial id
+  experimentId: string;       // the batch id (the name stays because tests/score.test.ts sets it)
+  taskId: string;
+  configId: string;
+  index: number;
+  status: TrialStatus;
+  startedAt?: string;
+  endedAt?: string;
+  durationMs?: number;        // agent wall-clock only
+  result?: AgentResult;
+  grades?: Grades;
+  error?: string;
+}
+
+/**
+ * The input summarize() in score.ts takes: the tasks, suites and configs of one view (view.ts builds it).
+ * It is also the shape of the old `experiment.json` that migrate.ts imports. Nothing writes it any more.
+ * The name stays because score.ts and tests/score.test.ts use it, and both stay as they are.
+ */
+export interface Experiment {
+  schema: 2;
+  id: string;
+  suiteIds: string[];
+  tasks: Record<string, number | null>;   // task id → version at run time (null: the task file has none)
+  configIds: string[];
+  trialsPerConfig: number;
+  judge: JudgeConfig & { votes: number };
+  createdAt: string;
+  status: "running" | "done";
+  configs: Record<string, AgentConfig>;  // snapshot at run time
+  suites: Record<string, Suite>;         // snapshot at run time
+  userSetup?: { files: Record<string, string> };         // the user-level setup trials ran with (path → hash)
+  blockedTools?: Partial<Record<AgentHarness, string[]>>;      // outside-service tools blocked in trials
+  gradedAt?: string;                     // when the current grading started; createdAt before any regrade
+  taskDefs?: Record<string, TaskSnapshot>; // each task's criteria as the current grading used them
+}
+
+/** A task's definition as one grading setup used it, so an earlier grading setup shows its own criteria. */
+export type TaskSnapshot = Pick<Task, "id" | "title" | "description" | "version" | "gates" | "measures">;
+
+// ---- Summaries (what the dashboard and `static-fire show` read) -------------------------------------
+
+export interface Stat { mean: number | null; ci: number | null; n: number }
+
+export interface TaskSummary {
+  score: Stat;
+  measures: Record<string, number>;      // mean 0–100 per measure
+  criteria: Record<string, number>;      // mean pass rate 0–1 per criterion
+  costUsd: number | null;
+  costSource: AgentResult["costSource"];
+  durationMs: number | null;
+  valid: number; invalid: number; failed: number; total: number;
+}
+
+export interface ConfigSummary {
+  configId: string;
+  label: string;
+  harness: Harness;
+  model: string;
+  family: string;
+  effort?: string;
+  profile?: string;
+  tasks: Record<string, TaskSummary>;
+  suites: Record<string, Stat>;          // Suite Index per suite
+  index: Stat;                           // Agent Performance Index
+  costUsd: number | null;                // mean cost per task over all tasks
+  durationMs: number | null;             // mean time per task
+}
+
+export interface ExperimentSummary {
+  configs: ConfigSummary[];
+  deadCriteria: Record<string, string[]>; // task id → criterion ids every config passed or failed
+}

package/src/core/view-query.ts

@@ -0,0 +1,102 @@
+// The query of GET /api/view, shared by the server and the dashboard (no Node imports here).
+// One table of parameters: a filter added to the view is one more field in ViewQuery and one more entry in
+// SINGLE (one value) or CONFIG_FACETS (repeated, several values), and in PARAMS for its place in the URL. Both
+// the route and the dashboard's loading code pick it up. This file is the one home of the config facet list.
+import type { ViewFilters } from "./view";
+
+/**
+ * What the chart is about: the Agent Performance Index over the suites in view (the picked suites, else the default
+ * set), or one task.
+ */
+export type Scope = { kind: "index" } | { kind: "task"; id: string };
+
+/** The view a page asks for. Every field is optional; an empty object is the default view. */
+export interface ViewQuery {
+  scope?: Scope;
+  /**
+   * The picked suites of the index, as the repeated `suite` parameter (`?suite=short&suite=writing`). None picked:
+   * the view chooses the default set. Only with the index scope.
+   */
+  suites?: string[];
+  judge?: string;
+  grading?: string;
+  /**
+   * Config facets: the picked values of each. A trial matches a facet when its value is any of them.
+   * `none` selects configs without an effort or without a Profile.
+   */
+  harness?: string[];
+  model?: string[];
+  effort?: string[];
+  profile?: string[];
+  since?: string;
+  until?: string;
+}
+
+/** Scope in a URL: absent or `index`, `task:<id>`. A link from before #80 can say `suite:<id>`: that is `suite=<id>`. */
+export const scopeParam = (s: Scope): string => (s.kind === "index" ? "index" : `task:${s.id}`);
+
+function parseScope(v: string): { scope: Scope; suite?: string } {
+  if (v === "index") return { scope: { kind: "index" } };
+  const m = /^(suite|task):(.+)$/.exec(v);
+  if (!m) throw new Error(`scope ${v} is not index, suite:<id> or task:<id>`);
+  return m[1] === "task" ? { scope: { kind: "task", id: m[2]! } } : { scope: { kind: "index" }, suite: m[2]! };
+}
+
+/** Parameters that take one value each. A view has one judge, one grading version and one date range. */
+type Single = "judge" | "grading" | "since" | "until";
+const SINGLE: Single[] = ["judge", "grading", "since", "until"];
+/** The config facets, each a field of the trial row: the parameter repeats for each picked value (`?model=opus&model=haiku`). */
+export const CONFIG_FACETS = ["harness", "model", "effort", "profile"] as const;
+export type ConfigFacet = (typeof CONFIG_FACETS)[number];
+/** Every parameter, in the order a URL the page writes lists them. */
+const PARAMS: readonly ("scope" | "suite" | Single | ConfigFacet)[] = ["scope", "suite", "judge", "grading", "harness", "model", "effort", "profile", "since", "until"];
+const KNOWN = new Set<string>(["scope", "suite", ...SINGLE, ...CONFIG_FACETS]);
+const isConfig = (k: string): k is ConfigFacet => (CONFIG_FACETS as readonly string[]).includes(k);
+
+/** The values in one stable order, each once, with empty ones dropped: a set of picks has one URL. */
+export const valueSet = (values: Iterable<string>): string[] => [...new Set(values)].filter(Boolean).sort();
+
+/** Reads a query string. Throws on an unknown or repeated parameter or a bad scope, so a filter is never silently ignored. */
+export function parseViewQuery(params: URLSearchParams): ViewQuery {
+  const unknown = [...new Set([...params.keys()].filter((k) => !KNOWN.has(k)))];
+  if (unknown.length) throw new Error(`unknown view parameter(s): ${unknown.join(", ")} (known: ${PARAMS.join(", ")})`);
+  const repeated = (["scope", ...SINGLE] as string[]).filter((k) => params.getAll(k).length > 1);
+  if (repeated.length) throw new Error(`view parameter(s) ${repeated.join(", ")} can take only one value (repeat only ${CONFIG_FACETS.join(", ")})`);
+  const q: ViewQuery = {};
+  const scope = params.get("scope");
+  const suites = params.getAll("suite");
+  if (scope) {
+    const parsed = parseScope(scope);
+    q.scope = parsed.scope;
+    if (parsed.suite) suites.push(parsed.suite);
+  }
+  const picked = valueSet(suites);
+  if (picked.length && q.scope?.kind === "task") throw new Error(`suite cannot be picked with scope ${scope}: a task's view has no suites`);
+  if (picked.length) q.suites = picked;
+  for (const k of SINGLE) { const v = params.get(k); if (v) q[k] = v; }
+  for (const k of CONFIG_FACETS) { const v = valueSet(params.getAll(k)); if (v.length) q[k] = v; }
+  return q;
+}
+
+/** The query string for `q`: only the fields that are set; the index scope is the default, so it is left out. */
+export function viewQueryParams(q: ViewQuery): URLSearchParams {
+  const p = new URLSearchParams();
+  if (q.scope && q.scope.kind !== "index") p.set("scope", scopeParam(q.scope));
+  else for (const v of valueSet(q.suites ?? [])) p.append("suite", v);
+  for (const k of PARAMS) {
+    if (k === "scope" || k === "suite") continue;
+    if (isConfig(k)) for (const v of valueSet(q[k] ?? [])) p.append(k, v);
+    else if (q[k]) p.set(k, q[k]!);
+  }
+  return p;
+}
+
+/** The view function's filters for `q`: the picked suites of the index, or a task scope's task. */
+export function toViewFilters(q: ViewQuery): ViewFilters {
+  const f: ViewFilters = {};
+  if (q.scope?.kind === "task") f.tasks = [q.scope.id];
+  else if (q.suites?.length) f.suites = valueSet(q.suites);
+  for (const k of SINGLE) if (q[k]) f[k] = q[k];
+  for (const k of CONFIG_FACETS) { const v = valueSet(q[k] ?? []); if (v.length) f[k] = v; }
+  return f;
+}