static-fire 0.0.0-stage → 0.44.0

package/src/api.ts

@@ -0,0 +1,218 @@
+// The JSON API the dashboard reads. `serve` (src/server.ts) mounts it next to the page; tests mount it on
+// their own server with a temporary store and catalog, so they never touch runs/.
+import { parseRunRequest, planBatch, runBatch, startBatch, type BatchPlan } from "./core/batch";
+import { batchState } from "./core/batch-state";
+import { listBatchSummaries, summarizeBatch, type BatchSummary } from "./core/batches";
+import { openStore } from "./core/db";
+import type { Store } from "./core/db";
+import { listTasks, loadConfigs, loadJudges } from "./core/authored";
+import { projectRoot } from "./core/project";
+import { readTrialOutputFile, trialDetail } from "./core/trial-detail";
+import type { Task } from "./core/types";
+import { compareGradings } from "./core/compare";
+import { buildFacets } from "./core/facets";
+import { buildView } from "./core/view";
+import type { Catalog } from "./core/view";
+import { parseViewQuery, toViewFilters } from "./core/view-query";
+import { failureMessage, planRegrade, regrade, type RegradePlan } from "./core/regrade";
+
+export interface ApiOptions {
+  /** The store the routes read (and, from later routes, write). */
+  store: Store;
+  /** The path of that store's database. The event stream opens its own connection to it. */
+  dbPath: string;
+  /** The authored definitions, read per request so an edited task or suite file shows on the next load. */
+  catalog: () => Catalog;
+  /** How often the event stream checks the database, in ms. Default 1000. */
+  pollMs?: number;
+  /** The project folder POST /api/runs plans against, and the configs, judges and tasks lists read. Default: the project found from the current folder. */
+  root?: string;
+  /** Where a started batch keeps its Profile snapshots. Default the project's .cache. */
+  cacheDir?: string;
+  /** Called on a checked plan before anything is written; throws to refuse it. Test servers refuse paid calls with it. */
+  checkPlan?: (plan: BatchPlan) => void;
+  /** The clock that decides whether a batch has stalled. */
+  now?: () => Date;
+  /**
+   * Called on a regrade's plan, with the tasks it grades, before anything is written; throws to refuse it.
+   * Test servers refuse a plan with a judge criterion in it (a real judge call).
+   */
+  checkRegrade?: (plan: RegradePlan, tasks: Task[]) => void;
+  /** The function a regrade runs. Default regrade.ts's; tests pass one that fails. */
+  runRegrade?: typeof regrade;
+}
+
+/** The regrade this server is running: one at a time. A page reload and a second tab read it from GET /api/regrade. */
+export interface RegradeRun {
+  /** The view query it was started from, as /api/view's query string. */
+  query: string;
+  taskIds: string[];
+  /** The judge's label. */
+  judge: string;
+  startedAt: string;
+  /** Trials to grade, and gradings added so far. */
+  total: number;
+  graded: number;
+}
+
+/** The last regrade that ended here: its counts, or the error that stopped it. */
+export interface RegradeEnded extends RegradeRun {
+  endedAt: string; done: number; invalid: number;
+  /** Trials that could not be graded; `error` names each one. */
+  failed: number;
+  error: string | null;
+}
+
+const bad = (e: unknown) => Response.json({ error: e instanceof Error ? e.message : String(e) }, { status: 400 });
+
+/** The API routes, and `close` to release the event stream's database connection. */
+export function apiRoutes({ store, dbPath, catalog, pollMs = 1000, root: givenRoot, cacheDir, checkPlan, now = () => new Date(), checkRegrade, runRegrade = regrade }: ApiOptions) {
+  const root = () => givenRoot ?? projectRoot();   // looked up per request, so a server built with only a catalog needs no project
+  // A connection of its own that only reads PRAGMA data_version. The value moves when any other connection
+  // commits, including the one the routes write through, so every write to the store emits an event.
+  const watcher = openStore(dbPath);
+  const dataVersion = () => Number(watcher.pragma("data_version"));
+  // Batches this process is running. A batch here is running even when no trial has moved for a while.
+  const live = new Set<string>();
+  let regrading: RegradeRun | null = null, lastRegrade: RegradeEnded | null = null;
+  const withState = (s: BatchSummary) => ({ ...s, ...batchState(s.batch, store.listTrials({ batchId: s.batch.id }), { live: live.has(s.batch.id), now: now() }) });
+
+  const routes = {
+    "/api/configs": () => Response.json(loadConfigs(root())),
+    "/api/judges": () => Response.json(loadJudges(root())),
+    "/api/suites": () => Response.json(catalog().suites),   // the suites the view reads, so the scope selector matches it
+    "/api/tasks": () => Response.json(listTasks(root())),
+    "/api/view": (req: Request) => {
+      try { return Response.json(buildView(store, toViewFilters(parseViewQuery(new URL(req.url).searchParams)), catalog())); } catch (e) { return bad(e); }
+    },
+    "/api/facets": (req: Request) => {
+      try { return Response.json(buildFacets(store, toViewFilters(parseViewQuery(new URL(req.url).searchParams)), catalog())); } catch (e) { return bad(e); }
+    },
+    /** Two grading setups of one task compared over the trials valid under both: the view parameters plus `a` and `b`. */
+    "/api/compare": (req: Request) => {
+      try {
+        const params = new URL(req.url).searchParams;
+        const a = params.get("a") ?? "", b = params.get("b") ?? "";
+        params.delete("a"); params.delete("b");
+        return Response.json(compareGradings(store, toViewFilters(parseViewQuery(params)), a, b, catalog()));
+      } catch (e) { return bad(e); }
+    },
+    /** Starts a batch: checks the request (nothing is written when it is bad), records the batch, returns its id, then runs it here. */
+    "/api/runs": {
+      POST: async (req: Request) => {
+        let plan: BatchPlan;
+        try {
+          const body = await req.json().catch(() => { throw new Error("the request body is not JSON"); });
+          plan = planBatch(parseRunRequest(body), root());   // the shape first: nothing is chosen from a malformed field
+          checkPlan?.(plan);
+        } catch (e) { return bad(e); }
+        const started = await startBatch(store, plan, { dbPath, root: root(), cacheDir });
+        const id = started.batch.id;
+        live.add(id);
+        runBatch(store, started, { onEvent: (m) => console.log(m) })
+          .catch((e) => console.error(`batch ${id}: ${e instanceof Error ? e.message : String(e)}`))
+          .finally(() => live.delete(id));
+        return Response.json({ id });
+      },
+    },
+    /**
+     * Regrades the trials a view lists as gradable (view.leftOut.gradable), with judge calls only. The query is
+     * /api/view's. Refused (400, nothing written) for a view on an earlier grading setup, a view with nothing to
+     * regrade, or a plan checkRegrade refuses; 409 while another regrade runs. Trials of a running batch are
+     * left to that batch, which is grading them. Returns 202, then grades in this process.
+     */
+    "/api/regrade": {
+      GET: () => Response.json({ running: regrading, last: lastRegrade }),
+      POST: (req: Request) => {
+        // No await before `regrading` is set: two requests cannot both pass this check.
+        if (regrading) return Response.json({ error: "A regrade is already running.", running: regrading }, { status: 409 });
+        let plan: RegradePlan, query: string, taskIds: string[], cat: Catalog, leftForBatch: number;
+        try {
+          const params = new URL(req.url).searchParams;
+          cat = catalog();
+          const view = buildView(store, toViewFilters(parseViewQuery(params)), cat);
+          query = params.toString();
+          taskIds = view.filters.taskIds;
+          const earlier = Object.entries(view.tasks).filter(([, t]) => !t.current).map(([id]) => id);
+          if (earlier.length) throw new Error(`This view is on an earlier grading version (${earlier.join(", ")}). A regrade always uses today's criteria.`);
+          const runningBatch = new Map<string, boolean>();
+          const isRunning = (batchId: string) => {
+            if (!runningBatch.has(batchId)) {
+              const b = store.getBatch(batchId);
+              runningBatch.set(batchId, !!b && batchState(b, store.listTrials({ batchId }), { live: live.has(batchId), now: now() }).state === "running");
+            }
+            return runningBatch.get(batchId)!;
+          };
+          const ids = view.leftOut.gradable.filter((id) => !isRunning(store.getTrial(id)!.batchId));
+          leftForBatch = view.leftOut.gradable.length - ids.length;
+          if (!ids.length) throw new Error(leftForBatch ? "Nothing in this view to regrade: its trials with no grading belong to a running batch." : "Nothing in this view to regrade.");
+          plan = planRegrade(store, { tasks: taskIds, trials: ids }, { dbPath, catalog: cat, judge: view.filters.judge.id });
+          if (!plan.todo.length) throw new Error("Nothing in this view to regrade.");
+          checkRegrade?.(plan, [...new Set(plan.todo.map((x) => x.taskId))].map((id) => cat.loadTask(id)));
+        } catch (e) { return bad(e); }
+        const run: RegradeRun = { query, taskIds, judge: plan.judge.label, startedAt: now().toISOString(), total: plan.todo.length, graded: 0 };
+        regrading = run;
+        const end = (c: { done: number; invalid: number; failed: number }, error: string | null) => {
+          lastRegrade = { ...run, endedAt: now().toISOString(), done: c.done, invalid: c.invalid, failed: c.failed, error };
+          regrading = null;
+        };
+        // An async function runs to its first await at once; a throw there must still clear the running state.
+        Promise.resolve()
+          .then(() => runRegrade(store, { tasks: taskIds, trials: plan.todo.map((x) => x.trialId) }, {
+            dbPath, catalog: cat, judge: plan.judge.id, votes: plan.votes,
+            onEvent: (m) => console.log(m), onGraded: () => { run.graded++; },
+          }))
+          .then((c) => {
+            const error = failureMessage(c);
+            if (error) console.error(`regrade: ${error}`);
+            end(c, error);
+          }, (e) => {
+            const error = e instanceof Error ? e.message : String(e);
+            console.error(`regrade: ${error}`);
+            end({ done: 0, invalid: 0, failed: 0 }, error);
+          });
+        return Response.json({ total: run.total, leftForBatch }, { status: 202 });
+      },
+    },
+    /** Every batch, newest first, with its progress and whether it is running, stalled or ended. */
+    "/api/batches": () => Response.json(listBatchSummaries(store).map(withState)),
+    /** One batch with the status of each trial. */
+    "/api/batches/:id": (req: Request & { params: { id: string } }) => {
+      const batch = store.getBatch(req.params.id);
+      if (!batch) return Response.json({ error: `no batch ${req.params.id}` }, { status: 404 });
+      const trials = store.listTrials({ batchId: batch.id }).map(({ id, taskId, configId, n, status, startedAt, endedAt, error }) => ({ id, taskId, configId, n, status, startedAt, endedAt, error }));
+      return Response.json({ ...withState(summarizeBatch(store, batch)), trials });
+    },
+    /** One trial with its run conditions and every grading it has, each with its verdicts. */
+    "/api/trials/:id": (req: Request & { params: { id: string } }) => {
+      const d = trialDetail(store, req.params.id);
+      return d ? Response.json(d) : Response.json({ error: `no trial ${req.params.id}` }, { status: 404 });
+    },
+    /** The trial's output file from its saved workspace: an image with its media type, else Markdown text; 404 with the reason when there is none. */
+    "/api/trials/:id/output": (req: Request & { params: { id: string } }) => {
+      const out = readTrialOutputFile(store, dbPath, req.params.id);
+      return "missing" in out ? new Response(out.missing, { status: 404, headers: { "content-type": "text/plain; charset=utf-8" } })
+        : "image" in out ? new Response(out.image, { headers: { "content-type": out.mediaType } })
+        : new Response(out.text, { headers: { "content-type": "text/markdown; charset=utf-8" } });
+    },
+    /** Server-sent events: one on connect, then one each time the database changes. The data is the data_version. */
+    "/api/events": () => {
+      let last = -1, timer: Timer;
+      const stream = new ReadableStream({
+        start(ctrl) {
+          const tick = () => {
+            try {
+              const v = dataVersion();
+              if (v !== last) { last = v; ctrl.enqueue(`data: ${v}\n\n`); }
+            } catch { clearInterval(timer); }
+          };
+          tick();
+          timer = setInterval(tick, pollMs);
+        },
+        cancel() { clearInterval(timer); },
+      });
+      return new Response(stream, { headers: { "content-type": "text/event-stream", "cache-control": "no-cache" } });
+    },
+  };
+  return { routes, close: () => watcher.close() };
+}

package/src/cli.ts

@@ -0,0 +1,297 @@
+#!/usr/bin/env bun
+// static-fire — the same primitives the dashboard uses, for people and agents.
+//   static-fire --version                   the installed version
+//   static-fire init [--json]               make a project in the current folder (core/init.ts); never overwrites a file
+//   static-fire guide                       the guide for agents (src/guide.md); needs no project
+//   static-fire serve [--port 4477]
+//   static-fire configs | judges | suites | tasks
+//   static-fire new task <id>               a new task folder: evals/<id>/task.yaml (commented, to fill in) and inputs/
+//   static-fire run --suites writing[,visual] | --tasks a[,b]  [--configs a,b|all] [--trials 3] [--judge id] [--votes 3] [--concurrency 4]
+//                                           one batch in the store; trial artifacts in the project's runs/trials/<trial-id>/
+//   static-fire regrade [--tasks t] [--configs a,b] [--batch id] [--judge id] [--votes n] [--only-invalid] [--concurrency 4]
+//                                           grade stored outputs again (judge calls only) where the current grading is missing
+//   static-fire rerun <batch>               run its failed trials again, as new trials in the same batch
+//   static-fire list                        every batch in the store, newest first, with its trial counts
+//   static-fire show [--suites s | --tasks t] [--judge id] [--grading hash] [--since date] [--until date] [--json]
+//                                           the comparable trials, pooled across batches, from the store
+//   static-fire export [path]               every record in the store as JSON (to stdout without a path)
+//   static-fire migrate [--from <folder>]   one-off import of the 2026-11-06 experiment into the store
+//
+// The commands and the flags each takes are listed in core/commands.ts; a flag a command does not take is an error.
+// The project is the folder found by walking up from the current folder to static-fire.yaml (core/project.ts).
+// Each command that needs one calls needProject() before it reads or writes anything; outside a project it
+// stops with an error that names `static-fire init`. The record store is the project's runs/static-fire.db,
+// or STATIC_FIRE_DB when set; trial artifacts go next to the store, and the Profile cache in the project's .cache/.
+import { readFileSync, renameSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+import { parseArgs } from "node:util";
+import { configLabel } from "./core/label";
+import { listTasks, loadConfigs, loadJudges, loadSuites } from "./core/authored";
+import { COMMANDS, OPTIONS, unknownFlags, USAGE } from "./core/commands";
+import { ENGINE_ROOT, NoProjectError, projectPaths, projectRoot, type ProjectPaths } from "./core/project";
+
+const LABEL_WIDTH = 52;   // fits the longest built config label, so `show` columns line up
+
+const [cmd, ...rest] = Bun.argv.slice(2);
+if (cmd === "--version" || cmd === "-v") {
+  console.log((await import("../package.json")).default.version);
+  process.exit(0);
+}
+const { values, positionals, tokens } = parseArgs({ args: rest, allowPositionals: true, tokens: true, options: OPTIONS });
+// A flag the command does not take stops it, so a flag is never ignored in silence.
+if (cmd && COMMANDS[cmd]) {
+  const extra = unknownFlags(cmd, tokens.flatMap((t) => (t.kind === "option" ? [t.name] : [])));
+  if (extra.length) {
+    const takes = COMMANDS[cmd]!.flags;
+    console.error(`${cmd}: ${extra.join(", ")} ${extra.length === 1 ? "is not a flag" : "are not flags"} of \`static-fire ${cmd}\`; it takes ${takes.length ? takes.map((f) => `--${f}`).join(", ") : "no flags"}`);
+    process.exit(1);
+  }
+}
+
+const fmt = (n: number | null | undefined, d = 1) => (n == null ? "—" : n.toFixed(d));
+const log = (m: string) => console.log(m);
+
+/** The project of this command, or exit 1 with an error that names `static-fire init`, before anything is written. */
+function needProject(): ProjectPaths {
+  try {
+    return projectPaths(projectRoot());
+  } catch (e) {
+    if (!(e instanceof NoProjectError)) throw e;
+    console.error(`${cmd}: ${e.message}`);
+    process.exit(1);
+  }
+}
+
+switch (cmd) {
+  case "init": {
+    const { formatReport, initProject } = await import("./core/init");
+    const report = initProject();
+    process.stdout.write(values.json ? `${JSON.stringify(report, null, 2)}\n` : formatReport(report));
+    break;
+  }
+  case "guide":
+    // An engine file, so it matches the installed version. It needs no project.
+    process.stdout.write(readFileSync(join(ENGINE_ROOT, "src", "guide.md"), "utf8"));
+    break;
+  case "serve": {
+    const project = needProject();
+    const { serve } = await import("./server");
+    serve(Number(values.port), project);
+    break;
+  }
+  case "configs":
+    for (const c of loadConfigs(needProject().root)) console.log(`${c.id.padEnd(24)} ${configLabel(c)}`);
+    break;
+  case "judges":
+    loadJudges(needProject().root).forEach((j, i) => console.log(`${j.id.padEnd(24)} ${j.label}${i === 0 ? "  (default)" : ""}`));
+    break;
+  case "suites":
+    for (const s of loadSuites(needProject().root)) console.log(`${s.id.padEnd(24)} ${s.title}: ${s.tasks.join(", ")}`);
+    break;
+  case "tasks":
+    for (const t of listTasks(needProject().root)) {
+      console.log(`${t.id.padEnd(24)} ${t.title}`);
+      for (const e of t.errors) console.log(`  error: ${e}`);
+      for (const w of t.warnings) console.log(`  warning: ${w}`);
+    }
+    break;
+  case "new": {
+    const [what, id] = positionals;
+    if (what !== "task" || !id || positionals.length > 2) {
+      console.error("usage: static-fire new task <id>");
+      process.exit(1);
+    }
+    const { root } = needProject();
+    const { BadTaskId, TaskExists, newTask } = await import("./core/new-task");
+    try {
+      newTask(root, id);
+    } catch (e) {
+      if (!(e instanceof BadTaskId || e instanceof TaskExists)) throw e;
+      console.error(`new task: ${e.message}`);
+      process.exit(1);
+    }
+    log(`made evals/${id}/task.yaml and evals/${id}/inputs/`);
+    log(`next: put the files the agent works from in evals/${id}/inputs/, fill in task.yaml, then run \`static-fire tasks\` to check it`);
+    break;
+  }
+  case "run": {
+    const { root, dbPath, cache } = needProject();
+    const { planBatch, runBatch, startBatch } = await import("./core/batch");
+    const { openStore } = await import("./core/db");
+    const list = (v?: string) => (v === undefined ? undefined : v.split(",").map((x) => x.trim()).filter(Boolean));
+    const plan = planBatch({
+      tasks: list(values.tasks), suites: values.suites === "all" ? undefined : list(values.suites), configs: list(values.configs),
+      trials: Number(values.trials), judge: values.judge, votes: values.votes ? Number(values.votes) : undefined,
+    }, root);
+    const store = openStore(dbPath);
+    try {
+      const started = await startBatch(store, plan, { dbPath, root, cacheDir: cache });
+      log(`batch ${started.batch.id}: ${plan.tasks.length} task(s) × ${plan.configs.length} config(s), ${started.trials.length} trial(s), judge ${plan.judge.label} × ${plan.votes} votes`);
+      await runBatch(store, started, { concurrency: Number(values.concurrency), onEvent: log });
+    } finally {
+      store.close();
+    }
+    break;
+  }
+  case "regrade": {
+    const { root, dbPath } = needProject();
+    const { formatCounts, regrade } = await import("./core/regrade");
+    const { openStore } = await import("./core/db");
+    const { projectCatalog } = await import("./core/view");
+    const list = (v?: string) => (v === undefined ? undefined : v.split(",").map((x) => x.trim()).filter(Boolean));
+    const store = openStore(dbPath);
+    try {
+      const counts = await regrade(store, { tasks: list(values.tasks), configs: list(values.configs), batch: values.batch }, {
+        dbPath, catalog: projectCatalog(root), judge: values.judge, votes: values.votes ? Number(values.votes) : undefined, onlyInvalid: values["only-invalid"],
+        concurrency: Number(values.concurrency), onEvent: log,
+      });
+      log(formatCounts(counts));
+      if (counts.failed) {
+        console.error(`regrade: ${counts.failed} trial(s) could not be graded (listed above)`);
+        process.exitCode = 1;
+      }
+    } catch (e) {
+      console.error(`regrade: ${e instanceof Error ? e.message : String(e)}`);
+      process.exitCode = 1;
+    } finally {
+      store.close();
+    }
+    break;
+  }
+  case "rerun": {
+    const id = positionals[0];
+    if (!id) throw new Error("usage: static-fire rerun <batch>");
+    const { root, dbPath, cache } = needProject();
+    const { runBatch } = await import("./core/batch");
+    const { RerunRefused, planRerun, startRerun } = await import("./core/batches");
+    const { openStore } = await import("./core/db");
+    const store = openStore(dbPath);
+    try {
+      let plan;
+      try { plan = planRerun(store, id, root); } catch (e) {
+        if (!(e instanceof RerunRefused)) throw e;
+        console.error(`rerun: ${e.message}`);
+        process.exitCode = 1;
+        break;
+      }
+      if (plan.busy.length) log(`still queued or running, not rerun: ${plan.busy.join(", ")}`);
+      if (!plan.slots.length) { log(`no failed trials to rerun in ${id}`); break; }
+      const started = await startRerun(store, plan, { dbPath, root, cacheDir: cache });
+      for (const t of started.trials) log(`+ ${t.id} (for ${started.replaces[t.id]})`);
+      await runBatch(store, started, { concurrency: Number(values.concurrency), onEvent: log });
+    } finally {
+      store.close();
+    }
+    break;
+  }
+  case "list": {
+    const { dbPath } = needProject();
+    const { listBatchSummaries } = await import("./core/batches");
+    const { openStore } = await import("./core/db");
+    const store = openStore(dbPath);
+    let all;
+    try { all = listBatchSummaries(store); } finally { store.close(); }
+    if (!all.length) { log(`no batches in ${dbPath}`); break; }
+    const when = (iso: string) => `${iso.slice(0, 10)} ${iso.slice(11, 16)} UTC`;
+    for (const { batch, request: r, counts: c, toRerun, ended } of all) {
+      log(`${batch.id}  ${when(batch.startedAt)}  ${ended ? `ended ${when(batch.endedAt!)}` : "not ended"}`);
+      const what = r.tasks ? `tasks ${r.tasks.join(", ")}` : `suites ${r.suites?.join(", ") ?? "all"}`;
+      log(`  ${what} · configs ${r.configs.join(", ")} · ${r.trials} trial${r.trials === 1 ? "" : "s"} · judge ${r.judge} × ${r.votes} votes`);
+      const busy = [c.queued ? `queued ${c.queued}` : "", c.running ? `running ${c.running}` : ""].filter(Boolean);
+      log(`  ${[`done ${c.done}`, `failed ${c.failed}`, `invalid ${c.invalid}`, ...busy, ...(toRerun ? [`${toRerun} to rerun`] : [])].join(" · ")}`);
+    }
+    break;
+  }
+  case "show": {
+    const { root, dbPath } = needProject();
+    const { openStore } = await import("./core/db");
+    const { buildView, projectCatalog } = await import("./core/view");
+    const list = (v: string | undefined) => v?.split(",").map((x) => x.trim()).filter(Boolean);
+    const store = openStore(dbPath);
+    let view;
+    try {
+      view = buildView(store, { suites: list(values.suites), tasks: list(values.tasks), judge: values.judge, grading: values.grading, since: values.since, until: values.until }, projectCatalog(root));
+    } finally { store.close(); }
+    if (values.json) { console.log(JSON.stringify(view, null, 2)); break; }
+    printView(view);
+    break;
+  }
+  case "export": {
+    const { dbPath } = needProject();
+    const { openStore } = await import("./core/db");
+    const store = openStore(dbPath);
+    let doc;
+    try { doc = store.exportAll(); } finally { store.close(); }
+    const json = JSON.stringify(doc, null, 2);
+    const path = positionals[0];
+    if (!path) { console.log(json); break; }
+    const tmp = `${path}.tmp`;   // a reader never sees half a file
+    writeFileSync(tmp, json);
+    renameSync(tmp, path);
+    const records = Object.values(doc.tables).reduce((n, rows) => n + rows.length, 0);
+    console.log(`exported ${records} records to ${path}`);
+    break;
+  }
+  case "migrate": {
+    // The store is STATIC_FIRE_DB (else the project's runs/static-fire.db); the artifacts go next to it, in trials/.
+    const { root, runs, dbPath } = needProject();
+    const { DEFAULT_EXPERIMENT, migrateExperiment } = await import("./core/migrate");
+    try {
+      migrateExperiment({ source: values.from ?? `${runs}/${DEFAULT_EXPERIMENT}`, dbPath, root, log });
+    } catch (e) {
+      console.error(`migrate: ${e instanceof Error ? e.message : String(e)}`);
+      process.exit(1);
+    }
+    break;
+  }
+  default:
+    console.log(USAGE);
+}
+
+type View = import("./core/view").View;
+
+/** `show` as text: what is in view, one row per config, partial suites, left-out counts, dead criteria. */
+function printView(v: View) {
+  const { filters: f } = v;
+  const range = f.since || f.until ? ` · ${f.since?.slice(0, 10) ?? "…"} to ${f.until?.slice(0, 10) ?? "…"}` : "";
+  log(`${f.suiteIds.length ? `suites ${f.suiteIds.join(", ")}` : `tasks ${f.taskIds.join(", ")}`} · judge ${f.judge.label}${range}`);
+  for (const [id, t] of Object.entries(v.tasks)) {
+    const grading = t.gradingName ? `${t.gradingName} (${t.gradingHash.slice(0, 8)})` : t.gradingHash.slice(0, 8);
+    log(`  ${id}${t.label != null ? ` v${t.label}` : ""} · task ${t.taskHash.slice(0, 8)} · grading ${grading}${t.current ? "" : " (earlier)"}`);
+  }
+  const bySuites = f.suiteIds.length > 0;
+  const day = (s: string | null) => s?.slice(0, 10) ?? "";
+  log(`${"config".padEnd(LABEL_WIDTH)} ${bySuites ? "index" : "score"}   ±95%     n  dates                    $/task   time   valid`);
+  const headline = (c: View["summary"]["configs"][number]) => (bySuites ? c.index : c.tasks[f.taskIds[0]!]!.score);
+  for (const c of [...v.summary.configs].sort((a, b) => (headline(b).mean ?? -1) - (headline(a).mean ?? -1))) {
+    const pooled = Object.values(c.pooled);
+    const n = pooled.reduce((a, p) => a + p.n, 0);
+    const from = pooled.map((p) => p.from).filter((x): x is string => !!x).sort()[0] ?? null;
+    const to = pooled.map((p) => p.to).filter((x): x is string => !!x).sort().at(-1) ?? null;
+    const dates = from ? (day(from) === day(to) ? day(from) : `${day(from)} to ${day(to)}`) : "—";
+    const ts = Object.values(c.tasks);
+    const valid = `${ts.reduce((a, t) => a + t.valid, 0)}/${ts.reduce((a, t) => a + t.total, 0)}`;
+    const h = headline(c);
+    log(`${c.label.padEnd(LABEL_WIDTH)} ${fmt(h.mean).padStart(5)}  ${fmt(h.ci).padStart(5)}  ${String(n).padStart(4)}  ${dates.padEnd(24)} ${fmt(c.costUsd, 3).padStart(6)}  ${(c.durationMs == null ? "—" : `${fmt(c.durationMs / 1000, 0)}s`).padStart(5)}  ${valid}`);
+    if (bySuites && f.suiteIds.length > 1) {
+      log(`  ${f.suiteIds.map((s) => `${s} ${c.partial[s] ? `partial (${c.partial[s]!.have} of ${c.partial[s]!.of} tasks)` : fmt(c.suites[s]!.mean)}`).join(" · ")}`);
+    } else if (bySuites) {
+      for (const s of f.suiteIds) if (c.partial[s]) log(`  ${s}: partial (${c.partial[s]!.have} of ${c.partial[s]!.of} tasks)`);
+    } else if (f.taskIds.length > 1) {
+      log(`  ${f.taskIds.map((t) => `${t} ${fmt(c.tasks[t]!.score.mean)}`).join(" · ")}`);
+    }
+  }
+  if (!v.summary.configs.length) log("no comparable trials in this view");
+  const { otherGrading: og, otherTask: ot, ungraded: ug } = v.leftOut;
+  const of = (x: { trials: number; configs: number }) => `${x.trials} trial${x.trials === 1 ? "" : "s"} from ${x.configs} config${x.configs === 1 ? "" : "s"}`;
+  if (og.trials) log(`left out: ${of(og)} graded under other criteria or another judge (regrade to bring them in)`);
+  if (ot.trials) {
+    log(`left out: ${of(ot)} ran on another version of the task`);
+    for (const ver of ot.versions) {
+      const versions = ver.gradingSetups.map((g) => `${g.hash.slice(0, 12)} (${g.name}, judge ${g.judgeLabel})`).join(", ") || "none recorded";
+      log(`  ${ver.taskId}${ver.label != null ? ` v${ver.label}` : ""} · task ${ver.taskHash.slice(0, 8)} · ${ver.trials} trial(s) · show it with --grading: ${versions}`);
+    }
+  }
+  if (ug.trials) log(`left out: ${of(ug)} not graded yet`);
+  for (const [t, ids] of Object.entries(v.summary.deadCriteria)) log(`dead criteria in ${t}: ${ids.join(", ")}`);
+}

package/src/core/apkit-install.ts

@@ -0,0 +1,47 @@
+// Builds one Profile snapshot in a process of its own, which holds the apkit lock for the whole sequence:
+// install into a scratch folder, copy it to the snapshot, remove the install. The `run` that starts it
+// (profiles.ts) waits for it. If that `run` is killed, this process goes on to the end of the sequence and only
+// then lets go of the lock, so another install never overlaps a running one.
+//   bun src/core/apkit-install.ts <lock file> <profile> <apkit agent> <snapshot folder>
+import { cpSync, existsSync, mkdirSync, renameSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { basename, dirname, join } from "node:path";
+import { withFileLock } from "./lock";
+
+async function apkit(args: string[]) {
+  const p = Bun.spawn(["apkit", ...args], { stdout: "pipe", stderr: "pipe", stdin: "ignore" });
+  const [out, err, code] = await Promise.all([new Response(p.stdout).text(), new Response(p.stderr).text(), p.exited]);
+  if (code !== 0) throw new Error(`apkit ${args[0]} failed (exit ${code}): ${(err || out).trim().slice(-400)}`);
+}
+
+async function build(profile: string, agent: string, snap: string) {
+  if (existsSync(snap)) return;
+  const scratch = join(tmpdir(), "static-fire-apkit", `${process.pid}`, basename(dirname(snap)), basename(snap));
+  rmSync(scratch, { recursive: true, force: true });
+  mkdirSync(scratch, { recursive: true });
+  try {
+    // --yes: apkit asks before creating AGENTS.md; this scratch folder is ours.
+    await apkit(["install", profile, scratch, "--agents", agent, "--yes"]);
+    // Copied beside the snapshot, then renamed: a snapshot folder is always complete.
+    const part = `${snap}.part`;
+    rmSync(part, { recursive: true, force: true });
+    mkdirSync(part, { recursive: true });
+    cpSync(scratch, part, { recursive: true, dereference: true });
+    renameSync(part, snap);
+  } finally {
+    await apkit(["remove", scratch, "--yes"]).catch(() => {});
+    rmSync(scratch, { recursive: true, force: true });
+  }
+}
+
+const [lock, profile, agent, snap] = Bun.argv.slice(2);
+if (!lock || !profile || !agent || !snap) {
+  console.error("usage: apkit-install.ts <lock file> <profile> <apkit agent> <snapshot folder>");
+  process.exit(2);
+}
+try {
+  await withFileLock(lock, () => build(profile, agent, snap));
+} catch (e) {
+  console.error(e instanceof Error ? e.message : String(e));
+  process.exit(1);
+}

package/src/core/authored.ts

@@ -0,0 +1,47 @@
+// The authored files, read as YAML (records are in the SQLite record store, src/core/db.ts):
+//   configs/<id>.yaml · judges.yaml · suites/<id>.yaml · evals/<task>/task.yaml
+import { existsSync, readdirSync, readFileSync } from "node:fs";
+import { join } from "node:path";
+import { normalizeConfig } from "./config";
+import { projectRoot } from "./project";
+import { validateTask } from "./task-file";
+import type { AgentConfig, JudgeConfig, Suite, Task } from "./types";
+
+const readYaml = (p: string) => Bun.YAML.parse(readFileSync(p, "utf8")) as any;
+const yamlIds = (dir: string) => (existsSync(dir) ? readdirSync(dir).filter((f) => f.endsWith(".yaml")).map((f) => f.slice(0, -5)).sort() : []);
+
+// Each loader reads from the project root (`root`: configs/, judges.yaml, suites/, evals/). The default is the
+// project found from the current folder (project.ts); it throws outside a project. Tests pass a temp project.
+export function loadConfigs(root = projectRoot()): AgentConfig[] {
+  return yamlIds(join(root, "configs")).map((id) => normalizeConfig(id, readYaml(join(root, "configs", `${id}.yaml`)), root));
+}
+
+export function loadJudges(root = projectRoot()): JudgeConfig[] {
+  return (readYaml(join(root, "judges.yaml")).judges ?? []) as JudgeConfig[];
+}
+
+export function loadSuites(root = projectRoot()): Suite[] {
+  return yamlIds(join(root, "suites")).map((id) => ({ id, ...readYaml(join(root, "suites", `${id}.yaml`)) }));
+}
+
+export interface TaskListing { id: string; title: string; description: string; errors: string[]; warnings: string[] }
+
+/** Loads and validates a task from `<root>/evals/<id>/task.yaml`. Throws when the file has errors. */
+export function loadTask(id: string, root = projectRoot()): Task & { warnings: string[] } {
+  const dir = join(root, "evals", id);
+  const raw = readYaml(join(dir, "task.yaml"));
+  const { errors, warnings } = validateTask(raw, dir);
+  if (errors.length) throw new Error(`task ${id} is invalid:\n- ${errors.join("\n- ")}`);
+  // YAML `derived:` and `derived: null` mean no derived evidence: drop the key here, so the grader sees one form.
+  for (const c of [...(raw.gates ?? []), ...raw.measures.flatMap((m: any) => m.criteria)]) if (c.derived == null) delete c.derived;
+  return { gates: [], ...raw, dir, warnings };
+}
+
+export function listTasks(root = projectRoot()): TaskListing[] {
+  const dir = join(root, "evals");
+  return readdirSync(dir).filter((d) => existsSync(join(dir, d, "task.yaml"))).map((id) => {
+    const raw = readYaml(join(dir, id, "task.yaml"));
+    const v = validateTask(raw, join(dir, id));
+    return { id, title: raw.title ?? id, description: raw.description ?? "", ...v };
+  });
+}

package/src/core/batch-state.ts

@@ -0,0 +1,25 @@
+// Whether a batch is still running, for the batches panel. A batch whose process died never gets its end
+// time, so "not ended" alone does not mean "running": a batch also needs a trial that moved recently, or
+// this server process running it.
+import type { BatchRecord, TrialRecord } from "./db";
+
+/** The agent timeout (20 min) plus a margin: a long agent trial or a slow grading is not called stopped. */
+export const STALLED_AFTER_MS = 25 * 60_000;
+
+export type BatchRunState = "running" | "stalled" | "ended";
+
+export interface BatchState {
+  total: number;
+  /** The latest start or end time of its trials; the batch start when no trial has moved. */
+  lastMoveAt: string;
+  /** ended: it has its end time. running: this server runs it, or a trial moved in the last 25 min. stalled: neither; its process may have stopped. */
+  state: BatchRunState;
+}
+
+export function batchState(batch: BatchRecord, trials: TrialRecord[], o: { live: boolean; now: Date }): BatchState {
+  let lastMoveAt = batch.startedAt;
+  for (const t of trials) for (const at of [t.startedAt, t.endedAt]) if (at && at > lastMoveAt) lastMoveAt = at;
+  const state: BatchRunState = batch.endedAt != null ? "ended"
+    : o.live || o.now.getTime() - Date.parse(lastMoveAt) < STALLED_AFTER_MS ? "running" : "stalled";
+  return { total: trials.length, lastMoveAt, state };
+}