static-fire 0.0.0-stage → 0.44.0

package/src/core/batch.ts

@@ -0,0 +1,435 @@
+// A batch is one `run`: what was asked for, and its trials. Every trial is its own record in the store, with
+// its task hash, config key and run conditions, and zero or more gradings. Artifacts go to the trial's own
+// folder (db.ts trialDir): workspace/, agent.log and gradings/<grading-id>/ for judge logs.
+//
+// Running a trial and grading it are two separate steps. A trial's row ends (done, failed or invalid)
+// before any grading starts and never changes after that; a grading is a new row. So a crash between the
+// two leaves a done trial with no finished grading, which regrade can grade later.
+//
+// The agent works in a workspace in the OS temp folder, outside any repo (ADR-0004); it is copied to the
+// trial's folder before the row ends, so an ended trial always has its artifacts.
+import { cpSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { adapters } from "../adapters";
+import { COPY_LIMIT, copiedShare } from "./copying";
+import { trialDir, type BatchRecord, type GradingRecord, type GradingResult, type Store, type TrialRecord } from "./db";
+import { grade } from "./grade";
+import { canonicalJson, configKey, folderHash, gradingIdentity, sha256, taskIdentity } from "./identity";
+import { profileSnapshot } from "./profiles";
+import { familyOf } from "./score";
+import { loadSettings, userSetupFiles } from "./settings";
+import { loadConfigs, loadJudges, loadSuites, loadTask } from "./authored";
+import { projectPaths, projectRoot } from "./project";
+import type { AgentConfig, AgentHarness, AgentResult, JudgeConfig, Task } from "./types";
+
+export const DEFAULT_VOTES = 3;
+const AGENT_TIMEOUT_MS = 20 * 60_000;
+const VERSION_TIMEOUT_MS = 10_000;
+
+// ---- Planning: what was asked for, checked before anything is written ----------------------------
+
+/** What `run` was asked for. Give `tasks` or `suites`, not both; neither means every suite. */
+export interface RunRequest {
+  tasks?: string[];
+  suites?: string[];
+  configs?: string[];   // left out or ["all"]: every config
+  trials?: number;      // per agent config and task (a human config gets one); default 3
+  judge?: string;       // judge id; default the first in judges.yaml
+  votes?: number;       // default DEFAULT_VOTES
+}
+
+export interface BatchPlan {
+  /** Stored as the batch's `request`: what was asked for, with the task list it resolved to. */
+  request: { tasks?: string[]; suites?: string[]; configs: string[]; trials: number; judge: string; votes: number; taskIds: string[] };
+  tasks: Task[];
+  configs: AgentConfig[];
+  judge: JudgeConfig;
+  votes: number;
+  trials: number;
+}
+
+const positiveInt = (n: number, what: string) => {
+  if (!Number.isInteger(n) || n < 1) throw new Error(`${what} must be a whole number of 1 or more (got ${JSON.stringify(n)})`);
+  return n;
+};
+
+const REQUEST_FIELDS = ["tasks", "suites", "configs", "trials", "judge", "votes"];
+
+/**
+ * Checks the shape of a request that came from outside the type checker (the JSON body of POST /api/runs):
+ * a JSON object with only RunRequest's fields, each list a list of strings, each count a whole number of 1 or
+ * more. A field left out or undefined takes its default. Throws a plain message; returns the request as typed.
+ */
+export function parseRunRequest(body: unknown): RunRequest {
+  if (!body || typeof body !== "object" || Array.isArray(body)) throw new Error("the request body must be a JSON object");
+  const r = body as Record<string, unknown>;
+  const unknown = Object.keys(r).filter((k) => !REQUEST_FIELDS.includes(k));
+  if (unknown.length) throw new Error(`unknown request field(s): ${unknown.join(", ")} (known: ${REQUEST_FIELDS.join(", ")})`);
+  for (const k of ["tasks", "suites", "configs"]) {
+    const v = r[k];
+    if (v !== undefined && !(Array.isArray(v) && v.every((x) => typeof x === "string"))) throw new Error(`${k} must be a list of ids (strings), got ${JSON.stringify(v)}`);
+  }
+  for (const k of ["trials", "votes"]) if (r[k] !== undefined) positiveInt(r[k] as number, k);
+  if (r.judge !== undefined && typeof r.judge !== "string") throw new Error(`judge must be a judge id (a string), got ${JSON.stringify(r.judge)}`);
+  return r as RunRequest;
+}
+
+/** Resolves and checks a request against the project at `root`. Throws on anything unknown, so no batch is started. */
+export function planBatch(req: RunRequest, root = projectRoot()): BatchPlan {
+  parseRunRequest(req);
+  if (req.tasks && req.suites) throw new Error("give --tasks or --suites, not both");
+  let taskIds: string[];
+  if (req.tasks) {
+    taskIds = [...new Set(req.tasks)];
+    const unknown = taskIds.filter((id) => !existsSync(join(root, "evals", id, "task.yaml")));
+    if (!taskIds.length || unknown.length) throw new Error(`unknown or no task(s): ${unknown.join(", ") || "none given"}`);
+  } else {
+    const suites = Object.fromEntries(loadSuites(root).map((s) => [s.id, s]));
+    const suiteIds = req.suites ?? Object.keys(suites);
+    const unknown = suiteIds.filter((id) => !suites[id]);
+    if (!suiteIds.length || unknown.length) throw new Error(`unknown or no suite(s): ${unknown.join(", ") || "none given"}`);
+    taskIds = [...new Set(suiteIds.flatMap((s) => suites[s].tasks))];
+  }
+  const all = loadConfigs(root);
+  const byId = Object.fromEntries(all.map((c) => [c.id, c]));
+  const configIds = !req.configs || req.configs.includes("all") ? all.map((c) => c.id) : [...new Set(req.configs)];
+  const missing = configIds.filter((id) => !byId[id]);
+  if (!configIds.length || missing.length) throw new Error(`unknown or no config(s): ${missing.join(", ") || "none given"}`);
+  const judges = loadJudges(root);
+  const judge = req.judge ? judges.find((j) => j.id === req.judge) : judges[0];
+  if (!judge) throw new Error(`unknown judge ${req.judge ?? "(judges.yaml lists none)"}`);
+  const trials = positiveInt(req.trials ?? 3, "trials");
+  const votes = positiveInt(req.votes ?? DEFAULT_VOTES, "votes");
+  const tasks = taskIds.map((id) => loadTask(id, root));   // throws on an invalid task
+  return {
+    // Neither --tasks nor --suites asked for every suite: recorded as suites ["all"].
+    request: { ...(req.tasks ? { tasks: req.tasks } : { suites: req.suites ?? ["all"] }), configs: configIds, trials, judge: judge.id, votes, taskIds },
+    tasks, configs: configIds.map((id) => byId[id]), judge, votes, trials,
+  };
+}
+
+// ---- Starting: the batch record, its hashes and run conditions, and every trial queued ------------
+
+/** What running and grading the trials of one batch needs. */
+export interface BatchContext {
+  batchId: string;
+  dbPath: string;                                  // trial folders are beside the database
+  tasks: Record<string, { task: Task; taskHash: string; gradingHash: string }>;
+  configs: Record<string, AgentConfig>;
+  judge: JudgeConfig;
+  votes: number;
+  setupHash: string | null;                        // the user setup agent trials run with
+  blockedTools: Partial<Record<AgentHarness, string[]>>;
+  harnessVersions: Record<string, string | null>;
+  cacheDir: string;                                // Profile snapshots and the apkit lock
+}
+
+export interface StartOptions {
+  dbPath: string;
+  root?: string;
+  cacheDir?: string;
+  /** Reads `<harness> --version`; tests replace it. */
+  readVersion?: (harness: AgentHarness) => Promise<string | null>;
+  /** The user setup file list (path → hash); tests replace it. */
+  userSetup?: () => Record<string, string>;
+  now?: () => Date;
+}
+
+export interface StartedBatch { batch: BatchRecord; trials: TrialRecord[]; ctx: BatchContext }
+
+/** UTC YYYYMMDD-HHMMSS. */
+export const batchStamp = (d: Date) => d.toISOString().replace(/[-:]/g, "").replace(/\..+/, "").replace("T", "-");
+
+/** The first line `<harness> --version` prints, or null when it cannot be read. */
+export async function readHarnessVersion(harness: AgentHarness): Promise<string | null> {
+  try {
+    const p = Bun.spawn([harness, "--version"], { stdin: "ignore", stdout: "pipe", stderr: "pipe" });
+    const timer = setTimeout(() => p.kill(), VERSION_TIMEOUT_MS);
+    const [out, code] = await Promise.all([new Response(p.stdout).text(), p.exited]);
+    clearTimeout(timer);
+    const line = out.split("\n").map((l) => l.trim()).find(Boolean);
+    return code === 0 && line ? line : null;
+  } catch {
+    return null;   // not installed
+  }
+}
+
+/** Each agent harness's CLI version, read once. */
+export async function harnessVersions(harnesses: AgentHarness[], read = readHarnessVersion): Promise<Record<string, string | null>> {
+  const unique = [...new Set(harnesses)].sort();
+  return Object.fromEntries(await Promise.all(unique.map(async (h) => [h, await read(h)] as const)));
+}
+
+/** A human wrote one file per task, so it gets one trial, and only for tasks it has a file for. */
+const trialCount = (c: AgentConfig, taskId: string, n: number) => (c.harness !== "human" ? n : c.outputs?.[taskId] ? 1 : 0);
+
+/**
+ * Records a new batch in one transaction: the batch (id = UTC timestamp, with -2, -3, ... when a batch
+ * started in the same second has it), each task version and grading setup when first seen, the user setup,
+ * and every planned trial as queued. Reads the harness versions and the user setup first.
+ */
+export async function startBatch(store: Store, plan: BatchPlan, o: StartOptions): Promise<StartedBatch> {
+  const root = o.root ?? projectRoot();
+  const now = o.now ?? (() => new Date());
+  const agentHarnesses = plan.configs.flatMap((c) => (c.harness === "human" ? [] : [c.harness]));
+  const versions = await harnessVersions(agentHarnesses, o.readVersion);
+  const files = agentHarnesses.length ? (o.userSetup ?? userSetupFiles)() : null;
+  const setupHash = files ? sha256(canonicalJson(files)) : null;
+  const settings = loadSettings(root);
+  const ids = plan.tasks.map((task) => ({ task, t: taskIdentity(task), g: gradingIdentity(task, plan.judge) }));
+
+  const started = now();
+  const at = started.toISOString();
+  const { batch, trials } = store.transaction(() => {
+    const stamp = batchStamp(started);
+    let id = stamp;
+    for (let k = 2; store.getBatch(id); k++) id = `${stamp}-${k}`;
+    const batch: BatchRecord = { id, startedAt: at, endedAt: null, request: plan.request, harnessVersions: versions };
+    store.createBatch(batch);
+    store.startBatchRun(id);
+    for (const { task, t, g } of ids) {
+      store.putTaskVersion({ hash: t.hash, taskId: task.id, label: task.version ?? null, snapshot: t.snapshot as unknown as Record<string, unknown>, firstSeen: at });
+      store.putGradingSetup({
+        hash: g.hash, taskId: task.id, taskHash: t.hash,
+        judgeHarness: plan.judge.harness, judgeModel: plan.judge.model, judgeEffort: plan.judge.effort ?? null, judgeLabel: plan.judge.label,
+        snapshot: g.snapshot as unknown as Record<string, unknown>, firstSeen: at,
+      });
+    }
+    if (files && setupHash) store.putUserSetup({ hash: setupHash, files });
+    const trials: TrialRecord[] = [];
+    for (const { task, t } of ids) for (const c of plan.configs) for (let n = 1; n <= trialCount(c, task.id, plan.trials); n++) {
+      const tid = `${id}--${task.id}--${c.id}--${n}`;
+      const r: TrialRecord = {
+        id: tid, batchId: id, taskId: task.id, taskHash: t.hash, configKey: configKey(c), configId: c.id,
+        harness: c.harness, model: c.model, effort: c.effort ?? null, profile: c.profile ?? null, provider: c.provider ?? null,
+        family: c.family ?? familyOf(c.model), n, status: "queued",
+        startedAt: null, endedAt: null, durationMs: null,
+        inputTokens: null, outputTokens: null, cacheReadTokens: null, cacheWriteTokens: null, costUsd: null, costSource: null, turns: null,
+        setupHash: null, profileHash: null, blockedTools: null, harnessVersion: null, copiedShare: null, error: null,
+        dir: `trials/${tid}`,
+      };
+      store.insertTrial(r);
+      trials.push(r);
+    }
+    return { batch, trials };
+  });
+
+  const ctx: BatchContext = {
+    batchId: batch.id, dbPath: o.dbPath,
+    tasks: Object.fromEntries(ids.map(({ task, t, g }) => [task.id, { task, taskHash: t.hash, gradingHash: g.hash }])),
+    configs: Object.fromEntries(plan.configs.map((c) => [c.id, c])),
+    judge: plan.judge, votes: plan.votes, setupHash, blockedTools: settings.blockedTools, harnessVersions: versions,
+    cacheDir: o.cacheDir ?? projectPaths(root).cache,
+  };
+  return { batch, trials, ctx };
+}
+
+// ---- Running a batch ------------------------------------------------------------------------------
+
+export interface RunHooks { concurrency?: number; onEvent?: (msg: string) => void }
+
+/**
+ * Runs every trial of a started batch and grades each one that ends done. Then this run of the batch is over: the
+ * batch ends unless another run of it (a rerun, in this process or another) is still in flight, or a trial this
+ * run did not start is still queued or running.
+ */
+export async function runBatch(store: Store, started: StartedBatch, o: RunHooks = {}) {
+  const log = o.onEvent ?? (() => {});
+  const { ctx } = started;
+  const queue = started.trials.map((t) => t.id);
+  const worker = async () => {
+    for (let id = queue.shift(); id; id = queue.shift()) {
+      try {
+        const t = await runTrial(store, ctx, id, log);
+        if (t.status !== "done") continue;
+        const { task, gradingHash } = ctx.tasks[t.taskId];
+        const g = await gradeTrial(store, ctx.dbPath, t, { task, judge: ctx.judge, votes: ctx.votes, gradingHash });
+        log(`${g.status === "done" ? "✓" : "!"} ${t.id} score ${g.score?.toFixed(1) ?? "—"}${g.error ? ` (${g.error})` : ""} cost $${t.costUsd?.toFixed(3) ?? "?"}`);
+      } catch (e) {
+        log(`✗ ${id}: ${e instanceof Error ? e.message : String(e)}`);   // a store error: the other trials go on
+      }
+    }
+  };
+  await Promise.all(Array.from({ length: Math.max(1, o.concurrency ?? 4) }, worker));
+  // Another run of this batch may still run or grade its trials, here or in another process: then the last one
+  // to finish ends it. A run whose process died is never finished, so its batch stays not ended.
+  const mine = new Set(started.trials.map((t) => t.id));
+  const ended = store.transaction(() => {
+    const others = store.finishBatchRun(ctx.batchId);
+    const busy = others > 0 || store.listTrials({ batchId: ctx.batchId }).some((t) => !mine.has(t.id) && (t.status === "queued" || t.status === "running"));
+    if (!busy) store.endBatch(ctx.batchId, new Date().toISOString());
+    return !busy;
+  });
+  log(ended ? `batch ${ctx.batchId} done` : `batch ${ctx.batchId}: these trials are done and graded; another run of it is still running or grading trials`);
+}
+
+const message = (e: unknown) => (e instanceof Error ? e.message : String(e));
+
+/**
+ * Runs one queued trial: workspace, agent run (or the human's file), held-out check, artifacts saved, then
+ * the row ends once as done, failed (the harness could not run) or invalid (copied the held-out original).
+ * It never grades. Returns the ended row.
+ */
+export async function runTrial(store: Store, ctx: BatchContext, trialId: string, log: (m: string) => void = () => {}): Promise<TrialRecord> {
+  // Claim it: queued -> running is a compare-and-set in one write transaction (BEGIN IMMEDIATE), so two
+  // callers, in this process or another, can never both start the same trial.
+  const { trial, task, config, human } = store.transaction(() => {
+    const trial = store.getTrial(trialId);
+    if (!trial) throw new Error(`no trial ${trialId}`);
+    if (trial.status !== "queued") throw new Error(`trial ${trialId} is ${trial.status}; only a queued trial can run`);
+    const entry = ctx.tasks[trial.taskId], config = ctx.configs[trial.configId];
+    if (!entry || !config) throw new Error(`trial ${trialId}: its task or config is not part of this batch`);
+    const human = config.harness === "human";
+    // Run conditions. A human trial had no agent setup, Profile, blocked tools or harness.
+    store.updateTrial(trial.id, {
+      status: "running", startedAt: new Date().toISOString(),
+      setupHash: human ? null : ctx.setupHash,
+      blockedTools: human ? null : (ctx.blockedTools[config.harness as AgentHarness] ?? []),
+      harnessVersion: human ? null : (ctx.harnessVersions[config.harness] ?? null),
+    });
+    return { trial, task: entry.task, config, human };
+  });
+  const dir = trialDir(ctx.dbPath, trial.id);
+  let ws = "";   // a new folder in the OS temp folder, so no two trials (or two stores) share one
+  log(`▶ ${trial.id}`);
+
+  const end: Partial<TrialRecord> = {};
+  let status: "done" | "failed" | "invalid";
+  let error: string | null = null;
+  try {
+    ws = mkdtempSync(join(tmpdir(), "static-fire-trial-"));
+    mkdirSync(join(ws, "out"), { recursive: true });
+    mkdirSync(dir, { recursive: true });
+    fillWorkspace(task, trial.taskHash, ws);
+    if (config.profile && !human) {
+      const snap = await profileSnapshot(config.profile, config.harness as AgentHarness, ctx.batchId, ctx.cacheDir);
+      end.profileHash = folderHash(snap);
+      cpSync(snap, ws, { recursive: true });
+    }
+
+    const t0 = Date.now();
+    const result: AgentResult = human
+      ? copyHumanOutput(config, trial.taskId, join(ws, task.output))
+      : await adapters[config.harness as AgentHarness]({
+        config, prompt: task.prompt, cwd: ws, logPath: join(dir, "agent.log"), timeoutMs: AGENT_TIMEOUT_MS,
+        guard: { blockedTools: ctx.blockedTools[config.harness as AgentHarness] ?? [] },
+      });
+    if (!human) end.durationMs = Date.now() - t0;
+    Object.assign(end, {
+      inputTokens: result.usage.inputTokens, outputTokens: result.usage.outputTokens,
+      cacheReadTokens: result.usage.cacheReadTokens, cacheWriteTokens: result.usage.cacheWriteTokens,
+      costUsd: result.costUsd, costSource: result.costSource, turns: result.turns ?? null,
+    });
+
+    const outPath = join(ws, task.output);
+    const ran = result.ok || (result.turns ?? 0) > 0 || existsSync(outPath);
+    if (!ran) throw new Error(result.error ?? "the harness did not run");
+    if (!result.ok) error = result.error ?? null;
+    // An agent that found the published original (for example on the web) and copied it did not do the work.
+    if (task.heldOut && !human) {
+      end.copiedShare = existsSync(outPath) ? copiedShare(readFileSync(outPath, "utf8"), readFileSync(join(task.dir, task.heldOut), "utf8")) : 0;
+    }
+    if ((end.copiedShare ?? 0) > COPY_LIMIT) {
+      status = "invalid";
+      error = `copied the held-out original (${Math.round(end.copiedShare! * 100)}% of its 8-word runs)`;
+    } else status = "done";
+  } catch (e) {
+    status = "failed";
+    error = message(e);
+  }
+
+  // The artifacts are saved before the row ends, so an ended trial always has them.
+  try {
+    if (ws && existsSync(ws)) {
+      cpSync(ws, join(dir, "workspace"), { recursive: true });
+      rmSync(ws, { recursive: true, force: true });
+    }
+  } catch (e) {
+    if (status === "done") { status = "failed"; error = `could not save the workspace: ${message(e)}`; }
+  }
+
+  error = error?.trim() || null;   // harness messages often end with a newline
+  store.updateTrial(trial.id, { ...end, status, error, endedAt: new Date().toISOString() });
+  if (status !== "done") log(`${status === "invalid" ? "!" : "✗"} ${trial.id}: ${error}`);
+  return store.getTrial(trial.id)!;
+}
+
+/**
+ * Copies the task's inputs and project into the workspace, first to a private folder whose files are hashed:
+ * when they no longer give the trial's recorded task hash (a file changed after the batch started), it throws
+ * and the workspace stays empty. The workspace gets the copy that was hashed, so the record names what the agent got.
+ */
+function fillWorkspace(task: Task, want: string, ws: string) {
+  const stage = mkdtempSync(join(tmpdir(), "static-fire-task-"));
+  try {
+    cpSync(join(task.dir, task.inputs), join(stage, task.inputs), { recursive: true });
+    if (task.project) cpSync(join(task.dir, task.project), join(stage, task.project), { recursive: true });
+    const now = taskIdentity({ ...task, dir: stage }).hash;
+    if (now !== want) {
+      throw new Error(`task ${task.id}: its files changed after the batch started (task hash ${want.slice(0, 8)} then, ${now.slice(0, 8)} now), `
+        + "so this trial did not run. Run the task again in a new batch.");
+    }
+    cpSync(join(stage, task.inputs), join(ws, task.inputs), { recursive: true });
+    if (task.project) cpSync(join(stage, task.project), ws, { recursive: true });
+  } finally {
+    rmSync(stage, { recursive: true, force: true });
+  }
+}
+
+export function copyHumanOutput(config: AgentConfig, taskId: string, out: string): AgentResult {
+  const src = config.outputs?.[taskId]?.path;
+  const none = { usage: { inputTokens: 0, outputTokens: 0, cacheReadTokens: 0, cacheWriteTokens: 0 }, costUsd: null, costSource: "unknown" as const };
+  if (!src || !existsSync(src)) return { ok: false, error: `no file for ${taskId} at ${src ?? "(none)"}`, ...none };
+  mkdirSync(join(out, ".."), { recursive: true });
+  cpSync(src, out);
+  return { ok: true, ...none };
+}
+
+// ---- Grading one trial ----------------------------------------------------------------------------
+
+/** One grading setup: the task and judge whose grading hash it is, and the vote count. */
+export interface GradingSetup { task: Task; judge: JudgeConfig; votes: number; gradingHash: string }
+
+/**
+ * Grades a done trial's saved output under one grading setup, with no agent run: adds a grading row, runs the
+ * graders on a hashed private copy of the task folder with judge logs in the trial's gradings/<grading-id>/
+ * folder, and finishes the row with its verdicts (invalid when a grader returned an error, or when the copy
+ * does not give the grading hash). Never changes the trial row or an earlier grading.
+ */
+export async function gradeTrial(store: Store, dbPath: string, trial: TrialRecord, s: GradingSetup): Promise<GradingRecord> {
+  if (trial.status !== "done") throw new Error(`trial ${trial.id} is ${trial.status}; only a done trial is graded`);
+  const gradingId = store.startGrading({ trialId: trial.id, gradingHash: s.gradingHash, votes: s.votes, gradedAt: new Date().toISOString() });
+  const dir = trialDir(dbPath, trial.id);
+  const ws = join(dir, "workspace");
+  const logs = join(dir, "gradings", String(gradingId));
+  let result: GradingResult;
+  // The grader reads a private copy of the task folder, made now and hashed: when it no longer gives the
+  // recorded grading hash, nothing is graded under it (the row ends invalid, and a regrade grades the trial
+  // under the current hash). An edit to the task folder while grading runs does not reach the grader.
+  let stage = "";
+  try {
+    mkdirSync(logs, { recursive: true });
+    stage = mkdtempSync(join(tmpdir(), "static-fire-grading-"));
+    cpSync(s.task.dir, stage, { recursive: true });
+    const task: Task = { ...s.task, dir: stage };
+    const now = gradingIdentity(task, s.judge).hash;
+    if (now !== s.gradingHash) {
+      throw new Error(`task ${s.task.id}: its grading files changed after this grading was planned (grading hash ${s.gradingHash.slice(0, 8)} then, `
+        + `${now.slice(0, 8)} now), so it was not graded. Regrade to grade it under the current files.`);
+    }
+    const g = await grade({ task, outputPath: join(ws, s.task.output), workspace: ws, judge: { ...s.judge, votes: s.votes }, dir: logs });
+    result = {
+      status: g.errors.length ? "invalid" : "done",
+      score: g.score, gatesPassed: g.gatesPassed, measures: g.measures, judgeCostUsd: g.judgeCostUsd,
+      error: g.errors.length ? `grader errors: ${g.errors.join(", ")}` : null,
+      verdicts: Object.entries(g.verdicts).map(([criterionId, v]) => ({ criterionId, score: v.score, error: v.error ?? null, votes: v.votes })),
+    };
+  } catch (e) {
+    result = { status: "invalid", score: null, gatesPassed: null, measures: null, judgeCostUsd: null, error: `grading failed: ${message(e)}`, verdicts: [] };
+  } finally {
+    if (stage) rmSync(stage, { recursive: true, force: true });
+  }
+  store.finishGrading(gradingId, result);
+  return store.listGradings(trial.id, s.gradingHash).find((g) => g.id === gradingId)!;
+}

package/src/core/batches.ts

@@ -0,0 +1,221 @@
+// Batches after they start: what `list` prints for each one, and `rerun <batch>`, which runs a batch's failed
+// trials again in the same batch.
+//
+// A failed trial's row never changes. A rerun adds a new trial with the next free n for that task and config.
+// Which failed trials still need a rerun is worked out from the rows, with no link column:
+//   - the batch asked for `want` trials of each task and config (request.trials for an agent, 1 for a human);
+//   - every trial that did not fail (queued, running, done, invalid) fills one of them;
+//   - each later trial replaces the earliest failed trial not yet replaced, so the failed trials still to rerun
+//     are the latest ones. A trial that failed and then passed on a rerun is never offered again.
+// A queued or running trial fills its place and is never touched: the store holds no evidence that its process
+// died (no pid, no heartbeat), and a long time alone is not proof (a laptop that sleeps keeps a live run going).
+import { existsSync } from "node:fs";
+import { join } from "node:path";
+import { harnessVersions, type StartOptions, type StartedBatch, type BatchContext } from "./batch";
+import type { BatchRecord, Store, TrialRecord, TrialRecordStatus } from "./db";
+import { canonicalJson, configKey, gradingIdentity, sha256, taskIdentity } from "./identity";
+import { loadSettings, userSetupFiles } from "./settings";
+import { loadConfigs, loadJudges, loadTask } from "./authored";
+import { projectPaths, projectRoot } from "./project";
+import type { AgentConfig, JudgeConfig, Task } from "./types";
+
+/** What the batch stored as its request (batch.ts BatchPlan.request). */
+export interface BatchRequest {
+  tasks?: string[]; suites?: string[]; configs: string[]; trials: number; judge: string; votes: number; taskIds: string[];
+}
+
+export interface BatchSummary {
+  batch: BatchRecord;
+  request: BatchRequest;
+  counts: Record<TrialRecordStatus, number>;
+  /** Failed trials that no later trial has replaced yet: what `rerun` would run. */
+  toRerun: number;
+  ended: boolean;
+}
+
+/** One failed trial still to rerun. */
+export interface RerunSlot { taskId: string; configId: string; replaces: string }
+
+const ENDED_OK: TrialRecordStatus[] = ["queued", "running", "done", "invalid"];   // every status but failed fills a place
+
+/** The failed trials of a batch that no later trial replaced, oldest task/config first. */
+export function rerunSlots(request: BatchRequest, trials: TrialRecord[]): RerunSlot[] {
+  const groups = new Map<string, TrialRecord[]>();
+  for (const t of trials) {
+    const k = `${t.taskId}\0${t.configId}`;
+    groups.set(k, [...(groups.get(k) ?? []), t]);
+  }
+  const slots: RerunSlot[] = [];
+  for (const group of groups.values()) {
+    const want = group[0]!.harness === "human" ? 1 : request.trials;
+    const filled = group.filter((t) => ENDED_OK.includes(t.status)).length;
+    const failed = group.filter((t) => t.status === "failed").sort((a, b) => a.n - b.n);
+    const short = Math.min(Math.max(0, want - filled), failed.length);
+    for (const t of failed.slice(failed.length - short)) slots.push({ taskId: t.taskId, configId: t.configId, replaces: t.id });
+  }
+  return slots.sort((a, b) => (a.replaces < b.replaces ? -1 : a.replaces > b.replaces ? 1 : 0));
+}
+
+export function summarizeBatch(store: Store, batch: BatchRecord): BatchSummary {
+  const trials = store.listTrials({ batchId: batch.id });
+  const counts: Record<TrialRecordStatus, number> = { queued: 0, running: 0, done: 0, failed: 0, invalid: 0 };
+  for (const t of trials) counts[t.status]++;
+  const request = batch.request as unknown as BatchRequest;
+  return { batch, request, counts, toRerun: rerunSlots(request, trials).length, ended: batch.endedAt !== null };
+}
+
+/** Every batch, newest first. */
+export const listBatchSummaries = (store: Store): BatchSummary[] => store.listBatches().map((b) => summarizeBatch(store, b));
+
+// ---- rerun ----------------------------------------------------------------------------------------
+
+/** `rerun` will not run this batch's failed trials as they were: something they ran with has changed or is gone. */
+export class RerunRefused extends Error {}
+
+export interface RerunPlan {
+  batch: BatchRecord;
+  request: BatchRequest;
+  slots: RerunSlot[];
+  /** Trials still queued or running: never rerun, never changed. */
+  busy: string[];
+  tasks: Record<string, { task: Task; taskHash: string }>;
+  configs: Record<string, AgentConfig>;
+  judge: JudgeConfig | null;
+}
+
+const short = (h: string) => h.slice(0, 8);
+
+/**
+ * Works out what `rerun <batch>` would run and checks it against today's files. Writes nothing. Throws
+ * RerunRefused when a task's hash or a config's key differs from what the batch ran with, or the batch's judge
+ * is gone. With no failed trial to rerun, returns no slots and checks nothing.
+ */
+export function planRerun(store: Store, batchId: string, root = projectRoot()): RerunPlan {
+  const batch = store.getBatch(batchId);
+  if (!batch) throw new RerunRefused(`no batch ${batchId}`);
+  const request = batch.request as unknown as BatchRequest;
+  const trials = store.listTrials({ batchId });
+  const slots = rerunSlots(request, trials);
+  const busy = trials.filter((t) => t.status === "queued" || t.status === "running").map((t) => t.id);
+  const plan: RerunPlan = { batch, request, slots, busy, tasks: {}, configs: {}, judge: null };
+  if (!slots.length) return plan;
+
+  const byId = new Map(trials.map((t) => [t.id, t]));
+  const configIds = [...new Set(slots.map((s) => s.configId))];
+  const useRun = (taskIds: string[]) =>
+    `Use \`static-fire run --tasks ${taskIds.join(",")} --configs ${configIds.join(",")}\` to run it on the current files.`;
+  for (const taskId of [...new Set(slots.map((s) => s.taskId))]) {
+    const was = byId.get(slots.find((s) => s.taskId === taskId)!.replaces)!.taskHash;
+    if (!existsSync(join(root, "evals", taskId, "task.yaml"))) {
+      throw new RerunRefused(`task ${taskId} is gone since batch ${batchId} (no evals/${taskId}/task.yaml), so its failed trials cannot run again as they were.`);
+    }
+    let task: Task;
+    try { task = loadTask(taskId, root); } catch (e) {
+      throw new RerunRefused(`task ${taskId} cannot be read: ${e instanceof Error ? e.message : String(e)}`);
+    }
+    const now = taskIdentity(task).hash;
+    if (now !== was) {
+      throw new RerunRefused(`task ${taskId} changed since batch ${batchId} (task hash ${short(was)} then, ${short(now)} now). `
+        + `A rerun keeps a batch on the task version it ran. ${useRun([taskId])}`);
+    }
+    plan.tasks[taskId] = { task, taskHash: now };
+  }
+  const configs = new Map(loadConfigs(root).map((c) => [c.id, c]));
+  const taskIds = Object.keys(plan.tasks);
+  for (const configId of configIds) {
+    const was = byId.get(slots.find((s) => s.configId === configId)!.replaces)!.configKey;
+    const c = configs.get(configId);
+    if (!c) throw new RerunRefused(`config ${configId} is gone since batch ${batchId} (no configs/${configId}.yaml). ${useRun(taskIds)}`);
+    if (configKey(c) !== was) {
+      throw new RerunRefused(`config ${configId} changed since batch ${batchId} (${was} then, ${configKey(c)} now). `
+        + `A rerun uses the configs the batch recorded. ${useRun(taskIds)}`);
+    }
+    plan.configs[configId] = c;
+  }
+  const judge = loadJudges(root).find((j) => j.id === request.judge);
+  if (!judge) throw new RerunRefused(`judge ${request.judge}, which batch ${batchId} used, is no longer in judges.yaml. ${useRun(taskIds)}`);
+  plan.judge = judge;
+  return plan;
+}
+
+export interface StartedRerun extends StartedBatch {
+  /** New trial id → the failed trial it replaces. */
+  replaces: Record<string, string>;
+}
+
+/**
+ * Adds one queued trial per slot to the batch, with the next free n for its task and config, counts this rerun
+ * as a run of the batch in flight (which clears its end time), and builds the BatchContext to run them (pass the
+ * result to runBatch). The slots are worked out again inside one write transaction, so two reruns of one batch at the same time never both add a
+ * trial for the same failure; one that finds no slot left throws and writes nothing.
+ * Run conditions (user setup, blocked tools, harness version) are today's, as each trial records its own.
+ */
+export async function startRerun(store: Store, plan: RerunPlan, o: StartOptions): Promise<StartedRerun> {
+  const root = o.root ?? projectRoot();
+  const judge = plan.judge;
+  if (!judge || !plan.slots.length) throw new Error(`batch ${plan.batch.id}: nothing to rerun`);
+  const configs = Object.values(plan.configs);
+  const agentHarnesses = configs.flatMap((c) => (c.harness === "human" ? [] : [c.harness]));
+  const versions = await harnessVersions(agentHarnesses, o.readVersion);
+  const files = agentHarnesses.length ? (o.userSetup ?? userSetupFiles)() : null;
+  const setupHash = files ? sha256(canonicalJson(files)) : null;
+  const settings = loadSettings(root);
+  const grading = Object.fromEntries(Object.entries(plan.tasks).map(([id, { task }]) => [id, gradingIdentity(task, judge)]));
+  const at = (o.now ?? (() => new Date()))().toISOString();
+
+  const { trials, replaces } = store.transaction(() => {
+    const rows = store.listTrials({ batchId: plan.batch.id });
+    const slots = rerunSlots(plan.request, rows).filter((s) => plan.tasks[s.taskId] && plan.configs[s.configId]);
+    // Another rerun took them since the plan: nothing is written, and no runBatch follows to end the batch.
+    if (!slots.length) throw new Error(`batch ${plan.batch.id}: nothing to rerun`);
+    for (const [id, { task, taskHash }] of Object.entries(plan.tasks)) {
+      const g = grading[id]!;
+      store.putTaskVersion({ hash: taskHash, taskId: id, label: task.version ?? null, snapshot: taskIdentity(task).snapshot as unknown as Record<string, unknown>, firstSeen: at });
+      store.putGradingSetup({
+        hash: g.hash, taskId: id, taskHash,
+        judgeHarness: judge.harness, judgeModel: judge.model, judgeEffort: judge.effort ?? null, judgeLabel: judge.label,
+        snapshot: g.snapshot as unknown as Record<string, unknown>, firstSeen: at,
+      });
+    }
+    if (files && setupHash) store.putUserSetup({ hash: setupHash, files });
+    const next = new Map<string, number>();
+    for (const t of rows) {
+      const k = `${t.taskId}\0${t.configId}`;
+      next.set(k, Math.max(next.get(k) ?? 1, t.n + 1));
+    }
+    const byId = new Map(rows.map((t) => [t.id, t]));
+    const trials: TrialRecord[] = [];
+    const replaces: Record<string, string> = {};
+    for (const s of slots) {
+      const k = `${s.taskId}\0${s.configId}`;
+      const n = next.get(k)!;
+      next.set(k, n + 1);
+      const failed = byId.get(s.replaces)!;
+      const id = `${plan.batch.id}--${s.taskId}--${s.configId}--${n}`;
+      const r: TrialRecord = {
+        id, batchId: plan.batch.id, taskId: s.taskId, taskHash: failed.taskHash, configKey: failed.configKey, configId: s.configId,
+        harness: failed.harness, model: failed.model, effort: failed.effort, profile: failed.profile, provider: failed.provider,
+        family: failed.family, n, status: "queued",
+        startedAt: null, endedAt: null, durationMs: null,
+        inputTokens: null, outputTokens: null, cacheReadTokens: null, cacheWriteTokens: null, costUsd: null, costSource: null, turns: null,
+        setupHash: null, profileHash: null, blockedTools: null, harnessVersion: null, copiedShare: null, error: null,
+        dir: `trials/${id}`,
+      };
+      store.insertTrial(r);
+      trials.push(r);
+      replaces[id] = s.replaces;
+    }
+    store.startBatchRun(plan.batch.id);   // not ended while this run's trials run and are graded; runBatch ends it again
+    return { trials, replaces };
+  });
+
+  const ctx: BatchContext = {
+    batchId: plan.batch.id, dbPath: o.dbPath,
+    tasks: Object.fromEntries(Object.entries(plan.tasks).map(([id, { task, taskHash }]) => [id, { task, taskHash, gradingHash: grading[id]!.hash }])),
+    configs: plan.configs,
+    judge, votes: plan.request.votes, setupHash, blockedTools: settings.blockedTools, harnessVersions: versions,
+    cacheDir: o.cacheDir ?? projectPaths(root).cache,
+  };
+  return { batch: plan.batch, trials, ctx, replaces };
+}
+