static-fire 0.0.0-stage → 0.54.1

package/src/core/apkit-install.ts

@@ -0,0 +1,47 @@
+// Builds one Profile snapshot in a process of its own, which holds the apkit lock for the whole sequence:
+// install into a scratch folder, copy it to the snapshot, remove the install. The `run` that starts it
+// (profiles.ts) waits for it. If that `run` is killed, this process goes on to the end of the sequence and only
+// then lets go of the lock, so another install never overlaps a running one.
+//   bun src/core/apkit-install.ts <lock file> <profile> <apkit agent> <snapshot folder>
+import { cpSync, existsSync, mkdirSync, renameSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { basename, dirname, join } from "node:path";
+import { withFileLock } from "./lock";
+
+async function apkit(args: string[]) {
+  const p = Bun.spawn(["apkit", ...args], { stdout: "pipe", stderr: "pipe", stdin: "ignore" });
+  const [out, err, code] = await Promise.all([new Response(p.stdout).text(), new Response(p.stderr).text(), p.exited]);
+  if (code !== 0) throw new Error(`apkit ${args[0]} failed (exit ${code}): ${(err || out).trim().slice(-400)}`);
+}
+
+async function build(profile: string, agent: string, snap: string) {
+  if (existsSync(snap)) return;
+  const scratch = join(tmpdir(), "static-fire-apkit", `${process.pid}`, basename(dirname(snap)), basename(snap));
+  rmSync(scratch, { recursive: true, force: true });
+  mkdirSync(scratch, { recursive: true });
+  try {
+    // --yes: apkit asks before creating AGENTS.md; this scratch folder is ours.
+    await apkit(["install", profile, scratch, "--agents", agent, "--yes"]);
+    // Copied beside the snapshot, then renamed: a snapshot folder is always complete.
+    const part = `${snap}.part`;
+    rmSync(part, { recursive: true, force: true });
+    mkdirSync(part, { recursive: true });
+    cpSync(scratch, part, { recursive: true, dereference: true });
+    renameSync(part, snap);
+  } finally {
+    await apkit(["remove", scratch, "--yes"]).catch(() => {});
+    rmSync(scratch, { recursive: true, force: true });
+  }
+}
+
+const [lock, profile, agent, snap] = Bun.argv.slice(2);
+if (!lock || !profile || !agent || !snap) {
+  console.error("usage: apkit-install.ts <lock file> <profile> <apkit agent> <snapshot folder>");
+  process.exit(2);
+}
+try {
+  await withFileLock(lock, () => build(profile, agent, snap));
+} catch (e) {
+  console.error(e instanceof Error ? e.message : String(e));
+  process.exit(1);
+}

package/src/core/authored.ts

@@ -0,0 +1,47 @@
+// The authored files, read as YAML (records are in the SQLite record store, src/core/db.ts):
+//   configs/<id>.yaml · judges.yaml · suites/<id>.yaml · evals/<task>/task.yaml
+import { existsSync, readdirSync, readFileSync } from "node:fs";
+import { join } from "node:path";
+import { normalizeConfig } from "./config";
+import { projectRoot } from "./project";
+import { validateTask } from "./task-file";
+import type { AgentConfig, JudgeConfig, Suite, Task } from "./types";
+
+const readYaml = (p: string) => Bun.YAML.parse(readFileSync(p, "utf8")) as any;
+const yamlIds = (dir: string) => (existsSync(dir) ? readdirSync(dir).filter((f) => f.endsWith(".yaml")).map((f) => f.slice(0, -5)).sort() : []);
+
+// Each loader reads from the project root (`root`: configs/, judges.yaml, suites/, evals/). The default is the
+// project found from the current folder (project.ts); it throws outside a project. Tests pass a temp project.
+export function loadConfigs(root = projectRoot()): AgentConfig[] {
+  return yamlIds(join(root, "configs")).map((id) => normalizeConfig(id, readYaml(join(root, "configs", `${id}.yaml`)), root));
+}
+
+export function loadJudges(root = projectRoot()): JudgeConfig[] {
+  return (readYaml(join(root, "judges.yaml")).judges ?? []) as JudgeConfig[];
+}
+
+export function loadSuites(root = projectRoot()): Suite[] {
+  return yamlIds(join(root, "suites")).map((id) => ({ id, ...readYaml(join(root, "suites", `${id}.yaml`)) }));
+}
+
+export interface TaskListing { id: string; title: string; description: string; errors: string[]; warnings: string[] }
+
+/** Loads and validates a task from `<root>/evals/<id>/task.yaml`. Throws when the file has errors. */
+export function loadTask(id: string, root = projectRoot()): Task & { warnings: string[] } {
+  const dir = join(root, "evals", id);
+  const raw = readYaml(join(dir, "task.yaml"));
+  const { errors, warnings } = validateTask(raw, dir);
+  if (errors.length) throw new Error(`task ${id} is invalid:\n- ${errors.join("\n- ")}`);
+  // YAML `derived:` and `derived: null` mean no derived evidence: drop the key here, so the grader sees one form.
+  for (const c of [...(raw.gates ?? []), ...raw.measures.flatMap((m: any) => m.criteria)]) if (c.derived == null) delete c.derived;
+  return { gates: [], ...raw, dir, warnings };
+}
+
+export function listTasks(root = projectRoot()): TaskListing[] {
+  const dir = join(root, "evals");
+  return readdirSync(dir).filter((d) => existsSync(join(dir, d, "task.yaml"))).map((id) => {
+    const raw = readYaml(join(dir, id, "task.yaml"));
+    const v = validateTask(raw, join(dir, id));
+    return { id, title: raw.title ?? id, description: raw.description ?? "", ...v };
+  });
+}

package/src/core/batch-state.ts

@@ -0,0 +1,25 @@
+// Whether a batch is still running, for the batches panel. A batch whose process died never gets its end
+// time, so "not ended" alone does not mean "running": a batch also needs a trial that moved recently, or
+// this server process running it.
+import type { BatchRecord, TrialRecord } from "./db";
+
+/** The agent timeout (20 min) plus a margin: a long agent trial or a slow grading is not called stopped. */
+export const STALLED_AFTER_MS = 25 * 60_000;
+
+export type BatchRunState = "running" | "stalled" | "ended";
+
+export interface BatchState {
+  total: number;
+  /** The latest start or end time of its trials; the batch start when no trial has moved. */
+  lastMoveAt: string;
+  /** ended: it has its end time. running: this server runs it, or a trial moved in the last 25 min. stalled: neither; its process may have stopped. */
+  state: BatchRunState;
+}
+
+export function batchState(batch: BatchRecord, trials: TrialRecord[], o: { live: boolean; now: Date }): BatchState {
+  let lastMoveAt = batch.startedAt;
+  for (const t of trials) for (const at of [t.startedAt, t.endedAt]) if (at && at > lastMoveAt) lastMoveAt = at;
+  const state: BatchRunState = batch.endedAt != null ? "ended"
+    : o.live || o.now.getTime() - Date.parse(lastMoveAt) < STALLED_AFTER_MS ? "running" : "stalled";
+  return { total: trials.length, lastMoveAt, state };
+}

package/src/core/batch.ts

@@ -0,0 +1,450 @@
+// A batch is one `run`: what was asked for, and its trials. Every trial is its own record in the store, with
+// its task hash, config key and run conditions, and zero or more gradings. Artifacts go to the trial's own
+// folder (db.ts trialDir): workspace/, agent.log and gradings/<grading-id>/ for judge logs.
+//
+// Running a trial and grading it are two separate steps. A trial's row ends (done, failed or invalid)
+// before any grading starts and never changes after that; a grading is a new row. So a crash between the
+// two leaves a done trial with no finished grading, which regrade can grade later.
+//
+// The agent works in a workspace in the OS temp folder, outside any repo (ADR-0004); it is copied to the
+// trial's folder before the row ends, so an ended trial always has its artifacts.
+import { cpSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { adapters } from "../adapters";
+import { COPY_LIMIT, copiedShare } from "./copying";
+import { trialDir, type BatchRecord, type GradingRecord, type GradingResult, type Store, type TrialRecord } from "./db";
+import { grade } from "./grade";
+import { canonicalJson, configKey, folderHash, gradingIdentity, sha256, taskIdentity } from "./identity";
+import { profileSnapshot } from "./profiles";
+import { familyOf } from "./score";
+import { loadSettings, userSetupFiles } from "./settings";
+import { loadConfigs, loadJudges, loadSuites, loadTask } from "./authored";
+import { projectPaths, projectRoot } from "./project";
+import type { AgentConfig, AgentHarness, AgentResult, JudgeConfig, Task } from "./types";
+
+export const DEFAULT_VOTES = 3;
+const AGENT_TIMEOUT_MS = 20 * 60_000;
+const VERSION_TIMEOUT_MS = 10_000;
+
+// ---- Planning: what was asked for, checked before anything is written ----------------------------
+
+/** What `run` was asked for. Give `tasks` or `suites`, not both; neither means every suite. */
+export interface RunRequest {
+  tasks?: string[];
+  suites?: string[];
+  configs?: string[];   // left out or ["all"]: every config
+  trials?: number;      // per agent config and task (a human config gets one); default 3
+  judge?: string;       // judge id; default the first in judges.yaml
+  votes?: number;       // default DEFAULT_VOTES
+}
+
+export interface BatchPlan {
+  /** Stored as the batch's `request`: what was asked for, with the task list it resolved to. */
+  request: { tasks?: string[]; suites?: string[]; configs: string[]; trials: number; judge: string; votes: number; taskIds: string[] };
+  tasks: Task[];
+  configs: AgentConfig[];
+  judge: JudgeConfig;
+  votes: number;
+  trials: number;
+}
+
+const positiveInt = (n: number, what: string) => {
+  if (!Number.isInteger(n) || n < 1) throw new Error(`${what} must be a whole number of 1 or more (got ${JSON.stringify(n)})`);
+  return n;
+};
+
+const REQUEST_FIELDS = ["tasks", "suites", "configs", "trials", "judge", "votes"];
+
+/**
+ * Checks the shape of a request that came from outside the type checker (the JSON body of POST /api/runs):
+ * a JSON object with only RunRequest's fields, each list a list of strings, each count a whole number of 1 or
+ * more. A field left out or undefined takes its default. Throws a plain message; returns the request as typed.
+ */
+export function parseRunRequest(body: unknown): RunRequest {
+  if (!body || typeof body !== "object" || Array.isArray(body)) throw new Error("the request body must be a JSON object");
+  const r = body as Record<string, unknown>;
+  const unknown = Object.keys(r).filter((k) => !REQUEST_FIELDS.includes(k));
+  if (unknown.length) throw new Error(`unknown request field(s): ${unknown.join(", ")} (known: ${REQUEST_FIELDS.join(", ")})`);
+  for (const k of ["tasks", "suites", "configs"]) {
+    const v = r[k];
+    if (v !== undefined && !(Array.isArray(v) && v.every((x) => typeof x === "string"))) throw new Error(`${k} must be a list of ids (strings), got ${JSON.stringify(v)}`);
+  }
+  for (const k of ["trials", "votes"]) if (r[k] !== undefined) positiveInt(r[k] as number, k);
+  if (r.judge !== undefined && typeof r.judge !== "string") throw new Error(`judge must be a judge id (a string), got ${JSON.stringify(r.judge)}`);
+  return r as RunRequest;
+}
+
+/** Resolves and checks a request against the project at `root`. Throws on anything unknown, so no batch is started. */
+export function planBatch(req: RunRequest, root = projectRoot()): BatchPlan {
+  parseRunRequest(req);
+  if (req.tasks && req.suites) throw new Error("give --tasks or --suites, not both");
+  let taskIds: string[];
+  if (req.tasks) {
+    taskIds = [...new Set(req.tasks)];
+    const unknown = taskIds.filter((id) => !existsSync(join(root, "evals", id, "task.yaml")));
+    if (!taskIds.length || unknown.length) throw new Error(`unknown or no task(s): ${unknown.join(", ") || "none given"}`);
+  } else {
+    const suites = Object.fromEntries(loadSuites(root).map((s) => [s.id, s]));
+    const suiteIds = req.suites ?? Object.keys(suites);
+    const unknown = suiteIds.filter((id) => !suites[id]);
+    if (!suiteIds.length || unknown.length) throw new Error(`unknown or no suite(s): ${unknown.join(", ") || "none given"}`);
+    taskIds = [...new Set(suiteIds.flatMap((s) => suites[s].tasks))];
+  }
+  const all = loadConfigs(root);
+  const byId = Object.fromEntries(all.map((c) => [c.id, c]));
+  const configIds = !req.configs || req.configs.includes("all") ? all.map((c) => c.id) : [...new Set(req.configs)];
+  const missing = configIds.filter((id) => !byId[id]);
+  if (!configIds.length || missing.length) throw new Error(`unknown or no config(s): ${missing.join(", ") || "none given"}`);
+  const judges = loadJudges(root);
+  const judge = req.judge ? judges.find((j) => j.id === req.judge) : judges[0];
+  if (!judge) throw new Error(`unknown judge ${req.judge ?? "(judges.yaml lists none)"}`);
+  const trials = positiveInt(req.trials ?? 3, "trials");
+  const votes = positiveInt(req.votes ?? DEFAULT_VOTES, "votes");
+  const tasks = taskIds.map((id) => loadTask(id, root));   // throws on an invalid task
+  return {
+    // Neither --tasks nor --suites asked for every suite: recorded as suites ["all"].
+    request: { ...(req.tasks ? { tasks: req.tasks } : { suites: req.suites ?? ["all"] }), configs: configIds, trials, judge: judge.id, votes, taskIds },
+    tasks, configs: configIds.map((id) => byId[id]), judge, votes, trials,
+  };
+}
+
+// ---- Starting: the batch record, its hashes and run conditions, and every trial queued ------------
+
+/** What running and grading the trials of one batch needs. */
+export interface BatchContext {
+  batchId: string;
+  dbPath: string;                                  // trial folders are beside the database
+  tasks: Record<string, { task: Task; taskHash: string; gradingHash: string }>;
+  configs: Record<string, AgentConfig>;
+  judge: JudgeConfig;
+  votes: number;
+  setupHash: string | null;                        // the user setup agent trials run with
+  blockedTools: Partial<Record<AgentHarness, string[]>>;
+  harnessVersions: Record<string, string | null>;
+  cacheDir: string;                                // Profile snapshots and the apkit lock
+}
+
+export interface StartOptions {
+  dbPath: string;
+  root?: string;
+  cacheDir?: string;
+  /** Reads `<harness> --version`; tests replace it. */
+  readVersion?: (harness: AgentHarness) => Promise<string | null>;
+  /** The user setup file list (path → hash); tests replace it. */
+  userSetup?: () => Record<string, string>;
+  now?: () => Date;
+}
+
+export interface StartedBatch { batch: BatchRecord; trials: TrialRecord[]; ctx: BatchContext }
+
+/** UTC YYYYMMDD-HHMMSS. */
+export const batchStamp = (d: Date) => d.toISOString().replace(/[-:]/g, "").replace(/\..+/, "").replace("T", "-");
+
+/** The first line `<harness> --version` prints, or null when it cannot be read. */
+export async function readHarnessVersion(harness: AgentHarness): Promise<string | null> {
+  try {
+    const p = Bun.spawn([harness, "--version"], { stdin: "ignore", stdout: "pipe", stderr: "pipe" });
+    const timer = setTimeout(() => p.kill(), VERSION_TIMEOUT_MS);
+    const [out, code] = await Promise.all([new Response(p.stdout).text(), p.exited]);
+    clearTimeout(timer);
+    const line = out.split("\n").map((l) => l.trim()).find(Boolean);
+    return code === 0 && line ? line : null;
+  } catch {
+    return null;   // not installed
+  }
+}
+
+/** Each agent harness's CLI version, read once. */
+export async function harnessVersions(harnesses: AgentHarness[], read = readHarnessVersion): Promise<Record<string, string | null>> {
+  const unique = [...new Set(harnesses)].sort();
+  return Object.fromEntries(await Promise.all(unique.map(async (h) => [h, await read(h)] as const)));
+}
+
+/** A human wrote one file per task, so it gets one trial, and only for tasks it has a file for. */
+const trialCount = (c: AgentConfig, taskId: string, n: number) => (c.harness !== "human" ? n : c.outputs?.[taskId] ? 1 : 0);
+
+/** One trial a batch starts: its task, config and number. */
+export interface PlannedTrial { task: Task; config: AgentConfig; n: number }
+
+/**
+ * Every trial of a planned batch, in the order startBatch queues them: each task, each config, n = 1..trials (one
+ * for a human config with a file for the task, none without). startBatch inserts this list, and the plan in runs
+ * (plan.ts) counts it, so the counts are what starts.
+ */
+export function plannedTrials(plan: Pick<BatchPlan, "tasks" | "configs" | "trials">): PlannedTrial[] {
+  return plan.tasks.flatMap((task) => plan.configs.flatMap((config) =>
+    Array.from({ length: trialCount(config, task.id, plan.trials) }, (_, i) => ({ task, config, n: i + 1 }))));
+}
+
+/**
+ * Records a new batch in one transaction: the batch (id = UTC timestamp, with -2, -3, ... when a batch
+ * started in the same second has it), each task version and grading setup when first seen, the user setup,
+ * and every planned trial as queued. Reads the harness versions and the user setup first.
+ */
+export async function startBatch(store: Store, plan: BatchPlan, o: StartOptions): Promise<StartedBatch> {
+  const root = o.root ?? projectRoot();
+  const now = o.now ?? (() => new Date());
+  const agentHarnesses = plan.configs.flatMap((c) => (c.harness === "human" ? [] : [c.harness]));
+  const versions = await harnessVersions(agentHarnesses, o.readVersion);
+  const files = agentHarnesses.length ? (o.userSetup ?? userSetupFiles)() : null;
+  const setupHash = files ? sha256(canonicalJson(files)) : null;
+  const settings = loadSettings(root);
+  const ids = plan.tasks.map((task) => ({ task, t: taskIdentity(task), g: gradingIdentity(task, plan.judge) }));
+
+  const started = now();
+  const at = started.toISOString();
+  const { batch, trials } = store.transaction(() => {
+    const stamp = batchStamp(started);
+    let id = stamp;
+    for (let k = 2; store.getBatch(id); k++) id = `${stamp}-${k}`;
+    const batch: BatchRecord = { id, startedAt: at, endedAt: null, request: plan.request, harnessVersions: versions };
+    store.createBatch(batch);
+    store.startBatchRun(id);
+    for (const { task, t, g } of ids) {
+      store.putTaskVersion({ hash: t.hash, taskId: task.id, label: task.version ?? null, snapshot: t.snapshot as unknown as Record<string, unknown>, firstSeen: at });
+      store.putGradingSetup({
+        hash: g.hash, taskId: task.id, taskHash: t.hash,
+        judgeHarness: plan.judge.harness, judgeModel: plan.judge.model, judgeEffort: plan.judge.effort ?? null, judgeLabel: plan.judge.label,
+        snapshot: g.snapshot as unknown as Record<string, unknown>, firstSeen: at,
+      });
+    }
+    if (files && setupHash) store.putUserSetup({ hash: setupHash, files });
+    const trials: TrialRecord[] = [];
+    const hashes = new Map(ids.map(({ task, t }) => [task.id, t]));
+    for (const { task, config: c, n } of plannedTrials(plan)) {
+      const t = hashes.get(task.id)!;
+      const tid = `${id}--${task.id}--${c.id}--${n}`;
+      const r: TrialRecord = {
+        id: tid, batchId: id, taskId: task.id, taskHash: t.hash, configKey: configKey(c), configId: c.id,
+        harness: c.harness, model: c.model, effort: c.effort ?? null, profile: c.profile ?? null, provider: c.provider ?? null,
+        family: c.family ?? familyOf(c.model), n, status: "queued",
+        startedAt: null, endedAt: null, durationMs: null,
+        inputTokens: null, outputTokens: null, cacheReadTokens: null, cacheWriteTokens: null, costUsd: null, costSource: null, turns: null,
+        setupHash: null, profileHash: null, blockedTools: null, harnessVersion: null, copiedShare: null, error: null,
+        dir: `trials/${tid}`,
+      };
+      store.insertTrial(r);
+      trials.push(r);
+    }
+    return { batch, trials };
+  });
+
+  const ctx: BatchContext = {
+    batchId: batch.id, dbPath: o.dbPath,
+    tasks: Object.fromEntries(ids.map(({ task, t, g }) => [task.id, { task, taskHash: t.hash, gradingHash: g.hash }])),
+    configs: Object.fromEntries(plan.configs.map((c) => [c.id, c])),
+    judge: plan.judge, votes: plan.votes, setupHash, blockedTools: settings.blockedTools, harnessVersions: versions,
+    cacheDir: o.cacheDir ?? projectPaths(root).cache,
+  };
+  return { batch, trials, ctx };
+}
+
+// ---- Running a batch ------------------------------------------------------------------------------
+
+export interface RunHooks { concurrency?: number; onEvent?: (msg: string) => void }
+
+/**
+ * Runs every trial of a started batch and grades each one that ends done. Then this run of the batch is over: the
+ * batch ends unless another run of it (a rerun, in this process or another) is still in flight, or a trial this
+ * run did not start is still queued or running.
+ */
+export async function runBatch(store: Store, started: StartedBatch, o: RunHooks = {}) {
+  const log = o.onEvent ?? (() => {});
+  const { ctx } = started;
+  const queue = started.trials.map((t) => t.id);
+  const worker = async () => {
+    for (let id = queue.shift(); id; id = queue.shift()) {
+      try {
+        const t = await runTrial(store, ctx, id, log);
+        if (t.status !== "done") continue;
+        const { task, gradingHash } = ctx.tasks[t.taskId];
+        const g = await gradeTrial(store, ctx.dbPath, t, { task, judge: ctx.judge, votes: ctx.votes, gradingHash });
+        log(`${g.status === "done" ? "✓" : "!"} ${t.id} score ${g.score?.toFixed(1) ?? "—"}${g.error ? ` (${g.error})` : ""} cost $${t.costUsd?.toFixed(3) ?? "?"}`);
+      } catch (e) {
+        log(`✗ ${id}: ${e instanceof Error ? e.message : String(e)}`);   // a store error: the other trials go on
+      }
+    }
+  };
+  await Promise.all(Array.from({ length: Math.max(1, o.concurrency ?? 4) }, worker));
+  // Another run of this batch may still run or grade its trials, here or in another process: then the last one
+  // to finish ends it. A run whose process died is never finished, so its batch stays not ended.
+  const mine = new Set(started.trials.map((t) => t.id));
+  const ended = store.transaction(() => {
+    const others = store.finishBatchRun(ctx.batchId);
+    const busy = others > 0 || store.listTrials({ batchId: ctx.batchId }).some((t) => !mine.has(t.id) && (t.status === "queued" || t.status === "running"));
+    if (!busy) store.endBatch(ctx.batchId, new Date().toISOString());
+    return !busy;
+  });
+  log(ended ? `batch ${ctx.batchId} done` : `batch ${ctx.batchId}: these trials are done and graded; another run of it is still running or grading trials`);
+}
+
+const message = (e: unknown) => (e instanceof Error ? e.message : String(e));
+
+/**
+ * Runs one queued trial: workspace, agent run (or the human's file), held-out check, artifacts saved, then
+ * the row ends once as done, failed (the harness could not run) or invalid (copied the held-out original).
+ * It never grades. Returns the ended row.
+ */
+export async function runTrial(store: Store, ctx: BatchContext, trialId: string, log: (m: string) => void = () => {}): Promise<TrialRecord> {
+  // Claim it: queued -> running is a compare-and-set in one write transaction (BEGIN IMMEDIATE), so two
+  // callers, in this process or another, can never both start the same trial.
+  const { trial, task, config, human } = store.transaction(() => {
+    const trial = store.getTrial(trialId);
+    if (!trial) throw new Error(`no trial ${trialId}`);
+    if (trial.status !== "queued") throw new Error(`trial ${trialId} is ${trial.status}; only a queued trial can run`);
+    const entry = ctx.tasks[trial.taskId], config = ctx.configs[trial.configId];
+    if (!entry || !config) throw new Error(`trial ${trialId}: its task or config is not part of this batch`);
+    const human = config.harness === "human";
+    // Run conditions. A human trial had no agent setup, Profile, blocked tools or harness.
+    store.updateTrial(trial.id, {
+      status: "running", startedAt: new Date().toISOString(),
+      setupHash: human ? null : ctx.setupHash,
+      blockedTools: human ? null : (ctx.blockedTools[config.harness as AgentHarness] ?? []),
+      harnessVersion: human ? null : (ctx.harnessVersions[config.harness] ?? null),
+    });
+    return { trial, task: entry.task, config, human };
+  });
+  const dir = trialDir(ctx.dbPath, trial.id);
+  let ws = "";   // a new folder in the OS temp folder, so no two trials (or two stores) share one
+  log(`▶ ${trial.id}`);
+
+  const end: Partial<TrialRecord> = {};
+  let status: "done" | "failed" | "invalid";
+  let error: string | null = null;
+  try {
+    ws = mkdtempSync(join(tmpdir(), "static-fire-trial-"));
+    mkdirSync(join(ws, "out"), { recursive: true });
+    mkdirSync(dir, { recursive: true });
+    fillWorkspace(task, trial.taskHash, ws);
+    if (config.profile && !human) {
+      const snap = await profileSnapshot(config.profile, config.harness as AgentHarness, ctx.batchId, ctx.cacheDir);
+      end.profileHash = folderHash(snap);
+      cpSync(snap, ws, { recursive: true });
+    }
+
+    const t0 = Date.now();
+    const result: AgentResult = human
+      ? copyHumanOutput(config, trial.taskId, join(ws, task.output))
+      : await adapters[config.harness as AgentHarness]({
+        config, prompt: task.prompt, cwd: ws, logPath: join(dir, "agent.log"), timeoutMs: AGENT_TIMEOUT_MS,
+        guard: { blockedTools: ctx.blockedTools[config.harness as AgentHarness] ?? [] },
+      });
+    if (!human) end.durationMs = Date.now() - t0;
+    Object.assign(end, {
+      inputTokens: result.usage.inputTokens, outputTokens: result.usage.outputTokens,
+      cacheReadTokens: result.usage.cacheReadTokens, cacheWriteTokens: result.usage.cacheWriteTokens,
+      costUsd: result.costUsd, costSource: result.costSource, turns: result.turns ?? null,
+    });
+
+    const outPath = join(ws, task.output);
+    const ran = result.ok || (result.turns ?? 0) > 0 || existsSync(outPath);
+    if (!ran) throw new Error(result.error ?? "the harness did not run");
+    if (!result.ok) error = result.error ?? null;
+    // An agent that found the published original (for example on the web) and copied it did not do the work.
+    if (task.heldOut && !human) {
+      end.copiedShare = existsSync(outPath) ? copiedShare(readFileSync(outPath, "utf8"), readFileSync(join(task.dir, task.heldOut), "utf8")) : 0;
+    }
+    if ((end.copiedShare ?? 0) > COPY_LIMIT) {
+      status = "invalid";
+      error = `copied the held-out original (${Math.round(end.copiedShare! * 100)}% of its 8-word runs)`;
+    } else status = "done";
+  } catch (e) {
+    status = "failed";
+    error = message(e);
+  }
+
+  // The artifacts are saved before the row ends, so an ended trial always has them.
+  try {
+    if (ws && existsSync(ws)) {
+      cpSync(ws, join(dir, "workspace"), { recursive: true });
+      rmSync(ws, { recursive: true, force: true });
+    }
+  } catch (e) {
+    if (status === "done") { status = "failed"; error = `could not save the workspace: ${message(e)}`; }
+  }
+
+  error = error?.trim() || null;   // harness messages often end with a newline
+  store.updateTrial(trial.id, { ...end, status, error, endedAt: new Date().toISOString() });
+  if (status !== "done") log(`${status === "invalid" ? "!" : "✗"} ${trial.id}: ${error}`);
+  return store.getTrial(trial.id)!;
+}
+
+/**
+ * Copies the task's inputs and project into the workspace, first to a private folder whose files are hashed:
+ * when they no longer give the trial's recorded task hash (a file changed after the batch started), it throws
+ * and the workspace stays empty. The workspace gets the copy that was hashed, so the record names what the agent got.
+ */
+function fillWorkspace(task: Task, want: string, ws: string) {
+  const stage = mkdtempSync(join(tmpdir(), "static-fire-task-"));
+  try {
+    cpSync(join(task.dir, task.inputs), join(stage, task.inputs), { recursive: true });
+    if (task.project) cpSync(join(task.dir, task.project), join(stage, task.project), { recursive: true });
+    const now = taskIdentity({ ...task, dir: stage }).hash;
+    if (now !== want) {
+      throw new Error(`task ${task.id}: its files changed after the batch started (task hash ${want.slice(0, 8)} then, ${now.slice(0, 8)} now), `
+        + "so this trial did not run. Run the task again in a new batch.");
+    }
+    cpSync(join(stage, task.inputs), join(ws, task.inputs), { recursive: true });
+    if (task.project) cpSync(join(stage, task.project), ws, { recursive: true });
+  } finally {
+    rmSync(stage, { recursive: true, force: true });
+  }
+}
+
+export function copyHumanOutput(config: AgentConfig, taskId: string, out: string): AgentResult {
+  const src = config.outputs?.[taskId]?.path;
+  const none = { usage: { inputTokens: 0, outputTokens: 0, cacheReadTokens: 0, cacheWriteTokens: 0 }, costUsd: null, costSource: "unknown" as const };
+  if (!src || !existsSync(src)) return { ok: false, error: `no file for ${taskId} at ${src ?? "(none)"}`, ...none };
+  mkdirSync(join(out, ".."), { recursive: true });
+  cpSync(src, out);
+  return { ok: true, ...none };
+}
+
+// ---- Grading one trial ----------------------------------------------------------------------------
+
+/** One grading setup: the task and judge whose grading hash it is, and the vote count. */
+export interface GradingSetup { task: Task; judge: JudgeConfig; votes: number; gradingHash: string }
+
+/**
+ * Grades a done trial's saved output under one grading setup, with no agent run: adds a grading row, runs the
+ * graders on a hashed private copy of the task folder with judge logs in the trial's gradings/<grading-id>/
+ * folder, and finishes the row with its verdicts (invalid when a grader returned an error, or when the copy
+ * does not give the grading hash). Never changes the trial row or an earlier grading.
+ */
+export async function gradeTrial(store: Store, dbPath: string, trial: TrialRecord, s: GradingSetup): Promise<GradingRecord> {
+  if (trial.status !== "done") throw new Error(`trial ${trial.id} is ${trial.status}; only a done trial is graded`);
+  const gradingId = store.startGrading({ trialId: trial.id, gradingHash: s.gradingHash, votes: s.votes, gradedAt: new Date().toISOString() });
+  const dir = trialDir(dbPath, trial.id);
+  const ws = join(dir, "workspace");
+  const logs = join(dir, "gradings", String(gradingId));
+  let result: GradingResult;
+  // The grader reads a private copy of the task folder, made now and hashed: when it no longer gives the
+  // recorded grading hash, nothing is graded under it (the row ends invalid, and a regrade grades the trial
+  // under the current hash). An edit to the task folder while grading runs does not reach the grader.
+  let stage = "";
+  try {
+    mkdirSync(logs, { recursive: true });
+    stage = mkdtempSync(join(tmpdir(), "static-fire-grading-"));
+    cpSync(s.task.dir, stage, { recursive: true });
+    const task: Task = { ...s.task, dir: stage };
+    const now = gradingIdentity(task, s.judge).hash;
+    if (now !== s.gradingHash) {
+      throw new Error(`task ${s.task.id}: its grading files changed after this grading was planned (grading hash ${s.gradingHash.slice(0, 8)} then, `
+        + `${now.slice(0, 8)} now), so it was not graded. Regrade to grade it under the current files.`);
+    }
+    const g = await grade({ task, outputPath: join(ws, s.task.output), workspace: ws, judge: { ...s.judge, votes: s.votes }, dir: logs });
+    result = {
+      status: g.errors.length ? "invalid" : "done",
+      score: g.score, gatesPassed: g.gatesPassed, measures: g.measures, judgeCostUsd: g.judgeCostUsd,
+      error: g.errors.length ? `grader errors: ${g.errors.join(", ")}` : null,
+      verdicts: Object.entries(g.verdicts).map(([criterionId, v]) => ({ criterionId, score: v.score, error: v.error ?? null, votes: v.votes })),
+    };
+  } catch (e) {
+    result = { status: "invalid", score: null, gatesPassed: null, measures: null, judgeCostUsd: null, error: `grading failed: ${message(e)}`, verdicts: [] };
+  } finally {
+    if (stage) rmSync(stage, { recursive: true, force: true });
+  }
+  store.finishGrading(gradingId, result);
+  return store.listGradings(trial.id, s.gradingHash).find((g) => g.id === gradingId)!;
+}