static-fire 0.0.0-stage → 0.54.1

package/src/core/batches.ts

@@ -0,0 +1,221 @@
+// Batches after they start: what `list` prints for each one, and `rerun <batch>`, which runs a batch's failed
+// trials again in the same batch.
+//
+// A failed trial's row never changes. A rerun adds a new trial with the next free n for that task and config.
+// Which failed trials still need a rerun is worked out from the rows, with no link column:
+//   - the batch asked for `want` trials of each task and config (request.trials for an agent, 1 for a human);
+//   - every trial that did not fail (queued, running, done, invalid) fills one of them;
+//   - each later trial replaces the earliest failed trial not yet replaced, so the failed trials still to rerun
+//     are the latest ones. A trial that failed and then passed on a rerun is never offered again.
+// A queued or running trial fills its place and is never touched: the store holds no evidence that its process
+// died (no pid, no heartbeat), and a long time alone is not proof (a laptop that sleeps keeps a live run going).
+import { existsSync } from "node:fs";
+import { join } from "node:path";
+import { harnessVersions, type StartOptions, type StartedBatch, type BatchContext } from "./batch";
+import type { BatchRecord, Store, TrialRecord, TrialRecordStatus } from "./db";
+import { canonicalJson, configKey, gradingIdentity, sha256, taskIdentity } from "./identity";
+import { loadSettings, userSetupFiles } from "./settings";
+import { loadConfigs, loadJudges, loadTask } from "./authored";
+import { projectPaths, projectRoot } from "./project";
+import type { AgentConfig, JudgeConfig, Task } from "./types";
+
+/** What the batch stored as its request (batch.ts BatchPlan.request). */
+export interface BatchRequest {
+  tasks?: string[]; suites?: string[]; configs: string[]; trials: number; judge: string; votes: number; taskIds: string[];
+}
+
+export interface BatchSummary {
+  batch: BatchRecord;
+  request: BatchRequest;
+  counts: Record<TrialRecordStatus, number>;
+  /** Failed trials that no later trial has replaced yet: what `rerun` would run. */
+  toRerun: number;
+  ended: boolean;
+}
+
+/** One failed trial still to rerun. */
+export interface RerunSlot { taskId: string; configId: string; replaces: string }
+
+const ENDED_OK: TrialRecordStatus[] = ["queued", "running", "done", "invalid"];   // every status but failed fills a place
+
+/** The failed trials of a batch that no later trial replaced, oldest task/config first. */
+export function rerunSlots(request: BatchRequest, trials: TrialRecord[]): RerunSlot[] {
+  const groups = new Map<string, TrialRecord[]>();
+  for (const t of trials) {
+    const k = `${t.taskId}\0${t.configId}`;
+    groups.set(k, [...(groups.get(k) ?? []), t]);
+  }
+  const slots: RerunSlot[] = [];
+  for (const group of groups.values()) {
+    const want = group[0]!.harness === "human" ? 1 : request.trials;
+    const filled = group.filter((t) => ENDED_OK.includes(t.status)).length;
+    const failed = group.filter((t) => t.status === "failed").sort((a, b) => a.n - b.n);
+    const short = Math.min(Math.max(0, want - filled), failed.length);
+    for (const t of failed.slice(failed.length - short)) slots.push({ taskId: t.taskId, configId: t.configId, replaces: t.id });
+  }
+  return slots.sort((a, b) => (a.replaces < b.replaces ? -1 : a.replaces > b.replaces ? 1 : 0));
+}
+
+export function summarizeBatch(store: Store, batch: BatchRecord): BatchSummary {
+  const trials = store.listTrials({ batchId: batch.id });
+  const counts: Record<TrialRecordStatus, number> = { queued: 0, running: 0, done: 0, failed: 0, invalid: 0 };
+  for (const t of trials) counts[t.status]++;
+  const request = batch.request as unknown as BatchRequest;
+  return { batch, request, counts, toRerun: rerunSlots(request, trials).length, ended: batch.endedAt !== null };
+}
+
+/** Every batch, newest first. */
+export const listBatchSummaries = (store: Store): BatchSummary[] => store.listBatches().map((b) => summarizeBatch(store, b));
+
+// ---- rerun ----------------------------------------------------------------------------------------
+
+/** `rerun` will not run this batch's failed trials as they were: something they ran with has changed or is gone. */
+export class RerunRefused extends Error {}
+
+export interface RerunPlan {
+  batch: BatchRecord;
+  request: BatchRequest;
+  slots: RerunSlot[];
+  /** Trials still queued or running: never rerun, never changed. */
+  busy: string[];
+  tasks: Record<string, { task: Task; taskHash: string }>;
+  configs: Record<string, AgentConfig>;
+  judge: JudgeConfig | null;
+}
+
+const short = (h: string) => h.slice(0, 8);
+
+/**
+ * Works out what `rerun <batch>` would run and checks it against today's files. Writes nothing. Throws
+ * RerunRefused when a task's hash or a config's key differs from what the batch ran with, or the batch's judge
+ * is gone. With no failed trial to rerun, returns no slots and checks nothing.
+ */
+export function planRerun(store: Store, batchId: string, root = projectRoot()): RerunPlan {
+  const batch = store.getBatch(batchId);
+  if (!batch) throw new RerunRefused(`no batch ${batchId}`);
+  const request = batch.request as unknown as BatchRequest;
+  const trials = store.listTrials({ batchId });
+  const slots = rerunSlots(request, trials);
+  const busy = trials.filter((t) => t.status === "queued" || t.status === "running").map((t) => t.id);
+  const plan: RerunPlan = { batch, request, slots, busy, tasks: {}, configs: {}, judge: null };
+  if (!slots.length) return plan;
+
+  const byId = new Map(trials.map((t) => [t.id, t]));
+  const configIds = [...new Set(slots.map((s) => s.configId))];
+  const useRun = (taskIds: string[]) =>
+    `Use \`static-fire run --tasks ${taskIds.join(",")} --configs ${configIds.join(",")}\` to run it on the current files.`;
+  for (const taskId of [...new Set(slots.map((s) => s.taskId))]) {
+    const was = byId.get(slots.find((s) => s.taskId === taskId)!.replaces)!.taskHash;
+    if (!existsSync(join(root, "evals", taskId, "task.yaml"))) {
+      throw new RerunRefused(`task ${taskId} is gone since batch ${batchId} (no evals/${taskId}/task.yaml), so its failed trials cannot run again as they were.`);
+    }
+    let task: Task;
+    try { task = loadTask(taskId, root); } catch (e) {
+      throw new RerunRefused(`task ${taskId} cannot be read: ${e instanceof Error ? e.message : String(e)}`);
+    }
+    const now = taskIdentity(task).hash;
+    if (now !== was) {
+      throw new RerunRefused(`task ${taskId} changed since batch ${batchId} (task hash ${short(was)} then, ${short(now)} now). `
+        + `A rerun keeps a batch on the task version it ran. ${useRun([taskId])}`);
+    }
+    plan.tasks[taskId] = { task, taskHash: now };
+  }
+  const configs = new Map(loadConfigs(root).map((c) => [c.id, c]));
+  const taskIds = Object.keys(plan.tasks);
+  for (const configId of configIds) {
+    const was = byId.get(slots.find((s) => s.configId === configId)!.replaces)!.configKey;
+    const c = configs.get(configId);
+    if (!c) throw new RerunRefused(`config ${configId} is gone since batch ${batchId} (no configs/${configId}.yaml). ${useRun(taskIds)}`);
+    if (configKey(c) !== was) {
+      throw new RerunRefused(`config ${configId} changed since batch ${batchId} (${was} then, ${configKey(c)} now). `
+        + `A rerun uses the configs the batch recorded. ${useRun(taskIds)}`);
+    }
+    plan.configs[configId] = c;
+  }
+  const judge = loadJudges(root).find((j) => j.id === request.judge);
+  if (!judge) throw new RerunRefused(`judge ${request.judge}, which batch ${batchId} used, is no longer in judges.yaml. ${useRun(taskIds)}`);
+  plan.judge = judge;
+  return plan;
+}
+
+export interface StartedRerun extends StartedBatch {
+  /** New trial id → the failed trial it replaces. */
+  replaces: Record<string, string>;
+}
+
+/**
+ * Adds one queued trial per slot to the batch, with the next free n for its task and config, counts this rerun
+ * as a run of the batch in flight (which clears its end time), and builds the BatchContext to run them (pass the
+ * result to runBatch). The slots are worked out again inside one write transaction, so two reruns of one batch at the same time never both add a
+ * trial for the same failure; one that finds no slot left throws and writes nothing.
+ * Run conditions (user setup, blocked tools, harness version) are today's, as each trial records its own.
+ */
+export async function startRerun(store: Store, plan: RerunPlan, o: StartOptions): Promise<StartedRerun> {
+  const root = o.root ?? projectRoot();
+  const judge = plan.judge;
+  if (!judge || !plan.slots.length) throw new Error(`batch ${plan.batch.id}: nothing to rerun`);
+  const configs = Object.values(plan.configs);
+  const agentHarnesses = configs.flatMap((c) => (c.harness === "human" ? [] : [c.harness]));
+  const versions = await harnessVersions(agentHarnesses, o.readVersion);
+  const files = agentHarnesses.length ? (o.userSetup ?? userSetupFiles)() : null;
+  const setupHash = files ? sha256(canonicalJson(files)) : null;
+  const settings = loadSettings(root);
+  const grading = Object.fromEntries(Object.entries(plan.tasks).map(([id, { task }]) => [id, gradingIdentity(task, judge)]));
+  const at = (o.now ?? (() => new Date()))().toISOString();
+
+  const { trials, replaces } = store.transaction(() => {
+    const rows = store.listTrials({ batchId: plan.batch.id });
+    const slots = rerunSlots(plan.request, rows).filter((s) => plan.tasks[s.taskId] && plan.configs[s.configId]);
+    // Another rerun took them since the plan: nothing is written, and no runBatch follows to end the batch.
+    if (!slots.length) throw new Error(`batch ${plan.batch.id}: nothing to rerun`);
+    for (const [id, { task, taskHash }] of Object.entries(plan.tasks)) {
+      const g = grading[id]!;
+      store.putTaskVersion({ hash: taskHash, taskId: id, label: task.version ?? null, snapshot: taskIdentity(task).snapshot as unknown as Record<string, unknown>, firstSeen: at });
+      store.putGradingSetup({
+        hash: g.hash, taskId: id, taskHash,
+        judgeHarness: judge.harness, judgeModel: judge.model, judgeEffort: judge.effort ?? null, judgeLabel: judge.label,
+        snapshot: g.snapshot as unknown as Record<string, unknown>, firstSeen: at,
+      });
+    }
+    if (files && setupHash) store.putUserSetup({ hash: setupHash, files });
+    const next = new Map<string, number>();
+    for (const t of rows) {
+      const k = `${t.taskId}\0${t.configId}`;
+      next.set(k, Math.max(next.get(k) ?? 1, t.n + 1));
+    }
+    const byId = new Map(rows.map((t) => [t.id, t]));
+    const trials: TrialRecord[] = [];
+    const replaces: Record<string, string> = {};
+    for (const s of slots) {
+      const k = `${s.taskId}\0${s.configId}`;
+      const n = next.get(k)!;
+      next.set(k, n + 1);
+      const failed = byId.get(s.replaces)!;
+      const id = `${plan.batch.id}--${s.taskId}--${s.configId}--${n}`;
+      const r: TrialRecord = {
+        id, batchId: plan.batch.id, taskId: s.taskId, taskHash: failed.taskHash, configKey: failed.configKey, configId: s.configId,
+        harness: failed.harness, model: failed.model, effort: failed.effort, profile: failed.profile, provider: failed.provider,
+        family: failed.family, n, status: "queued",
+        startedAt: null, endedAt: null, durationMs: null,
+        inputTokens: null, outputTokens: null, cacheReadTokens: null, cacheWriteTokens: null, costUsd: null, costSource: null, turns: null,
+        setupHash: null, profileHash: null, blockedTools: null, harnessVersion: null, copiedShare: null, error: null,
+        dir: `trials/${id}`,
+      };
+      store.insertTrial(r);
+      trials.push(r);
+      replaces[id] = s.replaces;
+    }
+    store.startBatchRun(plan.batch.id);   // not ended while this run's trials run and are graded; runBatch ends it again
+    return { trials, replaces };
+  });
+
+  const ctx: BatchContext = {
+    batchId: plan.batch.id, dbPath: o.dbPath,
+    tasks: Object.fromEntries(Object.entries(plan.tasks).map(([id, { task, taskHash }]) => [id, { task, taskHash, gradingHash: grading[id]!.hash }])),
+    configs: plan.configs,
+    judge, votes: plan.request.votes, setupHash, blockedTools: settings.blockedTools, harnessVersions: versions,
+    cacheDir: o.cacheDir ?? projectPaths(root).cache,
+  };
+  return { batch: plan.batch, trials, ctx, replaces };
+}
+

package/src/core/checks.ts

@@ -0,0 +1,63 @@
+// Deterministic checks. Each returns a Verdict with one vote (pass/fail) or an error.
+import type { Check, Verdict } from "./types";
+
+export interface CheckContext { output: string; outputPath: string; workspace: string; taskDir: string }
+
+export function splitFrontmatter(text: string): { fields: Record<string, unknown>; body: string } {
+  const m = text.match(/^---\r?\n([\s\S]*?)\r?\n---\r?\n?/);
+  if (!m) return { fields: {}, body: text };
+  let fields: Record<string, unknown> = {};
+  try { fields = (Bun.YAML.parse(m[1]) as Record<string, unknown>) ?? {}; } catch { /* treat as none */ }
+  return { fields, body: text.slice(m[0].length).replace(/^\s+/, "") };
+}
+
+const one = (pass: boolean, reason: string): Verdict => ({ score: pass ? 1 : 0, votes: [{ pass, reason }] });
+const err = (error: string): Verdict => ({ score: null, votes: [], error });
+
+const within = (n: number, min?: number, max?: number) => (min == null || n >= min) && (max == null || n <= max);
+const bounds = (min?: number, max?: number) => [min != null && `≥ ${min}`, max != null && `≤ ${max}`].filter(Boolean).join(" and ");
+
+export async function runCheck(check: Check, ctx: CheckContext): Promise<Verdict> {
+  try {
+    switch (check.type) {
+      // engine:check.regex:begin
+      case "regex": {
+        const flags = check.flags?.includes("g") ? check.flags : `${check.flags ?? ""}g`;
+        const n = [...ctx.output.matchAll(new RegExp(check.pattern, flags))].length;
+        const min = check.min ?? (check.max == null ? 1 : undefined);
+        return one(within(n, min, check.max), `${n} match${n === 1 ? "" : "es"} of /${check.pattern}/ (needs ${bounds(min, check.max)})`);
+      }
+      // engine:check.regex:end
+      // engine:check.words:begin
+      case "words": {
+        const n = splitFrontmatter(ctx.output).body.split(/\s+/).filter((w) => /[\p{L}\p{N}]/u.test(w)).length;
+        return one(within(n, check.min, check.max), `${n} words (needs ${bounds(check.min, check.max)})`);
+      }
+      // engine:check.words:end
+      // engine:check.frontmatter:begin
+      case "frontmatter": {
+        const v = splitFrontmatter(ctx.output).fields[check.field];
+        if (typeof v !== "string" || !v.trim()) return one(false, `frontmatter field "${check.field}" is missing`);
+        return one(within(v.length, check.minChars, check.maxChars), `"${check.field}" is ${v.length} characters (needs ${bounds(check.minChars, check.maxChars)})`);
+      }
+      // engine:check.frontmatter:end
+      // engine:check.command:begin
+      case "command": {
+        const proc = Bun.spawn(["sh", "-c", check.run], {
+          cwd: ctx.workspace, stdin: "ignore", stdout: "pipe", stderr: "pipe",
+          env: { ...process.env, OUTPUT: ctx.outputPath, WORKSPACE: ctx.workspace, TASK_DIR: ctx.taskDir },
+        });
+        let timedOut = false;
+        const timer = setTimeout(() => { timedOut = true; proc.kill(); }, (check.timeoutSec ?? 60) * 1000);
+        const [out, stderr, code] = await Promise.all([new Response(proc.stdout).text(), new Response(proc.stderr).text(), proc.exited]);
+        clearTimeout(timer);
+        if (timedOut) return err(`command timed out after ${check.timeoutSec ?? 60}s`);
+        const said = (out.trim() || stderr.trim()).split("\n").slice(-1)[0]?.slice(0, 300);
+        return one(code === 0, said || `exit ${code}`);
+      }
+      // engine:check.command:end
+    }
+  } catch (e) {
+    return err(e instanceof Error ? e.message : String(e));
+  }
+}

package/src/core/commands.ts

@@ -0,0 +1,43 @@
+// The commands of `static-fire` and the flags each one takes: the one list that the CLI (src/cli.ts) parses with,
+// rejects unknown flags by and prints as its usage. The guide for agents (src/guide.md) is checked against it
+// (tests/guide.test.ts), so the guide cannot name a command or a flag that the CLI does not have.
+
+/** Every flag, as `parseArgs` takes it. A default here is only used when the command takes the flag. */
+export const OPTIONS = {
+  suites: { type: "string" }, tasks: { type: "string" }, grading: { type: "string" }, since: { type: "string" }, until: { type: "string" },
+  configs: { type: "string" }, trials: { type: "string", default: "3" },
+  judge: { type: "string" }, votes: { type: "string" }, "only-invalid": { type: "boolean" },
+  concurrency: { type: "string", default: "4" }, port: { type: "string", default: "4477" }, dev: { type: "boolean" }, json: { type: "boolean" },
+  from: { type: "string" }, batch: { type: "string" }, plan: { type: "boolean" },
+} as const;
+
+export type Flag = keyof typeof OPTIONS;
+
+/** Each command: the flags its code reads, and its arguments for the usage text. */
+export const COMMANDS: Record<string, { flags: readonly Flag[]; args?: string }> = {
+  init: { flags: ["json"], args: "[<folder>]" },
+  guide: { flags: [] },
+  home: { flags: [], args: "[<folder>]" },
+  serve: { flags: ["port", "dev"] },
+  configs: { flags: [] },
+  judges: { flags: [] },
+  suites: { flags: [] },
+  tasks: { flags: [] },
+  new: { flags: [], args: "task <id>" },
+  run: { flags: ["suites", "tasks", "configs", "trials", "judge", "votes", "concurrency", "plan", "json"] },
+  regrade: { flags: ["tasks", "configs", "batch", "judge", "votes", "only-invalid", "concurrency", "plan", "json"] },
+  rerun: { flags: ["concurrency"], args: "<batch>" },
+  list: { flags: [] },
+  show: { flags: ["suites", "tasks", "judge", "grading", "since", "until", "json"] },
+  export: { flags: [], args: "[path]" },
+  migrate: { flags: ["from"] },
+};
+
+/** The flags given on the command line that `command` does not take, as written (`--json`). */
+export function unknownFlags(command: string, given: string[]): string[] {
+  const takes = new Set<string>(COMMANDS[command]?.flags ?? []);
+  return [...new Set(given)].filter((f) => !takes.has(f)).map((f) => `--${f}`);
+}
+
+/** The usage line the CLI prints for no command or an unknown one. */
+export const USAGE = `usage: static-fire <${Object.keys(COMMANDS).join("|")}>  (static-fire --version for the version)`;

package/src/core/compare.ts

@@ -0,0 +1,206 @@
+// Compare gradings: two grading setups of one task, over the trials that have a valid grading under both.
+// GET /api/compare and the dashboard's comparison card read it through compareGradings().
+//
+// Each side is the view (buildView) with that grading setup picked, so the scope, config facets, date range and
+// the human rule are the view's own. Only the trials valid on both sides are kept, and each side is summarised
+// over exactly those trials by the view's summary, so score.ts computes every number (DEC-2). Gradings are never
+// averaged together: each side reads only its own grading hash.
+import type { GradingSetupRecord, Store, TrialRecord } from "./db";
+import { gradingDiff, versionNames } from "./grading-versions";
+import type { GradingDiff } from "./grading-versions";
+import type { GradingContent } from "./identity";
+import type { Harness, TaskSnapshot } from "./types";
+import { buildView, projectCatalog, summarizeTrials } from "./view";
+import type { Catalog, View, ViewFilters, ViewTrial } from "./view";
+
+/** One of the two grading versions compared. */
+export interface CompareSide {
+  /** Full grading hash. */
+  hash: string;
+  /** Its name, from what changed since the version before it for the same task and judge ("Original grading" for the first). */
+  name: string;
+  /** What changed since that version, in short words; empty for the original grading. */
+  nameParts: string[];
+  /** The task hash it grades. */
+  taskHash: string;
+  /** The task file's `version` stored with that task hash (a label only), or null when the task has none. */
+  label: number | null;
+  judgeLabel: string;
+  /** When the setup was first recorded (ISO time). */
+  firstSeen: string;
+  /** True when it is today's grading setup of the task for the judge in view. */
+  current: boolean;
+  /** The task's criteria as this setup graded with them. */
+  def: TaskSnapshot;
+}
+
+/** One config's task score under each setup, over the compared trials. */
+export interface CompareConfig {
+  configKey: string;
+  label: string;
+  harness: Harness;
+  family: string;
+  profile?: string;
+  /** Compared trials of this config. */
+  n: number;
+  /** Task score (0–100) under setup a and setup b, and b − a. */
+  a: number | null;
+  b: number | null;
+  change: number | null;
+}
+
+/** One criterion's pass rate (0–1) for one config under each setup. Null when that setup's task has no such criterion. */
+export interface CompareCriterion {
+  criterionId: string;
+  configKey: string;
+  a: number | null;
+  b: number | null;
+  /**
+   * True when the criterion itself differs between the two versions: it was added, removed or changed (`diff.criteria`),
+   * or it is a command check whose `run` names a grader file that differs (`diff.files.grader`). A script the named
+   * file loads in its turn is not traced.
+   */
+  changed: boolean;
+  /** True when its pass rate differs between the versions (or it has one under one version only). */
+  moved: boolean;
+}
+
+export interface GradingComparison {
+  taskId: string;
+  a: CompareSide;
+  b: CompareSide;
+  /** Everything that differs from version a to version b, from their stored snapshots. */
+  diff: GradingDiff;
+  /** True when both setups grade the same task hash. Setups of two task versions share no trials. */
+  sameTask: boolean;
+  /** Trials with a valid grading under both setups: the only ones compared. */
+  trials: number;
+  /** Trials with a valid grading under only setup a, or only setup b. They are not counted. */
+  onlyA: number;
+  onlyB: number;
+  /** Configs with a compared trial, best under setup b first. */
+  configs: CompareConfig[];
+  /** Every criterion of either setup's task, for each config in `configs`. */
+  criteria: CompareCriterion[];
+  /** What the page's Reading line states (US-007, DEC-005). Null when no trial is compared, as with two task versions. */
+  reading: CompareReading | null;
+}
+
+/**
+ * The counts behind the Reading line. They count criteria and gates, not criterion-and-config rows: a criterion
+ * moved when its pass rate moved for at least one config.
+ */
+export interface CompareReading {
+  /** Changed criteria that moved. */
+  changedMoved: number;
+  /** Changed criteria that did not move. */
+  changedSteady: number;
+  /** Unchanged criteria that moved. */
+  unchangedMoved: number;
+  /**
+   * True when the difference is the judge varying between runs, not an effect of the change: no changed criterion
+   * moved, at least one unchanged criterion moved, and nothing outside criteria and gates differs (`diff.criteriaOnly`).
+   */
+  judgeVariation: boolean;
+}
+
+const valid = (t: ViewTrial) => t.status === "done" && !!t.grades;
+
+/**
+ * Compares grading setups `a` and `b` (full hashes or unique prefixes) of one task in the scope of `filters`.
+ * The view's own `grading` filter is replaced by each setup in turn. Throws on an unknown or ambiguous hash, the
+ * same setup twice, setups of two tasks, or a task outside the scope.
+ */
+export function compareGradings(store: Store, filters: ViewFilters, a: string, b: string, catalog: Catalog = projectCatalog()): GradingComparison {
+  const sa = resolve(store, a, "a"), sb = resolve(store, b, "b");
+  if (sa.hash === sb.hash) throw new Error(`a and b are the same grading version (${sa.hash}); pick two different ones`);
+  if (sa.taskId !== sb.taskId) throw new Error(`grading versions ${sa.hash.slice(0, 8)} and ${sb.hash.slice(0, 8)} are of two tasks (${sa.taskId}, ${sb.taskId}); compare two grading versions of one task`);
+  const taskId = sa.taskId;
+  // Each setup brings its own judge (the view refuses another judge with a picked setup).
+  // With no scope given, the scope is the task: only its trials are compared, and the default set of suites
+  // (DEC-019) may leave its suite out.
+  const { judge: _judge, ...given } = filters;
+  const rest = given.suites?.length || given.tasks?.length ? given : { ...given, tasks: [taskId] };
+  const va = buildView(store, { ...rest, grading: sa.hash }, catalog);   // throws when the task is not in scope
+  const vb = buildView(store, { ...rest, grading: sb.hash }, catalog);
+  const ofTask = (v: View) => new Map(v.trials.filter((t) => t.taskId === taskId && valid(t)).map((t) => [t.id, t]));
+  const ta = ofTask(va), tb = ofTask(vb);
+  const both = [...ta.keys()].filter((id) => tb.has(id));
+
+  const records = new Map<string, TrialRecord>(both.map((id) => [id, store.getTrial(id)!]));
+  const side = (v: View, trials: Map<string, ViewTrial>) =>
+    summarizeTrials(both.map((id) => trials.get(id)!), records, { [taskId]: v.tasks[taskId]! }, []).configs;
+  const ca = side(va, ta), cb = side(vb, tb);
+
+  const scoreOf = (list: typeof ca, key: string) => list.find((c) => c.configKey === key)?.tasks[taskId]?.score.mean ?? null;
+  const configs: CompareConfig[] = cb.map((c) => {
+    const x = scoreOf(ca, c.configKey), y = c.tasks[taskId]?.score.mean ?? null;
+    return {
+      configKey: c.configKey, label: c.label, harness: c.harness, family: c.family, ...(c.profile ? { profile: c.profile } : {}),
+      n: c.pooled[taskId]?.n ?? 0, a: x, b: y, change: x == null || y == null ? null : y - x,
+    };
+  }).sort((p, q) => (q.b ?? -1) - (p.b ?? -1) || (p.configKey < q.configKey ? -1 : 1));
+
+  const ids = (d: TaskSnapshot) => [...d.gates, ...d.measures.flatMap((m) => m.criteria)].map((c) => c.id);
+  const da = va.tasks[taskId]!.def, db = vb.tasks[taskId]!.def;
+  const inA = new Set(ids(da)), inB = new Set(ids(db));
+  const rate = (list: typeof ca, key: string, id: string, has: Set<string>) =>
+    has.has(id) ? list.find((c) => c.configKey === key)?.tasks[taskId]?.criteria[id] ?? null : null;
+  const diff = gradingDiff({ setup: sa, task: store.getTaskVersion(sa.taskHash) }, { setup: sb, task: store.getTaskVersion(sb.taskHash) });
+  const changed = changedCriteria(sa, sb, diff);
+  const criteria = [...new Set([...ids(da), ...ids(db)])].flatMap((criterionId) => configs.map((c): CompareCriterion => {
+    const x = rate(ca, c.configKey, criterionId, inA), y = rate(cb, c.configKey, criterionId, inB);
+    return { criterionId, configKey: c.configKey, a: x, b: y, changed: changed.has(criterionId), moved: x == null || y == null ? x !== y : Math.abs(y - x) >= 1e-9 };
+  }));
+
+  const names = versionNames(store, taskId);
+  const sideOf = (s: GradingSetupRecord, v: View): CompareSide => ({
+    hash: s.hash, name: names.get(s.hash)?.name ?? s.hash.slice(0, 8), nameParts: names.get(s.hash)?.nameParts ?? [], taskHash: s.taskHash, label: v.tasks[taskId]!.label, judgeLabel: s.judgeLabel, firstSeen: s.firstSeen,
+    current: v.tasks[taskId]!.current, def: v.tasks[taskId]!.def,
+  });
+  return {
+    taskId, a: sideOf(sa, va), b: sideOf(sb, vb),
+    diff,
+    sameTask: sa.taskHash === sb.taskHash,
+    trials: both.length, onlyA: ta.size - both.length, onlyB: tb.size - both.length, configs, criteria,
+    reading: both.length ? reading(criteria, diff) : null,
+  };
+}
+
+function reading(rows: CompareCriterion[], diff: GradingDiff): CompareReading {
+  const moved = new Map<string, { changed: boolean; moved: boolean }>();
+  for (const r of rows) {
+    const m = moved.get(r.criterionId);
+    moved.set(r.criterionId, { changed: r.changed, moved: r.moved || !!m?.moved });
+  }
+  const list = [...moved.values()];
+  const changedMoved = list.filter((c) => c.changed && c.moved).length;
+  const changedSteady = list.filter((c) => c.changed && !c.moved).length;
+  const unchangedMoved = list.filter((c) => !c.changed && c.moved).length;
+  return { changedMoved, changedSteady, unchangedMoved, judgeVariation: changedMoved === 0 && unchangedMoved > 0 && diff.criteriaOnly };
+}
+
+/**
+ * The ids of the gates and criteria that differ between the two versions: each one in the difference, and each
+ * command check whose `run` (under either version) names a grader file, by its path in the task folder, that was
+ * added, removed or changed.
+ */
+function changedCriteria(sa: GradingSetupRecord, sb: GradingSetupRecord, diff: GradingDiff): Set<string> {
+  const out = new Set(diff.criteria.map((c) => c.id));
+  const g = diff.files.grader, files = [...g.added, ...g.removed, ...g.changed];
+  if (!files.length) return out;
+  for (const snap of [sa.snapshot, sb.snapshot] as unknown as Partial<GradingContent>[]) {
+    for (const c of [...(snap.gates ?? []), ...(snap.criteria ?? [])]) {
+      if (c.check?.type === "command" && files.some((f) => (c.check as { run: string }).run.includes(f))) out.add(c.id);
+    }
+  }
+  return out;
+}
+
+function resolve(store: Store, prefix: string, which: string): GradingSetupRecord {
+  if (!prefix) throw new Error(`${which}: pick a grading version`);
+  const p = prefix.toLowerCase();
+  const found = store.listGradingSetups().filter((g) => g.hash.startsWith(p));
+  if (found.length === 1) return found[0]!;
+  throw new Error(found.length ? `${which}: grading ${prefix} matches ${found.length} grading versions; give more characters` : `${which}: no grading version ${prefix}`);
+}

package/src/core/config.ts

@@ -0,0 +1,25 @@
+// Agent and human configs: configs/<id>.yaml → AgentConfig, validated and with paths resolved.
+import { homedir } from "node:os";
+import { isAbsolute, join } from "node:path";
+import { HARNESSES, type AgentConfig, type Harness, type HumanOutput } from "./types";
+
+const expand = (p: string, root: string) => (p.startsWith("~/") ? join(homedir(), p.slice(2)) : isAbsolute(p) ? p : join(root, p));
+
+export function normalizeConfig(id: string, raw: any, root: string): AgentConfig {
+  const harness = raw?.harness as Harness;
+  if (!HARNESSES.includes(harness)) throw new Error(`configs/${id}.yaml: harness must be one of ${HARNESSES.join(", ")}`);
+  if ("label" in (raw ?? {})) throw new Error(`configs/${id}.yaml: remove label - it is built from harness, model, effort and profile`);
+  if (harness !== "human") {
+    if (!raw.model) throw new Error(`configs/${id}.yaml: an agent config needs a model`);
+    return { id, ...raw };
+  }
+  const entries = Object.entries(raw.outputs ?? {});
+  if (!entries.length) throw new Error(`configs/${id}.yaml: a human config needs outputs (task id → file)`);
+  const outputs: Record<string, HumanOutput> = {};
+  for (const [task, v] of entries) {
+    const o = typeof v === "string" ? { path: v } : (v as HumanOutput);
+    if (!o?.path) throw new Error(`configs/${id}.yaml: outputs.${task} needs a path`);
+    outputs[task] = { path: expand(o.path, root), ...(o.note ? { note: o.note } : {}) };
+  }
+  return { id, harness, model: "by hand", family: "human", outputs };
+}

package/src/core/copying.ts

@@ -0,0 +1,18 @@
+// Did an agent copy the held-out original (for example by finding the published version on the web)?
+// Share of the output's 8-word runs that also appear in the original. Independent writing on the same
+// topic shares almost none; a copy, even lightly edited, shares most.
+const N = 8;
+const words = (s: string) => s.toLowerCase().replace(/[^a-z0-9'\s]/g, " ").split(/\s+/).filter(Boolean);
+const runs = (ws: string[]) => { const out = new Set<string>(); for (let i = 0; i + N <= ws.length; i++) out.add(ws.slice(i, i + N).join(" ")); return out; };
+
+export function copiedShare(output: string, original: string): number {
+  const mine = runs(words(output));
+  if (!mine.size) return 0;
+  const theirs = runs(words(original));
+  let shared = 0;
+  for (const r of mine) if (theirs.has(r)) shared++;
+  return shared / mine.size;
+}
+
+/** Above this share, the trial is invalid: the output is a copy, not the agent's own work. */
+export const COPY_LIMIT = 0.3;