static-fire 0.0.0-stage → 0.44.0

package/src/core/regrade.ts

@@ -0,0 +1,233 @@
+// Regrade: grade saved outputs again, with judge calls only. No agent runs, and no trial or artifact is copied.
+//
+// It covers every matching trial that has the current task hash and no valid grading under the current grading
+// hash for the chosen judge. A different judge is just another grading hash on the same trials. A regrade adds a
+// grading row and never changes an earlier one; the latest finished one counts (db.ts currentGrading).
+//
+// A grading left in status `grading` (for example by a crashed process) is not finished, so it is not a valid
+// grading and the trial is graded again. The stale row is left as it is: the store cannot tell it from a grading
+// another process is doing right now.
+import { DEFAULT_VOTES, gradeTrial } from "./batch";
+import type { Store, TrialRecord } from "./db";
+import { gradingIdentity, taskIdentity, configKey } from "./identity";
+import { loadConfigs } from "./authored";
+import type { AgentConfig, JudgeConfig, Task } from "./types";
+import { projectCatalog, type Catalog } from "./view";
+
+/** Which trials to look at. Every field is optional; an empty object is every trial in the store. */
+export interface RegradeFilters {
+  /** Task ids. */
+  tasks?: string[];
+  /** Config ids (files in configs/, resolved to their config key) or config keys. Trials match by config key. "all": every config. */
+  configs?: string[];
+  /** One batch id. */
+  batch?: string;
+  /** Trial ids. A view's regrade passes the ids it counts as gradable (view.ts LeftOut.gradable). */
+  trials?: string[];
+}
+
+export interface RegradeOptions {
+  /** The store's database path: trial folders are beside it. */
+  dbPath: string;
+  /** Today's task and judge definitions. Default: the YAML files in the repo. */
+  catalog?: Catalog;
+  /** Today's config files, to resolve config ids in `filters.configs`. Default: configs/ in the repo. */
+  configs?: AgentConfig[];
+  /** Judge id. Default: the first in judges.yaml. */
+  judge?: string;
+  /** Votes per judge criterion. Default DEFAULT_VOTES. */
+  votes?: number;
+  /** Only trials whose latest finished grading under the current grading hash is invalid. */
+  onlyInvalid?: boolean;
+  /** Gradings at once. Default 4. */
+  concurrency?: number;
+  onEvent?: (msg: string) => void;
+  /** Called once per grading added, with how it ended. The server counts a running regrade's progress with it. */
+  onGraded?: (trialId: string, status: "done" | "invalid") => void;
+}
+
+/** Matching trials that are not graded, by reason. */
+export interface RegradeSkipped {
+  /** Ran on another task hash: a different job. Run the task again instead. */
+  otherTask: number;
+  /** Did not end done (failed, copied the held-out original, or not ended): there is no output to grade. */
+  notDone: number;
+  /** Already has a valid (done) grading under the current grading hash for the judge. */
+  alreadyGraded: number;
+  /** With onlyInvalid: its latest finished grading under the current grading hash is not invalid (or there is none). */
+  notInvalid: number;
+  /** Its task no longer loads from today's files. */
+  unknownTask: number;
+}
+
+/** What a regrade would do. Plain data: no write happens to make it. */
+export interface RegradePlan {
+  judge: JudgeConfig;
+  votes: number;
+  /** Trials that passed the filters. */
+  matched: number;
+  /** The trials to grade, in id order, with the grading hash each gets. */
+  todo: { trialId: string; taskId: string; gradingHash: string }[];
+  skipped: RegradeSkipped;
+}
+
+export interface RegradeCounts {
+  matched: number;
+  /** Gradings added, and how they ended. */
+  graded: number;
+  done: number;
+  invalid: number;
+  /** Trials that could not be graded (a store error); the other trials were graded. */
+  failed: number;
+  failures: { trialId: string; error: string }[];
+  skipped: RegradeSkipped;
+}
+
+/** The failures of a regrade as one message, or null when every trial was graded. */
+export function failureMessage(c: Pick<RegradeCounts, "failed" | "failures">): string | null {
+  if (!c.failed) return null;
+  return `${c.failed} trial${c.failed === 1 ? "" : "s"} could not be graded: ${c.failures.map((f) => `${f.trialId}: ${f.error}`).join("; ")}`;
+}
+
+const positiveInt = (n: number, what: string) => {
+  if (!Number.isInteger(n) || n < 1) throw new Error(`${what} must be a whole number of 1 or more (got ${n})`);
+  return n;
+};
+
+interface Resolved { plan: RegradePlan; trials: Map<string, TrialRecord>; tasks: Map<string, Task> }
+
+function resolve(store: Store, f: RegradeFilters, o: RegradeOptions): Resolved {
+  const catalog = o.catalog ?? projectCatalog();
+  const judge = o.judge ? catalog.judges.find((j) => j.id === o.judge) : catalog.judges[0];
+  if (!judge) throw new Error(`unknown judge ${o.judge ?? "(judges.yaml lists none)"}`);
+  const votes = positiveInt(o.votes ?? DEFAULT_VOTES, "votes");
+  if (f.batch !== undefined && !store.getBatch(f.batch)) throw new Error(`unknown batch ${f.batch}`);
+
+  let trials = store.listTrials(f.batch === undefined ? {} : { batchId: f.batch });
+  if (f.tasks) {
+    const want = new Set(f.tasks);
+    const known = new Set(store.listTrials().map((t) => t.taskId));
+    const unknown = [...want].filter((id) => !known.has(id) && !canLoad(catalog, id));
+    if (!want.size || unknown.length) throw new Error(`unknown or no task(s): ${unknown.join(", ") || "none given"}`);
+    trials = trials.filter((t) => want.has(t.taskId));
+  }
+  if (f.trials) {
+    const want = new Set(f.trials);
+    const known = new Set(store.listTrials().map((t) => t.id));
+    const unknown = [...want].filter((id) => !known.has(id));
+    if (!want.size || unknown.length) throw new Error(`unknown or no trial(s): ${unknown.join(", ") || "none given"}`);
+    trials = trials.filter((t) => want.has(t.id));
+  }
+  if (f.configs && !f.configs.includes("all")) {
+    const byId = new Map((o.configs ?? loadConfigs()).map((c) => [c.id, configKey(c)]));
+    const seenKeys = new Set(store.listTrials().map((t) => t.configKey));
+    const keys = new Set<string>(), unknown: string[] = [];
+    for (const v of f.configs) {
+      const key = byId.get(v) ?? (seenKeys.has(v) ? v : undefined);
+      if (key) keys.add(key); else unknown.push(v);
+    }
+    if (!f.configs.length || unknown.length) throw new Error(`unknown or no config(s): ${unknown.join(", ") || "none given"}`);
+    trials = trials.filter((t) => keys.has(t.configKey));
+  }
+
+  const skipped: RegradeSkipped = { otherTask: 0, notDone: 0, alreadyGraded: 0, notInvalid: 0, unknownTask: 0 };
+  const current = new Map<string, { task: Task; taskHash: string; gradingHash: string } | null>();
+  const hashesOf = (taskId: string) => {
+    if (!current.has(taskId)) {
+      try {
+        const task = catalog.loadTask(taskId);
+        current.set(taskId, { task, taskHash: taskIdentity(task).hash, gradingHash: gradingIdentity(task, judge).hash });
+      } catch {
+        current.set(taskId, null);
+      }
+    }
+    return current.get(taskId)!;
+  };
+
+  const todo: RegradePlan["todo"] = [];
+  for (const t of trials) {
+    const h = hashesOf(t.taskId);
+    if (!h) { skipped.unknownTask++; continue; }
+    if (t.taskHash !== h.taskHash) { skipped.otherTask++; continue; }
+    if (t.status !== "done") { skipped.notDone++; continue; }
+    const latest = store.currentGrading(t.id, h.gradingHash);
+    if (o.onlyInvalid) {
+      if (latest?.status !== "invalid") { skipped.notInvalid++; continue; }
+    } else if (latest?.status === "done") { skipped.alreadyGraded++; continue; }
+    todo.push({ trialId: t.id, taskId: t.taskId, gradingHash: h.gradingHash });
+  }
+  const tasks = new Map([...current].flatMap(([id, h]) => (h ? [[id, h.task] as const] : [])));
+  return { plan: { judge, votes, matched: trials.length, todo, skipped }, trials: new Map(trials.map((t) => [t.id, t])), tasks };
+}
+
+function canLoad(catalog: Catalog, id: string) {
+  try { catalog.loadTask(id); return true; } catch { return false; }
+}
+
+/**
+ * Which trials a regrade would grade, and why the others are skipped. Writes nothing. Throws on an unknown
+ * task, config, batch or judge, or votes below 1.
+ */
+export function planRegrade(store: Store, filters: RegradeFilters, opts: RegradeOptions): RegradePlan {
+  return resolve(store, filters, opts).plan;
+}
+
+/**
+ * Grades every trial planRegrade selects with gradeTrial (no agent run), and returns the counts. Records the
+ * task version and grading setup first when they are new (an edited criterion, or another judge). A trial that
+ * cannot be graded (a store error) does not stop the others: it is counted in `failed` with its error.
+ */
+export async function regrade(store: Store, filters: RegradeFilters, opts: RegradeOptions): Promise<RegradeCounts> {
+  const log = opts.onEvent ?? (() => {});
+  const { plan, trials, tasks } = resolve(store, filters, opts);
+  const counts: RegradeCounts = { matched: plan.matched, graded: 0, done: 0, invalid: 0, failed: 0, failures: [], skipped: plan.skipped };
+  const at = new Date().toISOString();
+  store.transaction(() => {
+    for (const taskId of new Set(plan.todo.map((x) => x.taskId))) {
+      const task = tasks.get(taskId)!;
+      const t = taskIdentity(task), g = gradingIdentity(task, plan.judge);
+      store.putTaskVersion({ hash: t.hash, taskId, label: task.version ?? null, snapshot: t.snapshot as unknown as Record<string, unknown>, firstSeen: at });
+      store.putGradingSetup({
+        hash: g.hash, taskId, taskHash: t.hash,
+        judgeHarness: plan.judge.harness, judgeModel: plan.judge.model, judgeEffort: plan.judge.effort ?? null, judgeLabel: plan.judge.label,
+        snapshot: g.snapshot as unknown as Record<string, unknown>, firstSeen: at,
+      });
+    }
+  });
+
+  const queue = [...plan.todo];
+  const worker = async () => {
+    for (let x = queue.shift(); x; x = queue.shift()) {
+      try {
+        const g = await gradeTrial(store, opts.dbPath, trials.get(x.trialId)!, { task: tasks.get(x.taskId)!, judge: plan.judge, votes: plan.votes, gradingHash: x.gradingHash });
+        counts.graded++;
+        if (g.status === "done") counts.done++; else counts.invalid++;
+        opts.onGraded?.(x.trialId, g.status === "done" ? "done" : "invalid");
+        log(`${g.status === "done" ? "✓" : "!"} ${x.trialId} score ${g.score?.toFixed(1) ?? "—"}${g.error ? ` (${g.error})` : ""}`);
+      } catch (e) {
+        // A store error: the other trials go on, and the failure is returned with the counts.
+        const error = e instanceof Error ? e.message : String(e);
+        counts.failed++;
+        counts.failures.push({ trialId: x.trialId, error });
+        log(`✗ ${x.trialId}: ${error}`);
+      }
+    }
+  };
+  await Promise.all(Array.from({ length: Math.max(1, opts.concurrency ?? 4) }, worker));
+  return counts;
+}
+
+/** What the CLI prints: one line of what was graded and skipped, by reason, then one line per trial that could not be graded. */
+export function formatCounts(c: RegradeCounts): string {
+  const s = c.skipped;
+  const parts = [
+    s.otherTask && `${s.otherTask} on another task version`,
+    s.alreadyGraded && `${s.alreadyGraded} already graded`,
+    s.notInvalid && `${s.notInvalid} not invalid`,
+    s.notDone && `${s.notDone} not done`,
+    s.unknownTask && `${s.unknownTask} with an unknown task`,
+  ].filter(Boolean);
+  const failed = c.failed ? ` · ${c.failed} could not be graded` : "";
+  return `regraded ${c.graded} trial(s) (${c.done} done, ${c.invalid} invalid) of ${c.matched} matching${failed} · skipped: ${parts.join(", ") || "none"}`
+    + c.failures.map((f) => `\n  ${f.trialId}: ${f.error}`).join("");
+}

package/src/core/score.ts

@@ -0,0 +1,148 @@
+// Pure scoring (ADR-0001). Verdicts → measure % → task score → Suite Index → Workflow Index.
+// No I/O here, so every rule is covered by tests/score.test.ts.
+import { configLabel, EFFORTS } from "./label";
+import type { ConfigSummary, Experiment, ExperimentSummary, Grades, Stat, Task, TaskSummary, Trial, Verdict, Vote } from "./types";
+
+// engine:judging:begin
+export function verdictFromVotes(votes: Vote[], error?: string): Verdict {
+  if (!votes.length) return { score: null, votes, error: error ?? "no votes" };
+  return { score: votes.filter((x) => x.pass).length / votes.length, votes };
+}
+
+/** Adaptive votes: run the smallest majority first; run the rest only if it did not settle every criterion. */
+export const firstRoundVotes = (planned: number) => Math.floor(planned / 2) + 1;
+
+export function needsMoreVotes(votes: Record<string, Vote[]>, planned: number): boolean {
+  const need = firstRoundVotes(planned);
+  if (need >= planned) return false;
+  return Object.values(votes).some((vs) => vs.length < need || vs.some((v) => v.pass !== vs[0].pass));
+}
+
+// engine:judging:end
+
+const passed = (v: Verdict | undefined) => v?.score != null && v.score > 0.5;
+
+export function scoreTrial(task: Task, verdicts: Record<string, Verdict>, judgeCostUsd = 0): Grades {
+  const errors: string[] = [];
+  const val = (id: string) => {
+    const v = verdicts[id];
+    if (!v || v.score == null) { errors.push(id); return null; }
+    return v.score;
+  };
+  const gatesPassed = task.gates.every((g) => { val(g.id); return passed(verdicts[g.id]); });
+
+  const measures: Record<string, number> = {};
+  let sum = 0, wsum = 0;
+  for (const m of task.measures) {
+    const scores = m.criteria.map((c) => val(c.id)).filter((s): s is number => s != null);
+    if (!scores.length) continue;
+    measures[m.id] = (scores.reduce((a, b) => a + b, 0) / scores.length) * 100;
+    const w = m.weight ?? 1;
+    sum += measures[m.id] * w;
+    wsum += w;
+  }
+  const score = gatesPassed && wsum ? sum / wsum : 0;
+  return { verdicts, gatesPassed, measures, score, errors: [...new Set(errors)], judgeCostUsd };
+}
+
+// ---- statistics ----
+
+const T95: Record<number, number> = { 1: 12.706, 2: 4.303, 3: 3.182, 4: 2.776, 5: 2.571, 6: 2.447, 7: 2.365, 8: 2.306, 9: 2.262, 10: 2.228, 15: 2.131, 20: 2.086, 30: 2.042 };
+const tCrit = (df: number) => {
+  if (T95[df]) return T95[df];
+  const keys = Object.keys(T95).map(Number).filter((k) => k <= df);
+  return df > 30 ? 1.96 : T95[Math.max(...keys)];
+};
+const mean = (xs: number[]) => (xs.length ? xs.reduce((a, b) => a + b, 0) / xs.length : null);
+
+/** Half-width of the 95% confidence interval of the mean. */
+export function ci95(xs: number[]): number | null {
+  if (xs.length < 2) return null;
+  const m = mean(xs)!;
+  const sd = Math.sqrt(xs.reduce((a, x) => a + (x - m) ** 2, 0) / (xs.length - 1));
+  return (tCrit(xs.length - 1) * sd) / Math.sqrt(xs.length);
+}
+
+/** Mean of independent means; the half-width combines in quadrature. */
+function combine(stats: Stat[], weights?: number[]): Stat {
+  if (!stats.length || stats.some((s) => s.mean == null)) return { mean: null, ci: null, n: 0 };
+  const w = weights ?? stats.map(() => 1);
+  const W = w.reduce((a, b) => a + b, 0);
+  const m = stats.reduce((a, s, i) => a + s.mean! * w[i], 0) / W;
+  const ci = stats.every((s) => s.ci != null) ? Math.sqrt(stats.reduce((a, s, i) => a + (s.ci! * w[i]) ** 2, 0)) / W : null;
+  return { mean: m, ci, n: Math.min(...stats.map((s) => s.n)) };
+}
+
+// ---- summaries ----
+
+export function familyOf(model: string) {
+  const parts = model.toLowerCase().split(/[-_/:]/).filter((p) => p && !/\d/.test(p) && !EFFORTS.has(p));
+  return parts.slice(0, 2).join("-") || model;
+}
+
+function taskSummary(trials: Trial[]): TaskSummary {
+  const valid = trials.filter((t) => t.status === "done" && t.grades);
+  const scores = valid.map((t) => t.grades!.score);
+  const measures: Record<string, number> = {};
+  const criteria: Record<string, number> = {};
+  for (const t of valid) {
+    for (const [k, s] of Object.entries(t.grades!.measures)) measures[k] = (measures[k] ?? 0) + s / valid.length;
+    for (const [k, v] of Object.entries(t.grades!.verdicts)) if (v.score != null) criteria[k] = (criteria[k] ?? 0) + v.score / valid.length;
+  }
+  const costs = valid.map((t) => t.result?.costUsd).filter((x): x is number => x != null);
+  return {
+    score: { mean: mean(scores), ci: ci95(scores), n: scores.length },
+    measures, criteria,
+    costUsd: mean(costs),
+    costSource: valid.find((t) => t.result)?.result?.costSource ?? "unknown",
+    durationMs: mean(valid.map((t) => t.durationMs).filter((x): x is number => x != null)),
+    valid: valid.length,
+    invalid: trials.filter((t) => t.status === "invalid").length,
+    failed: trials.filter((t) => t.status === "failed").length,
+    total: trials.length,
+  };
+}
+
+export function summarize(exp: Experiment, trials: Trial[], tasks: Task[] = []): ExperimentSummary {
+  const taskIds = Object.keys(exp.tasks);
+  const configs: ConfigSummary[] = exp.configIds.map((configId) => {
+    const c = exp.configs[configId];
+    const per: Record<string, TaskSummary> = {};
+    for (const tid of taskIds) per[tid] = taskSummary(trials.filter((t) => t.configId === configId && t.taskId === tid));
+    const suites: Record<string, Stat> = {};
+    for (const sid of exp.suiteIds) {
+      const s = exp.suites[sid];
+      suites[sid] = combine(s.tasks.map((tid) => per[tid]?.score ?? { mean: null, ci: null, n: 0 }), s.tasks.map((tid) => s.weights?.[tid] ?? 1));
+    }
+    const costs = taskIds.map((t) => per[t].costUsd).filter((x): x is number => x != null);
+    const times = taskIds.filter((t) => per[t].valid > 0).map((t) => per[t].durationMs).filter((x): x is number => x != null);
+    return {
+      configId, label: configLabel(c), harness: c.harness, model: c.model, effort: c.effort, profile: c.profile,
+      family: c.family ?? familyOf(c.model),
+      tasks: per, suites,
+      index: combine(exp.suiteIds.map((s) => suites[s])),
+      costUsd: mean(costs),
+      durationMs: mean(times),
+    };
+  });
+  return { configs, deadCriteria: deadCriteria(tasks, exp.configIds, trials) };
+}
+
+/**
+ * Criteria that carry no signal between these configs: a strong or ceiling criterion every config
+ * passed (pass rate ≥ 0.95), or any criterion every config failed (≤ 0.05). A floor criterion that
+ * every config passes is a guard working as designed, so it is not flagged.
+ */
+export function deadCriteria(tasks: Task[], configIds: string[], trials: Trial[]): Record<string, string[]> {
+  const out: Record<string, string[]> = {};
+  for (const task of tasks) {
+    const crits = task.measures.flatMap((m) => m.criteria);
+    const rates = configIds.map((cid) => taskSummary(trials.filter((t) => t.taskId === task.id && t.configId === cid)))
+      .filter((s) => s.valid > 0).map((s) => s.criteria);
+    if (rates.length < 2) continue;
+    const all = (id: string, ok: (r: number) => boolean) => rates.every((r) => r[id] != null && ok(r[id]));
+    const dead = crits.filter((c) => (c.level !== "floor" && all(c.id, (r) => r >= 0.95)) || all(c.id, (r) => r <= 0.05)).map((c) => c.id);
+    if (dead.length) out[task.id] = dead;
+  }
+  return out;
+}

package/src/core/settings.ts

@@ -0,0 +1,44 @@
+// static-fire.yaml: the user's own settings for trials, and a record of the user-level agent setup.
+import { createHash } from "node:crypto";
+import { existsSync, readdirSync, readFileSync, statSync } from "node:fs";
+import { homedir } from "node:os";
+import { join } from "node:path";
+import { projectRoot } from "./project";
+import type { AgentHarness } from "./types";
+
+export interface Settings {
+  blockedTools: Partial<Record<AgentHarness, string[]>>;      // claude: tool rules; codex: MCP server names
+}
+
+export function loadSettings(root = projectRoot()): Settings {
+  const p = join(root, "static-fire.yaml");
+  const raw = existsSync(p) ? (Bun.YAML.parse(readFileSync(p, "utf8")) as any) : {};
+  return { blockedTools: raw?.trials?.blockedTools ?? {} };
+}
+
+const hash = (p: string) => createHash("sha256").update(readFileSync(p)).digest("hex").slice(0, 12);
+
+/** Path → short content hash for each file (folders are walked), so a result can be traced to the setup it ran with. */
+export function snapshotFiles(paths: string[]): Record<string, string> {
+  const out: Record<string, string> = {};
+  const walk = (p: string) => {
+    if (!existsSync(p)) return;
+    if (statSync(p).isDirectory()) {
+      for (const f of readdirSync(p).sort()) if (f !== ".DS_Store") walk(join(p, f));
+    } else {
+      try { out[p] = hash(p); } catch { /* changed while we read it (a tool updating its files): skip */ }
+    }
+  };
+  for (const p of paths) walk(p);
+  return out;
+}
+
+/** The user-level files each harness reads (instructions, settings, skills). Login secrets are not included. */
+export function userSetupFiles() {
+  const h = homedir();
+  return snapshotFiles([
+    join(h, ".claude", "CLAUDE.md"), join(h, ".claude", "settings.json"), join(h, ".claude", "skills"),
+    join(h, ".codex", "AGENTS.md"), join(h, ".codex", "config.toml"), join(h, ".codex", "skills"),
+    join(h, ".gemini", "GEMINI.md"), join(h, ".gemini", "settings.json"),
+  ]);
+}

package/src/core/task-file.ts

@@ -0,0 +1,95 @@
+// Task file validation. Errors stop a task from running; warnings are the authoring guide's
+// advice (enough criteria, a mix of levels) and are shown wherever the task is listed.
+import { existsSync, statSync } from "node:fs";
+import { isAbsolute, join, normalize } from "node:path";
+import { DERIVED_RUNNERS, derivedExt } from "./derived";
+import type { Criterion } from "./types";
+
+const LEVELS = ["floor", "strong", "ceiling"];
+const GRADERS = ["check", "judge", "human"];
+const EVIDENCE = ["output", "inputs", "references", "project", "sources"];
+export const MIN_CRITERIA = 15;
+export const MIN_PER_MEASURE = 3;
+
+const filled = (v: unknown) => typeof v === "string" && v.trim() !== "";
+
+/** What to fill in for each required top-level part, for the error when it is empty (`static-fire tasks`). */
+const REQUIRED: Record<string, string> = {
+  id: "the task's folder name under evals/",
+  title: "a short name for the task",
+  prompt: "the instructions the agent gets, as the user would give them",
+  inputs: "the folder of files copied into the agent's workspace, for example ./inputs",
+  output: "the path the agent writes, relative to its workspace, for example ./out/answer.md",
+};
+
+function checkCriterion(c: any, where: string, errors: string[], task: any, dir?: string) {
+  const id = filled(c?.id) ? c.id : where;
+  if (c?.id == null || c.id === "") errors.push(`${where}: id is required: a kebab-case name, unique in the task`);
+  else if (typeof c.id !== "string" || !/^[a-z0-9][a-z0-9-]*$/.test(c.id)) errors.push(`${where}: criterion id must be kebab-case`);
+  if (!filled(c?.question)) errors.push(`${id}: question is required: a yes or no question, where yes means pass`);
+  if (!LEVELS.includes(c?.level)) errors.push(`${id}: level must be floor, strong or ceiling`);
+  if (!GRADERS.includes(c?.grader)) errors.push(`${id}: grader must be check, judge or human`);
+  if (c?.grader === "check") {
+    const k = c.check;
+    if (!k) errors.push(`${id}: a check criterion needs \`check\``);
+    else if (k.type === "regex") { try { new RegExp(k.pattern, k.flags); } catch { errors.push(`${id}: invalid regex`); } }
+    else if (k.type === "frontmatter") { if (!k.field) errors.push(`${id}: frontmatter check needs \`field\``); }
+    else if (k.type === "command") { if (!k.run) errors.push(`${id}: command check needs \`run\``); }
+    else if (k.type !== "words") errors.push(`${id}: unknown check type ${JSON.stringify(k.type)}`);
+  }
+  if (c?.evidence != null && (!Array.isArray(c.evidence) || c.evidence.some((e: string) => !EVIDENCE.includes(e))))
+    errors.push(`${id}: evidence must be a list of ${EVIDENCE.join(", ")}`);
+  else for (const k of ["references", "project", "sources"]) if (c?.evidence?.includes(k) && !task[k]) errors.push(`${id}: evidence ${k} needs \`${k}\` on the task`);
+  if (c?.derived != null) checkDerived(c, id, errors, dir);
+  else if (c?.grader === "judge" && Array.isArray(c?.evidence) && !c.evidence.length) errors.push(`${id}: evidence is empty, so the judge would see nothing; name evidence or \`derived\``);
+}
+
+/** `derived`: a judge criterion's script, a path under the task's grader/ folder (README, "Derived evidence"). */
+function checkDerived(c: any, id: string, errors: string[], dir?: string) {
+  const d = c.derived;
+  if (c.grader !== "judge") { errors.push(`${id}: only a judge criterion can have \`derived\` evidence`); return; }
+  if (typeof d !== "string" || !d.trim()) { errors.push(`${id}: derived must be a path under grader/`); return; }
+  if (isAbsolute(d) || normalize(d).split(/[\\/]/)[0] === "..") { errors.push(`${id}: derived ${d} must be a path inside grader/`); return; }
+  if (!derivedExt(d)) { errors.push(`${id}: derived ${d} must end in ${Object.keys(DERIVED_RUNNERS).join(", ")}`); return; }
+  if (dir) {
+    const p = join(dir, "grader", d);
+    if (!existsSync(p) || !statSync(p).isFile()) errors.push(`${id}: derived script grader/${d} does not exist`);
+  }
+}
+
+/** `dir` is the task folder; when given, a `derived` script must exist in its grader/ folder. */
+export function validateTask(raw: any, dir?: string): { errors: string[]; warnings: string[] } {
+  const errors: string[] = [], warnings: string[] = [];
+  for (const [k, what] of Object.entries(REQUIRED)) if (!filled(raw?.[k])) errors.push(`${k} is required: ${what}`);
+  // `version` is a readable label only, in neither hash (identity.ts). A task file can leave it out.
+  if (raw?.version != null && !Number.isInteger(raw.version)) errors.push("version, when given, must be an integer");
+  if (raw?.baseline) errors.push("`baseline` was replaced by human configs: list the file under `outputs` in configs/<you>.yaml");
+  const gates: Criterion[] = raw?.gates ?? [];
+  const measures: any[] = raw?.measures ?? [];
+  if (!Array.isArray(measures) || !measures.length) errors.push("at least one measure is required");
+
+  const seen = new Set<string>();
+  const note = (c: any) => {
+    if (c?.id === "output-exists") errors.push("output-exists is reserved (it is always added as a gate)");
+    else if (filled(c?.id) && seen.has(c.id)) errors.push(`duplicate criterion id ${c.id}`);
+    seen.add(c?.id);
+  };
+  gates.forEach((g, i) => { checkCriterion(g, `gates[${i}]`, errors, raw, dir); note(g); });
+  const mids = new Set<string>();
+  measures.forEach((m, i) => {
+    const mid = filled(m?.id) ? m.id : `measures[${i}]`;
+    if (!filled(m?.id)) errors.push(`measures[${i}]: id is required: a kebab-case name for the measure`);
+    else if (mids.has(m.id)) errors.push(`duplicate measure id ${m.id}`);
+    mids.add(m?.id);
+    if (!filled(m?.label)) errors.push(`${mid}: label is required: the measure's name on the dashboard`);
+    if (m?.weight != null && !(m.weight > 0)) errors.push(`${mid}: weight must be positive`);
+    if (!Array.isArray(m?.criteria) || !m.criteria.length) { errors.push(`${mid}: a measure needs criteria`); return; }
+    m.criteria.forEach((c: any, j: number) => { checkCriterion(c, `${mid}.criteria[${j}]`, errors, raw, dir); note(c); });
+    if (m.criteria.length < MIN_PER_MEASURE) warnings.push(`measure ${mid} has ${m.criteria.length} ${m.criteria.length === 1 ? "criterion" : "criteria"}; the guide asks for at least ${MIN_PER_MEASURE}`);
+  });
+
+  const crits = measures.flatMap((m) => (Array.isArray(m?.criteria) ? m.criteria : []));
+  if (crits.length < MIN_CRITERIA) warnings.push(`${crits.length} criteria in total; the guide asks for at least ${MIN_CRITERIA} so one criterion moves the score by ≤ ${Math.round(100 / MIN_CRITERIA)} points`);
+  for (const lvl of LEVELS) if (crits.length && !crits.some((c: any) => c.level === lvl)) warnings.push(`no ${lvl} criteria; mix floor, strong and ceiling so scores spread out`);
+  return { errors, warnings };
+}

package/src/core/trial-detail.ts

@@ -0,0 +1,73 @@
+// One trial as the trial drawer reads it (GET /api/trials/:id and its output): the trial row with its run
+// conditions, and every grading it has, each with its grading setup and verdicts. Read-only.
+import { existsSync, readFileSync, realpathSync } from "node:fs";
+import { extname, isAbsolute, join, relative, sep } from "node:path";
+import { trialDir } from "./db";
+import type { GradingRecord, Store, TrialRecord, UserSetupRecord, VerdictRecord } from "./db";
+import { namedSetup, versionNames } from "./grading-versions";
+import type { NamedGradingSetup } from "./grading-versions";
+import { IMAGE_MEDIA } from "./judge";
+
+export interface TrialGrading extends GradingRecord {
+  /** The grading version of its grading hash, with its name, or null when the store has no row for it. */
+  setup: NamedGradingSetup | null;
+  verdicts: VerdictRecord[];
+}
+
+export interface TrialDetail {
+  /** The trial row as stored, run conditions included (setupHash, profileHash, blockedTools, harnessVersion, startedAt). */
+  trial: TrialRecord;
+  /** The user setup of its setup hash (path → content hash of each file), or null. */
+  userSetup: UserSetupRecord | null;
+  /** Every grading of the trial, oldest first, finished or not. Which one a view counts is the view's ViewTrial.gradingId. */
+  gradings: TrialGrading[];
+}
+
+/** The trial and every grading it has, or null for an unknown id. */
+export function trialDetail(store: Store, trialId: string): TrialDetail | null {
+  const trial = store.getTrial(trialId);
+  if (!trial) return null;
+  const setups = new Map<string, NamedGradingSetup | null>();
+  let names: ReturnType<typeof versionNames> | undefined;   // built once, on the first grading that has a setup
+  const setupOf = (hash: string) => {
+    if (!setups.has(hash)) {
+      const s = store.getGradingSetup(hash);
+      // The task's grading versions include every one its trials were graded under, so each stored setup has a name.
+      setups.set(hash, s && namedSetup(s, names ??= versionNames(store, trial.taskId)));
+    }
+    return setups.get(hash)!;
+  };
+  return {
+    trial,
+    userSetup: trial.setupHash ? store.getUserSetup(trial.setupHash) : null,
+    gradings: store.listGradings(trialId).map((g) => ({ ...g, setup: setupOf(g.gradingHash), verdicts: store.listVerdicts(g.id) })),
+  };
+}
+
+/** The trial's output file as the drawer shows it: an image (by the output path's extension) or text. */
+export type TrialOutputFile = { image: Uint8Array<ArrayBuffer>; mediaType: string } | { text: string };
+
+/**
+ * The trial's output file: the task's `output` path, as recorded with the task version the trial ran on, inside
+ * the trial's saved workspace. An image type the judges take (IMAGE_MEDIA) is read as bytes with its media
+ * type; any other file as UTF-8 text. Otherwise a plain reason: the trial is unknown, its task version has no
+ * recorded output path (never guessed from today's task file), the file is missing, or the path (after
+ * symlinks) leaves the workspace.
+ */
+export function readTrialOutputFile(store: Store, dbPath: string, trialId: string): TrialOutputFile | { missing: string } {
+  const trial = store.getTrial(trialId);
+  if (!trial) return { missing: `no trial ${trialId}` };
+  const snap = store.getTaskVersion(trial.taskHash)?.snapshot as { output?: unknown } | undefined;
+  const output = typeof snap?.output === "string" && snap.output ? snap.output : null;
+  if (!output) return { missing: `trial ${trialId}: its task version ${trial.taskHash.slice(0, 8)} has no recorded output path` };
+  const none = { missing: `trial ${trialId} has no output file at ${output} in its saved workspace` };
+  const workspace = join(trialDir(dbPath, trial.id), "workspace");
+  const file = join(workspace, output);
+  if (!existsSync(file) || !existsSync(workspace)) return none;
+  let real: string;
+  try { real = realpathSync(file); } catch { return none; }
+  const rel = relative(realpathSync(workspace), real);
+  if (!rel || rel === ".." || rel.startsWith(`..${sep}`) || isAbsolute(rel)) return { missing: `trial ${trialId}: its output path ${output} leaves the workspace` };
+  const mediaType = IMAGE_MEDIA[extname(output).toLowerCase()];
+  try { return mediaType ? { image: readFileSync(real), mediaType } : { text: readFileSync(real, "utf8") }; } catch { return none; }   // a folder, or unreadable
+}