static-fire 0.0.0-stage → 0.54.1

package/src/core/derived.ts

@@ -0,0 +1,74 @@
+// Derived evidence: a judge criterion's script (`derived: <path under grader/>`) makes an image from the
+// output, and the judge sees that image. The contract for a script is in the README ("Derived evidence").
+import { existsSync, mkdirSync, readFileSync, statSync, writeFileSync } from "node:fs";
+import { extname, join } from "node:path";
+
+/**
+ * How a script is started, by its extension. `.ts` and `.js` use the Bun that runs the engine, not a `bun` on the
+ * PATH, so they run when the package brought Bun and the machine has none. `.py` uses `uv run --script`, so a
+ * script can declare its packages inline (PEP 723).
+ */
+export const DERIVED_RUNNERS: Record<string, (script: string) => string[]> = {
+  ".ts": (s) => [process.execPath, s],
+  ".js": (s) => [process.execPath, s],
+  ".py": (s) => ["uv", "run", "--quiet", "--script", s],
+  ".sh": (s) => ["sh", s],
+};
+
+export const derivedExt = (path: string) => {
+  const e = extname(path).toLowerCase();
+  return e in DERIVED_RUNNERS ? e : null;
+};
+
+/** The command for a script (a path under grader/), with the output file and the image path as its two arguments. */
+export function derivedCommand(taskDir: string, script: string, outputPath: string, imagePath: string): string[] {
+  const ext = derivedExt(script);
+  if (!ext) throw new Error(`no runner for ${script}`);
+  return [...DERIVED_RUNNERS[ext]!(join(taskDir, "grader", script)), outputPath, imagePath];
+}
+
+/**
+ * A name as one safe file or folder name, one to one: a character outside `A-Z a-z 0-9 + = . _ -` becomes
+ * `%` and its UTF-8 bytes in hex (so `/` is `%2F` and `%` is `%25`). Two different names never give the same result.
+ */
+export const safeName = (name: string) => name.replace(/[^A-Za-z0-9+=._-]/gu, (ch) => [...Buffer.from(ch)].map((b) => `%${b.toString(16).toUpperCase().padStart(2, "0")}`).join(""));
+
+/**
+ * Where a grading keeps the image a script made: `<grading dir>/_derived/<safeName(script)>.png` (for example
+ * `sub/overlay.ts` gives `sub%2Foverlay.ts.png`), with the script's output next to it as `.log`. The folder name
+ * starts with "_", which no judge batch folder does, so the two never meet.
+ */
+export const derivedImagePath = (gradingDir: string, script: string) => join(gradingDir, "_derived", `${safeName(script)}.png`);
+
+const PNG = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]);
+
+export interface DerivedContext { taskDir: string; outputPath: string; workspace: string }
+
+/**
+ * Runs one script in the workspace, as command checks run (same environment, plus DERIVED, the image path).
+ * Resolves to the image path, or to an error when the script fails, times out, or writes no PNG.
+ */
+export async function runDerived(script: string, ctx: DerivedContext, gradingDir: string, timeoutSec = 60): Promise<{ image: string } | { error: string }> {
+  const image = derivedImagePath(gradingDir, script);
+  mkdirSync(join(gradingDir, "_derived"), { recursive: true });
+  const fail = (why: string) => ({ error: `derived evidence grader/${script}: ${why}` });
+  try {
+    const proc = Bun.spawn(derivedCommand(ctx.taskDir, script, ctx.outputPath, image), {
+      cwd: ctx.workspace, stdin: "ignore", stdout: "pipe", stderr: "pipe",
+      env: { ...process.env, OUTPUT: ctx.outputPath, WORKSPACE: ctx.workspace, TASK_DIR: ctx.taskDir, DERIVED: image },
+    });
+    let timedOut = false;
+    const timer = setTimeout(() => { timedOut = true; proc.kill(); }, timeoutSec * 1000);
+    const [out, err, code] = await Promise.all([new Response(proc.stdout).text(), new Response(proc.stderr).text(), proc.exited]);
+    clearTimeout(timer);
+    writeFileSync(image.replace(/\.png$/, ".log"), `${out}${err ? `\n--- stderr ---\n${err}` : ""}`);
+    const said = (err.trim() || out.trim()).split("\n").slice(-1)[0]?.slice(0, 300);
+    if (timedOut) return fail(`timed out after ${timeoutSec}s`);
+    if (code !== 0) return fail(`exit ${code}${said ? `: ${said}` : ""}`);
+    if (!existsSync(image) || !statSync(image).size) return fail("wrote no image");
+    if (!readFileSync(image).subarray(0, 8).equals(PNG)) return fail("the image is not a PNG");
+    return { image };
+  } catch (e) {
+    return fail(e instanceof Error ? e.message : String(e));
+  }
+}

package/src/core/engine.ts

@@ -0,0 +1,85 @@
+// The engine revision: the parts of the grading code that can move a score. A grading hash holds a part only
+// when the task uses it, so a change to one part gives a new grading version only to the tasks that use it
+// (ADR-0007). Each revision here is raised by hand; the pin in tests/identity.test.ts fails when the code of a
+// part changes and names the revision to raise. The judge instructions are not here: the code hashes their wording.
+// The dashboard imports this file, so it must stay free of Node code.
+
+import { HARNESS_NAME } from "./label";
+import type { Check, JudgeConfig } from "./types";
+
+type CheckType = Check["type"];
+type JudgeHarness = JudgeConfig["harness"];
+
+export interface EngineRevisions {
+  /** How each check type decides pass or fail. In a grading hash when a gate or criterion uses the type; `command` always, because the output gate is one. */
+  checks: Record<CheckType, number>;
+  /** How a derived evidence script runs and how its image reaches the judge. In a grading hash when a criterion names a script. */
+  derived: number;
+  /** How a judge of each harness is called: the command, its flags, the retry, the answer schema. In a grading hash with a judge criterion, for its own judge's harness only. */
+  judgeCall: Record<JudgeHarness, number>;
+  /** How evidence is shown to a judge, how criteria are grouped into calls, how votes are counted. In a grading hash with a judge criterion. */
+  judging: number;
+  /** How verdicts become a score. In every grading hash. */
+  scoring: number;
+}
+
+export const ENGINE: EngineRevisions = {
+  checks: { regex: 1, words: 1, frontmatter: 1, command: 1 },
+  derived: 1,
+  judgeCall: { claude: 1, codex: 1, agy: 1 },
+  judging: 1,
+  scoring: 1,
+};
+
+/** The engine parts one task uses, as its grading hash holds them. A part the task does not use is left out. */
+export interface EngineParts {
+  instructions?: string;          // hash of the judge instruction wording every judged task gets
+  checks: Record<string, number>; // check type → revision
+  derived?: number;
+  derivedInstructions?: string;   // hash of the instructions as a batch with derived evidence gets them
+  judgeCall?: number;             // of the judge's own harness
+  judging?: number;
+  scoring: number;
+}
+
+/**
+ * The first value of every part: all revisions at 1, and the instruction wording of 0.37.0. A task whose parts are
+ * all at these values keeps the snapshot form of before the parts (`graderRev`), so its stored gradings stay current.
+ * Never change these: a part that changes gets a new current value, not a new first value.
+ */
+export const FIRST_REVISIONS: EngineRevisions = {
+  checks: { regex: 1, words: 1, frontmatter: 1, command: 1 },
+  derived: 1,
+  judgeCall: { claude: 1, codex: 1, agy: 1 },
+  judging: 1,
+  scoring: 1,
+};
+export const FIRST_INSTRUCTIONS = {
+  shared: "804b007a7160526b61fa7e56504aa9162344c48a964c520ec1a909398aa76145",
+  derived: "2697afc24b3201dd1b18a0ee5f1866bb2435cebce0328f8e2055dec79a0c61f8",
+};
+
+/**
+ * One changed part of the engine revision. `name` is the check type of a `check` part, the judge harness of a
+ * `judgeCall` part, and "derived" for the instructions as a batch with derived evidence gets them. `revision` is two snapshots of before the parts whose one number differs. The values are
+ * revisions, or hashes of the wording for `instructions`.
+ */
+export interface EngineChange {
+  part: "instructions" | "check" | "derived" | "judgeCall" | "judging" | "scoring" | "revision";
+  name: string | null;
+  a: number | string;
+  b: number | string;
+}
+
+/** One changed engine part in short plain words, for a name and for the Compare page. */
+export function engineChangeName(c: EngineChange): string {
+  switch (c.part) {
+    case "instructions": return "judge instructions changed";
+    case "check": return `check changed: ${c.name}`;
+    case "derived": return "derived evidence changed";
+    case "judgeCall": return `judge call changed: ${HARNESS_NAME[c.name as keyof typeof HARNESS_NAME] ?? c.name}`;
+    case "judging": return "judging changed";
+    case "scoring": return "scoring changed";
+    case "revision": return `engine revision ${c.a} to ${c.b}`;
+  }
+}

package/src/core/facets.ts

@@ -0,0 +1,143 @@
+// The values each filter of the view can take, with trial counts: what GET /api/facets returns and the
+// dashboard's filter bar lists. Every count comes from buildView, so a count is exactly the number of trials
+// the view would list (failed and invalid included) if that value were chosen. A count keeps every other
+// filter and drops the facet's own one (the usual faceted-search rule), so the other values of a facet show
+// what choosing them would give. Values are only those found in stored records: an empty store gives empty lists.
+import type { GradingSetupRecord, Store, TrialRecord } from "./db";
+import { versionNames } from "./grading-versions";
+import type { VersionName } from "./grading-versions";
+import { gradingHash as currentGradingHash } from "./identity";
+import { buildView, CONFIG_FACETS, facetValue, isSetupJudge, projectCatalog } from "./view";
+import type { Catalog, ConfigFacet, ViewFilters } from "./view";
+
+/** One value of a filter and the trials the view would list with it. */
+export interface FacetValue { value: string; count: number; /** A readable name, where the value is an id (judges). */ label?: string }
+
+/**
+ * One grading version of a task, as the grading version filter lists it: its name (built from what changed since
+ * the version before it for the same task and judge), with that version and the difference to it.
+ */
+export interface GradingOption extends VersionName {
+  /** Full grading hash: the value of the `grading` filter. */
+  hash: string;
+  /** The task hash it grades: a setup of an earlier task version reaches that version. */
+  taskHash: string;
+  /** The task file's `version` stored with that task hash, or null if the store has none or the task has no version. */
+  label: number | null;
+  /** The judges.yaml id of the judge that grades under this version, or null when no judge there matches it. */
+  judgeId: string | null;
+  judgeLabel: string;
+  /** When the version was first recorded (ISO time). */
+  firstSeen: string;
+  /** True for today's grading hash of the task for this version's own judge, whichever judge is in view. */
+  current: boolean;
+  /** Trials of the task the view would list with this version chosen. */
+  count: number;
+}
+
+export interface Facets {
+  /** The Agent Performance Index and every suite and task with a stored trial. A suite's count is the sum over its tasks. */
+  scope: { index: number; suites: FacetValue[]; tasks: FacetValue[] };
+  /** The judges in judges.yaml that have graded a task in scope. */
+  judge: FacetValue[];
+  /** Per task in scope, every grading version that graded a stored trial of that task, oldest first. */
+  grading: { taskId: string; setups: GradingOption[] }[];
+  harness: FacetValue[];
+  model: FacetValue[];
+  effort: FacetValue[];
+  profile: FacetValue[];
+  /** First and last start time of the trials the view would list without a date range. */
+  dates: { first: string | null; last: string | null };
+}
+
+const ENDED = new Set(["done", "failed", "invalid"]);
+const without = (f: ViewFilters, ...keys: (keyof ViewFilters)[]): ViewFilters => {
+  const g = { ...f };
+  for (const k of keys) delete g[k];
+  return g;
+};
+const tally = (values: string[]): Map<string, number> => {
+  const m = new Map<string, number>();
+  for (const v of values) m.set(v, (m.get(v) ?? 0) + 1);
+  return m;
+};
+
+/** The facets for `filters`. Throws on the same bad filters as buildView. */
+export function buildFacets(store: Store, given: ViewFilters = {}, catalog: Catalog = projectCatalog()): Facets {
+  const base = buildView(store, given, catalog);   // validates the filters and resolves scope, judge and grading
+  // The counts are over the suites the view resolved: without a pick, the default set moves with the filters,
+  // and a count without one filter must not count other suites than the view it is offered in.
+  const filters: ViewFilters = given.tasks?.length || !base.filters.suiteIds.length ? given : { ...given, suites: base.filters.suiteIds };
+  const { taskIds } = base.filters;
+  const picked = base.filters.grading ? store.getGradingSetup(base.filters.grading) : null;
+  const pickedTasks = picked ? store.gradingSetupTaskIds(picked.hash) : [];
+  const all = store.listTrials().filter((r) => ENDED.has(r.status));
+  const byId = new Map(all.map((r) => [r.id, r]));
+  const inScope = all.filter((r) => taskIds.includes(r.taskId));
+  const listed = (f: ViewFilters): TrialRecord[] => buildView(store, f, catalog).trials.map((t) => byId.get(t.id)!);
+
+  // Scope. A count is the number of trials the view would list with that value chosen (#80). A task: the view of
+  // that task, with the picked grading setup only when it is of that task (the page drops it for another task), so
+  // a task outside the default set of suites counts its own trials. A suite: the sum over its tasks. The index:
+  // the default set, or, with a picked grading setup, the suite of its task (the page picks it, web/filter-menu.ts
+  // withGradingPick).
+  const unscoped = without(filters, "suites", "tasks", "grading");
+  const stored = new Set(all.map((r) => r.taskId));
+  const loads = (t: string) => { try { catalog.loadTask(t); return true; } catch { return false; } };
+  const storedTasks = [...stored].filter(loads).sort();
+  const others = storedTasks.filter((t) => !pickedTasks.includes(t));
+  const ofTasks = [
+    ...(others.length ? listed({ ...unscoped, tasks: others }) : []),
+    ...(picked ? listed({ ...unscoped, tasks: pickedTasks, grading: picked.hash }) : []),
+  ];
+  const perTask = tally(ofTasks.map((r) => r.taskId));
+  const home = picked ? catalog.suites.find((s) => s.tasks.some((t) => pickedTasks.includes(t))) : undefined;
+  const index = home ? listed({ ...unscoped, suites: [home.id], grading: picked!.hash }) : listed(unscoped);
+  const scope = {
+    index: index.length,
+    suites: catalog.suites.filter((s) => s.tasks.some((t) => stored.has(t)))
+      .map((s) => ({ value: s.id, count: s.tasks.reduce((n, t) => n + (perTask.get(t) ?? 0), 0) })),
+    tasks: storedTasks.map((t) => ({ value: t, count: perTask.get(t) ?? 0 })),
+  };
+
+  // A grading setup is offered for a task only when a stored trial of that task carries a grading by it.
+  const setupsOf = (taskId: string) => store.listGradingSetups(taskId)
+    .filter((s: GradingSetupRecord) => all.some((r) => r.taskId === taskId && store.listGradings(r.id, s.hash).length > 0));
+
+  // Judge: the judges with a graded grading setup for a task in scope. A picked setup sets the judge, so another
+  // judge is counted over its own view without that setup.
+  const setups = taskIds.flatMap(setupsOf);
+  const judgeFacet = catalog.judges
+    .filter((j) => setups.some((s) => isSetupJudge(j, s)))
+    .map((j) => ({ value: j.id, label: j.label, count: listed({ ...(picked && !isSetupJudge(j, picked) ? without(filters, "grading") : filters), judge: j.id }).length }));
+
+  // Grading setup: per task in scope, each stored setup counted over that task alone, with its own judge.
+  const rest = without(filters, "suites", "tasks", "grading", "judge");
+  const grading = taskIds.map((taskId) => {
+    const task = catalog.loadTask(taskId);
+    const now = new Map(catalog.judges.map((j) => [j.id, currentGradingHash(task, j)]));
+    const names = versionNames(store, taskId);
+    return {
+      taskId,
+      setups: setupsOf(taskId).map((s): GradingOption => {
+        const judgeId = catalog.judges.find((j) => isSetupJudge(j, s))?.id ?? null;
+        return {
+          ...names.get(s.hash)!,
+          hash: s.hash, taskHash: s.taskHash, label: store.getTaskVersion(s.taskHash)?.label ?? null, judgeId, judgeLabel: s.judgeLabel,
+          firstSeen: s.firstSeen, current: !!judgeId && s.hash === now.get(judgeId),
+          count: listed({ ...rest, tasks: [taskId], grading: s.hash }).length,
+        };
+      }),
+    };
+  });
+
+  // Config facets: values from the stored trials of the tasks in scope, counts without the facet's own filter.
+  const config = Object.fromEntries(CONFIG_FACETS.map((k: ConfigFacet) => {
+    const counts = tally(listed(without(filters, k)).map((r) => facetValue(r, k)));
+    const values = [...new Set(inScope.map((r) => facetValue(r, k)))].sort();
+    return [k, values.map((value) => ({ value, count: counts.get(value) ?? 0 }))];
+  })) as Record<ConfigFacet, FacetValue[]>;
+
+  const starts = listed(without(filters, "since", "until")).map((r) => r.startedAt).filter((x): x is string => !!x).sort();
+  return { scope, judge: judgeFacet, grading, ...config, dates: { first: starts[0] ?? null, last: starts.at(-1) ?? null } };
+}

package/src/core/grade.ts

@@ -0,0 +1,197 @@
+// Grade one output against a Task: gates + checks + batched judge votes → Grades (ADR-0001).
+import { copyFileSync, existsSync, mkdirSync, readdirSync, readFileSync, statSync } from "node:fs";
+import { basename, dirname, extname, join, relative } from "node:path";
+import { runCheck, type CheckContext } from "./checks";
+import { runDerived, safeName } from "./derived";
+import { attachesImages, isImage, judgeVote, type JudgeFiles } from "./judge";
+import { firstRoundVotes, needsMoreVotes, scoreTrial, verdictFromVotes } from "./score";
+import type { Criterion, Evidence, JudgeConfig, Task, Verdict, Vote } from "./types";
+
+export const OUTPUT_GATE: Criterion = { id: "output-exists", question: "Did the agent write the output file?", level: "floor", grader: "check", check: { type: "command", run: 'test -s "$OUTPUT"' } };
+
+const TEXT = new Set([".md", ".mdx", ".txt", ".yaml", ".yml", ".json", ".html", ".css", ".ts", ".tsx", ".js", ".py", ".csv", ".svg"]);
+const files = (dir: string): string[] =>
+  !existsSync(dir) ? [] : statSync(dir).isFile() ? [dir]
+    : readdirSync(dir).flatMap((f) => files(join(dir, f))).filter((f) => !basename(f).startsWith("."));
+
+
+/**
+ * Text files are quoted inline. Images are attached to the judge's message when its Host can take them
+ * (their number in `files.images` is how the prompt names them), so seeing them is not left to the judge.
+ * Other files, and images for a Host that cannot attach them, are copied to `files.readDir` and named by
+ * path; the judge must open each one (judge.ts checks its log).
+ */
+export function evidenceBlock(tag: string, paths: string[], root: string, files: JudgeFiles) {
+  const parts: string[] = [];
+  for (const p of paths) {
+    const rel = relative(root, p) || basename(p);
+    if (TEXT.has(extname(p).toLowerCase())) parts.push(`<file path="${rel}">\n${readFileSync(p, "utf8")}\n</file>`);
+    else if (files.images && isImage(p)) {
+      files.images.push({ label: rel, path: p });
+      parts.push(`<file path="${rel}">Image ${files.images.length}, attached to this message.</file>`);
+    } else {
+      // The evidence kind is part of the name, so project/x.png and sources/x.png do not overwrite each other.
+      const local = join(files.readDir, `${tag}__${rel.replace(/[\\/]/g, "__")}`);
+      mkdirSync(files.readDir, { recursive: true });
+      copyFileSync(p, local);
+      parts.push(`<file path="${rel}">Binary file. Read it with the Read tool at: ${local}</file>`);
+      files.toRead.push(local);
+    }
+  }
+  return `<${tag}>\n${parts.join("\n\n")}\n</${tag}>`;
+}
+
+// engine:instructions:begin
+/** The instruction line about derived evidence. Its wording is hashed apart from the rest (instructionWording). */
+const derivedRule = (script: string) => `- The image in <derived> was made from the work by the grader script grader/${script}. It shows the work as a grader needs to see it, for example a smaller copy or a copy with marks drawn on it. It is part of the evidence.\n`;
+
+/** `derived` is the block of the batch's derived image and the script that made it, when it has one. */
+function judgePrompt(task: Pick<Task, "context" | "prompt">, criteria: Criterion[], evidence: Record<Evidence, string>, files: JudgeFiles, derived?: { block: string; script: string }) {
+  const attached = !!files.images?.length, needsRead = files.toRead.length > 0;
+  return `You are grading one piece of work against yes/no criteria.
+${task.context ? `\n${task.context.trim()}\n` : ""}
+Rules:
+- Judge each criterion on its own, as if it were the only question.
+- pass = true only when the work clearly meets the criterion. When it is unclear or only partly met, pass = false.
+${derived ? derivedRule(derived.script) : ""}${attached ? "- Files below marked \"Image N\" are attached to this message as images, in that order. Look at every one before you answer: they are part of the evidence.\n" : ""}${needsRead ? "- Some files below are images or other binary files. Open every one of them with the Read tool before you answer. Never answer that you could not see a file.\n" : ""}- For each criterion, write the reason first: one or two sentences that point to the specific evidence (quote it when you can). Then set pass so that it agrees with the reason.
+
+<assignment>
+${task.prompt.trim()}
+</assignment>
+
+${[evidence.inputs, evidence.project, evidence.sources, evidence.references, evidence.output, derived?.block].filter(Boolean).join("\n\n")}
+
+<criteria>
+${criteria.map((c) => `- id: ${c.id}\n  question: ${c.question}`).join("\n")}
+</criteria>
+
+Return one verdict for every criterion id above, exactly once each.`;
+}
+
+/**
+ * The fixed wording of the judge instructions, for the grading hash (identity.ts hashes it, so a change here
+ * needs no revision raised by hand). `shared` is what every judged task gets: the instructions with a fixed
+ * placeholder for the context, the assignment, each kind of evidence and one criterion, so the hash also covers
+ * where each of them goes and how a criterion is written. Both image lines are in it, because the files are not
+ * known before an output exists. `derived` is the same text as a batch with derived evidence gets it: with
+ * the line about the derived image, and with its block among the evidence. Only a task that uses derived evidence
+ * holds its hash, so a change to that line or to where the block goes moves only those tasks.
+ */
+export function instructionWording(): { shared: string; derived: string } {
+  const evidence: Record<Evidence, string> = { output: "<output>", inputs: "<inputs>", references: "<references>", project: "<project>", sources: "<sources>" };
+  const files = (): JudgeFiles => ({ images: [{ label: "", path: "" }], toRead: [""], readDir: "" });
+  const render = (derived?: { block: string; script: string }) =>
+    judgePrompt({ context: "<context>", prompt: "<assignment>" }, [{ id: "<id>", question: "<question>" } as Criterion], evidence, files(), derived);
+  return { shared: render(), derived: render({ block: "<derived>", script: "<script>" }) };
+}
+// engine:instructions:end
+
+const shuffle = <T>(xs: T[]) => { const a = [...xs]; for (let i = a.length - 1; i > 0; i--) { const j = Math.floor(Math.random() * (i + 1)); [a[i], a[j]] = [a[j], a[i]]; } return a; };
+
+export interface GradeArgs { task: Task; outputPath: string; workspace: string; judge: JudgeConfig & { votes: number }; dir: string; batching?: Batching }
+
+/**
+ * How judge criteria share calls. "evidence" (default): every criterion that needs the same evidence shares
+ * one call, across measures, so the call count does not grow with the number of measures. "measure": one
+ * call per measure. Either way, judged gates share a call and `isolate: true` criteria get their own.
+ * Criteria share a call only when they also name the same derived evidence script (or none).
+ */
+export type Batching = "evidence" | "measure";
+const EVIDENCE_ORDER: Evidence[] = ["output", "inputs", "references", "project", "sources"];
+
+export function planBatches(task: Pick<Task, "gates" | "measures">, mode: Batching = "evidence") {
+  const batches: { key: string; criteria: Criterion[]; derived?: string }[] = [];
+  const judged = (cs: Criterion[]) => cs.filter((c) => c.grader === "judge");
+  // One batch per derived script among `cs` (none first), its key marked with the script.
+  const push = (key: string, cs: Criterion[]) => {
+    const scripts = [...new Set(cs.flatMap((c) => (c.derived ? [c.derived] : [])))].sort();
+    for (const d of [undefined, ...scripts]) {
+      const part = cs.filter((c) => c.derived === d);
+      if (part.length) batches.push(d ? { key: [key, `derived=${d}`].filter(Boolean).join("+"), criteria: part, derived: d } : { key, criteria: part });
+    }
+  };
+  push("gates", judged(task.gates));
+  const shared = task.measures.map((m) => ({ id: m.id, cs: judged(m.criteria).filter((c) => !c.isolate) }));
+  if (mode === "measure") for (const m of shared) push(m.id, m.cs);
+  else {
+    const groups = new Map<string, Criterion[]>();
+    for (const c of shared.flatMap((m) => m.cs)) {
+      const kinds = new Set(c.evidence ?? ["output"]);
+      const key = EVIDENCE_ORDER.filter((k) => kinds.has(k)).join("+");
+      groups.set(key, [...(groups.get(key) ?? []), c]);
+    }
+    for (const [key, cs] of groups) push(key, cs);
+  }
+  for (const c of task.measures.flatMap((m) => judged(m.criteria)).filter((c) => c.isolate)) push(c.id, [c]);
+  return batches;
+}
+
+/** A batch's folder in the grading folder: its key as a safe name, so two keys never share a folder. */
+const batchFolder = (key: string) => safeName(key) || "_";
+
+export async function grade({ task, outputPath, workspace, judge, dir, batching = "evidence" }: GradeArgs) {
+  const output = existsSync(outputPath) ? readFileSync(outputPath, "utf8") : "";
+  const ctx: CheckContext = { output, outputPath, workspace, taskDir: task.dir };
+  const verdicts: Record<string, Verdict> = {};
+  const all = [OUTPUT_GATE, ...task.gates, ...task.measures.flatMap((m) => m.criteria)];
+
+  verdicts[OUTPUT_GATE.id] = await runCheck(OUTPUT_GATE.check!, ctx);
+  if (verdicts[OUTPUT_GATE.id].score !== 1) {
+    // Nothing to grade: everything fails for the same reason, and no judge is paid for.
+    for (const c of all.slice(1)) verdicts[c.id] = { score: 0, votes: [{ pass: false, reason: "no output" }] };
+    return scoreTrial({ ...task, gates: [OUTPUT_GATE, ...task.gates] }, verdicts, 0);
+  }
+
+  await Promise.all(all.slice(1).filter((c) => c.grader === "check").map(async (c) => { verdicts[c.id] = await runCheck(c.check!, ctx); }));
+  for (const c of all) if (c.grader === "human") verdicts[c.id] = { score: null, votes: [], error: "human grading is not available yet" };
+
+  const batches = planBatches(task, batching);
+  // Each derived evidence script runs once per grading; its image stays in <dir>/_derived/.
+  const derived = new Map([...new Set(batches.flatMap((b) => (b.derived ? [b.derived] : [])))]
+    .map((s) => [s, runDerived(s, { taskDir: task.dir, outputPath, workspace }, dir)] as const));
+
+  let judgeCost = 0;
+  await Promise.all(batches.map(async (b) => {
+    const made = b.derived ? await derived.get(b.derived)! : null;
+    if (made && "error" in made) {
+      // No image, no judge call: an error, never a silent pass or fail.
+      for (const c of b.criteria) verdicts[c.id] = { score: null, votes: [], error: made.error };
+      return;
+    }
+    const kinds = new Set(b.criteria.flatMap((c) => c.evidence ?? ["output"]));
+    const ev: Record<Evidence, string> = { output: "", inputs: "", references: "", project: "", sources: "" };
+    const batchDir = join(dir, batchFolder(b.key));
+    const jf: JudgeFiles = { images: attachesImages(judge.harness) ? [] : null, toRead: [], readDir: join(batchDir, "evidence") };
+    const put = (kind: Evidence, tag: string, paths: string[], root: string) => {
+      if (kinds.has(kind)) ev[kind] = evidenceBlock(tag, paths, root, jf);
+    };
+    put("output", "work", [outputPath], workspace);
+    put("inputs", "inputs", files(join(workspace, task.inputs)), workspace);
+    if (task.references) put("references", "references", files(join(task.dir, task.references)), join(task.dir, task.references));
+    // What the agent found in the project (for example past issues), and saved copies of the linked sources.
+    if (task.project) put("project", "project", files(join(task.dir, task.project)), join(task.dir, task.project));
+    if (task.sources) put("sources", "sources", files(join(task.dir, task.sources)), join(task.dir, task.sources));
+    const shown = made ? { block: evidenceBlock("derived", [made.image], dirname(made.image), jf), script: b.derived! } : undefined;
+
+    const ids = b.criteria.map((c) => c.id);
+    const votes: Record<string, Vote[]> = Object.fromEntries(ids.map((id) => [id, []]));
+    const errors: string[] = [];
+    const vote = async (k: number) => {
+      const order = shuffle(b.criteria);
+      const r = await judgeVote(judge, judgePrompt(task, order, ev, jf, shown), order.map((c) => c.id), join(batchDir, `vote-${k + 1}`), jf);
+      judgeCost += r.cost;
+      if (r.parsed.ok) for (const id of ids) votes[id].push(r.parsed.verdicts[id]);
+      else errors.push(r.parsed.error);
+    };
+    // Adaptive votes: a unanimous first-round majority already decides the outcome of all planned votes.
+    const first = firstRoundVotes(judge.votes);
+    await Promise.all(Array.from({ length: first }, (_, k) => vote(k)));
+    if (needsMoreVotes(votes, judge.votes)) await Promise.all(Array.from({ length: judge.votes - first }, (_, k) => vote(first + k)));
+    // A criterion is decided when a majority of the planned votes came back valid.
+    for (const id of ids) {
+      verdicts[id] = votes[id].length * 2 > judge.votes ? verdictFromVotes(votes[id]) : { score: null, votes: votes[id], error: errors[0] ?? "too few valid votes" };
+    }
+  }));
+
+  return scoreTrial({ ...task, gates: [OUTPUT_GATE, ...task.gates] }, verdicts, judgeCost);
+}