static-fire 0.0.0-stage → 0.54.1

package/src/core/grading-versions.ts

@@ -0,0 +1,281 @@
+// Grading versions: the difference between two of them, and the name each one gets from what changed since the
+// version before it. Built only from stored snapshots (the grading_setups and task_versions rows); today's task
+// files are never read. The name (facets, view, compare) and the Compare page's "What changed" both come from
+// one difference result, which is plain data.
+import type { GradingSetupRecord, Store, TaskVersionRecord } from "./db";
+import { canonicalJson, engineOf } from "./identity";
+import { engineChangeName, type EngineChange, type EngineParts } from "./engine";
+import type { CriterionContent, GradingContent, TaskContent } from "./identity";
+
+/** Files of one folder that differ: by path, as the snapshot keys them. */
+export interface FileChanges { added: string[]; removed: string[]; changed: string[] }
+
+/** A gate or measure criterion that differs. `fields` lists each changed field (question, check, evidence…) with both values. */
+export interface CriterionDiff {
+  id: string;
+  kind: "gate" | "criterion";
+  change: "added" | "removed" | "changed";
+  fields: { field: string; a: unknown; b: unknown }[];
+  /** The criterion under version a and version b; null on the side it is missing from. */
+  a: CriterionContent | null;
+  b: CriterionContent | null;
+}
+
+/** A measure that differs: added, removed, a changed weight, or criteria put in or taken out of it. */
+export interface MeasureDiff {
+  id: string;
+  change: "added" | "removed" | "changed";
+  /** Weight under each version (null on the side the measure is missing from), or null when it did not change. */
+  weight: { a: number | null; b: number | null } | null;
+  /** Criterion ids in the measure under b and not a, and under a and not b. */
+  criteria: { added: string[]; removed: string[] };
+  /** True when the only change is criteria added to or removed from the task as a whole (no weight change, no move). */
+  byCriteriaOnly: boolean;
+}
+
+export interface JudgeInfo { harness: string; model: string; effort: string | null; label: string }
+
+/** Everything that differs between grading versions a and b. Each part is null or empty when that part is the same. */
+export interface GradingDiff {
+  /** True when nothing differs. */
+  same: boolean;
+  /**
+   * True when something differs and every difference is in the gates and criteria. A measure whose criteria list
+   * changed only because criteria were added or removed counts as a criteria change. Any other difference (a
+   * measure or weight, context, judge, engine part, task hash or file) makes it false.
+   */
+  criteriaOnly: boolean;
+  /** Gates and measure criteria, sorted by kind then id. */
+  criteria: CriterionDiff[];
+  /** Measures, sorted by id. */
+  measures: MeasureDiff[];
+  context: { a: string | null; b: string | null } | null;
+  judge: { a: JudgeInfo; b: JudgeInfo } | null;
+  /**
+   * The parts of the engine revision that both versions use and that differ, in a fixed order. A part that only one
+   * version uses is not here: the criterion that brought it in or took it out is in `criteria`.
+   */
+  engine: EngineChange[];
+  /**
+   * What the agent was given, when the task hash differs. `prompt` and `output` say whether they changed; the file
+   * changes come from the stored task version snapshots (empty when one is missing). `label` is the task file's `version`.
+   */
+  task: {
+    a: string; b: string; label: { a: number | null; b: number | null };
+    prompt: boolean; output: boolean; inputs: FileChanges; project: FileChanges;
+  } | null;
+  /** Grader-only files: references, sources, the task's grader/ folder, and the held-out original. Only names, never content. */
+  files: { references: FileChanges; sources: FileChanges; grader: FileChanges; heldOut: boolean };
+}
+
+/** One stored grading version as the difference reads it. */
+export interface StoredVersion { setup: GradingSetupRecord; task: TaskVersionRecord | null }
+
+const same = (x: unknown, y: unknown) => canonicalJson(x ?? null) === canonicalJson(y ?? null);
+const sorted = (xs: Iterable<string>) => [...xs].sort();
+
+function fileChanges(a: Record<string, string> = {}, b: Record<string, string> = {}): FileChanges {
+  return {
+    added: sorted(Object.keys(b).filter((k) => !Object.hasOwn(a, k))),
+    removed: sorted(Object.keys(a).filter((k) => !Object.hasOwn(b, k))),
+    changed: sorted(Object.keys(a).filter((k) => Object.hasOwn(b, k) && a[k] !== b[k])),
+  };
+}
+const anyFile = (f: FileChanges) => f.added.length + f.removed.length + f.changed.length > 0;
+
+function criteriaDiff(kind: CriterionDiff["kind"], as: CriterionContent[] = [], bs: CriterionContent[] = []): CriterionDiff[] {
+  const ma = new Map(as.map((c) => [c.id, c])), mb = new Map(bs.map((c) => [c.id, c]));
+  return sorted(new Set([...ma.keys(), ...mb.keys()])).flatMap((id): CriterionDiff[] => {
+    const a = ma.get(id) ?? null, b = mb.get(id) ?? null;
+    if (!a) return [{ id, kind, change: "added", fields: [], a: null, b }];
+    if (!b) return [{ id, kind, change: "removed", fields: [], a, b: null }];
+    const keys = sorted(new Set([...Object.keys(a), ...Object.keys(b)].filter((k) => k !== "id")));
+    const fields = keys.filter((k) => !same((a as any)[k], (b as any)[k])).map((field) => ({ field, a: (a as any)[field] ?? null, b: (b as any)[field] ?? null }));
+    return fields.length ? [{ id, kind, change: "changed", fields, a, b }] : [];
+  });
+}
+
+function engineDiff(sa: Partial<GradingContent>, sb: Partial<GradingContent>): EngineChange[] {
+  const ea = engineOf(sa), eb = engineOf(sb), out: EngineChange[] = [];
+  const both = (part: EngineChange["part"], a: number | string | undefined, b: number | string | undefined, name: string | null = null) => {
+    if (a !== undefined && b !== undefined && a !== b) out.push({ part, name, a, b });
+  };
+  both("instructions", ea.instructions, eb.instructions);
+  for (const t of sorted(Object.keys(ea.checks))) both("check", ea.checks[t], eb.checks[t], t);
+  both("derived", ea.derived, eb.derived);
+  // The text for derived evidence holds the shared wording too, so it counts only when the shared wording is the same.
+  if (ea.instructions === eb.instructions) both("instructions", ea.derivedInstructions, eb.derivedInstructions, "derived");
+  // A judge call revision counts for one harness: with another judge harness, the judge itself is the change.
+  if (sa.judge?.harness === sb.judge?.harness) both("judgeCall", ea.judgeCall, eb.judgeCall, sa.judge?.harness ?? null);
+  both("judging", ea.judging, eb.judging);
+  both("scoring", ea.scoring, eb.scoring);
+  // Two snapshots of before the parts read as the same first values, so their one number is compared as it is.
+  // Not when only one side has derived evidence: its number is 2 for that reason, and the criterion is the change.
+  if (!sa.engine && !sb.engine && (ea.derived === undefined) === (eb.derived === undefined)) both("revision", sa.graderRev, sb.graderRev);
+  return out;
+}
+
+const judgeOf = (s: GradingSetupRecord): JudgeInfo => ({ harness: s.judgeHarness, model: s.judgeModel, effort: s.judgeEffort, label: s.judgeLabel });
+
+/** The difference between grading versions a and b, from their stored snapshots only. */
+export function gradingDiff(a: StoredVersion, b: StoredVersion): GradingDiff {
+  const sa = a.setup.snapshot as unknown as Partial<GradingContent>, sb = b.setup.snapshot as unknown as Partial<GradingContent>;
+
+  const criteria = [...criteriaDiff("gate", sa.gates, sb.gates), ...criteriaDiff("criterion", sa.criteria, sb.criteria)];
+  const addedOrRemoved = new Set(criteria.filter((c) => c.kind === "criterion" && c.change !== "changed").map((c) => c.id));
+
+  const ma = new Map((sa.measures ?? []).map((m) => [m.id, m])), mb = new Map((sb.measures ?? []).map((m) => [m.id, m]));
+  const measures = sorted(new Set([...ma.keys(), ...mb.keys()])).flatMap((id): MeasureDiff[] => {
+    const x = ma.get(id), y = mb.get(id);
+    const ca = new Set(x?.criteria ?? []), cb = new Set(y?.criteria ?? []);
+    const crit = { added: sorted([...cb].filter((c) => !ca.has(c))), removed: sorted([...ca].filter((c) => !cb.has(c))) };
+    const weight = x?.weight === y?.weight ? null : { a: x?.weight ?? null, b: y?.weight ?? null };
+    if (x && y && !weight && !crit.added.length && !crit.removed.length) return [];
+    const change = !x ? "added" : !y ? "removed" : "changed";
+    const byCriteriaOnly = change === "changed" && !weight && [...crit.added, ...crit.removed].every((c) => addedOrRemoved.has(c));
+    return [{ id, change, weight, criteria: crit, byCriteriaOnly }];
+  });
+
+  const context = same(sa.context, sb.context) ? null : { a: sa.context ?? null, b: sb.context ?? null };
+  const ja = judgeOf(a.setup), jb = judgeOf(b.setup);
+  const judge = ja.harness === jb.harness && ja.model === jb.model && ja.effort === jb.effort ? null : { a: ja, b: jb };
+  const engine = engineDiff(sa, sb);
+
+  let task: GradingDiff["task"] = null;
+  if (a.setup.taskHash !== b.setup.taskHash) {
+    const ta = a.task?.snapshot as Partial<TaskContent> | undefined, tb = b.task?.snapshot as Partial<TaskContent> | undefined;
+    const both = !!ta && !!tb;
+    task = {
+      a: a.setup.taskHash, b: b.setup.taskHash, label: { a: a.task?.label ?? null, b: b.task?.label ?? null },
+      prompt: both && ta!.prompt !== tb!.prompt, output: both && ta!.output !== tb!.output,
+      inputs: both ? fileChanges(ta!.inputs, tb!.inputs) : fileChanges(),
+      project: both ? fileChanges(ta!.project, tb!.project) : fileChanges(),
+    };
+  }
+
+  const files = {
+    references: fileChanges(sa.references, sb.references),
+    sources: fileChanges(sa.sources, sb.sources),
+    grader: fileChanges(sa.graderFiles, sb.graderFiles),
+    heldOut: (sa.heldOut ?? null) !== (sb.heldOut ?? null),
+  };
+
+  const otherThanCriteria = measures.some((m) => !m.byCriteriaOnly) || !!context || !!judge || engine.length > 0 || !!task
+    || anyFile(files.references) || anyFile(files.sources) || anyFile(files.grader) || files.heldOut;
+  const isSame = !criteria.length && !measures.length && !otherThanCriteria;
+  return { same: isSame, criteriaOnly: !isSame && !otherThanCriteria, criteria, measures, context, judge, engine, task, files };
+}
+
+const stored = (store: Store, setup: GradingSetupRecord): StoredVersion => ({ setup, task: store.getTaskVersion(setup.taskHash) });
+
+/** The difference between two stored grading versions (full hashes). Throws when either is not stored. */
+export function diffGradingVersions(store: Store, a: string, b: string): GradingDiff {
+  const get = (h: string) => store.getGradingSetup(h) ?? (() => { throw new Error(`no grading version ${h}`); })();
+  return gradingDiff(stored(store, get(a)), stored(store, get(b)));
+}
+
+// ---- Names -------------------------------------------------------------------------------------
+
+export const ORIGINAL = "Original grading";
+
+const count = (n: number, one: string, many: string) => `${n} ${n === 1 ? one : many}`;
+const fileName = (path: string) => path.replace(/^grader\//, "");
+
+/** The parts of a name, most telling first: what changed in `diff`, in short plain words. */
+export function nameParts(diff: GradingDiff): string[] {
+  const parts: string[] = [];
+  // Criteria and gates: grouped by what happened to them.
+  const group = (label: string, ids: string[], one: string, many: string) => {
+    if (ids.length) parts.push(`${count(ids.length, one, many)} ${label}${ids.length === 1 ? `: ${ids[0]}` : ""}`);
+  };
+  for (const kind of ["criterion", "gate"] as const) {
+    const [one, many] = kind === "gate" ? ["gate", "gates"] : ["criterion", "criteria"];
+    const of = (change: CriterionDiff["change"], only?: string) => diff.criteria
+      .filter((c) => c.kind === kind && c.change === change && (only === undefined || (c.fields.length === 1 && c.fields[0]!.field === only)))
+      .map((c) => c.id);
+    const reworded = of("changed", "question"), checks = of("changed", "check");
+    const other = of("changed").filter((id) => !reworded.includes(id) && !checks.includes(id));
+    group("reworded", reworded, kind === "gate" ? "gate question" : "question", kind === "gate" ? "gate questions" : "questions");
+    group("changed", checks, kind === "gate" ? "gate check" : "check", kind === "gate" ? "gate checks" : "checks");
+    group("changed", other, one, many);
+    group("added", of("added"), one, many);
+    group("removed", of("removed"), one, many);
+  }
+  // A weight change, where the measure's criteria list changed only through criteria added or removed (named above).
+  const addedOrRemoved = new Set(diff.criteria.filter((c) => c.kind === "criterion" && c.change !== "changed").map((c) => c.id));
+  const weights = diff.measures.filter((m) => m.change === "changed" && m.weight
+    && [...m.criteria.added, ...m.criteria.removed].every((c) => addedOrRemoved.has(c))).map((m) => m.id);
+  group("changed", weights, "weight", "weights");
+  if (diff.measures.some((m) => !m.byCriteriaOnly && !weights.includes(m.id))) parts.push("measures changed");
+  if (diff.context) parts.push("context changed");
+  const g = diff.files.grader, scripts = [...g.added, ...g.removed, ...g.changed];
+  if (scripts.length === 1) parts.push(`check script ${g.added.length ? "added" : g.removed.length ? "removed" : "changed"}: ${fileName(scripts[0]!)}`);
+  else if (scripts.length > 1) parts.push(`${scripts.length} check scripts changed`);
+  if (anyFile(diff.files.references)) parts.push("references changed");
+  if (anyFile(diff.files.sources)) parts.push("sources changed");
+  if (diff.files.heldOut) parts.push("held-out original changed");
+  if (diff.task) parts.push("task files changed");
+  if (diff.judge) parts.push("judge changed");
+  parts.push(...new Set(diff.engine.map(engineChangeName)));
+  return parts;
+}
+
+/** A name from its parts: at most three parts, else the first two and how many more. Starts with a capital. */
+export function nameFrom(parts: string[]): string {
+  const shown = parts.length > 3 ? [...parts.slice(0, 2), `${parts.length - 2} more changes`] : parts;
+  const text = shown.join(", ") || "No change";
+  return text[0]!.toUpperCase() + text.slice(1);
+}
+
+/** A grading version's name, with the parts it is built from and the version before it. */
+export interface VersionName {
+  name: string;
+  /** What changed since `previous`, in short words; empty for the original grading. */
+  nameParts: string[];
+  /** Full hash of the version before it for the same task and judge, or null when it is the first. */
+  previous: string | null;
+  /** The difference from `previous` to this version, or null when it is the first. */
+  diff: GradingDiff | null;
+}
+
+const sameJudge = (x: GradingSetupRecord, y: GradingSetupRecord) =>
+  x.judgeHarness === y.judgeHarness && x.judgeModel === y.judgeModel && x.judgeEffort === y.judgeEffort;
+
+/**
+ * The name of every stored grading version of a task, by full hash. The version before one is the one with the latest
+ * first_seen strictly earlier than its own, for the same task and judge (harness, model, effort); among versions first
+ * seen at that same time, the last in stored order (first_seen, then hash). With none, it is the "Original grading".
+ */
+export function versionNames(store: Store, taskId: string): Map<string, VersionName> {
+  const all = store.listGradingSetups(taskId);
+  const tasks = new Map<string, TaskVersionRecord | null>();
+  const of = (s: GradingSetupRecord): StoredVersion => {
+    if (!tasks.has(s.taskHash)) tasks.set(s.taskHash, store.getTaskVersion(s.taskHash));
+    return { setup: s, task: tasks.get(s.taskHash)! };
+  };
+  const out = new Map<string, VersionName>();
+  all.forEach((s, i) => {
+    const prev = all.slice(0, i).reverse().find((p) => p.firstSeen < s.firstSeen && sameJudge(p, s));
+    if (!prev) { out.set(s.hash, { name: ORIGINAL, nameParts: [], previous: null, diff: null }); return; }
+    const diff = gradingDiff(of(prev), of(s));
+    const parts = nameParts(diff);
+    out.set(s.hash, { name: nameFrom(parts), nameParts: parts, previous: prev.hash, diff });
+  });
+  return out;
+}
+
+/** A stored grading version as the dashboard and CLI list it: its name, the judge and the engine parts it uses, without the snapshot. */
+export type NamedGradingSetup = Pick<GradingSetupRecord, "hash" | "judgeHarness" | "judgeModel" | "judgeEffort" | "judgeLabel" | "firstSeen"> & {
+  /** The engine revision parts this version uses (a version of before the parts reads as their first values). */
+  engine: EngineParts;
+  /** The grading version name, for example "1 question reworded: subject-length". */
+  name: string;
+};
+
+/** `s` with its name from `names` (versionNames of its task). Throws when `names` has no entry for it. */
+export function namedSetup(s: GradingSetupRecord, names: Map<string, VersionName>): NamedGradingSetup {
+  const name = names.get(s.hash)?.name;
+  if (name === undefined) throw new Error(`grading version ${s.hash} has no name among its task's grading versions`);
+  const { hash, judgeHarness, judgeModel, judgeEffort, judgeLabel, firstSeen } = s;
+  return { hash, judgeHarness, judgeModel, judgeEffort, judgeLabel, engine: engineOf(s.snapshot as Partial<GradingContent>), firstSeen, name };
+}

package/src/core/identity.ts

@@ -0,0 +1,263 @@
+// Content identity: the hashes that decide whether two trials are comparable, and the config key.
+// Task hash = what the agent was given. Grading hash = how an output was graded. Both are SHA-256 over
+// canonical JSON, so only content counts: never a file name, the order of criteria, or the `version` label.
+import { createHash } from "node:crypto";
+import { existsSync, readdirSync, readFileSync, statSync } from "node:fs";
+import { join, relative, sep } from "node:path";
+import { ENGINE, FIRST_INSTRUCTIONS, FIRST_REVISIONS, type EngineParts, type EngineRevisions } from "./engine";
+import { instructionWording } from "./grade";
+import { ENGINE_ROOT } from "./project";
+import type { AgentConfig, Check, Criterion, Evidence, JudgeConfig, Task } from "./types";
+
+export const sha256 = (data: string | Uint8Array) => createHash("sha256").update(data).digest("hex");
+
+/** JSON with object keys sorted at every level and undefined values left out. */
+export function canonicalJson(value: unknown): string {
+  return JSON.stringify(value, (_k, v) =>
+    v && typeof v === "object" && !Array.isArray(v)
+      ? Object.fromEntries(Object.keys(v).sort().filter((k) => v[k] !== undefined).map((k) => [k, v[k]]))
+      : v);
+}
+
+type FileHashes = Record<string, string>;
+
+const noDotNames = (name: string) => !name.startsWith(".");
+
+/**
+ * Every file under `dir` (a file or a folder), keyed by its path relative to `root` with "/" separators,
+ * valued by the SHA-256 of its bytes. Names that start with "." (for example .DS_Store) are left out,
+ * unless `keep` says otherwise.
+ */
+function fileHashes(dir: string, root = dir, keep = noDotNames): FileHashes {
+  const out: FileHashes = {};
+  const walk = (p: string) => {
+    if (statSync(p).isFile()) { out[relative(root, p).split(sep).join("/")] = sha256(readFileSync(p)); return; }
+    for (const f of readdirSync(p).sort()) if (keep(f)) walk(join(p, f));
+  };
+  if (existsSync(dir)) walk(dir);
+  return out;
+}
+
+/**
+ * One hash over every file in a folder (path → content hash), dot-folders included, because a Profile
+ * snapshot keeps its skills in .claude/, .codex/ and .agents/. Only .DS_Store is left out.
+ */
+export const folderHash = (dir: string) => sha256(canonicalJson(fileHashes(dir, dir, (f) => f !== ".DS_Store")));
+
+const clean = (p: string) => p.replace(/^\.\//, "").replace(/\/+$/, "");
+
+// ---- Task hash ---------------------------------------------------------------------------------
+
+/** What the agent was given. Stored as a task version's snapshot. */
+export interface TaskContent {
+  prompt: string;
+  output: string;
+  inputs: FileHashes;     // path as the agent sees it in the workspace (inputs/brief.md) → hash
+  project: FileHashes;    // path from the workspace root, where the project is copied → hash
+}
+
+type TaskFields = Pick<Task, "prompt" | "output" | "inputs" | "project" | "dir">;
+
+export function taskIdentity(task: TaskFields): { hash: string; snapshot: TaskContent } {
+  const snapshot: TaskContent = {
+    prompt: task.prompt,
+    output: task.output,
+    inputs: fileHashes(join(task.dir, task.inputs), task.dir),
+    project: task.project ? fileHashes(join(task.dir, task.project)) : {},
+  };
+  return { hash: sha256(canonicalJson(snapshot)), snapshot };
+}
+
+export const taskHash = (task: TaskFields) => taskIdentity(task).hash;
+
+// ---- Grading hash ------------------------------------------------------------------------------
+
+/** A criterion as far as it can change a score: no `level`. Defaults are filled in so equal behaviour hashes equal. */
+export interface CriterionContent {
+  id: string;
+  question: string;
+  grader: Criterion["grader"];
+  check?: Check;
+  evidence: Evidence[];   // sorted; default ["output"]
+  isolate: boolean;
+  derived?: string;       // the derived evidence script, a path under grader/; left out when none, so such a criterion hashes as before
+}
+
+/** How an output was graded. Stored as a grading setup's snapshot. */
+export interface GradingContent {
+  taskHash: string;
+  context?: string;
+  gates: CriterionContent[];                                       // sorted by id
+  criteria: CriterionContent[];                                    // measure criteria, sorted by id
+  measures: { id: string; weight: number; criteria: string[] }[];  // sorted by id; no label
+  references: FileHashes;
+  sources: FileHashes;
+  heldOut: string | null;
+  graderFiles: FileHashes;  // the task's grader/ folder (helper scripts its command checks run), from the task folder
+  judge: { harness: string; model: string; effort?: string };     // no id, label or vote count
+  /**
+   * The engine revision, in one of two forms. `engine`: the parts this task uses. `graderRev`: the form of before
+   * the parts, kept while every part the task uses is at its first value (1 without derived evidence, 2 with it),
+   * so the gradings stored under it stay current. A snapshot has exactly one of the two.
+   */
+  engine?: EngineParts;
+  graderRev?: number;
+}
+
+/** The engine as a grading hash reads it: the hand-raised revisions and the hashes of the judge instruction wording. */
+export interface EngineState { revisions: EngineRevisions; instructions: { shared: string; derived: string } }
+
+export function currentEngine(): EngineState {
+  const w = instructionWording();
+  return { revisions: ENGINE, instructions: { shared: sha256(w.shared), derived: sha256(w.derived) } };
+}
+
+/** The first value of every part (engine.ts). */
+export const FIRST_ENGINE: EngineState = { revisions: FIRST_REVISIONS, instructions: FIRST_INSTRUCTIONS };
+
+type Graded = Pick<GradingContent, "gates" | "criteria" | "judge">;
+const usesDerived = (g: Pick<Graded, "gates" | "criteria">) => [...g.gates, ...g.criteria].some((c) => c.derived);
+
+/**
+ * The parts of `engine` that a task with these gates, criteria and judge uses. A check type or judge harness that
+ * `engine` has no revision for is an error, unless `stored`: a stored snapshot is read as it is, at revision 1.
+ */
+function engineParts(g: Graded, engine: EngineState, stored = false): EngineParts {
+  const all = [...g.gates, ...g.criteria], r = engine.revisions;
+  // The output gate that every grading adds is a command check.
+  const types = [...new Set(["command", ...all.flatMap((c) => (c.grader === "check" && c.check ? [c.check.type] : []))])].sort();
+  const judged = all.some((c) => c.grader === "judge"), derived = usesDerived(g);
+  const rev = (of: Record<string, number>, key: string) => {
+    if (of[key] === undefined && !stored) throw new Error(`the engine revision has no part for ${JSON.stringify(key)}: add it to ENGINE and FIRST_REVISIONS in src/core/engine.ts`);
+    return of[key] ?? 1;
+  };
+  return {
+    checks: Object.fromEntries(types.map((t) => [t, rev(r.checks, t)])),
+    scoring: r.scoring,
+    ...(judged ? { instructions: engine.instructions.shared, judgeCall: rev(r.judgeCall, g.judge.harness), judging: r.judging } : {}),
+    ...(derived ? { derived: r.derived, derivedInstructions: engine.instructions.derived } : {}),
+  };
+}
+
+/**
+ * The engine parts a stored snapshot stands for. A snapshot in the `graderRev` form reads as the first value of
+ * each part its task uses.
+ */
+export function engineOf(snapshot: Partial<GradingContent>): EngineParts {
+  if (snapshot.engine) return snapshot.engine;
+  return engineParts({ gates: snapshot.gates ?? [], criteria: snapshot.criteria ?? [], judge: snapshot.judge ?? { harness: "", model: "" } }, FIRST_ENGINE, true);
+}
+
+type GradedTask = TaskFields & Pick<Task, "context" | "gates" | "measures" | "references" | "sources" | "heldOut">;
+
+const byId = <T extends { id: string }>(xs: T[]) => [...xs].sort((a, b) => (a.id < b.id ? -1 : a.id > b.id ? 1 : 0));
+
+const criterionContent = (c: Criterion): CriterionContent => ({
+  id: c.id, question: c.question, grader: c.grader, check: c.check,
+  evidence: [...new Set<Evidence>(c.evidence ?? ["output"])].sort(),
+  isolate: c.isolate ?? false,
+  ...(c.derived ? { derived: c.derived } : {}),
+});
+
+/** `engine` is for tests only; every caller uses the current engine. */
+export function gradingIdentity(task: GradedTask, judge: Pick<JudgeConfig, "harness" | "model" | "effort">, engine: EngineState = currentEngine()): { hash: string; snapshot: GradingContent } {
+  const content: Omit<GradingContent, "engine" | "graderRev"> = {
+    taskHash: taskHash(task),
+    context: task.context,
+    gates: byId(task.gates).map(criterionContent),
+    criteria: byId(task.measures.flatMap((m) => m.criteria)).map(criterionContent),
+    measures: byId(task.measures).map((m) => ({ id: m.id, weight: m.weight ?? 1, criteria: m.criteria.map((c) => c.id).sort() })),
+    references: task.references ? fileHashes(join(task.dir, task.references)) : {},
+    sources: task.sources ? fileHashes(join(task.dir, task.sources)) : {},
+    heldOut: task.heldOut && existsSync(join(task.dir, task.heldOut)) ? sha256(readFileSync(join(task.dir, task.heldOut))) : null,
+    graderFiles: fileHashes(join(task.dir, "grader"), task.dir),
+    judge: { harness: judge.harness, model: judge.model, effort: judge.effort },
+  };
+  const parts = engineParts(content, engine);
+  const first = canonicalJson(parts) === canonicalJson(engineParts(content, FIRST_ENGINE));
+  const snapshot: GradingContent = { ...content, ...(first ? { graderRev: usesDerived(content) ? 2 : 1 } : { engine: parts }) };
+  return { hash: sha256(canonicalJson(snapshot)), snapshot };
+}
+
+export const gradingHash = (task: GradedTask, judge: Pick<JudgeConfig, "harness" | "model" | "effort">) => gradingIdentity(task, judge).hash;
+
+// ---- Config key --------------------------------------------------------------------------------
+
+/**
+ * A config's identity, from its fields and not its file name, so renaming configs/x.yaml keeps its history.
+ * `harness|model|effort|profile|provider` with an empty slot for an unset field; a human config is `human:<id>`.
+ */
+export function configKey(c: Pick<AgentConfig, "id" | "harness" | "model" | "effort" | "profile" | "provider">): string {
+  if (c.harness === "human") return `human:${c.id}`;
+  return [c.harness, c.model, c.effort, c.profile, c.provider].map((v) => v ?? "").join("|");
+}
+
+// ---- Engine pin --------------------------------------------------------------------------------
+
+const CHECKS = "the revision of each check type the change affects (ENGINE.checks)";
+const CALLS = "the judge call revision of each harness the change affects (ENGINE.judgeCall)";
+
+/**
+ * The files whose code can move a score, each with the revision to raise when its code outside any marked region
+ * changes (null: that code is not pinned). tests/identity.test.ts pins their hashes.
+ */
+export const ENGINE_FILES: Record<string, string | null> = {
+  "src/adapters/index.ts": null,
+  "src/core/checks.ts": CHECKS,
+  "src/core/derived.ts": "ENGINE.derived",
+  "src/core/grade.ts": "ENGINE.judging (or ENGINE.derived or ENGINE.checks.command, when the change is to how a derived evidence script or the output gate runs)",
+  "src/core/judge-schema.ts": CALLS,
+  "src/core/judge.ts": CALLS,
+  "src/core/score.ts": "ENGINE.scoring",
+};
+
+/**
+ * Marked regions: the lines between `// engine:<region>:begin` and `// engine:<region>:end` belong to one part.
+ * A region can be marked more than once in a file. Each has the revision to raise when its code changes.
+ */
+export const ENGINE_REGIONS: Record<string, string> = {
+  "check.regex": "ENGINE.checks.regex",
+  "check.words": "ENGINE.checks.words",
+  "check.frontmatter": "ENGINE.checks.frontmatter",
+  "check.command": "ENGINE.checks.command",
+  "judgeCall": CALLS,
+  "judgeCall.claude": "ENGINE.judgeCall.claude",
+  "judgeCall.codex": "ENGINE.judgeCall.codex",
+  "judgeCall.agy": "ENGINE.judgeCall.agy",
+  "judging": "ENGINE.judging",
+  "instructions": "nothing: the grading hash holds a hash of the instructions, with where the context, assignment, evidence and criteria go",
+};
+
+const MARKER = /^[ \t]*\/\/ engine:([A-Za-z.]+):(begin|end)\b.*$/;
+
+/**
+ * One hash per pinned piece of engine code: `<file>#<region>` for a marked region and `<file>` for the rest of a
+ * file. `edit` lets a test change a file's text in memory.
+ */
+export function engineSourceHashes(edit: (path: string, text: string) => string = (_p, t) => t): Record<string, string> {
+  const out: Record<string, string> = {};
+  for (const [file, rest] of Object.entries(ENGINE_FILES)) {
+    const regions = new Map<string, string[]>(), outside: string[] = [];
+    let open: string | null = null;
+    for (const line of edit(file, readFileSync(join(ENGINE_ROOT, file), "utf8")).split("\n")) {
+      const m = line.match(MARKER);
+      if (!m) { (open ? regions.get(open)! : outside).push(line); continue; }
+      const [, region, edge] = m as unknown as [string, string, string];
+      if (!(region in ENGINE_REGIONS)) throw new Error(`${file}: unknown engine region ${region}`);
+      if (edge === "begin" ? open !== null : open !== region) throw new Error(`${file}: the engine:${region}:${edge} line does not match the one before it`);
+      open = edge === "begin" ? region : null;
+      if (open && !regions.has(open)) regions.set(open, []);
+    }
+    if (open) throw new Error(`${file}: the engine:${open}:end line is missing`);
+    if (rest === null && !regions.size) throw new Error(`${file}: it has no engine region, so nothing of it is pinned`);
+    for (const [region, lines] of regions) out[`${file}#${region}`] = sha256(lines.join("\n"));
+    if (rest !== null) out[file] = sha256(outside.join("\n"));
+  }
+  return out;
+}
+
+/** What to raise when the pinned piece `key` (of engineSourceHashes) changed in a way that can move a score. */
+export function raiseFor(key: string): string {
+  const [file, region] = key.split("#") as [string, string?];
+  return (region ? ENGINE_REGIONS[region] : ENGINE_FILES[file]) ?? "(unknown)";
+}