simt-dev 0.6.3 → 0.7.1

package/src/bench/run.mjs

@@ -1,22 +1,21 @@
-import { spawn } from 'node:child_process';
 import { existsSync } from 'node:fs';
 import { lstat, mkdir, mkdtemp, readdir, readFile, rm, stat, unlink, writeFile } from 'node:fs/promises';
 import { tmpdir } from 'node:os';
 import { dirname, join, resolve, sep } from 'node:path';
 import { fileURLToPath } from 'node:url';
 
+import { assembleBrief } from '../brief/assemble.mjs';
+import { DEFAULT_BRIEF_BYTES } from '../brief/packet.mjs';
 import { loadRegistry } from '../graph/load.mjs';
 import { extractOpenedSkills, matchSkillsForPaths } from '../graph/compile.mjs';
-import { matchGlob } from '../index/govern.mjs';
-import { formatLessonParagraph, listPromotedLessons } from '../memory/lessons.mjs';
 import { MANIFEST_FILE, STATE_DIR } from '../name.mjs';
+import { LOCAL_DIR } from '../state.mjs';
 import { readManifest } from '../projection/manifest.mjs';
 import { plan } from '../projection/project.mjs';
 import { parse as parseRegion } from '../projection/markers.mjs';
-import { fingerprint } from '../resolve/fingerprint.mjs';
-import { rankSkills, tokenize } from '../resolve/resolve.mjs';
 import { gradeWorkspace } from './grade.mjs';
 import { resolveVaultDir } from './mine.mjs';
+import { spawnBounded } from './process.mjs';
 import { loadTasks } from './tasks.mjs';
 import { ablateSectionInWorkspace, parseAblationArmName } from './ablate.mjs';
 import { applyVariantInWorkspace, parseVariantArmName } from './variants.mjs';
@@ -53,48 +52,80 @@ function sanitizedEnv(extra = {}) {
   return env;
 }
 
-function runCmd(cmd, args, { cwd, env, input = null, timeoutMs = DEFAULT_AGENT_TIMEOUT_MS } = {}) {
-  return new Promise((resolveOutcome) => {
-    const child = spawn(cmd, args, {
-      cwd,
-      env: sanitizedEnv(env),
-      stdio: ['pipe', 'pipe', 'pipe'],
-    });
-    const out = [];
-    const err = [];
-    let timedOut = false;
-    const timer =
-      timeoutMs > 0
-        ? setTimeout(() => {
-            timedOut = true;
-            child.kill('SIGKILL');
-          }, timeoutMs)
-        : null;
-
-    child.stdout.on('data', (chunk) => out.push(chunk));
-    child.stderr.on('data', (chunk) => err.push(chunk));
-    child.on('error', (error) => {
-      if (timer) clearTimeout(timer);
-      resolveOutcome({
-        code: 1,
-        timedOut,
-        stdout: Buffer.concat(out),
-        stderr: Buffer.from(String(error.message ?? error)),
-      });
-    });
-    child.on('close', (code) => {
-      if (timer) clearTimeout(timer);
-      resolveOutcome({
-        code: timedOut ? 124 : (code ?? 1),
-        timedOut,
-        stdout: Buffer.concat(out),
-        stderr: Buffer.concat(err),
-      });
-    });
+/**
+ * What every benchmarked agent is allowed to inherit from the operator's shell.
+ *
+ * `PATH` and the locale are what a CLI needs to run at all. Nothing here names a
+ * credential or a state directory: those come from the adapter's own `env` list.
+ */
+const ENV_BASELINE = [
+  'PATH',
+  'LANG',
+  'LC_ALL',
+  'LC_CTYPE',
+  'TZ',
+  'TERM',
+  'SHELL',
+  'TMPDIR',
+  'TEMP',
+  'TMP',
+  'SystemRoot',
+  'WINDIR',
+  'USER',
+];
+
+/** The isolation levels a run may record; `sandbox` is declared, not implemented. */
+const ISOLATION_LEVELS = new Set(['none', 'env', 'sandbox']);
 
-    if (input !== null) child.stdin.end(input);
-    else child.stdin.end();
-  });
+/**
+ * The environment an arm's agent sees: the baseline, the adapter's declared keys,
+ * and nothing else.
+ *
+ * The agent used to inherit everything the operator had — `SIMT_DEV_VAULT_DIR`,
+ * `XDG_CACHE_HOME`, `HOME`, and every credential in the shell. A run that can read
+ * the vault can read the hidden tests it is graded against, and a run sharing the
+ * operator's HOME writes into their own configuration, so the number it produced
+ * described the machine rather than the arm (review 0003 Q49). `home` is the run's
+ * own slot, which is removed when the run ends.
+ *
+ * @param {object} adapter an entry from `adapters.json`
+ * @param {{home?: string, extra?: Record<string, string>, base?: NodeJS.ProcessEnv}} [options]
+ * @returns {Record<string, string>}
+ */
+export function agentEnv(adapter, { home, extra = {}, base = process.env } = {}) {
+  const allowed = new Set([...ENV_BASELINE, ...(adapter?.env ?? [])]);
+  const env = {};
+  for (const key of allowed) {
+    if (typeof base[key] === 'string') env[key] = base[key];
+  }
+  for (const [key, value] of Object.entries(extra ?? {})) {
+    if (typeof value === 'string') env[key] = value;
+  }
+  if (typeof home === 'string' && home.length > 0) env.HOME = home;
+  env.SIMT_DEV_WORKSPACE = 'off';
+  for (const key of Object.keys(env)) {
+    if (key.startsWith('NODE_TEST')) delete env[key];
+  }
+  return env;
+}
+
+/**
+ * The isolation an adapter's runs provide, as the record states it.
+ *
+ * `none` is a built-in that scripts the workspace, `env` is a CLI driven with a
+ * scrubbed environment and a slot-local HOME, and `sandbox` is reserved for a
+ * real filesystem sandbox — nothing claims it yet, so a record never says it.
+ *
+ * @param {object} adapter
+ * @returns {'none'|'env'|'sandbox'}
+ */
+export function adapterIsolation(adapter) {
+  const declared = adapter?.isolation;
+  return ISOLATION_LEVELS.has(declared) ? declared : 'none';
+}
+
+function runCmd(cmd, args, { cwd, env, input = null, timeoutMs = DEFAULT_AGENT_TIMEOUT_MS } = {}) {
+  return spawnBounded(cmd, args, { cwd, env: sanitizedEnv(env), input, timeoutMs });
 }
 
 function getByPath(obj, pathStr) {
@@ -119,7 +150,7 @@ function firstNumeric(obj, paths = []) {
 function emptyUsage() {
   return {
     usage: { input: 0, cacheWrite: 0, cacheRead: 0, output: 0, turns: 0, toolCalls: 0 },
-    costUsd: 0.0,
+    costUsd: null,
   };
 }
 
@@ -163,7 +194,7 @@ function parseJsonObject(text) {
  * @param {string|object} adapterOrId
  * @param {string} stdoutText
  * @param {string} [analyticsText]
- * @returns {{usage: {input: number, cacheWrite: number, cacheRead: number, output: number, turns: number, toolCalls: number}, costUsd: number}}
+ * @returns {{usage: {input: number, cacheWrite: number, cacheRead: number, output: number, turns: number, toolCalls: number}, costUsd: number|null}}
  */
 export function parseAdapterUsage(adapterOrId, stdoutText = '', analyticsText = '') {
   const result = emptyUsage();
@@ -192,7 +223,7 @@ export function parseAdapterUsage(adapterOrId, stdoutText = '', analyticsText =
     result.usage.output = firstNumeric(obj, fields.output) ?? 0;
     result.usage.turns = firstNumeric(obj, fields.turns) ?? 0;
     result.usage.toolCalls = firstNumeric(obj, fields.toolCalls) ?? 0;
-    result.costUsd = firstNumeric(obj, fields.costUsd) ?? 0.0;
+    result.costUsd = firstNumeric(obj, fields.costUsd) ?? null;
 
     // Gemini CLI also nests per-model token totals under `stats.models.<name>.tokens`.
     const models = obj?.stats?.models;
@@ -265,7 +296,7 @@ export function parseAdapterUsage(adapterOrId, stdoutText = '', analyticsText =
     }
   }
 
-  result.costUsd = Number(result.costUsd.toFixed(6));
+  if (result.costUsd !== null) result.costUsd = Number(result.costUsd.toFixed(6));
   return result;
 }
 
@@ -467,55 +498,29 @@ async function applyProjectionMode(workDir, mode) {
   }
 }
 
-async function writeTaskBrief(workDir, task) {
-  const prompt = String(task?.prompt ?? '');
-  const taskTokens = tokenize(prompt);
-  const taskTokenSet = new Set(taskTokens);
-  const [registry, fp, lessons] = await Promise.all([
-    loadRegistry().catch(() => ({ skills: [] })),
-    fingerprint(workDir).catch(() => ({ stacks: [], facts: new Set() })),
-    listPromotedLessons(workDir).catch(() => []),
-  ]);
-  const skillsList = Array.isArray(registry.skills)
-    ? registry.skills
-    : Object.values(registry.skills ?? {});
-  const { selected = [] } = rankSkills(skillsList, {
-    taskTokens,
-    paths: [],
-    repoStack: fp.stack ?? [],
-  });
-  const ranked = selected.slice(0, 5);
-
-  const taskPaths = Array.isArray(task?.paths) ? task.paths : [];
-  const matchingLessons = lessons.filter((lesson) => {
-    const lessonGlobs = Array.isArray(lesson.paths) ? lesson.paths : [];
-    if (taskPaths.some((p) => lessonGlobs.some((glob) => matchGlob(glob, p)))) return true;
-    if (lessonGlobs.some((glob) => prompt.includes(glob.replace(/\/\*\*$/, '')))) return true;
-    const kw = Array.isArray(lesson.keywords) ? lesson.keywords : [];
-    return kw.some((k) => taskTokenSet.has(String(k).toLowerCase()));
-  });
-
-  const lines = [
-    `# Task Brief: ${task?.id ?? 'task'}`,
-    '',
-    `Prompt: ${prompt}`,
-    '',
-    '## Ranked Skills',
-    ...(ranked.length > 0
-      ? ranked.map((r) => `- ${r.skill.name} (score ${r.score}): ${r.skill.description ?? ''}`)
-      : ['- (none matched)']),
-    ...(matchingLessons.length > 0
-      ? [
-          '',
-          '## Lessons',
-          ...matchingLessons.map((l) => `- ${formatLessonParagraph(l)}`),
-        ]
-      : []),
-    '',
-  ];
-  const briefPath = join(workDir, STATE_DIR, 'brief.md');
-  await mkdir(dirname(briefPath), { recursive: true });
-  await writeFile(briefPath, lines.join('\n'), 'utf8');
+/**
+ * The `brief` arm's context: the packet `simt brief` delivers, assembled by the same
+ * function the command and the prompt-submit hook call.
+ *
+ * The arm used to re-implement the packet from `registry.skills` — a property the
+ * registry does not have, so every run reported "(none matched)" while the real
+ * command resolved skills — and wrote it to a file no agent was told to read. An arm
+ * meant to measure the packet measured a different artefact (review 0003 Q49).
+ */
+async function buildBriefPrefix(workDir, task) {
+  // A packet is built from git's answer; a directory that is not a repository has none,
+  // and asking would print a failure into a bench log for no measurement.
+  if (!existsSync(join(workDir, '.git'))) return '';
+  const assembled = await assembleBrief({
+    root: workDir,
+    task: String(task?.prompt ?? ''),
+    paths: '',
+    budget: DEFAULT_BRIEF_BYTES,
+    build: true,
+    federate: false,
+  }).catch(() => null);
+  if (!assembled || assembled.error || assembled.cold) return '';
+  return assembled.packed?.text ?? '';
 }
 
 /**
@@ -524,9 +529,10 @@ async function writeTaskBrief(workDir, task) {
  * @param {string} workDir
  * @param {{name: string, kind?: string, base?: string|null, write?: Record<string, string>, remove?: string[], env?: Record<string, string>}} armSpec
  * @param {object} [task]
- * @returns {Promise<{env: Record<string, string>}>}
+ * @returns {Promise<{env: Record<string, string>, promptPrefix: string}>}
  */
 export async function materialiseArm(workDir, armSpec, task = {}) {
+  let promptPrefix = '';
   const ablate = armSpec.ablate ?? parseAblationArmName(armSpec.name)?.ablate ?? null;
   const variant = armSpec.variant ?? parseVariantArmName(armSpec.name)?.variant ?? null;
   const base = armSpec.base ?? (BUILTIN_ARMS.has(armSpec.name) ? armSpec.name : null);
@@ -542,7 +548,7 @@ export async function materialiseArm(workDir, armSpec, task = {}) {
     await applyProjectionMode(workDir, 'kernel');
   } else if (base === 'brief') {
     await applyProjectionMode(workDir, 'full');
-    await writeTaskBrief(workDir, task);
+    promptPrefix = await buildBriefPrefix(workDir, task);
   }
 
   for (const relPath of armSpec.remove ?? []) {
@@ -556,7 +562,7 @@ export async function materialiseArm(workDir, armSpec, task = {}) {
     await writeFile(target, String(content), 'utf8');
   }
 
-  return { env: { ...(armSpec.env ?? {}) } };
+  return { env: { ...(armSpec.env ?? {}) }, promptPrefix };
 }
 
 /**
@@ -569,7 +575,7 @@ export async function materialiseArm(workDir, armSpec, task = {}) {
  * @param {object} armSpec
  * @param {object} [task]
  * @param {{parentDir?: string}} [options]
- * @returns {Promise<{slotDir: string, workDir: string, initialHead: string, env: Record<string, string>}>}
+ * @returns {Promise<{slotDir: string, workDir: string, initialHead: string, env: Record<string, string>, promptPrefix: string}>}
  */
 export async function prepareIsolatedWorkspace(root, baseSha, armSpec, task = {}, { parentDir } = {}) {
   const slotDir = await mkdtemp(join(parentDir ?? tmpdir(), 'bench-run-'));
@@ -618,12 +624,18 @@ export async function prepareIsolatedWorkspace(root, baseSha, armSpec, task = {}
     }
   }
 
-  const { env } = await materialiseArm(workDir, armSpec, task);
-
   await runCmd('git', ['init', '-q'], { cwd: workDir });
   await runCmd('git', ['config', 'user.name', 'bench'], { cwd: workDir });
   await runCmd('git', ['config', 'user.email', 'bench@local'], { cwd: workDir });
   await runCmd('git', ['config', 'commit.gpgsign', 'false'], { cwd: workDir });
+
+  // The repository exists before an arm materialises: the `brief` arm assembles its
+  // packet from git's answer, which a directory that is not yet a repository cannot give.
+  const { env, promptPrefix } = await materialiseArm(workDir, armSpec, task);
+  // Assembling a packet writes tool state. It belongs to no arm, and leaving it in the
+  // base commit would make the `brief` workspace differ by an index as well as by context.
+  await rm(join(workDir, LOCAL_DIR), { recursive: true, force: true }).catch(() => {});
+
   await runCmd('git', ['add', '-A'], { cwd: workDir });
   const commitRes = await runCmd(
     'git',
@@ -638,7 +650,7 @@ export async function prepareIsolatedWorkspace(root, baseSha, armSpec, task = {}
   const headRes = await runCmd('git', ['rev-parse', 'HEAD'], { cwd: workDir });
   const initialHead = headRes.stdout.toString('utf8').trim();
 
-  return { slotDir, workDir, initialHead, env };
+  return { slotDir, workDir, initialHead, env, promptPrefix };
 }
 
 function buildAdapterArgs(adapter, { prompt, turns, model, analyticsLog }) {
@@ -656,25 +668,75 @@ function buildAdapterArgs(adapter, { prompt, turns, model, analyticsLog }) {
   return args;
 }
 
-async function executeAdapter(adapter, workDir, task, { vaultRoot, model, maxTurns, env, timeoutMs, slotDir }) {
-  if (adapter.id === 'noop') return { ...parseAdapterUsage(adapter, ''), transcript: '' };
+/**
+ * Apply a patch to a workspace, falling back to a three-way merge.
+ *
+ * The gold oracle and the regrade path land operator-supplied bytes the same way. They
+ * used to disagree (`--binary` on one side only), so a patch that applied under one read
+ * as unreproducible under the other (review 0003 Q59).
+ */
+async function applyPatch(workDir, patch, { timeoutMs = DEFAULT_AGENT_TIMEOUT_MS } = {}) {
+  if (!patch || patch.length === 0) return { code: 0 };
+  const applied = await runCmd('git', ['apply', '--binary', '-'], {
+    cwd: workDir,
+    input: patch,
+    timeoutMs,
+  });
+  if (applied.code === 0) return applied;
+  return runCmd('git', ['apply', '--3way', '-'], { cwd: workDir, input: patch, timeoutMs });
+}
+
+export async function executeAdapter(
+  adapter,
+  workDir,
+  task,
+  { vaultRoot, model, maxTurns = DEFAULT_MAX_TURNS, env, timeoutMs, slotDir, promptPrefix = '' },
+) {
+  if (adapter.id === 'noop') {
+    return {
+      ...parseAdapterUsage(adapter, ''),
+      transcript: '',
+      agent: { exitCode: 0, timedOut: false, wallMs: 0 },
+    };
+  }
   if (adapter.id === 'gold') {
+    const startedAt = Date.now();
     const goldPatch = await readFile(join(vaultRoot, task.id, 'gold.patch'));
-    if (goldPatch.length > 0) {
-      const applyRes = await runCmd('git', ['apply', '-'], { cwd: workDir, input: goldPatch, timeoutMs });
-      if (applyRes.code !== 0) {
-        await runCmd('git', ['apply', '--3way', '-'], { cwd: workDir, input: goldPatch, timeoutMs });
-      }
-    }
-    return { ...parseAdapterUsage(adapter, ''), transcript: '' };
+    await applyPatch(workDir, goldPatch, { timeoutMs });
+    return {
+      ...parseAdapterUsage(adapter, ''),
+      transcript: '',
+      agent: { exitCode: 0, timedOut: false, wallMs: Date.now() - startedAt },
+    };
   }
 
   const analyticsLog = join(slotDir, 'analytics.jsonl');
-  const args = buildAdapterArgs(adapter, { prompt: String(task.prompt ?? ''), turns: maxTurns, model, analyticsLog });
-  const outcome = await runCmd(adapter.bin, args, { cwd: workDir, env, timeoutMs });
+  // The arm's context arrives as a prefix rather than a file the agent must be told to
+  // read, so the prompt an adapter receives is the whole thing it was given. A `gold`
+  // or `noop` run never reaches this line: it scripts the workspace instead.
+  const prompt = [String(promptPrefix ?? ''), String(task.prompt ?? '')]
+    .filter((part) => part.length > 0)
+    .join('\n\n');
+  const args = buildAdapterArgs(adapter, { prompt, turns: maxTurns, model, analyticsLog });
+  const outcome = await runCmd(adapter.bin, args, {
+    cwd: workDir,
+    env: agentEnv(adapter, { home: slotDir, extra: env }),
+    timeoutMs,
+  });
+  // A binary that could not start produced no run. Grading its empty workspace would
+  // record an unresolved arm instead of the missing harness it is (review 0003 Q47).
+  if (outcome.spawnError) {
+    throw new Error(`harness ${adapter.id} could not start ${adapter.bin}: ${outcome.spawnError}`);
+  }
   const stdoutText = outcome.stdout.toString('utf8');
   const analyticsText = adapter.format === 'analytics-log' ? await readFile(analyticsLog, 'utf8').catch(() => '') : '';
-  return { ...parseAdapterUsage(adapter, stdoutText, analyticsText), transcript: `${stdoutText}\n${analyticsText}` };
+  return {
+    ...parseAdapterUsage(adapter, stdoutText, analyticsText),
+    transcript: `${stdoutText}\n${analyticsText}`,
+    // A run's own exit code, deadline and wall time belong in its record: the old one
+    // stored none, so a timed-out run was indistinguishable from a completed one.
+    agent: { exitCode: outcome.code, timedOut: outcome.timedOut, wallMs: outcome.wallMs },
+  };
 }
 
 // loadTasks lives in tasks.mjs (the advice engine became a second reader, so the loader
@@ -717,9 +779,10 @@ export async function runBench(
     maxTurns = DEFAULT_MAX_TURNS,
     concurrency = DEFAULT_CONCURRENCY,
     cwd = process.cwd(),
+    adapters: adapterTable,
   } = {},
 ) {
-  const adapters = await loadAdapters();
+  const adapters = adapterTable ?? (await loadAdapters());
   const adapter = adapters[harness];
   if (!adapter) {
     throw new Error(
@@ -741,11 +804,15 @@ export async function runBench(
     throw new Error(
       filterTaskIds.length > 0
         ? `no mined tasks matched: ${filterTaskIds.join(', ')}`
-        : `no mined tasks found in ${STATE_DIR}/bench/tasks`,
+        : `no mined tasks found in .simt/bench/tasks`,
     );
   }
 
   const resolvedModel = model ?? (adapter.kind === 'builtin' ? adapter.id : 'default');
+  // A built-in harness scripts the workspace instead of driving an agent, so its
+  // numbers calibrate the grader and are never an arm's measurement (review 0003 Q49).
+  const measured = adapter.kind !== 'builtin';
+  const isolation = adapterIsolation(adapter);
   const plannedRuns = [];
   for (const armSpec of armSpecs) {
     for (let rep = 1; rep <= reps; rep += 1) {
@@ -778,17 +845,19 @@ export async function runBench(
         harness,
         model: resolvedModel,
         rep: item.rep,
-        resolved: null,
-        gatesPassed: null,
-        usage: emptyUsage().usage,
-        costUsd: 0.0,
+        measured,
+        isolation,          status: 'planned',
+          resolved: null,
+          gatesPassed: null,
+          usage: emptyUsage().usage,
+        costUsd: null,
         diff: { files: 0, outsideBlastRadius: 0 },
         probe: null,
       })),
     };
   }
 
-  const runsBaseDir = join(root, STATE_DIR, 'bench', 'runs');
+  const runsBaseDir = join(root, LOCAL_DIR, 'bench', 'runs');
   for (const armSpec of armSpecs) {
     for (let rep = 1; rep <= reps; rep += 1) {
       const runDir = join(runsBaseDir, runIdFor(harness, armSpec.name, rep));
@@ -803,21 +872,25 @@ export async function runBench(
   const parentDir = await mkdtemp(join(tmpdir(), 'bench-batch-'));
   try {
     const records = await mapPool(plannedRuns, concurrency, async (item) => {
-      const { slotDir, workDir, initialHead, env } = await prepareIsolatedWorkspace(
-        root,
-        item.task.base,
-        item.armSpec,
-        item.task,
-        { parentDir },
-      );
+      let slotDir = null;
       try {
-        const { usage, costUsd, transcript } = await executeAdapter(adapter, workDir, item.task, {
+        const prepared = await prepareIsolatedWorkspace(
+          root,
+          item.task.base,
+          item.armSpec,
+          item.task,
+          { parentDir },
+        );
+        slotDir = prepared.slotDir;
+        const { workDir, initialHead, env, promptPrefix } = prepared;
+        const { usage, costUsd, transcript, agent } = await executeAdapter(adapter, workDir, item.task, {
           vaultRoot,
           model: resolvedModel,
           maxTurns,
           env,
           timeoutMs,
           slotDir,
+          promptPrefix,
         });
 
         const grade = await gradeWorkspace(workDir, item.task, {
@@ -840,8 +913,13 @@ export async function runBench(
           harness,
           model: resolvedModel,
           rep: item.rep,
+          measured,
+          isolation,
+          status: 'ok',
+          agent,
           resolved: grade.resolved,
           gatesPassed: grade.gatesPassed,
+          f2pPassed: grade.f2pPassed ?? [],
           usage,
           costUsd,
           diff: grade.diff,
@@ -856,8 +934,33 @@ export async function runBench(
         ]);
 
         return { ...record, runId: item.runId };
+      } catch (error) {
+        // One failed run is one exclusion, not a lost batch: the old path exited 3 and
+        // left a half-written run behind (review 0003 Q47).
+        return {
+          schemaVersion: 1,
+          runId: item.runId,
+          task: item.task.id,
+          arm: item.armSpec.name,
+          harness,
+          model: resolvedModel,
+          rep: item.rep,
+          measured,
+          isolation,
+          status: 'error',
+          error: String(error?.message ?? error),
+          resolved: false,
+          gatesPassed: false,
+          f2pPassed: [],
+          usage: emptyUsage().usage,
+          costUsd: null,
+          diff: { files: 0, outsideBlastRadius: 0 },
+          probe: null,
+        };
       } finally {
-        await rm(slotDir, { recursive: true, force: true }).catch(() => {});
+        if (slotDir !== null) {
+          await rm(slotDir, { recursive: true, force: true }).catch(() => {});
+        }
       }
     });
 
@@ -876,8 +979,15 @@ export async function runBench(
   }
 }
 
+/** Two string lists hold the same members, ignoring order and duplicates. */
+function sameStringSet(a, b) {
+  const left = [...new Set(a)].sort();
+  const right = [...new Set(b)].sort();
+  return left.length === right.length && left.every((value, index) => value === right[index]);
+}
+
 async function resolveRegradeTargets(root, target) {
-  const runsBaseDir = join(root, STATE_DIR, 'bench', 'runs');
+  const runsBaseDir = join(root, LOCAL_DIR, 'bench', 'runs');
   const direct = resolve(root, target);
   const candidate = existsSync(direct) ? direct : join(runsBaseDir, target);
 
@@ -933,7 +1043,7 @@ export async function regradeRun(
     throw new Error(`no stored task runs found under ${target}`);
   }
 
-  const tasksDir = join(root, STATE_DIR, 'bench', 'runs');
+  const tasksDir = join(root, LOCAL_DIR, 'bench', 'runs');
   const taskDefsDir = join(root, STATE_DIR, 'bench', 'tasks');
   const parentDir = await mkdtemp(join(tmpdir(), 'bench-regrade-'));
 
@@ -971,20 +1081,7 @@ export async function regradeRun(
         { parentDir },
       );
       try {
-        if (agentPatch.length > 0) {
-          const applyRes = await runCmd('git', ['apply', '--binary', '-'], {
-            cwd: workDir,
-            input: agentPatch,
-            timeoutMs,
-          });
-          if (applyRes.code !== 0) {
-            await runCmd('git', ['apply', '--3way', '-'], {
-              cwd: workDir,
-              input: agentPatch,
-              timeoutMs,
-            });
-          }
-        }
+        await applyPatch(workDir, agentPatch, { timeoutMs });
 
         const grade = await gradeWorkspace(workDir, task, {
           vaultRoot,
@@ -992,9 +1089,12 @@ export async function regradeRun(
           timeoutMs,
         });
 
+        // The verdicts alone cannot tell "the same tests passed" from "the same number
+        // of tests passed"; the fail-to-pass set is compared too (review 0003 Q59).
         const reproduced =
           grade.resolved === stored.resolved &&
           grade.gatesPassed === stored.gatesPassed &&
+          sameStringSet(grade.f2pPassed ?? [], stored.f2pPassed ?? []) &&
           grade.diff.files === stored.diff?.files &&
           grade.diff.outsideBlastRadius === stored.diff?.outsideBlastRadius;
 
@@ -1002,6 +1102,7 @@ export async function regradeRun(
           ...stored,
           resolved: grade.resolved,
           gatesPassed: grade.gatesPassed,
+          f2pPassed: grade.f2pPassed ?? [],
           diff: grade.diff,
         };