simt-dev 0.3.1 → 0.4.0

package/src/bench/run.mjs

@@ -6,6 +6,9 @@ import { dirname, join, resolve, sep } from 'node:path';
 import { fileURLToPath } from 'node:url';
 
 import { loadRegistry } from '../graph/load.mjs';
+import { extractOpenedSkills, matchSkillsForPaths } from '../graph/compile.mjs';
+import { matchGlob } from '../index/govern.mjs';
+import { formatLessonParagraph, listPromotedLessons } from '../memory/lessons.mjs';
 import { CONFIG_DIR, MANIFEST_FILE } from '../name.mjs';
 import { readManifest } from '../projection/manifest.mjs';
 import { plan } from '../projection/project.mjs';
@@ -15,6 +18,7 @@ import { rankSkills, tokenize } from '../resolve/resolve.mjs';
 import { gradeWorkspace } from './grade.mjs';
 import { resolveVaultDir } from './mine.mjs';
 import { ablateSectionInWorkspace, parseAblationArmName } from './ablate.mjs';
+import { applyVariantInWorkspace, parseVariantArmName } from './variants.mjs';
 
 const STATE_DIR = `.${CONFIG_DIR}`;
 
@@ -288,6 +292,15 @@ export async function parseArmSpec(rawSpec, cwd = process.cwd()) {
         env: {},
       };
     }
+    const variantSpec = parseVariantArmName(spec);
+    if (variantSpec) {
+      return {
+        ...variantSpec,
+        write: {},
+        remove: [],
+        env: {},
+      };
+    }
     if (!BUILTIN_ARMS.has(spec)) {
       throw new Error(
         `unknown arm "${spec}"; built-in arms are: ${[...BUILTIN_ARMS].join(', ')} (or pass --arm name=path/to/arm.json)`,
@@ -400,7 +413,7 @@ async function stripBareArm(workDir) {
     }
   }
 
-  for (const rel of ['.githooks', '.mcp.json', '.cursor/mcp.json', '.vscode/mcp.json']) {
+  for (const rel of ['.githooks', '.mcp.json', '.cursor/mcp.json', '.vscode/mcp.json', 'docs/lessons']) {
     await rm(join(workDir, rel), { recursive: true, force: true }).catch(() => {});
   }
 
@@ -457,20 +470,32 @@ async function applyProjectionMode(workDir, mode) {
 
 async function writeTaskBrief(workDir, task) {
   const prompt = String(task?.prompt ?? '');
-  const [registry, fp] = await Promise.all([
+  const taskTokens = tokenize(prompt);
+  const taskTokenSet = new Set(taskTokens);
+  const [registry, fp, lessons] = await Promise.all([
     loadRegistry().catch(() => ({ skills: [] })),
     fingerprint(workDir).catch(() => ({ stacks: [], facts: new Set() })),
+    listPromotedLessons(workDir).catch(() => []),
   ]);
   const skillsList = Array.isArray(registry.skills)
     ? registry.skills
     : Object.values(registry.skills ?? {});
   const { selected = [] } = rankSkills(skillsList, {
-    taskTokens: tokenize(prompt),
+    taskTokens,
     paths: [],
     repoStack: fp.stack ?? [],
   });
   const ranked = selected.slice(0, 5);
 
+  const taskPaths = Array.isArray(task?.paths) ? task.paths : [];
+  const matchingLessons = lessons.filter((lesson) => {
+    const lessonGlobs = Array.isArray(lesson.paths) ? lesson.paths : [];
+    if (taskPaths.some((p) => lessonGlobs.some((glob) => matchGlob(glob, p)))) return true;
+    if (lessonGlobs.some((glob) => prompt.includes(glob.replace(/\/\*\*$/, '')))) return true;
+    const kw = Array.isArray(lesson.keywords) ? lesson.keywords : [];
+    return kw.some((k) => taskTokenSet.has(String(k).toLowerCase()));
+  });
+
   const lines = [
     `# Task Brief: ${task?.id ?? 'task'}`,
     '',
@@ -480,6 +505,13 @@ async function writeTaskBrief(workDir, task) {
     ...(ranked.length > 0
       ? ranked.map((r) => `- ${r.skill.name} (score ${r.score}): ${r.skill.description ?? ''}`)
       : ['- (none matched)']),
+    ...(matchingLessons.length > 0
+      ? [
+          '',
+          '## Lessons',
+          ...matchingLessons.map((l) => `- ${formatLessonParagraph(l)}`),
+        ]
+      : []),
     '',
   ];
   const briefPath = join(workDir, STATE_DIR, 'brief.md');
@@ -497,9 +529,12 @@ async function writeTaskBrief(workDir, task) {
  */
 export async function materialiseArm(workDir, armSpec, task = {}) {
   const ablate = armSpec.ablate ?? parseAblationArmName(armSpec.name)?.ablate ?? null;
+  const variant = armSpec.variant ?? parseVariantArmName(armSpec.name)?.variant ?? null;
   const base = armSpec.base ?? (BUILTIN_ARMS.has(armSpec.name) ? armSpec.name : null);
   if (ablate) {
     await ablateSectionInWorkspace(workDir, ablate);
+  } else if (variant) {
+    await applyVariantInWorkspace(workDir, variant);
   } else if (base === 'bare') {
     await stripBareArm(workDir);
   } else if (base === 'current') {
@@ -559,6 +594,31 @@ export async function prepareIsolatedWorkspace(root, baseSha, armSpec, task = {}
     throw new Error(`tar unpack failed for ${baseSha}: ${tarRes.stderr.toString('utf8').trim()}`);
   }
 
+  const rootLessonsDir = join(root, 'docs', 'lessons');
+  const rootLessonFiles = (await readdir(rootLessonsDir).catch(() => [])).filter((f) => f.endsWith('.md'));
+  if (rootLessonFiles.length > 0) {
+    const destLessonsDir = join(workDir, 'docs', 'lessons');
+    await mkdir(destLessonsDir, { recursive: true });
+    for (const file of rootLessonFiles) {
+      const target = join(destLessonsDir, file);
+      if (!existsSync(target)) {
+        const text = await readFile(join(rootLessonsDir, file), 'utf8');
+        await writeFile(target, text, 'utf8');
+      }
+    }
+  }
+
+  const rootProfilesDir = join(root, STATE_DIR, 'profiles');
+  const rootProfileFiles = (await readdir(rootProfilesDir).catch(() => [])).filter((f) => f.endsWith('.md'));
+  if (rootProfileFiles.length > 0) {
+    const destProfilesDir = join(workDir, STATE_DIR, 'profiles');
+    await mkdir(destProfilesDir, { recursive: true });
+    for (const file of rootProfileFiles) {
+      const text = await readFile(join(rootProfilesDir, file), 'utf8');
+      await writeFile(join(destProfilesDir, file), text, 'utf8');
+    }
+  }
+
   const { env } = await materialiseArm(workDir, armSpec, task);
 
   await runCmd('git', ['init', '-q'], { cwd: workDir });
@@ -598,49 +658,24 @@ function buildAdapterArgs(adapter, { prompt, turns, model, analyticsLog }) {
 }
 
 async function executeAdapter(adapter, workDir, task, { vaultRoot, model, maxTurns, env, timeoutMs, slotDir }) {
-  if (adapter.id === 'noop') {
-    return parseAdapterUsage(adapter, '');
-  }
-
+  if (adapter.id === 'noop') return { ...parseAdapterUsage(adapter, ''), transcript: '' };
   if (adapter.id === 'gold') {
-    const goldPatchPath = join(vaultRoot, task.id, 'gold.patch');
-    const goldPatch = await readFile(goldPatchPath);
+    const goldPatch = await readFile(join(vaultRoot, task.id, 'gold.patch'));
     if (goldPatch.length > 0) {
-      const applyRes = await runCmd('git', ['apply', '-'], {
-        cwd: workDir,
-        input: goldPatch,
-        timeoutMs,
-      });
+      const applyRes = await runCmd('git', ['apply', '-'], { cwd: workDir, input: goldPatch, timeoutMs });
       if (applyRes.code !== 0) {
-        await runCmd('git', ['apply', '--3way', '-'], {
-          cwd: workDir,
-          input: goldPatch,
-          timeoutMs,
-        });
+        await runCmd('git', ['apply', '--3way', '-'], { cwd: workDir, input: goldPatch, timeoutMs });
       }
     }
-    return parseAdapterUsage(adapter, '');
+    return { ...parseAdapterUsage(adapter, ''), transcript: '' };
   }
 
   const analyticsLog = join(slotDir, 'analytics.jsonl');
-  const args = buildAdapterArgs(adapter, {
-    prompt: String(task.prompt ?? ''),
-    turns: maxTurns,
-    model,
-    analyticsLog,
-  });
-
-  const outcome = await runCmd(adapter.bin, args, {
-    cwd: workDir,
-    env,
-    timeoutMs,
-  });
-  const analyticsText =
-    adapter.format === 'analytics-log'
-      ? await readFile(analyticsLog, 'utf8').catch(() => '')
-      : '';
-
-  return parseAdapterUsage(adapter, outcome.stdout.toString('utf8'), analyticsText);
+  const args = buildAdapterArgs(adapter, { prompt: String(task.prompt ?? ''), turns: maxTurns, model, analyticsLog });
+  const outcome = await runCmd(adapter.bin, args, { cwd: workDir, env, timeoutMs });
+  const stdoutText = outcome.stdout.toString('utf8');
+  const analyticsText = adapter.format === 'analytics-log' ? await readFile(analyticsLog, 'utf8').catch(() => '') : '';
+  return { ...parseAdapterUsage(adapter, stdoutText, analyticsText), transcript: `${stdoutText}\n${analyticsText}` };
 }
 
 async function loadTasks(root, filterIds = []) {
@@ -776,6 +811,9 @@ export async function runBench(
     }
   }
 
+  const regRes = await loadRegistry().catch(() => ({ registry: { skills: [] } }));
+  const registrySkills = regRes?.registry?.skills ?? [];
+  const knownSkillNames = new Set(registrySkills.map((s) => s.name));
   const parentDir = await mkdtemp(join(tmpdir(), 'bench-batch-'));
   try {
     const records = await mapPool(plannedRuns, concurrency, async (item) => {
@@ -787,7 +825,7 @@ export async function runBench(
         { parentDir },
       );
       try {
-        const { usage, costUsd } = await executeAdapter(adapter, workDir, item.task, {
+        const { usage, costUsd, transcript } = await executeAdapter(adapter, workDir, item.task, {
           vaultRoot,
           model: resolvedModel,
           maxTurns,
@@ -801,6 +839,13 @@ export async function runBench(
           initialHead,
           timeoutMs,
         });
+        const expectedSkills = matchSkillsForPaths(registrySkills, grade.goldFiles ?? []);
+        const openedSkills =
+          adapter.id === 'gold'
+            ? expectedSkills
+            : adapter.id === 'noop'
+              ? []
+              : extractOpenedSkills(transcript, knownSkillNames);
 
         const record = {
           schemaVersion: 1,
@@ -814,6 +859,7 @@ export async function runBench(
           usage,
           costUsd,
           diff: grade.diff,
+          skills: { expected: expectedSkills, opened: openedSkills },
           probe: null,
         };
 

package/src/bench/variants.mjs

@@ -0,0 +1,456 @@
+// Variant selection for rule text (`simt bench variants`, plan 0003 R35).
+//
+// Discovers `simt-variants` slots across applicable rules, runs candidate
+// variants on the 10-task canary suite (escalating when significant), records
+// the winning variant with `{winner, variant, model, harness, dR, dC, n, p}`
+// in `simt-dev.lock.json`, and re-opens the experiment whenever the target
+// model or harness changes.
+
+import { mkdir, readdir, readFile, unlink, writeFile } from 'node:fs/promises';
+import { dirname, join, resolve } from 'node:path';
+
+import { extractVariantsFromRule, readLockedVariants } from '../kernel/variants.mjs';
+import { CONFIG_DIR, MANIFEST_FILE } from '../name.mjs';
+import { writeLockVariants } from '../projection/lock.mjs';
+import { readManifest } from '../projection/manifest.mjs';
+import { loadMdcRules } from '../projection/mdc.mjs';
+import { plan } from '../projection/project.mjs';
+import { isEligible, loadRules, readDeclared, select } from '../projection/rules.mjs';
+import { fingerprint } from '../resolve/fingerprint.mjs';
+import { CANARY_ALPHA, DEFAULT_CANARY_SIZE, DEL_R_TOLERANCE, shouldEscalateCanary } from './ablate.mjs';
+import {
+  DEFAULT_BOOTSTRAP_SAMPLES,
+  DEFAULT_BOOTSTRAP_SEED,
+  compareArms,
+  evaluateRunRecord,
+  loadPrices,
+  loadRunRecords,
+} from './report.mjs';
+
+const STATE_DIR = `.${CONFIG_DIR}`;
+
+/**
+ * Discover every `simt-variants` slot across applicable rules for `root`.
+ */
+export async function listVariantSlots(root, { includeAll = false } = {}) {
+  const resolvedRoot = resolve(root);
+  const [{ stack, facts }, allRules, mdcRules, declared] = await Promise.all([
+    fingerprint(resolvedRoot),
+    loadRules(resolvedRoot),
+    loadMdcRules(),
+    readDeclared(resolvedRoot),
+  ]);
+
+  const signals = new Set([...(stack ?? []), ...(facts ?? [])]);
+  const { selected } = select(allRules, stack ?? [], facts ?? []);
+  const rulePool = includeAll ? allRules : selected;
+  const mdcPool = includeAll
+    ? mdcRules.filter((r) => r.id && r.error === null)
+    : mdcRules.filter((r) => r.id && r.error === null && isEligible(r, signals, declared.optIn));
+
+  const slots = [];
+  for (const rule of [...rulePool, ...mdcPool]) {
+    if (!rule || rule.error !== null) continue;
+    slots.push(...extractVariantsFromRule(rule));
+  }
+  return slots.sort((a, b) => a.slotId.localeCompare(b.slotId));
+}
+
+/**
+ * Parse a variant benchmark arm name (`variant-<slotId>-<variantKey>`) or return null.
+ */
+export function parseVariantArmName(armName) {
+  const s = String(armName ?? '').trim();
+  if (!s.startsWith('variant-')) return null;
+  const rest = s.slice('variant-'.length);
+  const dash = rest.lastIndexOf('-');
+  if (dash <= 0 || dash === rest.length - 1) return null;
+  const slotId = rest.slice(0, dash);
+  const variantKey = rest.slice(dash + 1);
+  return {
+    name: s,
+    kind: 'variant',
+    base: 'current',
+    variant: { slotId, variantKey },
+  };
+}
+
+/**
+ * Format the arm name for a `(slotId, variantKey)` pair.
+ */
+export function variantArmName(slotId, variantKey) {
+  return `variant-${slotId}-${variantKey}`;
+}
+
+/**
+ * Determine whether a locked variant entry is still valid or re-opened by a model/harness change.
+ */
+export function isVariantExperimentOpen(
+  lockedEntry,
+  { harness, model, variants, force = false } = {},
+) {
+  if (force) return { open: true, reopened: Boolean(lockedEntry), reason: 'forced' };
+  if (!lockedEntry || typeof lockedEntry !== 'object') {
+    return { open: true, reopened: false, reason: 'unlocked' };
+  }
+  const winner = lockedEntry.winner ?? lockedEntry.variant;
+  if (typeof winner !== 'string' || winner === '') {
+    return { open: true, reopened: true, reason: 'missing-winner' };
+  }
+  if (variants && typeof variants === 'object' && !Object.hasOwn(variants, winner)) {
+    return { open: true, reopened: true, reason: 'unknown-variant' };
+  }
+  if (harness && lockedEntry.harness && lockedEntry.harness !== harness) {
+    return { open: true, reopened: true, reason: 'harness-changed' };
+  }
+  if (model && lockedEntry.model && lockedEntry.model !== model) {
+    return { open: true, reopened: true, reason: 'model-changed' };
+  }
+  return { open: false, reopened: false, reason: 'locked' };
+}
+
+/**
+ * Materialise a variant arm inside an isolated benchmark workspace `workDir`.
+ */
+export async function applyVariantInWorkspace(workDir, variant) {
+  if (!variant || typeof variant !== 'object') return;
+  const { slotId, variantKey } = variant;
+  if (!slotId || !variantKey) return;
+
+  const manifestRes = await readManifest(workDir);
+  const manifestPath = manifestRes.path ?? join(workDir, MANIFEST_FILE);
+  const hadManifest = manifestRes.exists;
+  const originalManifestBytes = hadManifest
+    ? await readFile(manifestPath, 'utf8').catch(() => null)
+    : null;
+
+  const baseManifest = manifestRes.data ?? {
+    schemaVersion: 1,
+    projection: { sha: '0000000000000000000000000000000000000000', sources: [] },
+    gates: { preCommit: [], prePush: [] },
+  };
+  const nextManifest = { ...baseManifest };
+  delete nextManifest.mode;
+  await writeFile(manifestPath, `${JSON.stringify(nextManifest, null, 2)}\n`, 'utf8');
+
+  const planned = await plan(workDir, {
+    variants: { [slotId]: { winner: variantKey } },
+  });
+  for (const t of planned.targets ?? []) {
+    if (t.action !== 'unchanged' && typeof t.text === 'string') {
+      await mkdir(dirname(t.path), { recursive: true });
+      await writeFile(t.path, t.text, 'utf8');
+    }
+  }
+
+  if (!hadManifest) {
+    await unlink(manifestPath).catch(() => {});
+  } else if (originalManifestBytes !== null) {
+    await writeFile(manifestPath, originalManifestBytes, 'utf8');
+  }
+}
+
+function summarizeEvaluated(evaluated) {
+  if (evaluated.length === 0) {
+    return { n: 0, R: 0, C: 0 };
+  }
+  const byTask = new Map();
+  for (const row of evaluated) {
+    const list = byTask.get(row.task) ?? [];
+    list.push(row);
+    byTask.set(row.task, list);
+  }
+  let sumR = 0;
+  let sumC = 0;
+  for (const list of byTask.values()) {
+    sumR += list.reduce((a, x) => a + x.R, 0) / list.length;
+    sumC += list.reduce((a, x) => a + x.costUsd, 0) / list.length;
+  }
+  const n = byTask.size;
+  return { n, R: sumR / n, C: sumC / n };
+}
+
+/**
+ * Run or evaluate variant selection on the canary suite and record winners in `simt-dev.lock.json`.
+ */
+export async function selectRuleVariants(
+  root,
+  {
+    apply = false,
+    force = false,
+    harness = 'gold',
+    model,
+    reps = 1,
+    canarySize = DEFAULT_CANARY_SIZE,
+    includeAll = false,
+    slots: customSlots,
+    prices: customPrices,
+    samples = DEFAULT_BOOTSTRAP_SAMPLES,
+    seed = DEFAULT_BOOTSTRAP_SEED,
+    vaultRoot,
+    runner,
+  } = {},
+) {
+  const resolvedRoot = resolve(root);
+  const resolvedModel = model ?? harness;
+  const [slots, existingLocks, prices] = await Promise.all([
+    customSlots ? Promise.resolve(customSlots) : listVariantSlots(resolvedRoot, { includeAll }),
+    readLockedVariants(resolvedRoot),
+    customPrices !== undefined ? Promise.resolve(customPrices) : loadPrices(resolvedRoot),
+  ]);
+
+  const tasksDir = join(resolvedRoot, STATE_DIR, 'bench', 'tasks');
+  let taskIds = [];
+  try {
+    const files = (await readdir(tasksDir)).filter((f) => f.endsWith('.json')).sort();
+    for (const file of files) {
+      const data = JSON.parse(await readFile(join(tasksDir, file), 'utf8'));
+      if (data?.id) taskIds.push(data.id);
+    }
+    taskIds.sort();
+  } catch {
+    taskIds = [];
+  }
+
+  const canaryTaskIds = taskIds.slice(0, Math.max(1, canarySize));
+  const remainingTaskIds = taskIds.slice(canaryTaskIds.length);
+
+  const slotStates = slots.map((slot) => {
+    const lockedEntry = existingLocks[slot.slotId] ?? null;
+    const check = isVariantExperimentOpen(lockedEntry, {
+      harness,
+      model: resolvedModel,
+      variants: slot.variants,
+      force,
+    });
+    return {
+      slot,
+      lockedEntry,
+      ...check,
+    };
+  });
+
+  const openStates = slotStates.filter((s) => s.open);
+  const escalatedArms = new Set();
+
+  if (apply && openStates.length > 0) {
+    if (taskIds.length === 0) {
+      throw new Error(`no mined benchmark tasks found in ${tasksDir}; run bench mine --yes first`);
+    }
+    const execRunBench = runner ?? (await import('./run.mjs')).runBench;
+    const canaryArms = [];
+    for (const { slot } of openStates) {
+      for (const key of Object.keys(slot.variants)) {
+        canaryArms.push(variantArmName(slot.slotId, key));
+      }
+    }
+
+    await execRunBench(resolvedRoot, {
+      apply: true,
+      arms: canaryArms,
+      harness,
+      model: resolvedModel,
+      reps,
+      tasks: canaryTaskIds,
+      ...(vaultRoot ? { vaultRoot } : {}),
+    });
+
+    if (remainingTaskIds.length > 0) {
+      const canaryEvaluated = (await loadRunRecords(resolvedRoot))
+        .filter(
+          (rec) =>
+            canaryTaskIds.includes(rec.task) &&
+            (!harness || rec.harness === harness) &&
+            (!model || rec.model === resolvedModel),
+        )
+        .map((rec) => evaluateRunRecord(rec, prices));
+      const armsToEscalate = [];
+      const includeBaseline = new Set();
+      for (const { slot } of openStates) {
+        const baseArm = variantArmName(slot.slotId, slot.baseline);
+        const baseRecs = canaryEvaluated.filter((r) => r.arm === baseArm);
+        for (const key of Object.keys(slot.variants)) {
+          if (key === slot.baseline) continue;
+          const candArm = variantArmName(slot.slotId, key);
+          const candRecs = canaryEvaluated.filter((r) => r.arm === candArm);
+          if (baseRecs.length > 0 && candRecs.length > 0) {
+            const comp = compareArms(baseArm, baseRecs, candArm, candRecs, { samples, seed });
+            if (shouldEscalateCanary(comp, { alpha: CANARY_ALPHA })) {
+              includeBaseline.add(baseArm);
+              armsToEscalate.push(candArm);
+              escalatedArms.add(candArm);
+            }
+          }
+        }
+      }
+      const fullEscalationArms = [...includeBaseline, ...armsToEscalate];
+      if (fullEscalationArms.length > 0) {
+        await execRunBench(resolvedRoot, {
+          apply: true,
+          arms: fullEscalationArms,
+          harness,
+          model: resolvedModel,
+          reps,
+          tasks: remainingTaskIds,
+          ...(vaultRoot ? { vaultRoot } : {}),
+        });
+      }
+    }
+  }
+
+  const allEvaluated = (await loadRunRecords(resolvedRoot))
+    .filter(
+      (rec) =>
+        (!harness || rec.harness === harness) &&
+        (!model || !rec.model || rec.model === resolvedModel),
+    )
+    .map((rec) => evaluateRunRecord(rec, prices));
+
+  const results = [];
+  const nextVariantsLock = { ...existingLocks };
+
+  for (const state of slotStates) {
+    const { slot, lockedEntry, open, reopened, reason } = state;
+    if (!open && lockedEntry) {
+      const winnerKey = lockedEntry.winner ?? lockedEntry.variant ?? slot.baseline;
+      results.push({
+        slotId: slot.slotId,
+        ruleId: slot.ruleId,
+        baseline: slot.baseline,
+        variants: slot.variants,
+        status: 'locked',
+        reopened: false,
+        reason,
+        measured: true,
+        winner: winnerKey,
+        model: lockedEntry.model ?? resolvedModel,
+        harness: lockedEntry.harness ?? harness,
+        dR: Number(lockedEntry.dR ?? 0),
+        dC: Number(lockedEntry.dC ?? 0),
+        n: Number(lockedEntry.n ?? 0),
+        p: Number(lockedEntry.p ?? 1),
+      });
+      continue;
+    }
+
+    const baseArm = variantArmName(slot.slotId, slot.baseline);
+    const baseRecs = allEvaluated.filter((r) => r.arm === baseArm);
+    const baseSummary = summarizeEvaluated(baseRecs);
+
+    const candidates = [];
+    for (const [key, text] of Object.entries(slot.variants)) {
+      const arm = variantArmName(slot.slotId, key);
+      const recs = allEvaluated.filter((r) => r.arm === arm);
+      const summary = summarizeEvaluated(recs);
+      const comp =
+        key === slot.baseline || baseRecs.length === 0 || recs.length === 0
+          ? null
+          : compareArms(baseArm, baseRecs, arm, recs, { samples, seed });
+      candidates.push({
+        key,
+        arm,
+        bytes: Buffer.byteLength(text, 'utf8'),
+        records: recs,
+        summary,
+        comparison: comp,
+        escalated: escalatedArms.has(arm),
+      });
+    }
+
+    const measured = baseSummary.n > 0 && candidates.every((c) => c.summary.n > 0);
+    if (!measured) {
+      results.push({
+        slotId: slot.slotId,
+        ruleId: slot.ruleId,
+        baseline: slot.baseline,
+        variants: slot.variants,
+        status: reopened ? 'reopened' : 'unmeasured',
+        reopened,
+        reason,
+        measured: false,
+        winner: slot.baseline,
+        model: resolvedModel,
+        harness,
+        dR: 0,
+        dC: 0,
+        n: 0,
+        p: 1,
+      });
+      continue;
+    }
+
+    // Rank variants: higher R (beyond tolerance), then lower C, then fewer bytes, then key.
+    const ranked = [...candidates].sort((a, b) => {
+      if (Math.abs(b.summary.R - a.summary.R) > DEL_R_TOLERANCE) {
+        return b.summary.R - a.summary.R;
+      }
+      if (Math.abs(a.summary.C - b.summary.C) > 1e-9) {
+        return a.summary.C - b.summary.C;
+      }
+      if (a.bytes !== b.bytes) return a.bytes - b.bytes;
+      return a.key.localeCompare(b.key);
+    });
+
+    const best = ranked[0];
+    let dR = 0;
+    let dC = 0;
+    let n = baseSummary.n;
+    let p = 1;
+
+    if (best.key !== slot.baseline && best.comparison) {
+      dR = Number(best.comparison.deltas.R.toFixed(4));
+      dC = Number(best.comparison.deltas.C.toFixed(4));
+      n = best.comparison.pairedTasks;
+      p = Number(
+        Math.min(best.comparison.mcnemar.pValue, best.comparison.wilcoxon.pValue).toFixed(4),
+      );
+    } else {
+      const runnerUp = ranked.find((c) => c.key !== slot.baseline && c.comparison);
+      if (runnerUp?.comparison) {
+        n = runnerUp.comparison.pairedTasks;
+      }
+    }
+
+    const lockEntry = {
+      winner: best.key,
+      variant: best.key,
+      model: resolvedModel,
+      harness,
+      dR,
+      dC,
+      n,
+      p,
+    };
+    nextVariantsLock[slot.slotId] = lockEntry;
+
+    results.push({
+      slotId: slot.slotId,
+      ruleId: slot.ruleId,
+      baseline: slot.baseline,
+      variants: slot.variants,
+      status: reopened ? 'reopened' : 'evaluated',
+      reopened,
+      reason,
+      measured: true,
+      ...lockEntry,
+    });
+  }
+
+  let lock = null;
+  if (apply && Object.keys(nextVariantsLock).length > 0) {
+    lock = await writeLockVariants(resolvedRoot, nextVariantsLock);
+  }
+
+  return {
+    root: resolvedRoot,
+    applied: Boolean(apply),
+    harness,
+    model: resolvedModel,
+    canarySize,
+    canaryTasks: canaryTaskIds.length,
+    totalTasks: taskIds.length,
+    slots: results,
+    variants: nextVariantsLock,
+    lock,
+  };
+}