simt-dev 0.3.1 → 0.5.0

package/src/bench/run.mjs

@@ -6,6 +6,9 @@ import { dirname, join, resolve, sep } from 'node:path';
 import { fileURLToPath } from 'node:url';
 
 import { loadRegistry } from '../graph/load.mjs';
+import { extractOpenedSkills, matchSkillsForPaths } from '../graph/compile.mjs';
+import { matchGlob } from '../index/govern.mjs';
+import { formatLessonParagraph, listPromotedLessons } from '../memory/lessons.mjs';
 import { CONFIG_DIR, MANIFEST_FILE } from '../name.mjs';
 import { readManifest } from '../projection/manifest.mjs';
 import { plan } from '../projection/project.mjs';
@@ -14,7 +17,9 @@ import { fingerprint } from '../resolve/fingerprint.mjs';
 import { rankSkills, tokenize } from '../resolve/resolve.mjs';
 import { gradeWorkspace } from './grade.mjs';
 import { resolveVaultDir } from './mine.mjs';
+import { loadTasks } from './tasks.mjs';
 import { ablateSectionInWorkspace, parseAblationArmName } from './ablate.mjs';
+import { applyVariantInWorkspace, parseVariantArmName } from './variants.mjs';
 
 const STATE_DIR = `.${CONFIG_DIR}`;
 
@@ -288,6 +293,15 @@ export async function parseArmSpec(rawSpec, cwd = process.cwd()) {
         env: {},
       };
     }
+    const variantSpec = parseVariantArmName(spec);
+    if (variantSpec) {
+      return {
+        ...variantSpec,
+        write: {},
+        remove: [],
+        env: {},
+      };
+    }
     if (!BUILTIN_ARMS.has(spec)) {
       throw new Error(
         `unknown arm "${spec}"; built-in arms are: ${[...BUILTIN_ARMS].join(', ')} (or pass --arm name=path/to/arm.json)`,
@@ -400,7 +414,7 @@ async function stripBareArm(workDir) {
     }
   }
 
-  for (const rel of ['.githooks', '.mcp.json', '.cursor/mcp.json', '.vscode/mcp.json']) {
+  for (const rel of ['.githooks', '.mcp.json', '.cursor/mcp.json', '.vscode/mcp.json', 'docs/lessons']) {
     await rm(join(workDir, rel), { recursive: true, force: true }).catch(() => {});
   }
 
@@ -457,20 +471,32 @@ async function applyProjectionMode(workDir, mode) {
 
 async function writeTaskBrief(workDir, task) {
   const prompt = String(task?.prompt ?? '');
-  const [registry, fp] = await Promise.all([
+  const taskTokens = tokenize(prompt);
+  const taskTokenSet = new Set(taskTokens);
+  const [registry, fp, lessons] = await Promise.all([
     loadRegistry().catch(() => ({ skills: [] })),
     fingerprint(workDir).catch(() => ({ stacks: [], facts: new Set() })),
+    listPromotedLessons(workDir).catch(() => []),
   ]);
   const skillsList = Array.isArray(registry.skills)
     ? registry.skills
     : Object.values(registry.skills ?? {});
   const { selected = [] } = rankSkills(skillsList, {
-    taskTokens: tokenize(prompt),
+    taskTokens,
     paths: [],
     repoStack: fp.stack ?? [],
   });
   const ranked = selected.slice(0, 5);
 
+  const taskPaths = Array.isArray(task?.paths) ? task.paths : [];
+  const matchingLessons = lessons.filter((lesson) => {
+    const lessonGlobs = Array.isArray(lesson.paths) ? lesson.paths : [];
+    if (taskPaths.some((p) => lessonGlobs.some((glob) => matchGlob(glob, p)))) return true;
+    if (lessonGlobs.some((glob) => prompt.includes(glob.replace(/\/\*\*$/, '')))) return true;
+    const kw = Array.isArray(lesson.keywords) ? lesson.keywords : [];
+    return kw.some((k) => taskTokenSet.has(String(k).toLowerCase()));
+  });
+
   const lines = [
     `# Task Brief: ${task?.id ?? 'task'}`,
     '',
@@ -480,6 +506,13 @@ async function writeTaskBrief(workDir, task) {
     ...(ranked.length > 0
       ? ranked.map((r) => `- ${r.skill.name} (score ${r.score}): ${r.skill.description ?? ''}`)
       : ['- (none matched)']),
+    ...(matchingLessons.length > 0
+      ? [
+          '',
+          '## Lessons',
+          ...matchingLessons.map((l) => `- ${formatLessonParagraph(l)}`),
+        ]
+      : []),
     '',
   ];
   const briefPath = join(workDir, STATE_DIR, 'brief.md');
@@ -497,9 +530,12 @@ async function writeTaskBrief(workDir, task) {
  */
 export async function materialiseArm(workDir, armSpec, task = {}) {
   const ablate = armSpec.ablate ?? parseAblationArmName(armSpec.name)?.ablate ?? null;
+  const variant = armSpec.variant ?? parseVariantArmName(armSpec.name)?.variant ?? null;
   const base = armSpec.base ?? (BUILTIN_ARMS.has(armSpec.name) ? armSpec.name : null);
   if (ablate) {
     await ablateSectionInWorkspace(workDir, ablate);
+  } else if (variant) {
+    await applyVariantInWorkspace(workDir, variant);
   } else if (base === 'bare') {
     await stripBareArm(workDir);
   } else if (base === 'current') {
@@ -559,6 +595,31 @@ export async function prepareIsolatedWorkspace(root, baseSha, armSpec, task = {}
     throw new Error(`tar unpack failed for ${baseSha}: ${tarRes.stderr.toString('utf8').trim()}`);
   }
 
+  const rootLessonsDir = join(root, 'docs', 'lessons');
+  const rootLessonFiles = (await readdir(rootLessonsDir).catch(() => [])).filter((f) => f.endsWith('.md'));
+  if (rootLessonFiles.length > 0) {
+    const destLessonsDir = join(workDir, 'docs', 'lessons');
+    await mkdir(destLessonsDir, { recursive: true });
+    for (const file of rootLessonFiles) {
+      const target = join(destLessonsDir, file);
+      if (!existsSync(target)) {
+        const text = await readFile(join(rootLessonsDir, file), 'utf8');
+        await writeFile(target, text, 'utf8');
+      }
+    }
+  }
+
+  const rootProfilesDir = join(root, STATE_DIR, 'profiles');
+  const rootProfileFiles = (await readdir(rootProfilesDir).catch(() => [])).filter((f) => f.endsWith('.md'));
+  if (rootProfileFiles.length > 0) {
+    const destProfilesDir = join(workDir, STATE_DIR, 'profiles');
+    await mkdir(destProfilesDir, { recursive: true });
+    for (const file of rootProfileFiles) {
+      const text = await readFile(join(rootProfilesDir, file), 'utf8');
+      await writeFile(join(destProfilesDir, file), text, 'utf8');
+    }
+  }
+
   const { env } = await materialiseArm(workDir, armSpec, task);
 
   await runCmd('git', ['init', '-q'], { cwd: workDir });
@@ -598,66 +659,28 @@ function buildAdapterArgs(adapter, { prompt, turns, model, analyticsLog }) {
 }
 
 async function executeAdapter(adapter, workDir, task, { vaultRoot, model, maxTurns, env, timeoutMs, slotDir }) {
-  if (adapter.id === 'noop') {
-    return parseAdapterUsage(adapter, '');
-  }
-
+  if (adapter.id === 'noop') return { ...parseAdapterUsage(adapter, ''), transcript: '' };
   if (adapter.id === 'gold') {
-    const goldPatchPath = join(vaultRoot, task.id, 'gold.patch');
-    const goldPatch = await readFile(goldPatchPath);
+    const goldPatch = await readFile(join(vaultRoot, task.id, 'gold.patch'));
     if (goldPatch.length > 0) {
-      const applyRes = await runCmd('git', ['apply', '-'], {
-        cwd: workDir,
-        input: goldPatch,
-        timeoutMs,
-      });
+      const applyRes = await runCmd('git', ['apply', '-'], { cwd: workDir, input: goldPatch, timeoutMs });
       if (applyRes.code !== 0) {
-        await runCmd('git', ['apply', '--3way', '-'], {
-          cwd: workDir,
-          input: goldPatch,
-          timeoutMs,
-        });
+        await runCmd('git', ['apply', '--3way', '-'], { cwd: workDir, input: goldPatch, timeoutMs });
       }
     }
-    return parseAdapterUsage(adapter, '');
+    return { ...parseAdapterUsage(adapter, ''), transcript: '' };
   }
 
   const analyticsLog = join(slotDir, 'analytics.jsonl');
-  const args = buildAdapterArgs(adapter, {
-    prompt: String(task.prompt ?? ''),
-    turns: maxTurns,
-    model,
-    analyticsLog,
-  });
-
-  const outcome = await runCmd(adapter.bin, args, {
-    cwd: workDir,
-    env,
-    timeoutMs,
-  });
-  const analyticsText =
-    adapter.format === 'analytics-log'
-      ? await readFile(analyticsLog, 'utf8').catch(() => '')
-      : '';
-
-  return parseAdapterUsage(adapter, outcome.stdout.toString('utf8'), analyticsText);
+  const args = buildAdapterArgs(adapter, { prompt: String(task.prompt ?? ''), turns: maxTurns, model, analyticsLog });
+  const outcome = await runCmd(adapter.bin, args, { cwd: workDir, env, timeoutMs });
+  const stdoutText = outcome.stdout.toString('utf8');
+  const analyticsText = adapter.format === 'analytics-log' ? await readFile(analyticsLog, 'utf8').catch(() => '') : '';
+  return { ...parseAdapterUsage(adapter, stdoutText, analyticsText), transcript: `${stdoutText}\n${analyticsText}` };
 }
 
-async function loadTasks(root, filterIds = []) {
-  const tasksDir = join(root, STATE_DIR, 'bench', 'tasks');
-  const entries = (await readdir(tasksDir).catch(() => []))
-    .filter((f) => f.endsWith('.json'))
-    .sort();
-
-  const filterSet = new Set(filterIds);
-  const tasks = [];
-  for (const entry of entries) {
-    const data = JSON.parse(await readFile(join(tasksDir, entry), 'utf8'));
-    if (filterSet.size > 0 && !filterSet.has(data.id)) continue;
-    tasks.push(data);
-  }
-  return tasks.sort((a, b) => a.id.localeCompare(b.id));
-}
+// loadTasks lives in tasks.mjs (the advice engine became a second reader, so the loader
+// was extracted rather than copied — ADR-002).
 
 function runIdFor(harness, armName, rep) {
   return `${harness}-${armName}-r${rep}`;
@@ -776,6 +799,9 @@ export async function runBench(
     }
   }
 
+  const regRes = await loadRegistry().catch(() => ({ registry: { skills: [] } }));
+  const registrySkills = regRes?.registry?.skills ?? [];
+  const knownSkillNames = new Set(registrySkills.map((s) => s.name));
   const parentDir = await mkdtemp(join(tmpdir(), 'bench-batch-'));
   try {
     const records = await mapPool(plannedRuns, concurrency, async (item) => {
@@ -787,7 +813,7 @@ export async function runBench(
         { parentDir },
       );
       try {
-        const { usage, costUsd } = await executeAdapter(adapter, workDir, item.task, {
+        const { usage, costUsd, transcript } = await executeAdapter(adapter, workDir, item.task, {
           vaultRoot,
           model: resolvedModel,
           maxTurns,
@@ -801,6 +827,13 @@ export async function runBench(
           initialHead,
           timeoutMs,
         });
+        const expectedSkills = matchSkillsForPaths(registrySkills, grade.goldFiles ?? []);
+        const openedSkills =
+          adapter.id === 'gold'
+            ? expectedSkills
+            : adapter.id === 'noop'
+              ? []
+              : extractOpenedSkills(transcript, knownSkillNames);
 
         const record = {
           schemaVersion: 1,
@@ -814,6 +847,7 @@ export async function runBench(
           usage,
           costUsd,
           diff: grade.diff,
+          skills: { expected: expectedSkills, opened: openedSkills },
           probe: null,
         };
 

package/src/bench/tasks.mjs

@@ -0,0 +1,38 @@
+// Mined task descriptors, read back from the tool state's `bench/tasks/` directory (plan 0003 R01).
+//
+// Extracted from bench/run.mjs on its second real reader (ADR-002): the R40 advice engine
+// needs the two facts a run record does not carry — the prompt, which classifies the task,
+// and the gold stat, which says how many files the change touched. A second private loader
+// beside the first would drift the first time one learned a filter the other lacked.
+//
+// Deterministic by construction: the directory is read sorted, the filter is exact, and no
+// part of this touches a model, a network or a clock.
+import { readdir, readFile } from 'node:fs/promises';
+import { join, resolve } from 'node:path';
+import { CONFIG_DIR } from '../name.mjs';
+
+const STATE_DIR = `.${CONFIG_DIR}`;
+
+/**
+ * Load mined benchmark task descriptors.
+ *
+ * @param {string} root the repository
+ * @param {string[]} [filterIds] exact task ids to keep; empty means load them all
+ * @returns descriptor objects sorted by id (id, base, sourceCommit, prompt, testCmd,
+ *   f2p, p2p, gold {files, linesAdded, linesRemoved}, vault)
+ */
+export async function loadTasks(root, filterIds = []) {
+  const tasksDir = join(resolve(root), STATE_DIR, 'bench', 'tasks');
+  const entries = (await readdir(tasksDir).catch(() => []))
+    .filter((f) => f.endsWith('.json'))
+    .sort();
+
+  const filterSet = new Set(filterIds);
+  const tasks = [];
+  for (const entry of entries) {
+    const data = JSON.parse(await readFile(join(tasksDir, entry), 'utf8'));
+    if (filterSet.size > 0 && !filterSet.has(data.id)) continue;
+    tasks.push(data);
+  }
+  return tasks.sort((a, b) => a.id.localeCompare(b.id));
+}