simt-dev 0.6.3 → 0.7.1

package/src/agents/render.mjs

@@ -0,0 +1,277 @@
+// One renderer per harness for the agent file, and the plan the install writes.
+//
+// Six harnesses read a subagent and no two agree on the file. Five write a
+// markdown file with YAML frontmatter and a body that becomes the system prompt;
+// Codex writes a TOML file whose `developer_instructions` holds the same body.
+// The path, the extension, the frontmatter keys and the document shape are all
+// different, and none of them is the same fact. So this module owns the
+// differences and the command owns none of them:
+//
+//   opencode     .opencode/agents/simt-agent.md   markdown   description
+//   claude       .claude/agents/simt-agent.md     markdown   name, description
+//   cursor       .cursor/agents/simt-agent.md     markdown   name, description
+//   gemini       .gemini/agents/simt-agent.md     markdown   name, description
+//   codex        .codex/agents/simt-agent.toml    TOML       name, description, developer_instructions
+//   antigravity  .agents/agents/simt-agent.md     markdown   name, description
+//
+// The source of truth is ONE file in this repository, `agents/simt-agent.md`:
+// neutral frontmatter plus a harness-agnostic body. The renderer strips that
+// frontmatter and re-emits the keys each harness documents, so the body — the
+// part that says how to act — is written once and cannot drift between
+// harnesses. An agent is a file the tool owns whole, not a config entry to
+// merge, so `rendered` compares bytes and reports created, unchanged or updated.
+
+import { mkdir, readFile } from 'node:fs/promises';
+import { dirname, join } from 'node:path';
+
+import { PACKAGE_ROOT } from '../workspace.mjs';
+import { splitFrontmatter } from '../memory/decide.mjs';
+import { LOADERS } from '../projection/loaders.mjs';
+import { journaledWriteFile } from '../journal.mjs';
+
+/** The one agent this tool ships, and the filename stem every harness shares. */
+const AGENT_FILE = 'simt-agent.md';
+export const AGENT_STEM = 'simt-agent';
+
+/** The companion skill: the command reference the agent reads on demand. */
+const SKILL_SOURCE = join('skills', 'simt-commands', 'SKILL.md');
+export const SKILL_STEM = 'simt-commands';
+
+/**
+ * The per-harness differences. `frontmatter` returns the ordered [key, value]
+ * pairs a markdown harness documents; Codex has none because its shape is TOML,
+ * not frontmatter. The body is identical everywhere and is not a difference.
+ */
+const ADAPTERS = {
+  opencode: {
+    id: 'opencode',
+    format: 'md',
+    frontmatter: (agent) => [
+      ['description', agent.description],
+    ],
+  },
+  claude: {
+    id: 'claude',
+    format: 'md',
+    frontmatter: (agent) => [
+      ['name', agent.name],
+      ['description', agent.description],
+    ],
+  },
+  cursor: {
+    id: 'cursor',
+    format: 'md',
+    frontmatter: (agent) => [
+      ['name', agent.name],
+      ['description', agent.description],
+    ],
+  },
+  gemini: {
+    id: 'gemini',
+    format: 'md',
+    frontmatter: (agent) => [
+      ['name', agent.name],
+      ['description', agent.description],
+    ],
+  },
+  codex: {
+    id: 'codex',
+    format: 'toml',
+  },
+  antigravity: {
+    id: 'antigravity',
+    format: 'md',
+    frontmatter: (agent) => [
+      ['name', agent.name],
+      ['description', agent.description],
+    ],
+  },
+};
+
+/** The reader-table row for a harness's agents directory, or null when none is documented. */
+function agentInfo(id) {
+  const row = LOADERS.find((l) => l.id === id);
+  return row?.agents ?? null;
+}
+
+/**
+ * The path a harness reads the agent from, sourced from the reader table rather
+ * than spelled here. Null for a harness that documents no agents directory.
+ */
+export function agentPath(id) {
+  const info = agentInfo(id);
+  return info ? join(info.dir, `${AGENT_STEM}${info.ext}`) : null;
+}
+
+/**
+ * Every reader, with the agents path it documents and the note for the ones that
+ * do not. The command reports these so a harness this tool does not write is
+ * named rather than silent.
+ */
+export function agentRows() {
+  return LOADERS.map((l) => ({
+    id: l.id,
+    label: l.label,
+    path: l.agents ? join(l.agents.dir, `${AGENT_STEM}${l.agents.ext}`) : null,
+    note: l.agents ? l.agents.verified : (l.agentsNote ?? 'no agents file is documented'),
+  }));
+}
+
+/** The adapters that both have a reader-table path and a renderer, in reader order. */
+export function adapters() {
+  return agentRows()
+    .filter((r) => r.path !== null)
+    .map((r) => ADAPTERS[r.id])
+    .filter(Boolean);
+}
+
+/** One adapter by id, or null when the harness has no path or no renderer. */
+export function adapter(id) {
+  return ADAPTERS[id] && agentInfo(id) !== null ? ADAPTERS[id] : null;
+}
+
+/** Parse the neutral frontmatter of the source file: `key: value` lines. */
+function parseSimpleFront(front) {
+  const out = {};
+  for (const line of front.split('\n')) {
+    const idx = line.indexOf(':');
+    if (idx < 0) continue;
+    const key = line.slice(0, idx).trim();
+    const val = line.slice(idx + 1).trim();
+    if (key) out[key] = val;
+  }
+  return out;
+}
+
+/**
+ * The one agent, read from this repository's `agents/` directory. The body is
+ * the system prompt; `name` and `description` are re-emitted per harness.
+ */
+export async function loadAgent() {
+  const raw = await readFile(join(PACKAGE_ROOT, 'agents', AGENT_FILE), 'utf8');
+  const { front, body } = splitFrontmatter(raw);
+  const meta = parseSimpleFront(front ?? '');
+  if (!meta.name || !meta.description) {
+    throw new Error(`${AGENT_FILE} must declare a name and a description in its frontmatter`);
+  }
+  return { name: meta.name, description: meta.description, body: body.trim() };
+}
+
+/** The companion skill file, read verbatim from this repository. */
+async function loadSkill() {
+  return readFile(join(PACKAGE_ROOT, SKILL_SOURCE), 'utf8');
+}
+
+/**
+ * Where the skill lands in the receiving repo. The same heuristic `skills
+ * install` uses: respect an existing skills directory, default to `.agents/skills`.
+ */
+async function skillDestPath(root) {
+  let entries = [];
+  try {
+    const { readdir } = await import('node:fs/promises');
+    entries = await readdir(root);
+  } catch { /* fresh repo, no entries */ }
+  let dir;
+  if (entries.includes('.agents')) dir = '.agents/skills';
+  else if (entries.includes('.cursor')) dir = '.cursor/skills';
+  else if (entries.includes('.claude')) dir = '.claude/skills';
+  else dir = '.agents/skills';
+  return join(dir, SKILL_STEM, 'SKILL.md');
+}
+
+/** A basic TOML string: double-quoted, with backslashes and quotes escaped. */
+function tomlString(value) {
+  return '"' + String(value).replace(/\\/g, '\\\\').replace(/"/g, '\\"') + '"';
+}
+
+/** The markdown form: frontmatter the harness documents, a blank line, the body. */
+function renderMarkdown(adapterRow, agent) {
+  const fm = adapterRow.frontmatter(agent).map(([k, v]) => `${k}: ${v}`).join('\n');
+  return `---\n${fm}\n---\n\n${agent.body}\n`;
+}
+
+/**
+ * The Codex form: a TOML file. The body goes in `developer_instructions` as a
+ * multi-line basic string; a literal `"""` in the body would end the string, so
+ * one of its quotes is escaped.
+ */
+function renderToml(agent) {
+  const body = agent.body.replace(/"""/g, '\\"""');
+  return `name = ${tomlString(agent.name)}\ndescription = ${tomlString(agent.description)}\ndeveloper_instructions = """\n${body}\n"""\n`;
+}
+
+/**
+ * Render the agent for a harness and compare it to what is on disk. The tool
+ * owns the file whole, so the comparison is by bytes and the action is one of
+ * created, unchanged, updated.
+ */
+export function rendered(adapterRow, agent, existing) {
+  const text = adapterRow.format === 'toml' ? renderToml(agent) : renderMarkdown(adapterRow, agent);
+  if (existing === null) return { text, action: 'created' };
+  if (existing === text) return { text, action: 'unchanged' };
+  return { text, action: 'updated' };
+}
+
+/** Read a file as UTF-8, or null when it is not there. */
+async function readIfExists(path) {
+  try {
+    return await readFile(path, 'utf8');
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * The agent files a repository should carry, one per harness that documents an
+ * agents directory, plus the companion skill file. Reads each existing file and
+ * renders what install would write, so a dry run and a run describe the same set
+ * of files. `harness` narrows the agent files to one; the skill is always included.
+ */
+export async function planAgents(root, { harness } = {}) {
+  const agent = await loadAgent();
+  const rows = harness === undefined ? adapters() : adapters().filter((a) => a.id === harness);
+  const plan = [];
+  for (const row of rows) {
+    const path = agentPath(row.id);
+    const existing = await readIfExists(join(root, path));
+    const result = rendered(row, agent, existing);
+    plan.push({ id: row.id, path, action: result.action, text: result.text });
+  }
+  // The companion skill: one file, not per-harness.
+  const skillText = await loadSkill();
+  const skillPath = await skillDestPath(root);
+  const skillExisting = await readIfExists(join(root, skillPath));
+  const skillAction = skillExisting === null ? 'created' : skillExisting === skillText ? 'unchanged' : 'updated';
+  plan.push({ id: 'skill', path: skillPath, action: skillAction, text: skillText });
+  return plan;
+}
+
+/**
+ * Write the agent files a plan names, journaled so `undo` restores the run.
+ * Skips files that are already unchanged. Returns the paths written.
+ */
+export async function writeAgentPlan(root, journal, plan) {
+  const written = [];
+  for (const p of plan.filter((x) => x.action !== 'unchanged')) {
+    await mkdir(join(root, dirname(p.path)), { recursive: true });
+    await journaledWriteFile(root, journal, join(root, p.path), p.text);
+    written.push(p.path);
+  }
+  return written;
+}
+
+/**
+ * Write just the skill file so `installedSkills` can find it before `plan()`
+ * computes the region. Called by `init` before the region is rendered.
+ */
+export async function writeSkillFile(root, journal) {
+  const text = await loadSkill();
+  const path = await skillDestPath(root);
+  const full = join(root, path);
+  const existing = await readIfExists(full);
+  if (existing === text) return null;
+  await mkdir(dirname(full), { recursive: true });
+  await journaledWriteFile(root, journal, full, text);
+  return path;
+}

package/src/api.mjs

@@ -0,0 +1,29 @@
+// The library surface: the read commands as data (review 0003 Q63).
+//
+// Every function here is the one the CLI and the MCP tools already call. Nothing in
+// this file renders: a caller that wants text, JSON or an MCP result renders these
+// results itself, which is the whole point - a UI, a harness and a terminal asking the
+// same question must not get three answers, and they cannot disagree if they are
+// reading one function's return value.
+//
+// This is the read half only. The commands that write (`init`, `sync`, `hooks
+// install`, `skills install`, `upgrade`) still print their plans from the command
+// module, because their output is a diff a person reviews before it is applied
+// (ADR-001). They migrate in the next slice, when `run(ctx, input)` exists to carry
+// a plan back to a caller that can show it.
+//
+// The exports map in package.json points `import 'simt-dev'` here, and keeps
+// `./src/*` because the plugin runtime and the harness shims import deep paths.
+// Removing that subpath would break them to make the surface look tidier.
+
+export { checkRepo, checkTargets, checkTarget, frozenFindings } from './commands/sync.mjs';
+export { diagnose, serialisedFindings } from './commands/doctor.mjs';
+export { rulesCatalogue, rulesWhyFacts } from './commands/rules.mjs';
+export { skillsBriefFacts, skillsFindFacts, skillsShowFacts, skillsWhyFacts } from './commands/skills.mjs';
+export { assembleBrief } from './brief/assemble.mjs';
+export { loadRegistry } from './graph/load.mjs';
+export { discover } from './compare/discover.mjs';
+export { buildRows, resolveConcurrency, summarise } from './compare/row.mjs';
+export { planNext } from './plan/index.mjs';
+export { buildPrBody } from './prbody.mjs';
+export { federationAccess, readFederation } from './federation/config.mjs';

package/src/argv.mjs

@@ -26,6 +26,11 @@ import { EXIT } from './output/index.mjs';
 // mean something, which it could not while three accepted flags had no description.
 
 export const BOOL_FLAGS = new Set([
+  // bench ablate: compare across harness, model and experiment groups. The command
+  // read `flags.crossGroup` while the parser accepted no such flag, so the override
+  // existed only in the error message that told the reader to pass it (review 0003
+  // Q62, found by the spec-vs-parser drift test).
+  'cross-group',
   'json',
   'all',
   'dry-run',
@@ -89,6 +94,10 @@ export const BOOL_FLAGS = new Set([
   // bench.json marked unmeasured, and a lint that failed it on arrival would make
   // every new pack red before its author wrote the rule (plan 0003 R45).
   'require-measured',
+  // bench mine: restore the old "reflect exactly this history" semantics by
+  // deleting task files the current history does not reproduce. Off by default
+  // because mining is append-only (review 0003 Q48).
+  'replace',
   // distill: propose the summary from the configured local chat model instead of
   // the deterministic outline. Off by default: no model in the default path (P1),
   // and a model that is not asked for should not be opened against (ADR-090).
@@ -119,6 +128,14 @@ export const VALUE_FLAGS = new Set([
   'task',
   'workspace',
   'root',
+  // skills/graph: which registry to read. A value rather than a boolean because
+  // "a registry" without a path would have to guess, and the guess is the
+  // defect Q37 exists to remove (review 0003).
+  'registry',
+  // compare: how many repository rows to build at once (review 0003 Q38).
+  // A count, not a switch, because "parallel" without a cap is the burst this
+  // flag exists to prevent.
+  'concurrency',
   // skills install: which body of a multi-variant skill, and which source when the
   // bytes alone do not identify one.
   'variant',
@@ -158,6 +175,9 @@ export const VALUE_FLAGS = new Set([
   // lease: how long the claim lasts, in seconds. A value, because "a lease is short"
   // is not a number any reader can act on, and an expiry nobody chose is an expiry
   // nobody can plan around (plan 0003 R54, ADR-101).
+  // lessons capture: how long a captured candidate stays eligible. A value, because a
+  // default would be a deadline nobody chose (review 0003 Q62, found by the drift test).
+  'expires',
   'ttl',
   // map: outline (paths), names (exported symbols) or signatures (the line).
   // A boolean would not say which of the three the caller wanted.
@@ -273,9 +293,23 @@ export function parse(argv) {
         flags.reps = n;
       } else {
         const key = camel(raw);
-        flags[key] = REPEATABLE_FLAGS.has(raw)
-          ? [...(Array.isArray(flags[key]) ? flags[key] : []), value]
-          : value;
+        if (REPEATABLE_FLAGS.has(raw)) {
+          flags[key] = [...(Array.isArray(flags[key]) ? flags[key] : []), value];
+        } else if (flags[key] !== undefined) {
+          // `--root a --root b` kept `b`, so half of what the caller asked for vanished
+          // with no error and no way to tell which half. Only a flag declared repeatable
+          // accumulates (review 0003 Q62); a second value for any other is a usage error.
+          return {
+            ok: false,
+            error: usage(
+              `--${raw} was given twice; it takes one value (repeatable: ${[...REPEATABLE_FLAGS]
+                .map((name) => `--${name}`)
+                .join(', ')})`,
+            ),
+          };
+        } else {
+          flags[key] = value;
+        }
       }
       continue;
     }

package/src/bench/ablate.mjs

@@ -11,6 +11,7 @@ import { mkdir, readFile, rm, unlink, writeFile } from 'node:fs/promises';
 import { dirname, join, resolve } from 'node:path';
 
 import { MANIFEST_FILE, STATE_DIR } from '../name.mjs';
+import { LOCAL_DIR } from '../state.mjs';
 import { readManifest } from '../projection/manifest.mjs';
 import { inject, parse as parseRegion } from '../projection/markers.mjs';
 import { loadMdcRules, renderMdc, ruleName } from '../projection/mdc.mjs';
@@ -23,6 +24,7 @@ import {
   DEFAULT_BOOTSTRAP_SEED,
   compareArms,
   evaluateRunRecord,
+  holmAdjust,
   loadPrices,
   loadRunRecords,
 } from './report.mjs';
@@ -340,11 +342,14 @@ export async function ablateSectionInWorkspace(workDir, ablate) {
 
 /**
  * Decide whether a canary comparison warrants escalating the section arm to the full task suite.
+ *
+ * `pValue` overrides the comparison's raw p-values when the caller supplies a
+ * multiplicity-adjusted value (Holm over the family of sections tested in one run).
  */
-export function shouldEscalateCanary(comparison, { alpha = CANARY_ALPHA } = {}) {
+export function shouldEscalateCanary(comparison, { alpha = CANARY_ALPHA, pValue } = {}) {
   if (!comparison || !comparison.pairedTasks) return false;
-  if (comparison.mcnemar.pValue < alpha) return true;
-  if (comparison.wilcoxon.pValue < alpha) return true;
+  const p = pValue ?? Math.min(comparison.rTest.pValue, comparison.wilcoxon.pValue);
+  if (p < alpha) return true;
   if (comparison.ci95.R[0] > 0 || comparison.ci95.R[1] < 0) return true;
   if (Math.abs(comparison.deltas.R) >= 0.2) return true;
   return false;
@@ -469,15 +474,27 @@ export async function ablateBench(
       const canaryEvaluated = canaryRecords.map((rec) => evaluateRunRecord(rec, prices));
       const currentCanary = canaryEvaluated.filter((r) => r.arm === 'current');
 
+      const canaryComparisons = [];
       for (const sec of sections) {
         const armCanary = canaryEvaluated.filter((r) => r.arm === sec.arm);
         if (currentCanary.length > 0 && armCanary.length > 0) {
-          const cmp = compareArms(sec.arm, armCanary, 'current', currentCanary, { samples, seed });
-          if (shouldEscalateCanary(cmp)) {
-            escalatedArms.add(sec.arm);
-          }
+          canaryComparisons.push({
+            sec,
+            cmp: compareArms(sec.arm, armCanary, 'current', currentCanary, { samples, seed }),
+          });
         }
       }
+      // Family = every section tested in this one run: testing ~N sections at α 0.05
+      // each inflates the false-positive rate, so the p-values are Holm-adjusted
+      // before the escalation decision (review 0003 Q46).
+      const adjusted = holmAdjust(
+        canaryComparisons.map(({ cmp }) => Math.min(cmp.rTest.pValue, cmp.wilcoxon.pValue)),
+      );
+      canaryComparisons.forEach(({ sec, cmp }, i) => {
+        if (shouldEscalateCanary(cmp, { pValue: adjusted[i] })) {
+          escalatedArms.add(sec.arm);
+        }
+      });
 
       if (escalatedArms.size > 0) {
         await execRunBench(resolvedRoot, {
@@ -529,7 +546,7 @@ export async function ablateBench(
           dC: [0, 0],
           valuePerKb: [0, 0],
         },
-        mcnemar: { statistic: 0, pValue: 1, b: 0, c: 0 },
+        rTest: { statistic: 0, pValue: 1, plus: 0, minus: 0 },
         wilcoxon: { statistic: 0, pValue: 1, wPlus: 0, wMinus: 0 },
         deletionCandidate: false,
       };
@@ -566,11 +583,11 @@ export async function ablateBench(
         dC: cmp.ci95.C,
         valuePerKb: vCi,
       },
-      mcnemar: {
-        statistic: cmp.mcnemar.statistic,
-        pValue: cmp.mcnemar.pValue,
-        b: cmp.mcnemar.b,
-        c: cmp.mcnemar.c,
+      rTest: {
+        statistic: cmp.rTest.statistic,
+        pValue: cmp.rTest.pValue,
+        plus: cmp.rTest.plus,
+        minus: cmp.rTest.minus,
       },
       wilcoxon: {
         statistic: cmp.wilcoxon.statistic,
@@ -616,7 +633,7 @@ export async function ablateBench(
   const markdown = formatAblationMarkdown(result);
 
   if (apply) {
-    const outPath = join(resolvedRoot, STATE_DIR, 'bench', 'ablation.json');
+    const outPath = join(resolvedRoot, LOCAL_DIR, 'bench', 'ablation.json');
     await mkdir(dirname(outPath), { recursive: true });
     await writeFile(outPath, `${JSON.stringify({ schemaVersion: 1, ...result }, null, 2)}\n`, 'utf8');
   }

package/src/bench/adapters.json

@@ -10,6 +10,8 @@
       "args": ["-p", "{prompt}", "--output-format", "json", "--max-turns", "{turns}"],
       "modelArgs": ["--model", "{model}"],
       "format": "json",
+      "isolation": "env",
+      "env": ["ANTHROPIC_API_KEY"],
       "usageSource": "usage.*, total_cost_usd, num_turns",
       "fields": {
         "input": ["usage.input_tokens", "usage.input"],
@@ -30,6 +32,8 @@
       "args": ["exec", "--json", "{prompt}"],
       "modelArgs": ["--model", "{model}"],
       "format": "jsonl",
+      "isolation": "env",
+      "env": ["OPENAI_API_KEY"],
       "usageSource": "token events in JSONL",
       "fields": {
         "input": ["usage.input_tokens", "input_tokens", "usage.input"],
@@ -50,6 +54,8 @@
       "args": ["-p", "{prompt}", "--output-format", "json"],
       "modelArgs": ["--model", "{model}"],
       "format": "json",
+      "isolation": "env",
+      "env": ["GEMINI_API_KEY", "GOOGLE_API_KEY"],
       "usageSource": "stats block",
       "fields": {
         "input": ["stats.inputTokens", "stats.tokens.prompt", "stats.prompt_tokens", "usage.input_tokens"],
@@ -70,6 +76,8 @@
       "args": ["run", "--format", "json", "{prompt}"],
       "modelArgs": ["--model", "{model}"],
       "format": "jsonl",
+      "isolation": "env",
+      "env": ["OPENAI_API_KEY", "ANTHROPIC_API_KEY"],
       "usageSource": "session export",
       "fields": {
         "input": ["tokens.input", "usage.input_tokens", "part.tokens.input"],
@@ -90,6 +98,8 @@
       "args": ["-p", "{prompt}", "--output-format", "json"],
       "modelArgs": ["--model", "{model}"],
       "format": "json",
+      "isolation": "env",
+      "env": ["CURSOR_API_KEY"],
       "usageSource": "result JSON",
       "fields": {
         "input": ["usage.input_tokens", "usage.input", "stats.inputTokens"],
@@ -110,6 +120,8 @@
       "args": ["--message", "{prompt}", "--yes-always", "--analytics-log", "{analyticsLog}"],
       "modelArgs": ["--model", "{model}"],
       "format": "analytics-log",
+      "isolation": "env",
+      "env": ["OPENAI_API_KEY", "ANTHROPIC_API_KEY"],
       "usageSource": "--analytics-log file",
       "fields": {
         "input": ["prompt_tokens", "properties.prompt_tokens", "usage.input_tokens"],
@@ -130,6 +142,8 @@
       "args": [],
       "modelArgs": [],
       "format": "builtin",
+      "isolation": "none",
+      "env": [],
       "usageSource": "vault gold.patch",
       "fields": {}
     },
@@ -142,6 +156,8 @@
       "args": [],
       "modelArgs": [],
       "format": "builtin",
+      "isolation": "none",
+      "env": [],
       "usageSource": "none",
       "fields": {}
     }