simt-dev 0.6.3 → 0.7.0

package/src/bench/ablate.mjs

@@ -11,6 +11,7 @@ import { mkdir, readFile, rm, unlink, writeFile } from 'node:fs/promises';
 import { dirname, join, resolve } from 'node:path';
 
 import { MANIFEST_FILE, STATE_DIR } from '../name.mjs';
+import { LOCAL_DIR } from '../state.mjs';
 import { readManifest } from '../projection/manifest.mjs';
 import { inject, parse as parseRegion } from '../projection/markers.mjs';
 import { loadMdcRules, renderMdc, ruleName } from '../projection/mdc.mjs';
@@ -23,6 +24,7 @@ import {
   DEFAULT_BOOTSTRAP_SEED,
   compareArms,
   evaluateRunRecord,
+  holmAdjust,
   loadPrices,
   loadRunRecords,
 } from './report.mjs';
@@ -340,11 +342,14 @@ export async function ablateSectionInWorkspace(workDir, ablate) {
 
 /**
  * Decide whether a canary comparison warrants escalating the section arm to the full task suite.
+ *
+ * `pValue` overrides the comparison's raw p-values when the caller supplies a
+ * multiplicity-adjusted value (Holm over the family of sections tested in one run).
  */
-export function shouldEscalateCanary(comparison, { alpha = CANARY_ALPHA } = {}) {
+export function shouldEscalateCanary(comparison, { alpha = CANARY_ALPHA, pValue } = {}) {
   if (!comparison || !comparison.pairedTasks) return false;
-  if (comparison.mcnemar.pValue < alpha) return true;
-  if (comparison.wilcoxon.pValue < alpha) return true;
+  const p = pValue ?? Math.min(comparison.rTest.pValue, comparison.wilcoxon.pValue);
+  if (p < alpha) return true;
   if (comparison.ci95.R[0] > 0 || comparison.ci95.R[1] < 0) return true;
   if (Math.abs(comparison.deltas.R) >= 0.2) return true;
   return false;
@@ -469,15 +474,27 @@ export async function ablateBench(
       const canaryEvaluated = canaryRecords.map((rec) => evaluateRunRecord(rec, prices));
       const currentCanary = canaryEvaluated.filter((r) => r.arm === 'current');
 
+      const canaryComparisons = [];
       for (const sec of sections) {
         const armCanary = canaryEvaluated.filter((r) => r.arm === sec.arm);
         if (currentCanary.length > 0 && armCanary.length > 0) {
-          const cmp = compareArms(sec.arm, armCanary, 'current', currentCanary, { samples, seed });
-          if (shouldEscalateCanary(cmp)) {
-            escalatedArms.add(sec.arm);
-          }
+          canaryComparisons.push({
+            sec,
+            cmp: compareArms(sec.arm, armCanary, 'current', currentCanary, { samples, seed }),
+          });
         }
       }
+      // Family = every section tested in this one run: testing ~N sections at α 0.05
+      // each inflates the false-positive rate, so the p-values are Holm-adjusted
+      // before the escalation decision (review 0003 Q46).
+      const adjusted = holmAdjust(
+        canaryComparisons.map(({ cmp }) => Math.min(cmp.rTest.pValue, cmp.wilcoxon.pValue)),
+      );
+      canaryComparisons.forEach(({ sec, cmp }, i) => {
+        if (shouldEscalateCanary(cmp, { pValue: adjusted[i] })) {
+          escalatedArms.add(sec.arm);
+        }
+      });
 
       if (escalatedArms.size > 0) {
         await execRunBench(resolvedRoot, {
@@ -529,7 +546,7 @@ export async function ablateBench(
           dC: [0, 0],
           valuePerKb: [0, 0],
         },
-        mcnemar: { statistic: 0, pValue: 1, b: 0, c: 0 },
+        rTest: { statistic: 0, pValue: 1, plus: 0, minus: 0 },
         wilcoxon: { statistic: 0, pValue: 1, wPlus: 0, wMinus: 0 },
         deletionCandidate: false,
       };
@@ -566,11 +583,11 @@ export async function ablateBench(
         dC: cmp.ci95.C,
         valuePerKb: vCi,
       },
-      mcnemar: {
-        statistic: cmp.mcnemar.statistic,
-        pValue: cmp.mcnemar.pValue,
-        b: cmp.mcnemar.b,
-        c: cmp.mcnemar.c,
+      rTest: {
+        statistic: cmp.rTest.statistic,
+        pValue: cmp.rTest.pValue,
+        plus: cmp.rTest.plus,
+        minus: cmp.rTest.minus,
       },
       wilcoxon: {
         statistic: cmp.wilcoxon.statistic,
@@ -616,7 +633,7 @@ export async function ablateBench(
   const markdown = formatAblationMarkdown(result);
 
   if (apply) {
-    const outPath = join(resolvedRoot, STATE_DIR, 'bench', 'ablation.json');
+    const outPath = join(resolvedRoot, LOCAL_DIR, 'bench', 'ablation.json');
     await mkdir(dirname(outPath), { recursive: true });
     await writeFile(outPath, `${JSON.stringify({ schemaVersion: 1, ...result }, null, 2)}\n`, 'utf8');
   }

package/src/bench/mine.mjs

@@ -29,11 +29,12 @@ export const DEFAULT_MINE_TARGET = 20;
 /**
  * Classify whether a repository-relative path is a test file.
  *
- * Covers the five ecosystems named in plan 0003 §5 R01:
+ * Covers the ecosystems named in plan 0003 §5 R01:
  *   - `test/`, `tests/`, `spec/`, `__tests__/`, `src/test/`
  *   - `*.test.*`, `*.spec.*`, `*_test.*`, `*_spec.*`
  *   - `*_test.go` (Go)
  *   - `test_*.py` (pytest)
+ *   - `tests/*.rs` (Cargo integration tests)
  *   - `*Test.java`, `*Tests.java`, `*Test.kt`, `*Tests.kt` (Gradle / Maven)
  *
  * @param {string} relPath forward-slash repository-relative path
@@ -259,6 +260,31 @@ export function parseGoTestJson(text) {
   };
 }
 
+/**
+ * Parse libtest's text output, which `cargo test` prints for every test binary.
+ *
+ * A line is a result only when it has the `test <name> ... <status>` shape.
+ * Harness summaries (`test result: ok.`), compile output and doctest prose are
+ * not results. `ignored` is neither a pass nor a failure: counting it as one
+ * would make a skipped test look like evidence the change works.
+ *
+ * @param {string} text
+ * @returns {{passed: string[], failed: string[]}}
+ */
+export function parseCargoTestOutput(text) {
+  const passed = [];
+  const failed = [];
+  for (const line of String(text ?? '').split('\n')) {
+    const match = /^test\s+(.+?)\s+\.\.\.\s+(ok|FAILED|ignored)\s*$/.exec(line.trimEnd());
+    if (!match || match[2] === 'ignored') continue;
+    (match[2] === 'ok' ? passed : failed).push(match[1]);
+  }
+  return {
+    passed: dedupeSorted(passed),
+    failed: dedupeSorted(failed),
+  };
+}
+
 /**
  * Detect the test runner for a repository and build the command for a set of
  * changed test files.
@@ -302,6 +328,27 @@ export async function detectTestRunner(root, testFiles) {
     };
   }
 
+  if (existsSync(join(root, 'Cargo.toml'))) {
+    // Cargo has no "run this file" argument. An integration test is a target
+    // named after its file in `tests/`, so it can be selected with `--test`.
+    // A source file carries inline unit tests and can only be reached by the
+    // crate's test binary, so one such file means the whole crate runs.
+    const targets = dedupeSorted(
+      files.filter((f) => /^tests\/[^/]+\.rs$/.test(f)).map((f) => basename(f, '.rs')),
+    );
+    const integrationOnly = files.length > 0 && files.every((f) => /^tests\/[^/]+\.rs$/.test(f));
+    const args = ['test'];
+    if (integrationOnly) {
+      for (const target of targets) args.push('--test', target);
+    }
+    return {
+      runner: 'cargo-test',
+      displayCmd: ['cargo', ...args].join(' '),
+      buildExec: () => ({ cmd: 'cargo', args }),
+      parse: (stdout) => parseCargoTestOutput(stdout),
+    };
+  }
+
   if (existsSync(join(root, 'go.mod'))) {
     const pkgs = dedupeSorted(
       files.map((f) => {

package/src/bench/probes.mjs

@@ -25,7 +25,7 @@ import { chmod, mkdir, mkdtemp, readdir, readFile, rm, writeFile } from 'node:fs
 import { tmpdir } from 'node:os';
 import { dirname, join, resolve } from 'node:path';
 
-import { STATE_DIR } from '../name.mjs';
+import { LOCAL_DIR } from '../state.mjs';
 import { parse as parseRegion } from '../projection/markers.mjs';
 import { materialiseArm, parseArmSpec } from './run.mjs';
 
@@ -92,7 +92,7 @@ function runNode(args, { cwd, env = {} } = {}) {
  * @returns {Promise<Array<object>>}
  */
 export async function readLedgerEntries(workDir) {
-  const ledgerPath = join(workDir, STATE_DIR, 'ledger.jsonl');
+  const ledgerPath = join(workDir, LOCAL_DIR, 'ledger.jsonl');
   try {
     const text = await readFile(ledgerPath, 'utf8');
     const entries = [];
@@ -115,7 +115,7 @@ export async function readLedgerEntries(workDir) {
  * Append a ledger entry to `.simt-dev/ledger.jsonl` in `workDir`.
  */
 export async function appendLedgerEntry(workDir, entry) {
-  const ledgerPath = join(workDir, STATE_DIR, 'ledger.jsonl');
+  const ledgerPath = join(workDir, LOCAL_DIR, 'ledger.jsonl');
   await mkdir(dirname(ledgerPath), { recursive: true });
   const treeRes = await runGit(['write-tree'], { cwd: workDir });
   const record = {
@@ -1148,8 +1148,7 @@ export async function gradeProbe(probeOrId, ctx) {
  * @param {string} root
  * @returns {Promise<Array<object>>}
  */
-export async function loadProbeRecords(root) {
-  const probesRoot = join(resolve(root), STATE_DIR, 'bench', 'probes');
+export async function loadProbeRecords(root) {    const probesRoot = join(resolve(root), LOCAL_DIR, 'bench', 'probes');
   if (!existsSync(probesRoot)) return [];
 
   const entries = await readdir(probesRoot, { withFileTypes: true }).catch(() => []);
@@ -1242,7 +1241,7 @@ export async function runBenchProbes(
       for (let rep = 1; rep <= reps; rep += 1) {
         const runDir = join(
           resolvedRoot,
-          STATE_DIR,
+          LOCAL_DIR,
           'bench',
           'probes',
           `${harness}-${armSpec.name}-r${rep}`,