simt-dev 0.7.1 → 0.8.0

package/src/bench/mine.mjs

@@ -10,22 +10,24 @@
 // outside every worktree (RepoTrials / PatchGym), while deterministic task
 // descriptors live in `.simt/bench/tasks/<id>.json`.
 
-import { execFile, execFileSync, spawn } from 'node:child_process';
+import { execFile, spawn } from 'node:child_process';
 import { createHash } from 'node:crypto';
 import { existsSync } from 'node:fs';
 import { chmod, mkdir, mkdtemp, readdir, readFile, rm, stat, writeFile } from 'node:fs/promises';
-import { homedir, platform as osPlatform, tmpdir } from 'node:os';
+import { tmpdir } from 'node:os';
 import { basename, dirname, join, resolve } from 'node:path';
 import { promisify } from 'node:util';
 
-import { CONFIG_DIR, STATE_DIR } from '../name.mjs';
-import { SCHEMA_VERSION } from '../output/index.mjs';
+import { STATE_DIR } from '../name.mjs';
 import { readManifest } from '../projection/manifest.mjs';
 import { packageVersion } from '../package-version.mjs';
+import { TASK_SCHEMA_VERSION, normaliseTask } from '../measure/task-schema.mjs';
+import { isTestPath } from '../testing/paths.mjs';
+import { cleanupReports, dedupeSorted, readReports } from '../testing/parsers.mjs';
+import { detectTestRunner } from '../testing/runners.mjs';
+import { resolveVaultDir } from '../testing/vault.mjs';
 
 const exec = promisify(execFile);
-
-/** Default target task count when mining a repository's history. */
 export const DEFAULT_MINE_TARGET = 20;
 
 /**
@@ -33,528 +35,6 @@ export const DEFAULT_MINE_TARGET = 20;
  * (review 0003 Q48: the pack integration test was hardcoded here, a repo fact, not a miner fact).
  */
 const DEFAULT_EXCLUDES = ['test/pack.test.mjs'];
-
-/**
- * Classify whether a repository-relative path is a test file.
- *
- * Covers the ecosystems named in plan 0003 §5 R01:
- *   - `test/`, `tests/`, `spec/`, `__tests__/`, `src/test/`
- *   - `*.test.*`, `*.spec.*`, `*_test.*`, `*_spec.*`
- *   - `*_test.go` (Go)
- *   - `test_*.py` (pytest)
- *   - `tests/*.rs` (Cargo integration tests)
- *   - `*Test.java`, `*Tests.java`, `*Test.kt`, `*Tests.kt` (Gradle / Maven)
- *
- * @param {string} relPath forward-slash repository-relative path
- * @returns {boolean}
- */
-export function isTestPath(relPath) {
-  const norm = String(relPath).replaceAll('\\', '/').replace(/^\.\//, '');
-  if (!norm) return false;
-  const segments = norm.split('/');
-  const base = segments[segments.length - 1];
-
-  if (
-    norm.startsWith('test/') ||
-    norm.startsWith('tests/') ||
-    norm.startsWith('spec/') ||
-    norm.startsWith('__tests__/') ||
-    norm.startsWith('src/test/') ||
-    segments.includes('__tests__') ||
-    segments.includes('test') ||
-    segments.includes('tests') ||
-    segments.includes('spec')
-  ) {
-    return true;
-  }
-
-  if (/\.(?:test|spec)\.[A-Za-z0-9]+$/.test(base)) return true;
-  if (/_(?:test|spec)\.[A-Za-z0-9]+$/.test(base)) return true;
-  if (/^test_[^/]+\.py$/.test(base)) return true;
-  if (/Tests?\.(?:java|kt)$/.test(base)) return true;
-
-  return false;
-}
-
-/**
- * Parse TAP version 13 output from `node --test --test-reporter=tap`.
- *
- * Leaf test cases carry `type: 'test'` in their YAML block; suite headers carry
- * `type: 'suite'` and are used only to build `parent > child` qualified IDs.
- *
- * @param {string} text
- * @returns {{passed: string[], failed: string[]}}
- */
-export function parseTapOutput(text) {
-  const passed = [];
-  const failed = [];
-  const lines = String(text ?? '').split('\n');
-  const stack = [];
-  let pending = null;
-
-  const flush = () => {
-    if (!pending) return;
-    if (pending.type !== 'suite' && !pending.hadChild && !pending.directive) {
-      const id = [...pending.ancestors, pending.name].join(' > ');
-      if (pending.ok) passed.push(id);
-      else failed.push(id);
-    }
-    pending = null;
-  };
-
-  for (const raw of lines) {
-    const sub = /^(\s*)# Subtest:\s+(.*)$/.exec(raw);
-    if (sub) {
-      flush();
-      const indent = sub[1].length;
-      while (stack.length > 0 && stack[stack.length - 1].indent >= indent) {
-        stack.pop();
-      }
-      if (stack.length > 0) {
-        stack[stack.length - 1].hadChild = true;
-      }
-      stack.push({ indent, name: sub[2].trim(), hadChild: false });
-      continue;
-    }
-
-    const res = /^(\s*)(not ok|ok)\s+\d+(?:\s+-\s+(.*?))?(?:\s+#\s*(SKIP|TODO)\b.*)?$/i.exec(raw);
-    if (res) {
-      flush();
-      const indent = res[1].length;
-      while (stack.length > 0 && stack[stack.length - 1].indent > indent) {
-        stack.pop();
-      }
-      let hadChild = false;
-      if (stack.length > 0 && stack[stack.length - 1].indent === indent) {
-        hadChild = stack.pop().hadChild;
-      }
-      const ancestors = stack.map((s) => s.name);
-      const ok = res[2].toLowerCase() === 'ok';
-      const name = (res[3] ?? '').trim();
-      const directive = res[4] ? res[4].toUpperCase() : null;
-      if (name) {
-        pending = { indent, ok, name, ancestors, hadChild, directive, type: null };
-      }
-      continue;
-    }
-
-    if (pending) {
-      const typeMatch = /^\s*type:\s*['"]?(test|suite)['"]?\s*$/.exec(raw);
-      if (typeMatch) {
-        pending.type = typeMatch[1];
-      } else if (/^\s*\.\.\.\s*$/.test(raw)) {
-        flush();
-      }
-    }
-  }
-  flush();
-
-  return {
-    passed: dedupeSorted(passed),
-    failed: dedupeSorted(failed),
-  };
-}
-
-/**
- * Parse Vitest `--reporter=json` output.
- *
- * @param {string} text
- * @returns {{passed: string[], failed: string[]}}
- */
-export function parseVitestJson(text) {
-  const passed = [];
-  const failed = [];
-  const raw = String(text ?? '').trim();
-  const start = raw.indexOf('{');
-  const end = raw.lastIndexOf('}');
-  if (start === -1 || end === -1 || end < start) return { passed, failed };
-
-  let doc;
-  try {
-    doc = JSON.parse(raw.slice(start, end + 1));
-  } catch {
-    return { passed, failed };
-  }
-
-  for (const suite of doc.testResults ?? []) {
-    for (const assertion of suite.assertionResults ?? []) {
-      const parts = [...(assertion.ancestorTitles ?? []), assertion.title].filter(Boolean);
-      const id = parts.length > 0 ? parts.join(' > ') : (assertion.fullName ?? '').trim();
-      if (!id) continue;
-      if (assertion.status === 'passed') passed.push(id);
-      else if (assertion.status === 'failed') failed.push(id);
-    }
-  }
-
-  return {
-    passed: dedupeSorted(passed),
-    failed: dedupeSorted(failed),
-  };
-}
-
-/**
- * Parse JUnit XML output (used by pytest `--junitxml`, Gradle and Maven).
- *
- * @param {string} xml
- * @returns {{passed: string[], failed: string[]}}
- */
-export function parseJunitXml(xml) {
-  const passed = [];
-  const failed = [];
-  const source = String(xml ?? '');
-  const tagRe = /<testcase\b([^>]*?)(?:\/>|>([\s\S]*?)<\/testcase>)/g;
-
-  for (const match of source.matchAll(tagRe)) {
-    const attrs = match[1] ?? '';
-    const inner = match[2] ?? '';
-    const name = xmlAttr(attrs, 'name');
-    const classname = xmlAttr(attrs, 'classname');
-    if (!name) continue;
-    if (/<skipped\b/i.test(inner)) continue;
-
-    const id = classname ? `${classname} > ${name}` : name;
-    if (/<(?:failure|error)\b/i.test(inner)) failed.push(id);
-    else passed.push(id);
-  }
-
-  return {
-    passed: dedupeSorted(passed),
-    failed: dedupeSorted(failed),
-  };
-}
-
-function xmlAttr(attrs, key) {
-  const m = new RegExp(`\\b${key}=["']([^"']*)["']`).exec(attrs);
-  if (!m) return '';
-  return m[1]
-    .replaceAll('&quot;', '"')
-    .replaceAll('&apos;', "'")
-    .replaceAll('&lt;', '<')
-    .replaceAll('&gt;', '>')
-    .replaceAll('&amp;', '&')
-    .trim();
-}
-
-/**
- * Parse `go test -json` JSONL event stream.
- *
- * Only leaf test events (carrying `Test`) are recorded; package-level summary
- * events without `Test` are ignored.
- *
- * @param {string} text
- * @returns {{passed: string[], failed: string[]}}
- */
-export function parseGoTestJson(text) {
-  const passed = [];
-  const failed = [];
-  for (const line of String(text ?? '').split('\n')) {
-    const trimmed = line.trim();
-    if (!trimmed.startsWith('{')) continue;
-    let ev;
-    try {
-      ev = JSON.parse(trimmed);
-    } catch {
-      continue;
-    }
-    if (typeof ev.Test !== 'string' || ev.Test.length === 0) continue;
-    const id = ev.Package ? `${ev.Package} > ${ev.Test}` : ev.Test;
-    if (ev.Action === 'pass') passed.push(id);
-    else if (ev.Action === 'fail') failed.push(id);
-  }
-
-  return {
-    passed: dedupeSorted(passed),
-    failed: dedupeSorted(failed),
-  };
-}
-
-/**
- * Parse libtest's text output, which `cargo test` prints for every test binary.
- *
- * A line is a result only when it has the `test <name> ... <status>` shape.
- * Harness summaries (`test result: ok.`), compile output and doctest prose are
- * not results. `ignored` is neither a pass nor a failure: counting it as one
- * would make a skipped test look like evidence the change works.
- *
- * @param {string} text
- * @returns {{passed: string[], failed: string[]}}
- */
-export function parseCargoTestOutput(text) {
-  const passed = [];
-  const failed = [];
-  for (const line of String(text ?? '').split('\n')) {
-    const match = /^test\s+(.+?)\s+\.\.\.\s+(ok|FAILED|ignored)\s*$/.exec(line.trimEnd());
-    if (!match || match[2] === 'ignored') continue;
-    (match[2] === 'ok' ? passed : failed).push(match[1]);
-  }
-  return {
-    passed: dedupeSorted(passed),
-    failed: dedupeSorted(failed),
-  };
-}
-
-/**
- * Read every JUnit XML a run left behind, from report files or report
- * *directories* (review 0003 Q48: Gradle writes one XML per class under
- * `build/test-results/test/`, Maven under `target/surefire-reports/`; reading
- * a single file saw none of them, so both graders read nothing). Directories
- * are walked recursively in sort order so the concatenation is deterministic.
- *
- * @param {string[]} paths report files and/or directories
- * @returns {Promise<string>} the concatenated XML, '' when nothing is there
- */
-export async function readReports(paths) {
-  const chunks = [];
-  for (const p of paths ?? []) {
-    let isDir = false;
-    try {
-      isDir = (await stat(p)).isDirectory();
-    } catch {
-      continue;
-    }
-    if (isDir) {
-      const files = await listXml(p);
-      for (const file of files) {
-        chunks.push(await readFile(file, 'utf8').catch(() => ''));
-      }
-    } else {
-      chunks.push(await readFile(p, 'utf8').catch(() => ''));
-    }
-  }
-  return chunks.join('\n');
-}
-
-/** Every `*.xml` under `dir`, sorted, for a deterministic read order. */
-async function listXml(dir) {
-  const out = [];
-  const entries = await readdir(dir, { withFileTypes: true }).catch(() => []);
-  for (const entry of entries.sort((a, b) => a.name.localeCompare(b.name))) {
-    const full = join(dir, entry.name);
-    if (entry.isDirectory()) out.push(...(await listXml(full)));
-    else if (entry.name.endsWith('.xml')) out.push(full);
-  }
-  return out;
-}
-
-/**
- * Detect the test runner for a repository and build the command for a set of
- * changed test files.
- *
- * @param {string} root
- * @param {string[]} testFiles
- * @returns {Promise<{runner: string, displayCmd: string, buildExec: (worktree: string) => {cmd: string, args: string[], reportPaths?: string[]}, parse: (stdout: string, reportText?: string) => {passed: string[], failed: string[]}}|null>}
- */
-export async function detectTestRunner(root, testFiles) {
-  const files = [...testFiles].sort();
-
-  if (existsSync(join(root, 'package.json'))) {
-    let pkg = {};
-    try {
-      pkg = JSON.parse(await readFile(join(root, 'package.json'), 'utf8'));
-    } catch {
-      pkg = {};
-    }
-    const deps = { ...(pkg.dependencies ?? {}), ...(pkg.devDependencies ?? {}) };
-    const testScript = String(pkg.scripts?.test ?? '');
-    if (deps.vitest || /\bvitest\b/.test(testScript)) {
-      const bin = join('node_modules', '.bin', 'vitest');
-      return {
-        runner: 'vitest',
-        displayCmd: `vitest run --reporter=json ${files.join(' ')}`,
-        buildExec: (wt) => ({
-          cmd: existsSync(join(wt, bin)) ? join(wt, bin) : 'vitest',
-          args: ['run', '--reporter=json', ...files],
-        }),
-        parse: (stdout) => parseVitestJson(stdout),
-      };
-    }
-    // Jest emits the same `testResults`/`assertionResults` JSON shape as
-    // Vitest (Vitest copied it), so the JSON parser serves both (review 0003
-    // Q48: a jest repository fell through to `node --test` and parsed TAP
-    // out of JSON).
-    if (deps.jest || /\bjest\b/.test(testScript)) {
-      const bin = join('node_modules', '.bin', 'jest');
-      return {
-        runner: 'jest',
-        displayCmd: `jest --json ${files.join(' ')}`,
-        buildExec: (wt) => ({
-          cmd: existsSync(join(wt, bin)) ? join(wt, bin) : 'jest',
-          args: ['--json', ...files],
-        }),
-        parse: (stdout) => parseVitestJson(stdout),
-      };
-    }
-    return {
-      runner: 'node-test',
-      displayCmd: `node --test ${files.join(' ')}`,
-      buildExec: () => ({
-        cmd: process.execPath,
-        args: ['--test', '--test-reporter=tap', ...files],
-      }),
-      parse: (stdout) => parseTapOutput(stdout),
-    };
-  }
-
-  if (existsSync(join(root, 'Cargo.toml'))) {
-    // Cargo has no "run this file" argument. An integration test is a target
-    // named after its file in `tests/`, so it can be selected with `--test`.
-    // A source file carries inline unit tests and can only be reached by the
-    // crate's test binary, so one such file means the whole crate runs.
-    const targets = dedupeSorted(
-      files.filter((f) => /^tests\/[^/]+\.rs$/.test(f)).map((f) => basename(f, '.rs')),
-    );
-    const integrationOnly = files.length > 0 && files.every((f) => /^tests\/[^/]+\.rs$/.test(f));
-    const args = ['test'];
-    if (integrationOnly) {
-      for (const target of targets) args.push('--test', target);
-    }
-    return {
-      runner: 'cargo-test',
-      displayCmd: ['cargo', ...args].join(' '),
-      buildExec: () => ({ cmd: 'cargo', args }),
-      parse: (stdout) => parseCargoTestOutput(stdout),
-    };
-  }
-
-  if (existsSync(join(root, 'go.mod'))) {
-    const pkgs = dedupeSorted(
-      files.map((f) => {
-        const dir = dirname(f);
-        return dir === '.' ? './...' : `./${dir}`;
-      }),
-    );
-    return {
-      runner: 'go-test',
-      displayCmd: `go test -json ${pkgs.join(' ')}`,
-      buildExec: () => ({
-        cmd: 'go',
-        args: ['test', '-json', ...pkgs],
-      }),
-      parse: (stdout) => parseGoTestJson(stdout),
-    };
-  }
-
-  if (
-    existsSync(join(root, 'pyproject.toml')) ||
-    existsSync(join(root, 'requirements.txt')) ||
-    existsSync(join(root, 'setup.py'))
-  ) {
-    return {
-      runner: 'pytest',
-      displayCmd: `pytest ${files.join(' ')}`,
-      buildExec: (wt) => {
-        const reportPath = join(wt, '.bench-junit.xml');
-        return {
-          cmd: 'pytest',
-          args: [`--junitxml=${reportPath}`, ...files],
-          reportPaths: [reportPath],
-        };
-      },
-      parse: (_stdout, reportText) => parseJunitXml(reportText ?? ''),
-    };
-  }
-
-  if (existsSync(join(root, 'build.gradle.kts')) || existsSync(join(root, 'build.gradle'))) {
-    const classes = files
-      .map((f) => basename(f).replace(/\.(?:java|kt)$/, ''))
-      .filter(Boolean);
-    const filterArgs = classes.flatMap((c) => ['--tests', `*${c}*`]);
-    return {
-      runner: 'gradle',
-      displayCmd: `./gradlew test ${filterArgs.join(' ')}`.trim(),
-      buildExec: (wt) => ({
-        cmd: existsSync(join(wt, 'gradlew')) ? join(wt, 'gradlew') : 'gradle',
-        args: ['test', ...filterArgs],
-        // One XML per test class; readReports walks the directory.
-        reportPaths: [join(wt, 'build', 'test-results', 'test')],
-      }),
-      parse: (_stdout, reportText) => parseJunitXml(reportText ?? ''),
-    };
-  }
-
-  if (existsSync(join(root, 'pom.xml'))) {
-    const classes = files
-      .map((f) => basename(f).replace(/\.(?:java|kt)$/, ''))
-      .filter(Boolean);
-    const arg = classes.length > 0 ? [`-Dtest=${classes.join(',')}`] : [];
-    return {
-      runner: 'maven',
-      displayCmd: `mvn test ${arg.join(' ')}`.trim(),
-      buildExec: (wt) => ({
-        cmd: 'mvn',
-        args: ['test', ...arg],
-        // Surefire writes one XML per class here; never read before (review 0003 Q48).
-        reportPaths: [join(wt, 'target', 'surefire-reports')],
-      }),
-      parse: (_stdout, reportText) => parseJunitXml(reportText ?? ''),
-    };
-  }
-
-  return null;
-}
-
-/**
- * Resolve the hidden verifier vault root outside every worktree.
- *
- * Precedence:
- *   1. `SIMT_DEV_VAULT_DIR` (explicit override for hermetic tests)
- *   2. `$XDG_CACHE_HOME/simt-dev/vault/<repo-id>`
- *   3. `~/Library/Caches/simt-dev/vault/<repo-id>` on macOS, or `~/.cache/simt-dev/vault/<repo-id>`
- *
- * @param {string} root
- * @param {{env?: NodeJS.ProcessEnv, home?: string, platform?: string}} [options]
- * @returns {string}
- */
-export function resolveVaultDir(root, { env = process.env, home = homedir(), platform = osPlatform() } = {}) {
-  const repoId = repoIdentifier(root);
-  if (env.SIMT_DEV_VAULT_DIR) {
-    return join(resolve(env.SIMT_DEV_VAULT_DIR), repoId);
-  }
-  const cacheBase = env.XDG_CACHE_HOME
-    ? resolve(env.XDG_CACHE_HOME)
-    : platform === 'darwin'
-      ? join(home, 'Library', 'Caches')
-      : join(home, '.cache');
-  return join(cacheBase, CONFIG_DIR, 'vault', repoId);
-}
-
-/**
- * The stable identity a repository's vault is keyed on: the basename for a
- * human-readable prefix, and the root commit (first commit in history) for the
- * digest. The root commit is identical in every clone and survives remote
- * renames, so the vault lands in the same directory on every machine with the
- * same history (review 0003 Q48: the absolute-path key made the committed
- * corpus unrunnable from any other checkout). Outside git it degrades to the
- * absolute-path digest: a root with no history has no better identity, and the
- * vault path must never fail because git is absent.
- *
- * Sync, like package-version.mjs: the vault path is computed on every read, and
- * a path that throws is worse than one that is occasionally less specific.
- */
-function repoIdentifier(root) {
-  const full = resolve(root);
-  const name = basename(full).replace(/[^A-Za-z0-9._-]/g, '-') || 'repo';
-  try {
-    const sha = execFileSync('git', ['rev-list', '--max-parents=0', 'HEAD'], {
-      cwd: root,
-      encoding: 'utf8',
-      stdio: ['ignore', 'pipe', 'ignore'],
-    }).trim().split(/\s/)[0];
-    if (/^[0-9a-f]{40}$/.test(sha)) return `${name}-root-${sha.slice(0, 8)}`;
-  } catch {
-    // Not a git root (or no commits): fall through to the path digest.
-  }
-  const digest = createHash('sha256').update(full).digest('hex').slice(0, 8);
-  return `${name}-${digest}`;
-}
-
-/**
- * Compute diff stats (`files`, `linesAdded`, `linesRemoved`) from a unified diff buffer.
- *
- * @param {Buffer|string} patch
- * @param {number} fileCount
- * @returns {{files: number, linesAdded: number, linesRemoved: number}}
- */
 export function patchStats(patch, fileCount) {
   const text = Buffer.isBuffer(patch) ? patch.toString('utf8') : String(patch ?? '');
   let linesAdded = 0;
@@ -690,6 +170,7 @@ async function executeRunner(runner, worktree, env, timeoutMs) {
   const res = await runProcess(spec.cmd, spec.args, { cwd: worktree, env, timeoutMs });
   const reportText = await readReports(spec.reportPaths);
   const parsed = runner.parse(res.stdout, reportText);
+  await cleanupReports(spec.reportPaths);
   return {
     code: res.code,
     timedOut: res.timedOut,
@@ -797,7 +278,7 @@ async function validateCandidate(
 
   return {
     task: {
-      schemaVersion: SCHEMA_VERSION,
+      schemaVersion: TASK_SCHEMA_VERSION,
       id,
       base: candidate.base,
       sourceCommit: candidate.sha,
@@ -968,7 +449,13 @@ export async function mineTasks(
           };
         }
       }
-      await writeFile(taskPath, `${JSON.stringify(entry.task, null, 2)}\n`, 'utf8');
+      // The miner is the writer, so it writes the schema a reader validates: a task file
+      // that only becomes valid on load is a file two readers can disagree about.
+      await writeFile(
+        taskPath,
+        `${JSON.stringify(normaliseTask(entry.task, { source: taskPath }), null, 2)}\n`,
+        'utf8',
+      );
 
       const taskVault = join(vaultRoot, entry.task.id);
       await mkdir(taskVault, { recursive: true, mode: 0o700 });
@@ -1008,11 +495,6 @@ export async function mineTasks(
     await rm(sandbox, { recursive: true, force: true }).catch(() => null);
   }
 }
-
-function dedupeSorted(items) {
-  return [...new Set(items)].sort();
-}
-
 function sameList(a, b) {
   if (a.length !== b.length) return false;
   for (let i = 0; i < a.length; i += 1) {