testguard-cli 0.5.0 → 0.7.0

package/src/probe/runners/python.mjs

@@ -0,0 +1,316 @@
+import { existsSync, readFileSync } from 'node:fs';
+import { spawn } from 'node:child_process';
+import { dirname, join } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { runProcess, listTestFiles, firstInformativeLine } from './shared.mjs';
+
+/**
+ * The Python runner. Selected **per file**: any `.py` defender runs here
+ * whatever the project runner is, which is what makes a repository holding
+ * both a JavaScript and a Python package probeable in one run. Selected for
+ * the whole project by `--runner python|pytest|unittest`, or by `auto` when
+ * neither vitest nor jest resolves.
+ *
+ * Two engines, one adapter. `pytest` when the project's interpreter can import
+ * it, stdlib `unittest` otherwise — and `unittest` is not a fallback for the
+ * poor: it is the only engine that runs on a project whose test dependencies
+ * are the standard library, which is a deliberate choice in exactly the
+ * codebases most worth probing. The engine that ran is what the evidence
+ * records as `runner.name`; TestGuard never says pytest ran when it did not.
+ *
+ * Neither engine is installed into the project. The reporters live in
+ * `runners/python/` inside this package and reach the interpreter through
+ * PYTHONPATH — `-p _testguard_pytest_plugin` for pytest, `-m
+ * _testguard_unittest_main` for unittest. The project's dependency set is
+ * never touched.
+ *
+ * Report protocol: TestGuard's own JSON shape (see `python/_testguard_report.py`),
+ * written to a file named by `TESTGUARD_REPORT`, never parsed from stdout —
+ * `unittest` writes its summary to stderr where it interleaves with whatever
+ * the tests print, and reading that stream is how hand-rolled harnesses come
+ * to report a red suite as green.
+ *
+ * Python outcomes map onto the spec's testRun as follows:
+ *
+ *   passed              → passed
+ *   failed              → failed, counts toward assertionFailures (can kill).
+ *                         An `AssertionError` and a `TypeError` raised by the
+ *                         test body are alike here, as they are under jest:
+ *                         both are the suite rejecting the behaviour. The
+ *                         exception class is reported, never scored.
+ *   failed, timed out   → failed, counts toward timeouts (never a kill)
+ *   xfail               → neither passed nor failed
+ *   xpass               → the suite's own expectation is wrong, so the run is
+ *                         NOT green — counted as failed with no assertion
+ *                         failure, so it reaches FLAKY-DEFENDER through the
+ *                         ordinary N-run rules rather than killing anything
+ *   skipped             → neither passed nor failed
+ *   collection error    → the suite could not load: `error`, never a kill.
+ *                         This is what a fault whose replacement does not
+ *                         parse looks like from Python, and it is how
+ *                         `fault-invalid` is reached.
+ */
+
+const HERE = dirname(fileURLToPath(import.meta.url));
+export const PYTHON_DIR = join(HERE, 'python');
+
+export const name = 'python';
+
+/**
+ * pytest's default `python_files`, plus unittest's default discovery pattern
+ * (`test*.py`). A helper named `testutils.py` matches and collects nothing,
+ * which costs a little; a test file that did not match would be invisible to
+ * discovery and report `nocover`, which costs the truth.
+ */
+export const testGlobs = ['**/test_*.py', '**/test*.py', '**/*_test.py'];
+
+const PY_FILE = /\.py$/;
+const TIMEOUT = /\bTimeout >|\btimed out\b|\bTimeoutError\b/i;
+
+/**
+ * Interpreters to try, best first: the active virtualenv, the project's own,
+ * then PATH.
+ *
+ * An explicit `--python` is NOT one of these. It is the only candidate, and it
+ * is returned alone: an interpreter the user named and TestGuard quietly
+ * replaced with another would change which dependencies the tests ran against
+ * without changing anything the evidence says.
+ */
+function candidates(projectDir, python) {
+  const out = [];
+  const win = process.platform === 'win32';
+  const binOf = (root) => (win ? join(root, 'Scripts', 'python.exe') : join(root, 'bin', 'python'));
+  const explicit = python ?? process.env.TESTGUARD_PYTHON;
+  if (explicit) return [{ path: explicit, source: 'project', explicit: true }];
+  if (process.env.VIRTUAL_ENV) out.push({ path: binOf(process.env.VIRTUAL_ENV), source: 'project' });
+  for (const dir of ['.venv', 'venv', '.env']) out.push({ path: binOf(join(projectDir, dir)), source: 'project' });
+  out.push({ path: win ? 'python.exe' : 'python3', source: 'path' });
+  out.push({ path: win ? 'py.exe' : 'python', source: 'path' });
+  return out;
+}
+
+const PROBE = [
+  'import json,sys',
+  'v=".".join(map(str,sys.version_info[:3]))',
+  'try:',
+  ' import pytest; p=pytest.__version__',
+  'except Exception: p=None',
+  'print(json.dumps({"python":v,"pytest":p}))',
+].join('\n');
+
+const interpreters = new Map(); // projectDir -> resolution promise
+
+/** Forget resolved interpreters (tests create and delete projects at the same paths). */
+export const resetInterpreterCache = () => interpreters.clear();
+
+/**
+ * Ask an interpreter what it is and whether it can import pytest.
+ *
+ * Deliberately NOT run with `-I`. Isolated mode ignores `PYTHONPATH` and the
+ * user site directory, so it would answer "no pytest" for an interpreter that
+ * will import pytest perfectly well during the actual run — and TestGuard
+ * would silently choose the stdlib engine for a suite written in pytest, which
+ * then collects nothing. The check has to see what the run will see.
+ */
+function ask(path, budgetMs) {
+  return new Promise((resolve) => {
+    let child;
+    try {
+      child = spawn(path, ['-c', PROBE], { stdio: ['ignore', 'pipe', 'pipe'], env: { ...process.env, CI: '1' } });
+    } catch {
+      return resolve(null);
+    }
+    let out = '';
+    child.on('error', () => resolve(null));
+    child.stdout.on('data', (d) => (out += d));
+    child.stderr.on('data', () => {});
+    const timer = setTimeout(() => child.kill('SIGKILL'), budgetMs);
+    child.on('close', (code) => {
+      clearTimeout(timer);
+      if (code !== 0) return resolve(null);
+      try {
+        resolve(JSON.parse(out.trim().split('\n').pop()));
+      } catch {
+        resolve(null);
+      }
+    });
+  });
+}
+
+/**
+ * The interpreter this project's tests run under, and whether it can import
+ * pytest. Resolved against the **original** project directory, never the
+ * scratch worktree: a virtualenv is gitignored, so it is not in the worktree
+ * checkout, and it is referenced by absolute path the way virtualenvs are
+ * meant to be — never copied and never symlinked, which would break the
+ * `pyvenv.cfg` prefix detection that makes one work at all.
+ */
+export function resolveInterpreter({ projectDir, python, budgetMs = 30_000 }) {
+  const key = `${projectDir}\0${python ?? ''}`;
+  if (interpreters.has(key)) return interpreters.get(key);
+  const task = (async () => {
+    const tried = [];
+    for (const candidate of candidates(projectDir, python)) {
+      if (!candidate.explicit && candidate.source === 'project' && !existsSync(candidate.path)) continue;
+      const answer = await ask(candidate.path, budgetMs);
+      if (answer) return { ...candidate, ...answer };
+      tried.push(candidate.path);
+    }
+    const explicit = python ?? process.env.TESTGUARD_PYTHON;
+    return { error: explicit
+      ? `${explicit} is not a usable Python interpreter`
+      : `no usable Python interpreter (tried ${tried.join(', ') || 'nothing'}); install Python 3.8+, or point --python / TESTGUARD_PYTHON at one` };
+  })();
+  interpreters.set(key, task);
+  return task;
+}
+
+/**
+ * Resolvable = an interpreter answers, and — when an engine is pinned — that
+ * engine is available. `--runner pytest` on a machine that has no pytest is a
+ * precondition failure, never a silent fall back to unittest: which engine ran
+ * changes what the evidence means.
+ */
+/**
+ * Which engine runs, given what the interpreter answered.
+ *
+ * Pure, and separate from the spawn, because the interesting branch is the one
+ * that is hard to stage: an interpreter that resolves but cannot import
+ * pytest. Testing that through `check()` would mean building a bare virtualenv
+ * inside a unit test, so the decision is lifted out and the spawn is left with
+ * nothing to decide.
+ */
+export function chooseEngine(pinned, found) {
+  const engine = pinned ?? (found.pytest ? 'pytest' : 'unittest');
+  if (engine === 'pytest' && !found.pytest) {
+    return { ok: false, message: `pytest is not importable from ${found.path} (pip install pytest, or use --runner unittest for the stdlib runner)` };
+  }
+  return { ok: true, engine, version: engine === 'pytest' ? found.pytest : `CPython ${found.python}` };
+}
+
+export function makeCheck(pinned) {
+  return async ({ projectDir, sourceDir, python, budgetMs = 30_000 }) => {
+    // Always the ORIGINAL project directory when there is one: a virtualenv is
+    // gitignored, so the scratch worktree does not contain it.
+    const found = await resolveInterpreter({ projectDir: sourceDir ?? projectDir, python, budgetMs });
+    if (found.error) return { ok: false, message: found.error };
+    const chosen = chooseEngine(pinned, found);
+    if (!chosen.ok) return chosen;
+    return { ...chosen, interpreter: found.path, source: found.source };
+  };
+}
+
+/** Any `.py` file. A Python test can never run under vitest, so ownership needs no configuration to read. */
+export const owns = (projectDir, file) => PY_FILE.test(file);
+
+export const tests = (projectDir) => listTestFiles(projectDir, testGlobs).filter((f) => owns(projectDir, f));
+
+/**
+ * TestGuard's Python report → the spec's testRun. Pure; exported for the
+ * sample-driven tests. `provenance` rides along for the import check that
+ * `probe` performs once the fault is live; it is not part of the testRun.
+ */
+export function parseReport(report, durationMs) {
+  const all = report.tests ?? [];
+  const collectionErrors = report.collectionErrors ?? [];
+  const failed = all.filter((t) => t.outcome === 'failed');
+  const timedOut = failed.filter((t) => TIMEOUT.test(t.message ?? ''));
+  const xpass = all.filter((t) => t.outcome === 'xpass');
+  const passed = all.filter((t) => t.outcome === 'passed').length;
+  const total = all.length;
+  const summary = { total, passed, failed: failed.length + xpass.length };
+
+  let outcome;
+  if (collectionErrors.length > 0 || total === 0) outcome = 'error';
+  else if (failed.length > 0 || xpass.length > 0) outcome = 'fail';
+  else outcome = 'pass';
+
+  const loadMessage = outcome === 'error'
+    ? (collectionErrors.length
+      ? firstInformativeLine(collectionErrors.map((e) => e.message ?? '').join('\n'))
+      : 'python collected no tests')
+    : undefined;
+
+  return {
+    run: { outcome, tests: summary, assertionFailures: failed.length - timedOut.length, durationMs },
+    timeouts: timedOut.length,
+    loadMessage,
+    failedTests: [...failed, ...xpass].map((t) => t.id),
+    provenance: report.provenance ?? {},
+  };
+}
+
+/**
+ * Environment for one run. Three things matter beyond finding the reporter:
+ *
+ *   PYTHONDONTWRITEBYTECODE + PYTHONPYCACHEPREFIX — `.pyc` files are validated
+ *     by the source's mtime **in whole seconds** and its size. A fault that
+ *     preserves the byte length and is applied within the same second as the
+ *     file it replaces can therefore be served from a stale cache and never
+ *     execute: a green run that proves nothing. The prefix moves every cache
+ *     lookup out of the tree, so an `--in-place` run cannot read the user's
+ *     existing `__pycache__` either, and DONTWRITEBYTECODE means the prefix
+ *     directory is never actually created.
+ *
+ *   TESTGUARD_TARGETS — the fault's file, so the reporter can say which file
+ *     the interpreter really loaded for it.
+ */
+export function environment(targets) {
+  const existing = process.env.PYTHONPATH;
+  return {
+    PYTHONPATH: existing ? `${PYTHON_DIR}${process.platform === 'win32' ? ';' : ':'}${existing}` : PYTHON_DIR,
+    PYTHONDONTWRITEBYTECODE: '1',
+    PYTHONPYCACHEPREFIX: '{out}.pycache',
+    PYTHONUNBUFFERED: '1',
+    TESTGUARD_REPORT: '{out}',
+    TESTGUARD_TARGETS: JSON.stringify(targets ?? []),
+  };
+}
+
+/**
+ * `--maxfail=0` is appended on every pytest run, and is not optional: a
+ * project whose `addopts` carries `-x` would otherwise stop at the first
+ * failure. The verdict would survive that (one assertion failure is still a
+ * kill) but the list of failing tests would not, and that list is what names
+ * an undeclared killer during escalation. `0` means no limit and overrides
+ * the configured value; the rest of the project's `addopts` is left alone,
+ * because TestGuard runs a project's tests the way the project runs them.
+ */
+export function argvFor({ engine, interpreter, files, serial }) {
+  if (engine === 'pytest') {
+    return [interpreter, '-m', 'pytest', '-p', '_testguard_pytest_plugin', '-p', 'no:cacheprovider',
+      '--maxfail=0', '-q', ...(serial ? ['-p', 'no:xdist'] : []), ...files];
+  }
+  return [interpreter, '-m', '_testguard_unittest_main', ...files];
+}
+
+export function makeRun(pinned) {
+  const check = makeCheck(pinned);
+  return async (opts) => {
+    const resolved = await check({ projectDir: opts.sourceDir ?? opts.projectDir, python: opts.python });
+    if (!resolved.ok) {
+      return { run: { outcome: 'error', tests: { total: 0, passed: 0, failed: 0 }, assertionFailures: 0, durationMs: 0 }, timeouts: 0, loadMessage: resolved.message, failedTests: [], provenance: {} };
+    }
+    return runProcess({
+      ...opts,
+      env: environment(opts.targets),
+      parse: parseReport,
+      argv: (files) => argvFor({ engine: resolved.engine, interpreter: resolved.interpreter, files, serial: opts.serial }),
+    });
+  };
+}
+
+export const check = makeCheck(null);
+export const run = makeRun(null);
+
+/** `--runner pytest` / `--runner unittest`: the same adapter with the engine pinned. */
+export function pinned(engine) {
+  return { name: engine, testGlobs, owns, tests, parseReport, fatalEdit, check: makeCheck(engine), run: makeRun(engine) };
+}
+
+/**
+ * The negative control's edit, for Python: an unterminated group, which is a
+ * `SyntaxError` at import time whatever the module contained. See the
+ * JavaScript runners for why this exists.
+ */
+export const fatalEdit = () => '# testguard negative control: this file must not parse\n(\n';

package/src/probe/runners/shared.mjs

@@ -1,5 +1,7 @@
 import { spawn } from 'node:child_process';
 import { existsSync, readFileSync, rmSync } from 'node:fs';
+import { createRequire } from 'node:module';
+import { dirname } from 'node:path';
 import { tmpdir } from 'node:os';
 import { join } from 'node:path';
 import { randomBytes } from 'node:crypto';
@@ -9,6 +11,9 @@ export const TEST_GLOBS = ['**/*.test.js', '**/*.test.mjs', '**/*.test.cjs', '**
   '**/*.spec.js', '**/*.spec.mjs', '**/*.spec.cjs', '**/*.spec.ts', '**/*.spec.mts', '**/*.spec.tsx', '**/*.spec.jsx'];
 export const npx = process.platform === 'win32' ? 'npx.cmd' : 'npx';
 
+/** Node's own advisory warnings about the probed project's configuration; never about the fault. */
+export const NODE_NOISE = /MODULE_TYPELESS_PACKAGE_JSON|ExperimentalWarning|--trace-warnings|Reparsing as ES module|To eliminate this warning/;
+
 /**
  * Split a runner command template into argv. Supports double and single
  * quotes; `{files}` expands to the test files (one argv entry each) and
@@ -43,28 +48,75 @@ export function parseCommandTemplate(template) {
 /** `{files}` must be a word of its own (one argv entry per file); `{out}` may be embedded, e.g. `--outputFile={out}`. */
 export const expandCommand = (words, files, outFile) => words.flatMap((w) => (w === '{files}' ? files : [w.replaceAll('{out}', outFile)]));
 
+const resolvedRunners = new Map(); // `${projectDir}\0${pkg}` → { argv, version, source }
+
 /**
- * Can `bin` be resolved at all from this directory? Run before any verdict
- * is attempted: an unresolvable runner is a precondition failure, never a
- * statement about the tests.
+ * Where does the runner come from? The PROJECT first: the runner package
+ * resolved from `projectDir` (its version, its `bin` script run with this
+ * node). Only when no package resolves, an executable on PATH — a global
+ * runner is a legitimate setup for some monorepos, and the evidence says so
+ * (`source: "path"`). Never `npx`: the npx cache and a same-named executable
+ * (a Python `playwright` shim, say) both answer `npx --no-install <bin>
+ * --version` from a project that has no such runner at all (#41).
  */
-export function checkBinary({ projectDir, bin, budgetMs = 60_000, env = process.env }) {
+export function resolveRunner({ projectDir, pkg, bin }) {
+  const key = `${projectDir}\0${pkg}`;
+  if (resolvedRunners.has(key)) return resolvedRunners.get(key);
+  let result = null;
+  try {
+    const req = createRequire(join(projectDir, 'noop.js'));
+    const pkgJsonPath = req.resolve(`${pkg}/package.json`);
+    const pkgJson = JSON.parse(readFileSync(pkgJsonPath, 'utf8'));
+    const binField = typeof pkgJson.bin === 'string' ? pkgJson.bin : pkgJson.bin?.[bin];
+    if (binField) result = { argv: [process.execPath, join(dirname(pkgJsonPath), binField)], version: pkgJson.version, source: 'project' };
+  } catch {}
+  resolvedRunners.set(key, result);
+  return result;
+}
+
+/** Forget cached resolutions (tests create and delete projects at the same paths). */
+export const resetRunnerCache = () => resolvedRunners.clear();
+
+/** The argv prefix that runs `pkg`'s `bin` for this project: the resolved local script, else the PATH executable. */
+export function runnerArgv(projectDir, pkg, bin) {
+  return resolveRunner({ projectDir, pkg, bin })?.argv ?? [bin];
+}
+
+/**
+ * Can the runner be resolved at all from this directory? Run before any
+ * verdict is attempted: an unresolvable runner is a precondition failure,
+ * never a statement about the tests. Project package first; PATH second;
+ * npx never.
+ */
+export function checkRunner({ projectDir, pkg, bin, budgetMs = 60_000, env = process.env }) {
+  const local = resolveRunner({ projectDir, pkg, bin });
+  if (local) return Promise.resolve({ ok: true, version: local.version, source: 'project' });
   return new Promise((resolve) => {
-    const child = spawn(npx, ['--no-install', bin, '--version'], { cwd: projectDir, stdio: ['ignore', 'pipe', 'pipe'], env: { ...env, CI: '1' } });
+    let child;
+    try {
+      child = spawn(process.platform === 'win32' ? `${bin}.cmd` : bin, ['--version'], { cwd: projectDir, stdio: ['ignore', 'pipe', 'pipe'], env: { ...env, CI: '1' } });
+    } catch (e) {
+      return resolve({ ok: false, message: e.message });
+    }
     let out = '';
     let err = '';
-    child.on('error', (e) => resolve({ ok: false, message: e.message }));
+    child.on('error', () => resolve({ ok: false, message: `${pkg} is not installed in the project and \`${bin}\` is not on PATH (npm i -D ${pkg})` }));
     child.stdout.on('data', (d) => (out += d));
     child.stderr.on('data', (d) => (err += d));
     const timer = setTimeout(() => child.kill('SIGKILL'), budgetMs);
     child.on('close', (code) => {
       clearTimeout(timer);
-      const version = out.trim().replace(/^vitest\//, '').split('\n').pop();
-      resolve(code === 0 && version ? { ok: true, version } : { ok: false, message: (err || out).trim().split('\n').filter(Boolean).slice(-1)[0] ?? `exit ${code}` });
+      const version = out.trim().replace(/^vitest\//, '').replace(/^Version\s+/i, '').split('\n').pop();
+      resolve(code === 0 && version
+        ? { ok: true, version, source: 'path' }
+        : { ok: false, message: `${pkg} is not installed in the project; \`${bin}\` on PATH answered: ${(err || out).trim().split('\n').filter(Boolean).slice(-1)[0] ?? `exit ${code}`}` });
     });
   });
 }
 
+/** @deprecated name kept for older callers; resolves the project package first, never npx. */
+export const checkBinary = ({ bin, ...rest }) => checkRunner({ ...rest, pkg: bin, bin });
+
 /** Defender globs → existing files. Empty result is the `nocover` signal. */
 export const resolveDefenders = (projectDir, globs) => matchGlobs(projectDir, globs ?? []);
 
@@ -78,12 +130,19 @@ export const listTestFiles = (projectDir, globs = TEST_GLOBS) => matchGlobs(proj
  * can kill a fault. A suite that fails to load, or a test that times out, is
  * not evidence that the suite defends the claim.
  */
-function firstInformativeLine(message) {
+export function firstInformativeLine(message) {
   const lines = message.split('\n').map((l) => l.replace(/\x1b\[[0-9;]*m/g, '').trim()).filter(Boolean);
   return lines.find((l) => /error|syntax|unexpected|expected .+ but found|cannot find|failed to parse|transform failed/i.test(l) && !/^●/.test(l)) ?? lines[0] ?? '';
 }
 
 export function parseReport(report, durationMs) {
+  // A report that parsed as JSON but carries no `testResults` is not a suite
+  // that ran zero tests — it is a report of a shape this parser does not
+  // understand, which is what `--runner-cmd` pointed at a non-jest reporter
+  // produces. The verdict is already safe (a non-green baseline makes the
+  // claim unverifiable rather than passing), but without this the user is told
+  // `defenders-failed-to-load` with no reason at all.
+  const unknownShape = report == null || typeof report !== 'object' || !Array.isArray(report.testResults);
   const files = report.testResults ?? [];
   const loadFailed = files.some((f) => f.status === 'failed' && (f.assertionResults?.length ?? 0) === 0);
   const results = files.flatMap((f) => (f.assertionResults ?? []).map((t) => ({ ...t, id: `${f.name}::${t.fullName ?? t.title ?? ''}` }))).filter((t) => t.status === 'failed');
@@ -101,7 +160,9 @@ export function parseReport(report, durationMs) {
   // jest opens a load failure with "● Test suite failed to run" and puts the
   // cause lines below; vitest puts it on the first line. Take the first line
   // that names an error, else the first non-empty one.
-  const loadMessage = loadFailed ? firstInformativeLine(files.find((f) => f.message)?.message ?? '') : undefined;
+  const loadMessage = unknownShape
+    ? 'the runner produced a report with no `testResults` array: this parser reads the jest-compatible JSON report, so a --runner-cmd must emit that shape (vitest --reporter=json, jest --json)'
+    : loadFailed ? firstInformativeLine(files.find((f) => f.message)?.message ?? '') : undefined;
   return { run, timeouts, loadMessage, failedTests };
 }
 
@@ -113,16 +174,21 @@ export function parseReport(report, durationMs) {
  * runner's command line; `command` (tests) or `commandTemplate` (--runner-cmd)
  * override it.
  */
-export function runProcess({ projectDir, files, budgetMs = 120_000, command, commandTemplate, argv }) {
+export function runProcess({ projectDir, files, budgetMs = 120_000, command, commandTemplate, argv, env = {}, parse = parseReport }) {
   const outFile = join(tmpdir(), `testguard-run-${randomBytes(6).toString('hex')}.json`);
   const [cmd, ...args] = command
     ?? (commandTemplate ? expandCommand(commandTemplate, files, outFile) : argv(files, outFile));
+  // A runner that names its report through the environment (Playwright) gets `{out}` substituted there too.
+  const extraEnv = Object.fromEntries(Object.entries(env).map(([k, v]) => [k, String(v).replaceAll('{out}', outFile)]));
   const started = Date.now();
 
   return new Promise((resolve) => {
-    const child = spawn(cmd, args, { cwd: projectDir, stdio: ['ignore', 'pipe', 'pipe'], env: { ...process.env, CI: '1', FORCE_COLOR: '0' } });
+    const child = spawn(cmd, args, { cwd: projectDir, stdio: ['ignore', 'pipe', 'pipe'], env: { ...process.env, CI: '1', FORCE_COLOR: '0', ...extraEnv } });
     let stderr = '';
-    child.stderr.on('data', (d) => (stderr += d));
+    // Node prints MODULE_TYPELESS_PACKAGE_JSON (and friends) for the PROBED
+    // project's config, not for anything the fault did. Keep it out of the
+    // stream and out of `loadMessage`, which names the cause of a load error.
+    child.stderr.on('data', (d) => (stderr += String(d).split('\n').filter((l) => !NODE_NOISE.test(l)).join('\n')));
     child.on('error', (e) => (stderr += e.message));
     let killed = false;
     const timer = setTimeout(() => {
@@ -140,7 +206,7 @@ export function runProcess({ projectDir, files, budgetMs = 120_000, command, com
         result = { run: { outcome: 'error', tests: { total: 0, passed: 0, failed: 0 }, assertionFailures: 0, durationMs }, timeouts: 0, loadMessage: stderr.trim().split('\n').filter(Boolean).slice(-1)[0] ?? 'runner produced no report', failedTests: [] };
       } else {
         try {
-          result = parseReport(JSON.parse(readFileSync(outFile, 'utf8')), durationMs);
+          result = parse(JSON.parse(readFileSync(outFile, 'utf8')), durationMs);
         } catch (e) {
           result = { run: { outcome: 'error', tests: { total: 0, passed: 0, failed: 0 }, assertionFailures: 0, durationMs }, timeouts: 0, loadMessage: `unreadable report: ${e.message}`, failedTests: [] };
         }

package/src/probe/runners/vitest.mjs

@@ -1,7 +1,20 @@
-import { TEST_GLOBS, npx, runProcess, checkBinary, listTestFiles } from './shared.mjs';
+import { TEST_GLOBS, runProcess, checkRunner, runnerArgv, listTestFiles } from './shared.mjs';
 
 export const name = 'vitest';
 export const testGlobs = TEST_GLOBS;
-export const check = (opts) => checkBinary({ ...opts, bin: 'vitest' });
+export const check = (opts) => checkRunner({ ...opts, pkg: 'vitest', bin: 'vitest' });
 export const tests = (projectDir) => listTestFiles(projectDir, testGlobs);
-export const run = (opts) => runProcess({ ...opts, argv: (files, outFile) => [npx, 'vitest', 'run', ...files, '--reporter=json', `--outputFile=${outFile}`] });
+// --serial: one file at a time in one process, so a contended machine cannot turn a slow suite into a TIMEOUT verdict.
+/** The command line, as data. Exported so it is falsifiable without spawning anything. */
+export const argvFor = (projectDir, files, outFile, { serial = false } = {}) =>
+  [...runnerArgv(projectDir, 'vitest', 'vitest'), 'run', ...files, '--reporter=json', `--outputFile=${outFile}`, ...(serial ? ['--no-file-parallelism'] : [])];
+export const run = (opts) => runProcess({ ...opts, argv: (files, outFile) => argvFor(opts.projectDir, files, outFile, opts) });
+
+/**
+ * The negative control's edit: content this runner's loader cannot possibly
+ * parse. Used to prove the defenders actually execute a file before a
+ * `survived` verdict about it is allowed to stand. An unterminated group is
+ * a syntax error at every JavaScript/TypeScript parser, whatever the file
+ * contained before.
+ */
+export const fatalEdit = () => '/* testguard negative control: this file must not parse */\n(\n';

package/src/render.mjs

@@ -5,7 +5,7 @@ export const formatVerdict = (v, provisional = false) => (v === 'killed' ? 'kill
 
 export const PROVISIONAL_WARNING = (n) => `PROVISIONAL — confirmRuns ${n} (< 3): nothing below is confirmed. Verdicts carry a "?"; this evidence cannot be frozen into a baseline. Re-run with --confirm 3 before trusting it.`;
 
-export function renderRecord(r, { provisional = false } = {}) {
+export function renderRecord(r, { provisional = false, showDiscovered = true } = {}) {
   const head = `${formatVerdict(r.verdict, provisional).padEnd(15)} ${r.claim.id}/${r.subject.id}`.padEnd(38);
   let why = '';
   if (r.detail.reason === 'killed-by-undeclared-tests' && r.detail.undeclaredKillers?.length) {
@@ -16,7 +16,7 @@ export function renderRecord(r, { provisional = false } = {}) {
   } else if (r.detail.reason) {
     why = `  [${r.detail.reason}]`;
   }
-  if (r.defenders.discovered) why += '  (defenders discovered by import)';
+  if (r.defenders.discovered && showDiscovered) why += '  (defenders discovered by import)';
   return `${head} ${r.claim.severity.padEnd(8)} ${r.subject.file}  ${r.subject.description}${why}`;
 }
 

package/src/replay/label.mjs

@@ -0,0 +1,64 @@
+/**
+ * Which injected-fault class does a real fix diff most resemble?
+ *
+ * This is the join key between ground truth and the fault model: a replayed
+ * bug is a real defect a human confirmed, so labelling it by fault class is
+ * what lets P(a surviving fault of this class is a real bug) be computed.
+ * Without it, a mutation score is a number nobody can interpret.
+ *
+ * Deterministic and shape-based, in the same spirit as the scaffold
+ * producers read in reverse: a fix ADDS what the bug lacked, so the lines a
+ * fix adds are the lines a fault would delete. No LLM, and `other` whenever
+ * the shape is not one the model has — an honest gap is worth more than a
+ * confident wrong label.
+ */
+const GUARD_RE = /^\s*if\s*\(.*\)\s*\{?\s*$/;
+const RETURN_RE = /^\s*return\s+/;
+const CALL_RE = /^\s*(?:await\s+)?(?:[\w.]*\.)?(?:verify|validate|check|assert|authorize|authorise|ensure|require)\w*\s*\(/i;
+const FIELD_RE = /^\s*['"`]?[\w$]+['"`]?\s*:\s*.+,\s*$/;
+const LITERAL_RE = /\b(?:httpOnly|secure|sameSite|rounds|cost|ttl|timeout|tolerance|window|limit|maxAge|expiresIn)\b\s*[:=]/i;
+const ASSIGN_RE = /^\s*(?:const |let |var )?[\w.[\]$]+\s*=\s*[^=]/;
+
+/** Most specific first; a tie between equally specific shapes is not a label. */
+const SPECIFICITY = ['literal-changed', 'call-removed', 'exception-swallowed', 'field-dropped', 'guard-removed', 'return-altered', 'statement-deleted'];
+
+function kind(line) {
+  if (LITERAL_RE.test(line)) return 'literal-changed';
+  if (/^\s*(?:\}\s*)?catch\b/.test(line) || /^\s*throw\b/.test(line)) return 'exception-swallowed';
+  if (GUARD_RE.test(line)) return 'guard-removed';
+  if (RETURN_RE.test(line)) return 'return-altered';
+  if (CALL_RE.test(line)) return 'call-removed';
+  if (FIELD_RE.test(line)) return 'field-dropped';
+  if (ASSIGN_RE.test(line)) return 'statement-deleted';
+  return null;
+}
+
+/**
+ * Label a unified diff of the fix's SOURCE changes (never its tests).
+ *
+ * Added lines carry the signal at double weight: the fix put back the guard,
+ * the field or the call that was missing. Removed lines are read too, for the
+ * fixes that delete a wrong condition rather than add a right one.
+ */
+export function labelDiff(diff) {
+  const added = [];
+  const removed = [];
+  for (const raw of diff.split('\n')) {
+    if (raw.startsWith('+++') || raw.startsWith('---')) continue;
+    if (raw.startsWith('+')) added.push(raw.slice(1));
+    else if (raw.startsWith('-')) removed.push(raw.slice(1));
+  }
+  const tally = new Map();
+  const add = (k, weight) => {
+    if (k) tally.set(k, (tally.get(k) ?? 0) + weight);
+  };
+  for (const l of added) add(kind(l), 2);
+  for (const l of removed) add(kind(l), 1);
+  if (tally.size === 0) return { faultClass: 'other', confident: false };
+
+  const ranked = [...tally].sort((a, b) => b[1] - a[1] || SPECIFICITY.indexOf(a[0]) - SPECIFICITY.indexOf(b[0]));
+  const [top, topWeight] = ranked[0];
+  const tiedSameSpecificity = ranked.length > 1 && ranked[1][1] === topWeight && SPECIFICITY.indexOf(ranked[1][0]) === SPECIFICITY.indexOf(top);
+  if (tiedSameSpecificity) return { faultClass: 'other', confident: false };
+  return { faultClass: top, confident: topWeight >= 2 };
+}