testguard-cli 0.4.0 → 0.6.0

package/src/probe/runners/jest.mjs

@@ -1,9 +1,9 @@
-import { TEST_GLOBS, npx, runProcess, checkBinary, listTestFiles } from './shared.mjs';
+import { TEST_GLOBS, runProcess, checkRunner, runnerArgv, listTestFiles } from './shared.mjs';
 
 export const name = 'jest';
 // jest's default testMatch also collects anything under __tests__/.
 export const testGlobs = [...TEST_GLOBS, '**/__tests__/**/*.js', '**/__tests__/**/*.mjs', '**/__tests__/**/*.cjs', '**/__tests__/**/*.ts', '**/__tests__/**/*.tsx', '**/__tests__/**/*.jsx'];
-export const check = (opts) => checkBinary({ ...opts, bin: 'jest' });
+export const check = (opts) => checkRunner({ ...opts, pkg: 'jest', bin: 'jest' });
 export const tests = (projectDir) => listTestFiles(projectDir, testGlobs);
 // --runTestsByPath: positionals are exact paths, not regexes — a path with `+` or `(` would otherwise silently match nothing.
-export const run = (opts) => runProcess({ ...opts, argv: (files, outFile) => [npx, 'jest', '--ci', '--json', `--outputFile=${outFile}`, '--runTestsByPath', ...files] });
+export const run = (opts) => runProcess({ ...opts, argv: (files, outFile) => [...runnerArgv(opts.projectDir, 'jest', 'jest'), '--ci', '--json', `--outputFile=${outFile}`, ...(opts.serial ? ['--runInBand'] : []), '--runTestsByPath', ...files] });

package/src/probe/runners/playwright.mjs

@@ -0,0 +1,133 @@
+import { existsSync, readFileSync } from 'node:fs';
+import { createRequire } from 'node:module';
+import { join, relative, resolve, sep } from 'node:path';
+import { npx, runProcess, listTestFiles, firstInformativeLine } from './shared.mjs';
+import { globToRegExp } from '../../util/glob.mjs';
+
+/**
+ * Playwright runner. Selected **per file**, not per project: a defender that
+ * lives under Playwright's `testDir` runs here, everything else runs under
+ * the project runner (vitest/jest). One claim may list both kinds. Selected
+ * for the whole project only with `--runner playwright`.
+ *
+ * Report protocol: `--reporter=json` with the JSON written to a file we name
+ * (PLAYWRIGHT_JSON_OUTPUT_FILE; the older PLAYWRIGHT_JSON_OUTPUT_NAME is set
+ * too), never parsed from stdout. Playwright's test statuses map onto the
+ * spec's testRun as follows, and the mapping is the point of the adapter:
+ *
+ *   expected            → passed
+ *   unexpected/failed   → failed, counts toward assertionFailures (can kill)
+ *   unexpected/timedOut → failed, counts toward timeouts (never a kill)
+ *   flaky               → failed on an attempt and passed on retry: the run
+ *                         is NOT green. Counted as failed with neither an
+ *                         assertion failure nor a timeout, so a green-looking
+ *                         exit code becomes FLAKY-DEFENDER through the
+ *                         ordinary N-run rules, exactly as GATE-SEMANTICS says.
+ *   skipped             → neither passed nor failed
+ */
+
+export const name = 'playwright';
+
+const CONFIG_FILES = ['playwright.config.ts', 'playwright.config.mts', 'playwright.config.js', 'playwright.config.mjs', 'playwright.config.cjs'];
+const SPEC_GLOBS = ['**/*.spec.js', '**/*.spec.mjs', '**/*.spec.cjs', '**/*.spec.ts', '**/*.spec.mts', '**/*.spec.tsx', '**/*.spec.jsx',
+  '**/*.test.js', '**/*.test.mjs', '**/*.test.cjs', '**/*.test.ts', '**/*.test.mts', '**/*.test.tsx', '**/*.test.jsx'];
+
+const configCache = new Map();
+
+/**
+ * The project's Playwright config, read as text (it may be TypeScript, so it
+ * is never imported): `{ path, testDir }` or null when there is none.
+ * `testDir` defaults to the config's own directory, as Playwright does.
+ */
+export function loadConfig(projectDir) {
+  if (configCache.has(projectDir)) return configCache.get(projectDir);
+  let out = null;
+  for (const f of CONFIG_FILES) {
+    const path = join(projectDir, f);
+    if (!existsSync(path)) continue;
+    const text = readFileSync(path, 'utf8');
+    const m = /\btestDir\s*:\s*(['"`])([^'"`]+)\1/.exec(text);
+    const testDir = relative(projectDir, resolve(projectDir, m ? m[2] : '.')).split(sep).join('/') || '.';
+    out = { path: f, testDir };
+    break;
+  }
+  configCache.set(projectDir, out);
+  return out;
+}
+
+/** Clear the config cache (tests create and delete projects at the same paths). */
+export const resetConfigCache = () => configCache.clear();
+
+export const testGlobs = SPEC_GLOBS;
+const SPEC_RES = SPEC_GLOBS.map(globToRegExp);
+
+/** Does this test file belong to Playwright: is there a config, and does the file live under its testDir? */
+export function owns(projectDir, file) {
+  const cfg = loadConfig(projectDir);
+  if (!cfg) return false;
+  if (!SPEC_RES.some((re) => re.test(file))) return false;
+  return cfg.testDir === '.' ? true : file.startsWith(cfg.testDir + '/');
+}
+
+/**
+ * Resolvable = a config exists AND `@playwright/test` resolves from the
+ * project. Deliberately not `npx --no-install playwright --version`: the npx
+ * cache and a Python `playwright` on PATH both answer that from a project
+ * that has no Playwright at all, and `playwright test` would then run
+ * against nothing. The package `playwright test` needs is the precondition.
+ */
+export async function check({ projectDir }) {
+  const cfg = loadConfig(projectDir);
+  if (!cfg) return { ok: false, message: 'no playwright.config.* in the project' };
+  try {
+    const pkg = JSON.parse(readFileSync(createRequire(join(projectDir, 'noop.js')).resolve('@playwright/test/package.json'), 'utf8'));
+    return { ok: true, version: pkg.version };
+  } catch {
+    return { ok: false, message: '@playwright/test is not installed in the project (npm i -D @playwright/test)' };
+  }
+}
+
+export const tests = (projectDir) => listTestFiles(projectDir, SPEC_GLOBS).filter((f) => owns(projectDir, f));
+
+/**
+ * Playwright's JSON report → the spec's testRun. Pure; exported for the
+ * sample-driven tests. `durationMs` is the wall clock measured by the runner.
+ */
+export function parseReport(report, durationMs) {
+  const tests = [];
+  const walk = (suite) => {
+    for (const spec of suite.specs ?? []) {
+      for (const t of spec.tests ?? []) {
+        const results = t.results ?? [];
+        const last = results[results.length - 1];
+        tests.push({ id: `${suite.file ?? spec.file ?? ''}::${spec.title}`, status: t.status, finalResult: last?.status, error: results.find((r) => r.error)?.error?.message });
+      }
+    }
+    for (const s of suite.suites ?? []) walk(s);
+  };
+  for (const s of report.suites ?? []) walk(s);
+
+  const expected = tests.filter((t) => t.status === 'expected').length;
+  const unexpected = tests.filter((t) => t.status === 'unexpected');
+  const flaky = tests.filter((t) => t.status === 'flaky').length;
+  const timeouts = unexpected.filter((t) => t.finalResult === 'timedOut' || t.finalResult === 'interrupted').length;
+  const assertionFailures = unexpected.length - timeouts;
+  const failedTests = [...unexpected, ...tests.filter((t) => t.status === 'flaky')].map((t) => t.id);
+  const total = tests.length;
+  const summary = { total, passed: expected, failed: unexpected.length + flaky };
+
+  const globalErrors = report.errors ?? [];
+  let outcome;
+  if (total === 0) outcome = 'error';
+  else if (unexpected.length > 0 || flaky > 0) outcome = 'fail';
+  else outcome = 'pass';
+  const loadMessage = outcome === 'error'
+    ? (globalErrors.length ? firstInformativeLine(globalErrors.map((e) => e.message ?? '').join('\n')) : 'playwright found no tests')
+    : undefined;
+  return { run: { outcome, tests: summary, assertionFailures, durationMs }, timeouts, loadMessage, failedTests, flaky };
+}
+
+export const run = (opts) => {
+  const env = { PLAYWRIGHT_JSON_OUTPUT_FILE: '{out}', PLAYWRIGHT_JSON_OUTPUT_NAME: '{out}' };
+  return runProcess({ ...opts, env, parse: parseReport, argv: (files) => [npx, 'playwright', 'test', '--reporter=json', ...(opts.serial ? ['--workers=1'] : []), ...files] });
+};

package/src/probe/runners/shared.mjs

@@ -1,5 +1,7 @@
 import { spawn } from 'node:child_process';
 import { existsSync, readFileSync, rmSync } from 'node:fs';
+import { createRequire } from 'node:module';
+import { dirname } from 'node:path';
 import { tmpdir } from 'node:os';
 import { join } from 'node:path';
 import { randomBytes } from 'node:crypto';
@@ -9,6 +11,9 @@ export const TEST_GLOBS = ['**/*.test.js', '**/*.test.mjs', '**/*.test.cjs', '**
   '**/*.spec.js', '**/*.spec.mjs', '**/*.spec.cjs', '**/*.spec.ts', '**/*.spec.mts', '**/*.spec.tsx', '**/*.spec.jsx'];
 export const npx = process.platform === 'win32' ? 'npx.cmd' : 'npx';
 
+/** Node's own advisory warnings about the probed project's configuration; never about the fault. */
+export const NODE_NOISE = /MODULE_TYPELESS_PACKAGE_JSON|ExperimentalWarning|--trace-warnings|Reparsing as ES module|To eliminate this warning/;
+
 /**
  * Split a runner command template into argv. Supports double and single
  * quotes; `{files}` expands to the test files (one argv entry each) and
@@ -43,28 +48,75 @@ export function parseCommandTemplate(template) {
 /** `{files}` must be a word of its own (one argv entry per file); `{out}` may be embedded, e.g. `--outputFile={out}`. */
 export const expandCommand = (words, files, outFile) => words.flatMap((w) => (w === '{files}' ? files : [w.replaceAll('{out}', outFile)]));
 
+const resolvedRunners = new Map(); // `${projectDir}\0${pkg}` → { argv, version, source }
+
 /**
- * Can `bin` be resolved at all from this directory? Run before any verdict
- * is attempted: an unresolvable runner is a precondition failure, never a
- * statement about the tests.
+ * Where does the runner come from? The PROJECT first: the runner package
+ * resolved from `projectDir` (its version, its `bin` script run with this
+ * node). Only when no package resolves, an executable on PATH — a global
+ * runner is a legitimate setup for some monorepos, and the evidence says so
+ * (`source: "path"`). Never `npx`: the npx cache and a same-named executable
+ * (a Python `playwright` shim, say) both answer `npx --no-install <bin>
+ * --version` from a project that has no such runner at all (#41).
  */
-export function checkBinary({ projectDir, bin, budgetMs = 60_000, env = process.env }) {
+export function resolveRunner({ projectDir, pkg, bin }) {
+  const key = `${projectDir}\0${pkg}`;
+  if (resolvedRunners.has(key)) return resolvedRunners.get(key);
+  let result = null;
+  try {
+    const req = createRequire(join(projectDir, 'noop.js'));
+    const pkgJsonPath = req.resolve(`${pkg}/package.json`);
+    const pkgJson = JSON.parse(readFileSync(pkgJsonPath, 'utf8'));
+    const binField = typeof pkgJson.bin === 'string' ? pkgJson.bin : pkgJson.bin?.[bin];
+    if (binField) result = { argv: [process.execPath, join(dirname(pkgJsonPath), binField)], version: pkgJson.version, source: 'project' };
+  } catch {}
+  resolvedRunners.set(key, result);
+  return result;
+}
+
+/** Forget cached resolutions (tests create and delete projects at the same paths). */
+export const resetRunnerCache = () => resolvedRunners.clear();
+
+/** The argv prefix that runs `pkg`'s `bin` for this project: the resolved local script, else the PATH executable. */
+export function runnerArgv(projectDir, pkg, bin) {
+  return resolveRunner({ projectDir, pkg, bin })?.argv ?? [bin];
+}
+
+/**
+ * Can the runner be resolved at all from this directory? Run before any
+ * verdict is attempted: an unresolvable runner is a precondition failure,
+ * never a statement about the tests. Project package first; PATH second;
+ * npx never.
+ */
+export function checkRunner({ projectDir, pkg, bin, budgetMs = 60_000, env = process.env }) {
+  const local = resolveRunner({ projectDir, pkg, bin });
+  if (local) return Promise.resolve({ ok: true, version: local.version, source: 'project' });
   return new Promise((resolve) => {
-    const child = spawn(npx, ['--no-install', bin, '--version'], { cwd: projectDir, stdio: ['ignore', 'pipe', 'pipe'], env: { ...env, CI: '1' } });
+    let child;
+    try {
+      child = spawn(process.platform === 'win32' ? `${bin}.cmd` : bin, ['--version'], { cwd: projectDir, stdio: ['ignore', 'pipe', 'pipe'], env: { ...env, CI: '1' } });
+    } catch (e) {
+      return resolve({ ok: false, message: e.message });
+    }
     let out = '';
     let err = '';
-    child.on('error', (e) => resolve({ ok: false, message: e.message }));
+    child.on('error', () => resolve({ ok: false, message: `${pkg} is not installed in the project and \`${bin}\` is not on PATH (npm i -D ${pkg})` }));
     child.stdout.on('data', (d) => (out += d));
     child.stderr.on('data', (d) => (err += d));
     const timer = setTimeout(() => child.kill('SIGKILL'), budgetMs);
     child.on('close', (code) => {
       clearTimeout(timer);
-      const version = out.trim().replace(/^vitest\//, '').split('\n').pop();
-      resolve(code === 0 && version ? { ok: true, version } : { ok: false, message: (err || out).trim().split('\n').filter(Boolean).slice(-1)[0] ?? `exit ${code}` });
+      const version = out.trim().replace(/^vitest\//, '').replace(/^Version\s+/i, '').split('\n').pop();
+      resolve(code === 0 && version
+        ? { ok: true, version, source: 'path' }
+        : { ok: false, message: `${pkg} is not installed in the project; \`${bin}\` on PATH answered: ${(err || out).trim().split('\n').filter(Boolean).slice(-1)[0] ?? `exit ${code}`}` });
     });
   });
 }
 
+/** @deprecated name kept for older callers; resolves the project package first, never npx. */
+export const checkBinary = ({ bin, ...rest }) => checkRunner({ ...rest, pkg: bin, bin });
+
 /** Defender globs → existing files. Empty result is the `nocover` signal. */
 export const resolveDefenders = (projectDir, globs) => matchGlobs(projectDir, globs ?? []);
 
@@ -78,7 +130,7 @@ export const listTestFiles = (projectDir, globs = TEST_GLOBS) => matchGlobs(proj
  * can kill a fault. A suite that fails to load, or a test that times out, is
  * not evidence that the suite defends the claim.
  */
-function firstInformativeLine(message) {
+export function firstInformativeLine(message) {
   const lines = message.split('\n').map((l) => l.replace(/\x1b\[[0-9;]*m/g, '').trim()).filter(Boolean);
   return lines.find((l) => /error|syntax|unexpected|expected .+ but found|cannot find|failed to parse|transform failed/i.test(l) && !/^●/.test(l)) ?? lines[0] ?? '';
 }
@@ -113,16 +165,21 @@ export function parseReport(report, durationMs) {
  * runner's command line; `command` (tests) or `commandTemplate` (--runner-cmd)
  * override it.
  */
-export function runProcess({ projectDir, files, budgetMs = 120_000, command, commandTemplate, argv }) {
+export function runProcess({ projectDir, files, budgetMs = 120_000, command, commandTemplate, argv, env = {}, parse = parseReport }) {
   const outFile = join(tmpdir(), `testguard-run-${randomBytes(6).toString('hex')}.json`);
   const [cmd, ...args] = command
     ?? (commandTemplate ? expandCommand(commandTemplate, files, outFile) : argv(files, outFile));
+  // A runner that names its report through the environment (Playwright) gets `{out}` substituted there too.
+  const extraEnv = Object.fromEntries(Object.entries(env).map(([k, v]) => [k, String(v).replaceAll('{out}', outFile)]));
   const started = Date.now();
 
   return new Promise((resolve) => {
-    const child = spawn(cmd, args, { cwd: projectDir, stdio: ['ignore', 'pipe', 'pipe'], env: { ...process.env, CI: '1', FORCE_COLOR: '0' } });
+    const child = spawn(cmd, args, { cwd: projectDir, stdio: ['ignore', 'pipe', 'pipe'], env: { ...process.env, CI: '1', FORCE_COLOR: '0', ...extraEnv } });
     let stderr = '';
-    child.stderr.on('data', (d) => (stderr += d));
+    // Node prints MODULE_TYPELESS_PACKAGE_JSON (and friends) for the PROBED
+    // project's config, not for anything the fault did. Keep it out of the
+    // stream and out of `loadMessage`, which names the cause of a load error.
+    child.stderr.on('data', (d) => (stderr += String(d).split('\n').filter((l) => !NODE_NOISE.test(l)).join('\n')));
     child.on('error', (e) => (stderr += e.message));
     let killed = false;
     const timer = setTimeout(() => {
@@ -140,7 +197,7 @@ export function runProcess({ projectDir, files, budgetMs = 120_000, command, com
         result = { run: { outcome: 'error', tests: { total: 0, passed: 0, failed: 0 }, assertionFailures: 0, durationMs }, timeouts: 0, loadMessage: stderr.trim().split('\n').filter(Boolean).slice(-1)[0] ?? 'runner produced no report', failedTests: [] };
       } else {
         try {
-          result = parseReport(JSON.parse(readFileSync(outFile, 'utf8')), durationMs);
+          result = parse(JSON.parse(readFileSync(outFile, 'utf8')), durationMs);
         } catch (e) {
           result = { run: { outcome: 'error', tests: { total: 0, passed: 0, failed: 0 }, assertionFailures: 0, durationMs }, timeouts: 0, loadMessage: `unreadable report: ${e.message}`, failedTests: [] };
         }

package/src/probe/runners/vitest.mjs

@@ -1,7 +1,8 @@
-import { TEST_GLOBS, npx, runProcess, checkBinary, listTestFiles } from './shared.mjs';
+import { TEST_GLOBS, runProcess, checkRunner, runnerArgv, listTestFiles } from './shared.mjs';
 
 export const name = 'vitest';
 export const testGlobs = TEST_GLOBS;
-export const check = (opts) => checkBinary({ ...opts, bin: 'vitest' });
+export const check = (opts) => checkRunner({ ...opts, pkg: 'vitest', bin: 'vitest' });
 export const tests = (projectDir) => listTestFiles(projectDir, testGlobs);
-export const run = (opts) => runProcess({ ...opts, argv: (files, outFile) => [npx, 'vitest', 'run', ...files, '--reporter=json', `--outputFile=${outFile}`] });
+// --serial: one file at a time in one process, so a contended machine cannot turn a slow suite into a TIMEOUT verdict.
+export const run = (opts) => runProcess({ ...opts, argv: (files, outFile) => [...runnerArgv(opts.projectDir, 'vitest', 'vitest'), 'run', ...files, '--reporter=json', `--outputFile=${outFile}`, ...(opts.serial ? ['--no-file-parallelism'] : [])] });

package/src/render.mjs

@@ -5,7 +5,7 @@ export const formatVerdict = (v, provisional = false) => (v === 'killed' ? 'kill
 
 export const PROVISIONAL_WARNING = (n) => `PROVISIONAL — confirmRuns ${n} (< 3): nothing below is confirmed. Verdicts carry a "?"; this evidence cannot be frozen into a baseline. Re-run with --confirm 3 before trusting it.`;
 
-export function renderRecord(r, { provisional = false } = {}) {
+export function renderRecord(r, { provisional = false, showDiscovered = true } = {}) {
   const head = `${formatVerdict(r.verdict, provisional).padEnd(15)} ${r.claim.id}/${r.subject.id}`.padEnd(38);
   let why = '';
   if (r.detail.reason === 'killed-by-undeclared-tests' && r.detail.undeclaredKillers?.length) {
@@ -16,7 +16,7 @@ export function renderRecord(r, { provisional = false } = {}) {
   } else if (r.detail.reason) {
     why = `  [${r.detail.reason}]`;
   }
-  if (r.defenders.discovered) why += '  (defenders discovered by import)';
+  if (r.defenders.discovered && showDiscovered) why += '  (defenders discovered by import)';
   return `${head} ${r.claim.severity.padEnd(8)} ${r.subject.file}  ${r.subject.description}${why}`;
 }
 

package/src/replay/label.mjs

@@ -0,0 +1,64 @@
+/**
+ * Which injected-fault class does a real fix diff most resemble?
+ *
+ * This is the join key between ground truth and the fault model: a replayed
+ * bug is a real defect a human confirmed, so labelling it by fault class is
+ * what lets P(a surviving fault of this class is a real bug) be computed.
+ * Without it, a mutation score is a number nobody can interpret.
+ *
+ * Deterministic and shape-based, in the same spirit as the scaffold
+ * producers read in reverse: a fix ADDS what the bug lacked, so the lines a
+ * fix adds are the lines a fault would delete. No LLM, and `other` whenever
+ * the shape is not one the model has — an honest gap is worth more than a
+ * confident wrong label.
+ */
+const GUARD_RE = /^\s*if\s*\(.*\)\s*\{?\s*$/;
+const RETURN_RE = /^\s*return\s+/;
+const CALL_RE = /^\s*(?:await\s+)?(?:[\w.]*\.)?(?:verify|validate|check|assert|authorize|authorise|ensure|require)\w*\s*\(/i;
+const FIELD_RE = /^\s*['"`]?[\w$]+['"`]?\s*:\s*.+,\s*$/;
+const LITERAL_RE = /\b(?:httpOnly|secure|sameSite|rounds|cost|ttl|timeout|tolerance|window|limit|maxAge|expiresIn)\b\s*[:=]/i;
+const ASSIGN_RE = /^\s*(?:const |let |var )?[\w.[\]$]+\s*=\s*[^=]/;
+
+/** Most specific first; a tie between equally specific shapes is not a label. */
+const SPECIFICITY = ['literal-changed', 'call-removed', 'exception-swallowed', 'field-dropped', 'guard-removed', 'return-altered', 'statement-deleted'];
+
+function kind(line) {
+  if (LITERAL_RE.test(line)) return 'literal-changed';
+  if (/^\s*(?:\}\s*)?catch\b/.test(line) || /^\s*throw\b/.test(line)) return 'exception-swallowed';
+  if (GUARD_RE.test(line)) return 'guard-removed';
+  if (RETURN_RE.test(line)) return 'return-altered';
+  if (CALL_RE.test(line)) return 'call-removed';
+  if (FIELD_RE.test(line)) return 'field-dropped';
+  if (ASSIGN_RE.test(line)) return 'statement-deleted';
+  return null;
+}
+
+/**
+ * Label a unified diff of the fix's SOURCE changes (never its tests).
+ *
+ * Added lines carry the signal at double weight: the fix put back the guard,
+ * the field or the call that was missing. Removed lines are read too, for the
+ * fixes that delete a wrong condition rather than add a right one.
+ */
+export function labelDiff(diff) {
+  const added = [];
+  const removed = [];
+  for (const raw of diff.split('\n')) {
+    if (raw.startsWith('+++') || raw.startsWith('---')) continue;
+    if (raw.startsWith('+')) added.push(raw.slice(1));
+    else if (raw.startsWith('-')) removed.push(raw.slice(1));
+  }
+  const tally = new Map();
+  const add = (k, weight) => {
+    if (k) tally.set(k, (tally.get(k) ?? 0) + weight);
+  };
+  for (const l of added) add(kind(l), 2);
+  for (const l of removed) add(kind(l), 1);
+  if (tally.size === 0) return { faultClass: 'other', confident: false };
+
+  const ranked = [...tally].sort((a, b) => b[1] - a[1] || SPECIFICITY.indexOf(a[0]) - SPECIFICITY.indexOf(b[0]));
+  const [top, topWeight] = ranked[0];
+  const tiedSameSpecificity = ranked.length > 1 && ranked[1][1] === topWeight && SPECIFICITY.indexOf(ranked[1][0]) === SPECIFICITY.indexOf(top);
+  if (tiedSameSpecificity) return { faultClass: 'other', confident: false };
+  return { faultClass: top, confident: topWeight >= 2 };
+}