testguard-cli 0.3.1 → 0.4.0

package/CHANGELOG.md

@@ -7,6 +7,26 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [0.4.0] - 2026-09-17
+
+### Added
+
+- **jest runner.** `--runner vitest|jest|auto` (default `auto`: the first
+  that resolves in the project). jest runs with `--ci --json --runTestsByPath`
+  so file arguments are exact paths, not regexes; its `Exceeded timeout`
+  wording is recognised as a timeout, never a kill; `__tests__/` files are
+  collected for escalation. `run.runner.name` records which runner produced
+  the evidence.
+- A CommonJS edition of the known-answer fixture (`fixtures/known-answer-jest`)
+  with the same claims and the same expected verdicts; the acceptance suite
+  and CI prove jest against it exactly as vitest is proven against the ESM
+  one.
+
+### Changed
+
+- Runners are now modules behind one interface (`src/probe/runners/`); the
+  vitest path is unchanged and still reproduces its oracle.
+
 ## [0.3.1] - 2026-09-17
 
 Dogfooding `init` and `status` on this repository.

package/README.md

@@ -52,7 +52,7 @@ tests were written against the survivors, 39/39 were killed.
 | npm | `npm i -D testguard-cli` then `npx testguard probe` |
 | pip | `pip install testguard-cli` then `testguard probe` (needs Node ≥ 20) |
 | Homebrew | `brew tap raccioly/tap && brew install testguard` |
-| GitHub Action | `uses: raccioly/testguard@v0.3.1` — see [`action.yml`](./action.yml) |
+| GitHub Action | `uses: raccioly/testguard@v0.4.0` — see [`action.yml`](./action.yml) |
 | pre-commit | `repo: https://github.com/raccioly/testguard`, hooks `testguard-claims`, `testguard-probe` |
 
 Projects that set `min-release-age` in `.npmrc` cannot see a version published
@@ -111,6 +111,11 @@ npx testguard-cli scaffold src/x.ts   # propose faults for a file, as a draft to
    files that import the fault's target, by relative path or resolved alias.
    `NOCOVER` then means exactly "no test file imports this source".
 
+   Runners: **vitest** and **jest** (`--runner auto` picks the first that
+   resolves; both read the same jest-compatible JSON report). Anything else
+   goes through `--runner-cmd`. Each runner is proven against its own copy
+   of the known-answer fixture.
+
    Practical loop: first pass `--no-escalate` (escalation re-runs the whole
    suite N times per survivor); iterate on one claim with `--claim <ID>` and
    either `--include-dirty` or `--in-place` (only fault target files must be
@@ -215,14 +220,14 @@ through every verdict.
 
 ## Status
 
-**v0.3.** Seven commands, vitest runner, hand-authored faults plus a
-mechanical scaffold, and an agent operating layer (`status`, `init`). The contract
+**v0.4.** Seven commands, vitest and jest runners, hand-authored faults plus
+a mechanical scaffold, and an agent operating layer (`status`, `init`). The contract
 spine — six JSON Schemas shared with the other Guard tools — is under
 [`spec/`](spec/). One exact-pinned runtime dependency (`ajv`, for schema validation); Node ≥ 20.
 
-Not yet: test generation (the two-gate acceptance loop), other runners,
-AST-aware producers, and calibration of fault classes against real escaped
-bugs. Each is designed for; none is claimed.
+Not yet: test generation (the two-gate acceptance loop), runners beyond
+vitest and jest, AST-aware producers, and calibration of fault classes
+against real escaped bugs. Each is designed for; none is claimed.
 
 ## Licence
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "testguard-cli",
-  "version": "0.3.1",
+  "version": "0.4.0",
   "description": "Proves a test suite defends the claims a project makes: injects the faults those claims forbid and reports every one the tests fail to detect.",
   "type": "module",
   "license": "MIT",
@@ -46,6 +46,7 @@
     "quality-gate"
   ],
   "devDependencies": {
+    "jest": "^30.5.1",
     "vitest": "^5.0.1"
   },
   "bin": {

package/src/cli.mjs

@@ -38,6 +38,7 @@ probe
   --include-dirty      probe the working tree (a snapshot commit) instead of HEAD; uncommitted tests count
   --verbose            also print each killed fault (default: only unproven ones, plus a count)
                        --confirm below 3 is PROVISIONAL: verdicts print with "?", evidence goes to .testguard/evidence-provisional.json
+  --runner <name>      vitest | jest | auto (default: auto — first of vitest, jest that resolves)
   --runner-cmd "<cmd>" custom runner; must contain {files} and {out}, e.g. "pnpm vitest run {files} --reporter=json --outputFile={out}"
   --node-modules <dir> node_modules to link into the scratch worktree (or TESTGUARD_NODE_MODULES)
   --in-place           mutate the working tree instead of a scratch worktree
@@ -82,6 +83,7 @@ export async function main(argv, io = { out: (s) => process.stdout.write(s + '\n
         force: { type: 'boolean', default: false },
         verbose: { type: 'boolean', default: false },
         'runner-cmd': { type: 'string' },
+        runner: { type: 'string', default: 'auto' },
         'node-modules': { type: 'string' },
         'in-place': { type: 'boolean', default: false },
         'no-escalate': { type: 'boolean', default: false },
@@ -114,6 +116,10 @@ export async function main(argv, io = { out: (s) => process.stdout.write(s + '\n
     io.err(USAGE);
     return 3;
   }
+  if (!['vitest', 'jest', 'auto'].includes(values.runner)) {
+    io.err('--runner must be vitest, jest or auto');
+    return 3;
+  }
   if (!['critical', 'high', 'medium', 'low'].includes(values.severity)) {
     io.err('--severity must be one of critical, high, medium, low');
     return 3;

package/src/commands/claims.mjs

@@ -1,7 +1,7 @@
 import { resolve } from 'node:path';
 import { loadClaims, defaultClaimsPath } from '../claims/load.mjs';
 import { scanAnnotations, reconcile } from '../claims/annotations.mjs';
-import { resolveDefenders } from '../probe/runner-vitest.mjs';
+import { resolveDefenders } from '../probe/runners/shared.mjs';
 import { discoverDefenders } from '../probe/discover.mjs';
 
 export async function claimsCommand({ projectDir, values }, io) {

package/src/commands/probe.mjs

@@ -42,6 +42,7 @@ export async function probeCommand({ projectDir, values, version }, io) {
     mode: values['in-place'] ? 'in-place' : 'worktree',
     ref: values.ref,
     runnerCommand: values['runner-cmd'],
+    runnerName: values.runner,
     nodeModules: values['node-modules'] ? resolve(values['node-modules']) : process.env.TESTGUARD_NODE_MODULES,
     only,
     escalate: !values['no-escalate'],

package/src/probe/discover.mjs

@@ -1,5 +1,5 @@
 import { join } from 'node:path';
-import { listTestFiles } from './runner-vitest.mjs';
+import { listTestFiles } from './runners/shared.mjs';
 import { fileImports } from './rank.mjs';
 
 /**

package/src/probe/probe.mjs

@@ -6,7 +6,8 @@ import { repoRoot as gitRoot, headSha, isDirty, snapshotWorkingTree } from '../g
 import { discoverDefenders } from './discover.mjs';
 import { createScratch, inPlace, PreconditionError } from './worktree.mjs';
 import { applyFault, locate } from './inject.mjs';
-import * as vitest from './runner-vitest.mjs';
+import { resolveDefenders, parseCommandTemplate } from './runners/shared.mjs';
+import { selectRunner, RUNNERS } from './runners/index.mjs';
 import { classify, shouldStopEarly } from './classify.mjs';
 import { blastRadius, rank } from './rank.mjs';
 import { hashFile, sha256 } from '../util/hash.mjs';
@@ -14,9 +15,9 @@ import { fingerprint } from '../../spec/lib/fingerprint.mjs';
 
 const isKill = (r) => r.outcome === 'fail' && r.assertionFailures > 0;
 
-function readRunnerVersion(projectDir) {
+function readRunnerVersion(projectDir, name = 'vitest') {
   try {
-    return JSON.parse(readFileSync(createRequire(join(projectDir, 'noop.js')).resolve('vitest/package.json'), 'utf8')).version;
+    return JSON.parse(readFileSync(createRequire(join(projectDir, 'noop.js')).resolve(`${name}/package.json`), 'utf8')).version;
   } catch {
     return undefined;
   }
@@ -34,6 +35,7 @@ export async function probe({
   ref = 'HEAD',
   budgetMs = 120_000,
   runnerCommand,
+  runnerName = 'auto',
   nodeModules,
   only,
   includeDirty = false,
@@ -75,7 +77,7 @@ export async function probe({
       const watched = new Set(targets);
       for (const claim of claims.claims) {
         if (selected && !selected.has(claim.id)) continue;
-        const declared = claim.defendedBy?.length ? vitest.resolveDefenders(projectDir, claim.defendedBy) : claim.faults.flatMap((f) => discoverDefenders(projectDir, f.file));
+        const declared = claim.defendedBy?.length ? resolveDefenders(projectDir, claim.defendedBy) : claim.faults.flatMap((f) => discoverDefenders(projectDir, f.file));
         for (const d of declared) watched.add(relative(root, join(projectDir, d)));
         for (const g of claim.defendedBy ?? []) if (!g.includes('*')) watched.add(relative(root, join(projectDir, g)));
       }
@@ -89,27 +91,29 @@ export async function probe({
   const iso = mode === 'worktree' ? createScratch({ repoRoot: root, projectDir, ref: snapshot ?? ref, scratchBase, nodeModules }) : inPlace({ repoRoot: root, projectDir });
   const records = [];
   let runnerVersion;
+  let runner = RUNNERS[runnerName === 'auto' ? 'vitest' : runnerName];
   try {
-    const commandTemplate = runnerCommand ? vitest.parseCommandTemplate(runnerCommand) : undefined;
+    const commandTemplate = runnerCommand ? parseCommandTemplate(runnerCommand) : undefined;
     if (!commandTemplate) {
       // An unresolvable runner is a precondition failure, not a flaky defender.
-      const check = await vitest.checkRunner({ projectDir: iso.projectDir });
-      if (!check.ok) {
-        throw new PreconditionError(`test runner is not resolvable in the ${mode === 'worktree' ? 'scratch worktree' : 'project'} (${check.message}). ` +
+      const sel = await selectRunner({ projectDir: iso.projectDir, name: runnerName });
+      if (sel.error) {
+        throw new PreconditionError(`test runner is not resolvable in the ${mode === 'worktree' ? 'scratch worktree' : 'project'} (${sel.error}). ` +
           (mode === 'worktree' ? 'No usable node_modules was linked: pass --node-modules <path>, or run with --in-place.' : 'Install dependencies first.'));
       }
-      runnerVersion = check.version;
+      runner = sel.runner;
+      runnerVersion = sel.version;
     }
-    const allTests = vitest.listTestFiles(iso.projectDir);
+    const allTests = runner.tests(iso.projectDir);
     const baselineCache = new Map();
     const prior = previous && previous.run.confirmRuns === confirmRuns
       ? new Map(previous.records.map((r) => [`${r.claim.id}/${r.subject.id}`, r]))
       : new Map();
-    const runDefenders = (files) => vitest.runVitest({ projectDir: iso.projectDir, files, budgetMs, commandTemplate });
+    const runDefenders = (files) => runner.run({ projectDir: iso.projectDir, files, budgetMs, commandTemplate });
 
     for (const claim of claims.claims) {
       if (selected && !selected.has(claim.id)) continue;
-      const declared = claim.defendedBy?.length ? vitest.resolveDefenders(iso.projectDir, claim.defendedBy) : null;
+      const declared = claim.defendedBy?.length ? resolveDefenders(iso.projectDir, claim.defendedBy) : null;
       for (const fault of claim.faults) {
         const defenders = declared ?? discoverDefenders(iso.projectDir, fault.file);
         const stage = (name, i, n) => onStage({ claimId: claim.id, faultId: fault.id, stage: name, i, n });
@@ -130,7 +134,7 @@ export async function probe({
       startedAt,
       finishedAt: new Date().toISOString(),
       repo: { head, dirty: isDirty(root), ...(snapshot ? { snapshot } : {}) },
-      runner: { name: vitest.name, ...((runnerVersion ?? readRunnerVersion(projectDir)) ? { version: runnerVersion ?? readRunnerVersion(projectDir) } : {}) },
+      runner: { name: runner.name, ...((runnerVersion ?? readRunnerVersion(projectDir, runner.name)) ? { version: runnerVersion ?? readRunnerVersion(projectDir, runner.name) } : {}) },
       confirmRuns,
       ...(confirmRuns < 3 ? { provisional: true } : {}),
       mode,

package/src/probe/runners/index.mjs

@@ -0,0 +1,20 @@
+import * as vitest from './vitest.mjs';
+import * as jest from './jest.mjs';
+
+export const RUNNERS = { vitest, jest };
+
+/**
+ * Pick the runner: an explicit name, or the first of vitest, jest that is
+ * resolvable from the project. Returns { runner, version } or { error }.
+ */
+export async function selectRunner({ projectDir, name = 'auto' }) {
+  const candidates = name === 'auto' ? [vitest, jest] : [RUNNERS[name]];
+  if (!candidates[0]) return { error: `unknown runner "${name}"; use vitest, jest or auto` };
+  const messages = [];
+  for (const r of candidates) {
+    const c = await r.check({ projectDir });
+    if (c.ok) return { runner: r, version: c.version };
+    messages.push(`${r.name}: ${c.message}`);
+  }
+  return { error: messages.join('; ') };
+}

package/src/probe/runners/jest.mjs

@@ -0,0 +1,9 @@
+import { TEST_GLOBS, npx, runProcess, checkBinary, listTestFiles } from './shared.mjs';
+
+export const name = 'jest';
+// jest's default testMatch also collects anything under __tests__/.
+export const testGlobs = [...TEST_GLOBS, '**/__tests__/**/*.js', '**/__tests__/**/*.mjs', '**/__tests__/**/*.cjs', '**/__tests__/**/*.ts', '**/__tests__/**/*.tsx', '**/__tests__/**/*.jsx'];
+export const check = (opts) => checkBinary({ ...opts, bin: 'jest' });
+export const tests = (projectDir) => listTestFiles(projectDir, testGlobs);
+// --runTestsByPath: positionals are exact paths, not regexes — a path with `+` or `(` would otherwise silently match nothing.
+export const run = (opts) => runProcess({ ...opts, argv: (files, outFile) => [npx, 'jest', '--ci', '--json', `--outputFile=${outFile}`, '--runTestsByPath', ...files] });

package/src/probe/{runner-vitest.mjs → runners/shared.mjs}

@@ -3,12 +3,11 @@ import { existsSync, readFileSync, rmSync } from 'node:fs';
 import { tmpdir } from 'node:os';
 import { join } from 'node:path';
 import { randomBytes } from 'node:crypto';
-import { matchGlobs } from '../util/glob.mjs';
+import { matchGlobs } from '../../util/glob.mjs';
 
-const TEST_GLOBS = ['**/*.test.js', '**/*.test.mjs', '**/*.test.cjs', '**/*.test.ts', '**/*.test.mts', '**/*.test.tsx', '**/*.test.jsx',
+export const TEST_GLOBS = ['**/*.test.js', '**/*.test.mjs', '**/*.test.cjs', '**/*.test.ts', '**/*.test.mts', '**/*.test.tsx', '**/*.test.jsx',
   '**/*.spec.js', '**/*.spec.mjs', '**/*.spec.cjs', '**/*.spec.ts', '**/*.spec.mts', '**/*.spec.tsx', '**/*.spec.jsx'];
-
-export const name = 'vitest';
+export const npx = process.platform === 'win32' ? 'npx.cmd' : 'npx';
 
 /**
  * Split a runner command template into argv. Supports double and single
@@ -45,14 +44,13 @@ export function parseCommandTemplate(template) {
 export const expandCommand = (words, files, outFile) => words.flatMap((w) => (w === '{files}' ? files : [w.replaceAll('{out}', outFile)]));
 
 /**
- * Can the runner be resolved at all from this directory? Run before any
- * verdict is attempted: an unresolvable runner is a precondition failure,
- * never a statement about the tests.
+ * Can `bin` be resolved at all from this directory? Run before any verdict
+ * is attempted: an unresolvable runner is a precondition failure, never a
+ * statement about the tests.
  */
-export function checkRunner({ projectDir, budgetMs = 60_000, env = process.env }) {
-  const npx = process.platform === 'win32' ? 'npx.cmd' : 'npx';
+export function checkBinary({ projectDir, bin, budgetMs = 60_000, env = process.env }) {
   return new Promise((resolve) => {
-    const child = spawn(npx, ['--no-install', 'vitest', '--version'], { cwd: projectDir, stdio: ['ignore', 'pipe', 'pipe'], env: { ...env, CI: '1' } });
+    const child = spawn(npx, ['--no-install', bin, '--version'], { cwd: projectDir, stdio: ['ignore', 'pipe', 'pipe'], env: { ...env, CI: '1' } });
     let out = '';
     let err = '';
     child.on('error', (e) => resolve({ ok: false, message: e.message }));
@@ -61,7 +59,7 @@ export function checkRunner({ projectDir, budgetMs = 60_000, env = process.env }
     const timer = setTimeout(() => child.kill('SIGKILL'), budgetMs);
     child.on('close', (code) => {
       clearTimeout(timer);
-      const version = out.trim().replace(/^vitest\//, '');
+      const version = out.trim().replace(/^vitest\//, '').split('\n').pop();
       resolve(code === 0 && version ? { ok: true, version } : { ok: false, message: (err || out).trim().split('\n').filter(Boolean).slice(-1)[0] ?? `exit ${code}` });
     });
   });
@@ -70,7 +68,7 @@ export function checkRunner({ projectDir, budgetMs = 60_000, env = process.env }
 /** Defender globs → existing files. Empty result is the `nocover` signal. */
 export const resolveDefenders = (projectDir, globs) => matchGlobs(projectDir, globs ?? []);
 
-export const listTestFiles = (projectDir) => matchGlobs(projectDir, TEST_GLOBS);
+export const listTestFiles = (projectDir, globs = TEST_GLOBS) => matchGlobs(projectDir, globs);
 
 /**
  * Turn vitest's JSON report into a spec `testRun`.
@@ -80,11 +78,16 @@ export const listTestFiles = (projectDir) => matchGlobs(projectDir, TEST_GLOBS);
  * can kill a fault. A suite that fails to load, or a test that times out, is
  * not evidence that the suite defends the claim.
  */
+function firstInformativeLine(message) {
+  const lines = message.split('\n').map((l) => l.replace(/\x1b\[[0-9;]*m/g, '').trim()).filter(Boolean);
+  return lines.find((l) => /error|syntax|unexpected|expected .+ but found|cannot find|failed to parse|transform failed/i.test(l) && !/^●/.test(l)) ?? lines[0] ?? '';
+}
+
 export function parseReport(report, durationMs) {
   const files = report.testResults ?? [];
   const loadFailed = files.some((f) => f.status === 'failed' && (f.assertionResults?.length ?? 0) === 0);
   const results = files.flatMap((f) => (f.assertionResults ?? []).map((t) => ({ ...t, id: `${f.name}::${t.fullName ?? t.title ?? ''}` }))).filter((t) => t.status === 'failed');
-  const timeouts = results.filter((t) => (t.failureMessages ?? []).some((m) => /timed out/i.test(m))).length;
+  const timeouts = results.filter((t) => (t.failureMessages ?? []).some((m) => /timed out|exceeded timeout/i.test(m))).length;
   const failedTests = results.map((t) => t.id);
   const assertionFailures = results.length - timeouts;
   const tests = { total: report.numTotalTests ?? 0, passed: report.numPassedTests ?? 0, failed: report.numFailedTests ?? 0 };
@@ -95,20 +98,25 @@ export function parseReport(report, durationMs) {
   else outcome = 'fail';
 
   const run = { outcome, tests, assertionFailures, durationMs };
-  const loadMessage = loadFailed ? (files.find((f) => f.message)?.message ?? '').split('\n')[0] : undefined;
+  // jest opens a load failure with "● Test suite failed to run" and puts the
+  // cause lines below; vitest puts it on the first line. Take the first line
+  // that names an error, else the first non-empty one.
+  const loadMessage = loadFailed ? firstInformativeLine(files.find((f) => f.message)?.message ?? '') : undefined;
   return { run, timeouts, loadMessage, failedTests };
 }
 
 /**
- * Run vitest on `files` inside `projectDir` with a hard wall-clock budget.
- * The budget matters because a synchronous infinite loop is immune to
- * vitest's own test timeout; only killing the process ends it.
+ * Run a test process on `files` inside `projectDir` with a hard wall-clock
+ * budget and read back its jest-compatible JSON report. The budget matters
+ * because a synchronous infinite loop is immune to any runner's own test
+ * timeout; only killing the process ends it. `argv(files, outFile)` is the
+ * runner's command line; `command` (tests) or `commandTemplate` (--runner-cmd)
+ * override it.
  */
-export function runVitest({ projectDir, files, budgetMs = 120_000, command, commandTemplate }) {
-  const outFile = join(tmpdir(), `testguard-vitest-${randomBytes(6).toString('hex')}.json`);
-  const npx = process.platform === 'win32' ? 'npx.cmd' : 'npx';
+export function runProcess({ projectDir, files, budgetMs = 120_000, command, commandTemplate, argv }) {
+  const outFile = join(tmpdir(), `testguard-run-${randomBytes(6).toString('hex')}.json`);
   const [cmd, ...args] = command
-    ?? (commandTemplate ? expandCommand(commandTemplate, files, outFile) : [npx, 'vitest', 'run', ...files, '--reporter=json', `--outputFile=${outFile}`]);
+    ?? (commandTemplate ? expandCommand(commandTemplate, files, outFile) : argv(files, outFile));
   const started = Date.now();
 
   return new Promise((resolve) => {

package/src/probe/runners/vitest.mjs

@@ -0,0 +1,7 @@
+import { TEST_GLOBS, npx, runProcess, checkBinary, listTestFiles } from './shared.mjs';
+
+export const name = 'vitest';
+export const testGlobs = TEST_GLOBS;
+export const check = (opts) => checkBinary({ ...opts, bin: 'vitest' });
+export const tests = (projectDir) => listTestFiles(projectDir, testGlobs);
+export const run = (opts) => runProcess({ ...opts, argv: (files, outFile) => [npx, 'vitest', 'run', ...files, '--reporter=json', `--outputFile=${outFile}`] });

package/src/status/status.mjs

@@ -4,7 +4,7 @@ import { readSpecDoc } from '../evidence/writer.mjs';
 import { loadClaims, defaultClaimsPath } from '../claims/load.mjs';
 import { gate } from '../baseline/baseline.mjs';
 import { hashFile, sha256 } from '../util/hash.mjs';
-import { resolveDefenders } from '../probe/runner-vitest.mjs';
+import { resolveDefenders } from '../probe/runners/shared.mjs';
 import { discoverDefenders } from '../probe/discover.mjs';
 import { sortForReport } from '../render.mjs';