testguard-cli 0.1.0

package/src/probe/probe.mjs

@@ -0,0 +1,182 @@
+import { existsSync, readFileSync } from 'node:fs';
+import { createRequire } from 'node:module';
+import { join, relative, resolve } from 'node:path';
+import { repoRoot as gitRoot, headSha, isDirty } from '../git.mjs';
+import { createScratch, inPlace, PreconditionError } from './worktree.mjs';
+import { applyFault, locate } from './inject.mjs';
+import * as vitest from './runner-vitest.mjs';
+import { classify, shouldStopEarly } from './classify.mjs';
+import { blastRadius, rank } from './rank.mjs';
+import { hashFile, sha256 } from '../util/hash.mjs';
+import { fingerprint } from '../../spec/lib/fingerprint.mjs';
+
+const isKill = (r) => r.outcome === 'fail' && r.assertionFailures > 0;
+
+function runnerVersion(projectDir) {
+  try {
+    return JSON.parse(readFileSync(createRequire(join(projectDir, 'noop.js')).resolve('vitest/package.json'), 'utf8')).version;
+  } catch {
+    return undefined;
+  }
+}
+
+/**
+ * Probe every fault of every claim. Returns a spec-conformant evidence
+ * document; writing it is the caller's job.
+ */
+export async function probe({
+  projectDir,
+  claims,
+  confirmRuns = 3,
+  mode = 'worktree',
+  ref = 'HEAD',
+  budgetMs = 120_000,
+  escalate = true,
+  scratchBase,
+  toolVersion = '0.0.0',
+  previous,
+  onProgress = () => {},
+}) {
+  projectDir = resolve(projectDir);
+  const root = gitRoot(projectDir);
+  if (mode === 'in-place' && ref !== 'HEAD') throw new PreconditionError('--ref needs a scratch worktree; drop --in-place');
+  const head = headSha(root, ref);
+  if (!head) throw new PreconditionError(ref === 'HEAD' ? 'repository has no commits; every verdict is tied to a commit' : `ref ${ref} does not resolve to a commit`);
+
+  const targets = [...new Set(claims.claims.flatMap((c) => c.faults.map((f) => relative(root, join(projectDir, f.file)))))];
+  if (mode === 'in-place' && isDirty(root, targets)) {
+    throw new PreconditionError(`uncommitted changes in target files (${targets.join(', ')}); commit or stash first, or drop --in-place`);
+  }
+
+  const startedAt = new Date().toISOString();
+  const iso = mode === 'worktree' ? createScratch({ repoRoot: root, projectDir, ref, scratchBase }) : inPlace({ repoRoot: root, projectDir });
+  const records = [];
+  try {
+    const allTests = vitest.listTestFiles(iso.projectDir);
+    const baselineCache = new Map();
+    const prior = previous && previous.run.confirmRuns === confirmRuns
+      ? new Map(previous.records.map((r) => [`${r.claim.id}/${r.subject.id}`, r]))
+      : new Map();
+    const runDefenders = (files) => vitest.runVitest({ projectDir: iso.projectDir, files, budgetMs });
+
+    for (const claim of claims.claims) {
+      const defenders = vitest.resolveDefenders(iso.projectDir, claim.defendedBy);
+      for (const fault of claim.faults) {
+        const record = await probeOne({ claim, fault, defenders, allTests, iso, confirmRuns, escalate, baselineCache, runDefenders, prior: prior.get(`${claim.id}/${fault.id}`), priorRunId: previous?.run.id });
+        records.push(record);
+        onProgress(record);
+      }
+    }
+  } finally {
+    iso.cleanup();
+  }
+
+  return {
+    schemaVersion: 1,
+    tool: { name: 'testguard', version: toolVersion },
+    run: {
+      id: `run-${startedAt.replace(/[-:.]/g, '').slice(0, 15)}`,
+      startedAt,
+      finishedAt: new Date().toISOString(),
+      repo: { head, dirty: isDirty(root) },
+      runner: { name: vitest.name, ...(runnerVersion(projectDir) ? { version: runnerVersion(projectDir) } : {}) },
+      confirmRuns,
+      mode,
+    },
+    records,
+  };
+}
+
+async function probeOne({ claim, fault, defenders, allTests, iso, confirmRuns, escalate, baselineCache, runDefenders, prior, priorRunId }) {
+  const targetPath = join(iso.projectDir, fault.file);
+  const targetExists = existsSync(targetPath);
+  const inputs = {
+    targetHash: targetExists ? hashFile(targetPath) : sha256(''),
+    defenderHashes: Object.fromEntries(defenders.map((f) => [f, hashFile(join(iso.projectDir, f))])),
+  };
+
+  // Same source, same defenders, same N: the verdict cannot have changed.
+  if (prior && sameInputs(prior, inputs, claim.defendedBy ?? [], defenders)) {
+    return { ...prior, reusedFrom: prior.reusedFrom ?? priorRunId };
+  }
+
+  const detail = { baselineRuns: [], probeRuns: [] };
+  const rawProbeRuns = []; // spec testRun + the runner's timeout count, which classify needs
+  let anchor = null;
+
+  if (defenders.length > 0) {
+    anchor = targetExists ? locate(readFileSync(targetPath, 'utf8'), fault) : { status: 'file-missing', hits: 0, expected: fault.expectHits ?? 1 };
+
+    if (anchor.status === 'ok') {
+      const key = defenders.join('\n');
+      if (!baselineCache.has(key)) {
+        const runs = [];
+        for (let i = 0; i < confirmRuns; i++) {
+          const { run } = await runDefenders(defenders);
+          runs.push(run);
+          if (run.outcome !== 'pass') break;
+        }
+        baselineCache.set(key, runs);
+      }
+      detail.baselineRuns = baselineCache.get(key);
+
+      if (detail.baselineRuns.every((r) => r.outcome === 'pass') && detail.baselineRuns.length === confirmRuns) {
+        const mutation = applyFault(iso.projectDir, fault);
+        try {
+          const probeRuns = rawProbeRuns;
+          for (let i = 0; i < confirmRuns; i++) {
+            const { run, timeouts, loadMessage } = await runDefenders(defenders);
+            probeRuns.push({ ...run, timeouts, loadMessage });
+            if (shouldStopEarly(probeRuns)) break;
+          }
+          detail.probeRuns = probeRuns.map(({ timeouts, loadMessage, ...run }) => run);
+          const provisional = classify({ defenders, anchor, baselineRuns: detail.baselineRuns, probeRuns, confirmRuns });
+
+          // Escalation: does anything *undeclared* catch it? A single run cannot
+          // say — a flaky test elsewhere in the suite would take the credit — so
+          // a test is an undeclared killer only if it fails in all N runs.
+          const broader = allTests.filter((t) => !defenders.includes(t));
+          if (provisional.verdict === 'survived' && escalate && broader.length > 0) {
+            const runs = [];
+            let killers = null;
+            for (let i = 0; i < confirmRuns; i++) {
+              const { run, failedTests } = await runDefenders(allTests);
+              runs.push(run);
+              killers = killers === null ? new Set(failedTests) : new Set(failedTests.filter((t) => killers.has(t)));
+              if (killers.size === 0) break;
+            }
+            detail.escalated = true;
+            detail.escalationRuns = runs;
+            if (killers.size > 0 && runs.length === confirmRuns && runs.every(isKill)) detail.reason = 'killed-by-undeclared-tests';
+          }
+        } finally {
+          mutation.restore();
+        }
+      }
+    }
+  }
+
+  const { verdict, reason } = classify({ defenders, anchor, baselineRuns: detail.baselineRuns, probeRuns: rawProbeRuns, confirmRuns });
+  if (reason && !detail.reason) detail.reason = reason;
+
+  const blast = targetExists ? blastRadius(iso.projectDir, fault.file) : 0;
+
+  return {
+    fingerprint: fingerprint({ claimId: claim.id, subjectId: fault.id, file: fault.file, verdict }),
+    claim: { id: claim.id, statement: claim.statement, severity: claim.severity, source: claim.source, producedBy: claim.producedBy },
+    subject: { kind: 'fault', id: fault.id, description: fault.description, file: fault.file, faultClass: fault.faultClass, producedBy: fault.producedBy },
+    verdict,
+    detail,
+    defenders: { requested: claim.defendedBy ?? [], resolved: defenders, nocover: defenders.length === 0 },
+    inputs,
+    rank: rank({ severity: claim.severity, sourceKind: claim.source.kind, blast }),
+  };
+}
+
+function sameInputs(prior, inputs, requested, resolved) {
+  const same = (a, b) => JSON.stringify(a) === JSON.stringify(b);
+  return prior.inputs.targetHash === inputs.targetHash
+    && same(prior.inputs.defenderHashes, inputs.defenderHashes)
+    && same(prior.defenders.requested, requested)
+    && same(prior.defenders.resolved, resolved);
+}

package/src/probe/rank.mjs

@@ -0,0 +1,51 @@
+import { readFileSync } from 'node:fs';
+import { dirname, join, resolve, extname } from 'node:path';
+import { walk } from '../util/glob.mjs';
+
+const SOURCE_EXT = new Set(['.js', '.mjs', '.cjs', '.ts', '.mts', '.cts', '.jsx', '.tsx']);
+const IMPORT_RE = /(?:from\s*|import\s*\(?\s*|require\s*\(\s*)['"]([^'"]+)['"]/g;
+const TEST_RE = /\.(test|spec)\.[cm]?[jt]sx?$/;
+
+const SEVERITY_WEIGHT = { critical: 8, high: 4, medium: 2, low: 1 };
+// A claim written by the same agent that wrote the tests carries less
+// independent evidence than one from a human-governed spec.
+const SOURCE_WEIGHT = { spec: 1, adr: 1, manual: 0.9, annotation: 0.75, comment: 0.75 };
+
+function resolvesTo(fromFile, specifier, targetAbs) {
+  if (!specifier.startsWith('.')) return false;
+  const base = resolve(dirname(fromFile), specifier);
+  const candidates = [base, ...[...SOURCE_EXT].map((e) => base + e), ...[...SOURCE_EXT].map((e) => join(base, 'index' + e))];
+  const stripped = base.replace(/\.(js|mjs|cjs)$/, '');
+  candidates.push(stripped + '.ts', stripped + '.mts', stripped + '.tsx');
+  return candidates.includes(targetAbs);
+}
+
+/** Number of non-test source files that import `targetRel` (direct imports only, documented as such). */
+export function blastRadius(projectDir, targetRel) {
+  const targetAbs = resolve(projectDir, targetRel);
+  let count = 0;
+  for (const rel of walk(projectDir)) {
+    if (!SOURCE_EXT.has(extname(rel)) || TEST_RE.test(rel) || rel === targetRel) continue;
+    const abs = join(projectDir, rel);
+    let src;
+    try {
+      src = readFileSync(abs, 'utf8');
+    } catch {
+      continue;
+    }
+    for (const m of src.matchAll(IMPORT_RE)) {
+      if (resolvesTo(abs, m[1], targetAbs)) {
+        count++;
+        break;
+      }
+    }
+  }
+  return count;
+}
+
+/** Additive ordering only. Never touches the verdict. */
+export function rank({ severity, sourceKind, blast }) {
+  const score = SEVERITY_WEIGHT[severity] * (SOURCE_WEIGHT[sourceKind] ?? 0.9) * (1 + Math.log2(1 + blast));
+  return { score: Number(score.toFixed(3)), blastRadius: blast, tier: severity };
+}
+

package/src/probe/runner-vitest.mjs

@@ -0,0 +1,85 @@
+import { spawn } from 'node:child_process';
+import { existsSync, readFileSync, rmSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { randomBytes } from 'node:crypto';
+import { matchGlobs } from '../util/glob.mjs';
+
+const TEST_GLOBS = ['**/*.test.js', '**/*.test.mjs', '**/*.test.cjs', '**/*.test.ts', '**/*.test.mts', '**/*.test.tsx', '**/*.test.jsx',
+  '**/*.spec.js', '**/*.spec.mjs', '**/*.spec.cjs', '**/*.spec.ts', '**/*.spec.mts', '**/*.spec.tsx', '**/*.spec.jsx'];
+
+export const name = 'vitest';
+
+/** Defender globs → existing files. Empty result is the `nocover` signal. */
+export const resolveDefenders = (projectDir, globs) => matchGlobs(projectDir, globs ?? []);
+
+export const listTestFiles = (projectDir) => matchGlobs(projectDir, TEST_GLOBS);
+
+/**
+ * Turn vitest's JSON report into a spec `testRun`.
+ *
+ * `assertionFailures` counts test-level failures that are neither timeouts nor
+ * load failures — the test body ran and rejected the behaviour. Only those
+ * can kill a fault. A suite that fails to load, or a test that times out, is
+ * not evidence that the suite defends the claim.
+ */
+export function parseReport(report, durationMs) {
+  const files = report.testResults ?? [];
+  const loadFailed = files.some((f) => f.status === 'failed' && (f.assertionResults?.length ?? 0) === 0);
+  const results = files.flatMap((f) => (f.assertionResults ?? []).map((t) => ({ ...t, id: `${f.name}::${t.fullName ?? t.title ?? ''}` }))).filter((t) => t.status === 'failed');
+  const timeouts = results.filter((t) => (t.failureMessages ?? []).some((m) => /timed out/i.test(m))).length;
+  const failedTests = results.map((t) => t.id);
+  const assertionFailures = results.length - timeouts;
+  const tests = { total: report.numTotalTests ?? 0, passed: report.numPassedTests ?? 0, failed: report.numFailedTests ?? 0 };
+
+  let outcome;
+  if (loadFailed || tests.total === 0) outcome = 'error';
+  else if (report.success) outcome = 'pass';
+  else outcome = 'fail';
+
+  const run = { outcome, tests, assertionFailures, durationMs };
+  const loadMessage = loadFailed ? (files.find((f) => f.message)?.message ?? '').split('\n')[0] : undefined;
+  return { run, timeouts, loadMessage, failedTests };
+}
+
+/**
+ * Run vitest on `files` inside `projectDir` with a hard wall-clock budget.
+ * The budget matters because a synchronous infinite loop is immune to
+ * vitest's own test timeout; only killing the process ends it.
+ */
+export function runVitest({ projectDir, files, budgetMs = 120_000, command }) {
+  const outFile = join(tmpdir(), `testguard-vitest-${randomBytes(6).toString('hex')}.json`);
+  const npx = process.platform === 'win32' ? 'npx.cmd' : 'npx';
+  const [cmd, ...args] = command ?? [npx, 'vitest', 'run', ...files, '--reporter=json', `--outputFile=${outFile}`];
+  const started = Date.now();
+
+  return new Promise((resolve) => {
+    const child = spawn(cmd, args, { cwd: projectDir, stdio: ['ignore', 'pipe', 'pipe'], env: { ...process.env, CI: '1', FORCE_COLOR: '0' } });
+    let stderr = '';
+    child.stderr.on('data', (d) => (stderr += d));
+    let killed = false;
+    const timer = setTimeout(() => {
+      killed = true;
+      child.kill('SIGKILL');
+    }, budgetMs);
+
+    child.on('close', () => {
+      clearTimeout(timer);
+      const durationMs = Date.now() - started;
+      let result;
+      if (killed) {
+        result = { run: { outcome: 'timeout', tests: { total: 0, passed: 0, failed: 0 }, assertionFailures: 0, durationMs }, timeouts: 1, loadMessage: `budget of ${budgetMs}ms exceeded`, failedTests: [] };
+      } else if (!existsSync(outFile)) {
+        result = { run: { outcome: 'error', tests: { total: 0, passed: 0, failed: 0 }, assertionFailures: 0, durationMs }, timeouts: 0, loadMessage: stderr.trim().split('\n').filter(Boolean).slice(-1)[0] ?? 'runner produced no report', failedTests: [] };
+      } else {
+        try {
+          result = parseReport(JSON.parse(readFileSync(outFile, 'utf8')), durationMs);
+        } catch (e) {
+          result = { run: { outcome: 'error', tests: { total: 0, passed: 0, failed: 0 }, assertionFailures: 0, durationMs }, timeouts: 0, loadMessage: `unreadable report: ${e.message}`, failedTests: [] };
+        }
+      }
+      rmSync(outFile, { force: true });
+      resolve(result);
+    });
+  });
+}

package/src/probe/worktree.mjs

@@ -0,0 +1,64 @@
+import { existsSync, mkdtempSync, readdirSync, symlinkSync, mkdirSync, rmSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join, relative, dirname } from 'node:path';
+import { addWorktree, removeWorktree, headSha } from '../git.mjs';
+
+export class PreconditionError extends Error {}
+
+/**
+ * Find every node_modules directory in the main tree (to a shallow depth) so
+ * the scratch worktree can borrow them instead of reinstalling.
+ */
+function findNodeModules(root, depth = 3) {
+  const found = [];
+  const visit = (dir, rel, d) => {
+    let entries;
+    try {
+      entries = readdirSync(dir, { withFileTypes: true });
+    } catch {
+      return;
+    }
+    for (const e of entries) {
+      if (!e.isDirectory()) continue;
+      const relPath = rel ? `${rel}/${e.name}` : e.name;
+      if (e.name === 'node_modules') {
+        found.push(relPath);
+      } else if (d < depth && e.name !== '.git' && !e.name.startsWith('.')) {
+        visit(join(dir, e.name), relPath, d + 1);
+      }
+    }
+  };
+  visit(root, '', 0);
+  return found;
+}
+
+/**
+ * A scratch git worktree at HEAD, with the main tree's node_modules linked in.
+ * Faults are applied here; the user's tree is never touched.
+ */
+export function createScratch({ repoRoot, projectDir, ref = 'HEAD', scratchBase = tmpdir() }) {
+  const sha = headSha(repoRoot, ref);
+  if (!sha) {
+    throw new PreconditionError(ref === 'HEAD' ? 'repository has no commits; commit first, or run with --in-place' : `ref ${ref} does not resolve to a commit`);
+  }
+  const dest = mkdtempSync(join(scratchBase, 'testguard-'));
+  addWorktree(repoRoot, dest, sha);
+  for (const rel of findNodeModules(repoRoot)) {
+    const target = join(dest, rel);
+    if (existsSync(target)) continue;
+    mkdirSync(dirname(target), { recursive: true });
+    symlinkSync(join(repoRoot, rel), target, 'dir');
+  }
+  return {
+    mode: 'worktree',
+    sha,
+    root: dest,
+    projectDir: join(dest, relative(repoRoot, projectDir)),
+    cleanup: () => removeWorktree(repoRoot, dest),
+  };
+}
+
+/** No isolation: mutate the user's tree and rely on restore. */
+export function inPlace({ repoRoot, projectDir }) {
+  return { mode: 'in-place', root: repoRoot, projectDir, cleanup: () => {} };
+}

package/src/render.mjs

@@ -0,0 +1,28 @@
+const ORDER = ['survived', 'nocover', 'unverifiable', 'fault-invalid', 'timeout', 'flaky-defender', 'killed'];
+
+/** Non-passing verdicts shout; the one pass does not. */
+export const formatVerdict = (v) => (v === 'killed' ? 'killed' : v.toUpperCase());
+
+export function renderRecord(r) {
+  const head = `${formatVerdict(r.verdict).padEnd(15)} ${r.claim.id}/${r.subject.id}`.padEnd(38);
+  const why = r.detail.reason ? `  [${r.detail.reason}]` : '';
+  return `${head} ${r.claim.severity.padEnd(8)} ${r.subject.file}  ${r.subject.description}${why}`;
+}
+
+export function summarize(records) {
+  const byVerdict = {};
+  for (const r of records) byVerdict[r.verdict] = (byVerdict[r.verdict] ?? 0) + 1;
+  return byVerdict;
+}
+
+export function renderSummary(records) {
+  const byVerdict = summarize(records);
+  const parts = ORDER.filter((v) => byVerdict[v]).map((v) => `${byVerdict[v]} ${formatVerdict(v)}`);
+  const gating = records.filter((r) => r.verdict !== 'killed').length;
+  return `${records.length} faults probed: ${parts.join(', ')}. ${gating} unproven claim${gating === 1 ? '' : 's'}.`;
+}
+
+/** Survivors first, then by rank score; killed last. */
+export function sortForReport(records) {
+  return [...records].sort((a, b) => ORDER.indexOf(a.verdict) - ORDER.indexOf(b.verdict) || (b.rank?.score ?? 0) - (a.rank?.score ?? 0));
+}

package/src/util/glob.mjs

@@ -0,0 +1,56 @@
+import { readdirSync } from 'node:fs';
+import { join } from 'node:path';
+
+const SKIP_DIRS = new Set(['node_modules', '.git', 'dist', 'coverage', '.testguard']);
+
+/** Translate a minimal glob (`**`, `*`, `?`) into an anchored RegExp over posix paths. */
+export function globToRegExp(glob) {
+  let re = '^';
+  for (let i = 0; i < glob.length; i++) {
+    const c = glob[i];
+    if (c === '*') {
+      if (glob[i + 1] === '*') {
+        i++;
+        if (glob[i + 1] === '/') {
+          i++;
+          re += '(?:.*/)?';
+        } else {
+          re += '.*';
+        }
+      } else {
+        re += '[^/]*';
+      }
+    } else if (c === '?') {
+      re += '[^/]';
+    } else if ('.+^${}()|[]\\'.includes(c)) {
+      re += '\\' + c;
+    } else {
+      re += c;
+    }
+  }
+  return new RegExp(re + '$');
+}
+
+/** Every file under `root` as a posix path relative to it, skipping tool and dependency directories. */
+export function walk(root) {
+  const out = [];
+  const visit = (dir, rel) => {
+    for (const entry of readdirSync(dir, { withFileTypes: true })) {
+      const relPath = rel ? `${rel}/${entry.name}` : entry.name;
+      if (entry.isDirectory()) {
+        if (!SKIP_DIRS.has(entry.name)) visit(join(dir, entry.name), relPath);
+      } else if (entry.isFile()) {
+        out.push(relPath);
+      }
+    }
+  };
+  visit(root, '');
+  return out.sort();
+}
+
+/** Files under `root` matching any of `globs`. A glob with no wildcard is an exact relative path. */
+export function matchGlobs(root, globs) {
+  if (!globs || globs.length === 0) return [];
+  const regexps = globs.map(globToRegExp);
+  return walk(root).filter((f) => regexps.some((r) => r.test(f)));
+}

package/src/util/hash.mjs

@@ -0,0 +1,5 @@
+import { createHash } from 'node:crypto';
+import { readFileSync } from 'node:fs';
+
+export const sha256 = (data) => createHash('sha256').update(data).digest('hex');
+export const hashFile = (path) => sha256(readFileSync(path));