testguard-cli 0.1.0

package/src/claims/annotations.mjs

@@ -0,0 +1,59 @@
+import { readFileSync, readdirSync, existsSync } from 'node:fs';
+import { join, extname } from 'node:path';
+
+const SCAN_EXT = new Set(['.js', '.mjs', '.cjs', '.ts', '.mts', '.cts', '.jsx', '.tsx', '.py', '.go', '.rs', '.java', '.kt', '.md']);
+const SKIP_DIRS = new Set(['node_modules', 'dist', 'coverage']);
+const TEST_FILE = /\.(test|spec)\.[cm]?[jt]sx?$|(^|\/)(tests?|__tests__)\//;
+// An annotation id must contain a hyphen (REDACT-001, TG-KILL-NEEDS-N). Prose
+// such as "@claim annotations" is not an annotation.
+const RE = /@claim\s+([A-Za-z0-9]+(?:[._]?[A-Za-z0-9]+)*-[A-Za-z0-9._-]*[A-Za-z0-9])\b/g;
+
+/**
+ * Source files worth scanning: skips hidden directories (tooling), dependency
+ * and build output, test files (a claim asserted by a test is the authorship
+ * trap this tool exists for), and nested projects that carry their own
+ * claims file.
+ */
+function sourceFiles(root) {
+  const out = [];
+  const visit = (dir, rel) => {
+    for (const e of readdirSync(dir, { withFileTypes: true })) {
+      const relPath = rel ? `${rel}/${e.name}` : e.name;
+      if (e.isDirectory()) {
+        if (e.name.startsWith('.') || SKIP_DIRS.has(e.name)) continue;
+        if (existsSync(join(dir, e.name, 'testguard.claims.json'))) continue;
+        visit(join(dir, e.name), relPath);
+      } else if (e.isFile() && SCAN_EXT.has(extname(e.name)) && !TEST_FILE.test(relPath)) {
+        out.push(relPath);
+      }
+    }
+  };
+  visit(root, '');
+  return out.sort();
+}
+
+/** Every `@claim <ID>` annotation in source, with where it was found. */
+export function scanAnnotations(projectDir) {
+  const found = [];
+  for (const rel of sourceFiles(projectDir)) {
+    readFileSync(join(projectDir, rel), 'utf8').split('\n').forEach((text, i) => {
+      for (const m of text.matchAll(RE)) found.push({ id: m[1], file: rel, line: i + 1 });
+    });
+  }
+  return found;
+}
+
+/**
+ * Drift between what the code annotates and what the claims file declares.
+ * `undeclared`: annotated in code, absent from the file — a claim with no
+ * fault model. `stale`: declared as annotation-sourced, but no code carries it.
+ */
+export function reconcile(claims, annotations) {
+  const declared = new Map(claims.claims.map((c) => [c.id, c]));
+  const annotated = new Set(annotations.map((a) => a.id));
+  return {
+    undeclared: annotations.filter((a) => !declared.has(a.id)),
+    stale: claims.claims.filter((c) => c.source.kind === 'annotation' && !annotated.has(c.id)),
+    annotated: [...declared.keys()].filter((id) => annotated.has(id)),
+  };
+}

package/src/claims/load.mjs

@@ -0,0 +1,23 @@
+import { readFileSync } from 'node:fs';
+import { join } from 'node:path';
+import { validate } from '../../spec/lib/validate.mjs';
+
+export class ClaimsError extends Error {}
+
+export const defaultClaimsPath = (projectDir) => join(projectDir, 'testguard.claims.json');
+
+/** Read and validate a claims file. Any defect is fatal: a claims file is code. */
+export function loadClaims(path) {
+  let doc;
+  try {
+    doc = JSON.parse(readFileSync(path, 'utf8'));
+  } catch (e) {
+    throw new ClaimsError(`cannot read claims file ${path}: ${e.message}`);
+  }
+  const result = validate('claims', doc);
+  if (!result.ok) {
+    const lines = result.errors.map((e) => `  ${e.path}: ${e.message}`).join('\n');
+    throw new ClaimsError(`claims file ${path} does not conform to the spec:\n${lines}`);
+  }
+  return doc;
+}

package/src/cli.mjs

@@ -0,0 +1,105 @@
+import { parseArgs } from 'node:util';
+import { readFileSync } from 'node:fs';
+import { join, resolve } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { ClaimsError } from './claims/load.mjs';
+import { PreconditionError } from './probe/worktree.mjs';
+import { GitError } from './git.mjs';
+import { SpecDocError } from './evidence/writer.mjs';
+import { probeCommand } from './commands/probe.mjs';
+import { claimsCommand } from './commands/claims.mjs';
+import { baselineCommand } from './commands/baseline.mjs';
+import { briefCommand } from './commands/brief.mjs';
+
+const VERSION = JSON.parse(readFileSync(join(fileURLToPath(import.meta.url), '..', '..', 'package.json'), 'utf8')).version;
+
+const USAGE = `testguard ${VERSION} — proves a test suite defends the claims a project makes
+
+  testguard claims [dir]     list the claims file and report drift against @claim annotations in code
+  testguard probe [dir]      inject each claim's faults, run its defenders, report what survived
+  testguard baseline [dir]   freeze today's unproven findings so only new ones gate
+  testguard brief [dir]      emit the blind-spot block for an agent's session-start context
+
+probe
+  --claims <path>      claims file             (default: <dir>/testguard.claims.json)
+  --confirm <n>        runs per verdict        (default: 3)
+  --budget <ms>        wall clock per run      (default: 120000)
+  --out <path>         evidence file           (default: <dir>/.testguard/evidence.json)
+  --baseline <path>    baseline to gate against (default: <dir>/.testguard/baseline.json if present)
+  --severity <level>   gate only at or above   (default: low)
+  --ref <commit>       probe this commit in the scratch worktree (default: HEAD)
+  --in-place           mutate the working tree instead of a scratch worktree
+  --no-escalate        do not re-run survivors against the whole suite
+  --no-reuse           re-probe claims whose inputs have not changed
+  --quiet              summary only
+
+claims     --json
+baseline   --evidence <path>  --out <path>
+brief      --evidence <path>  --baseline <path>  --max <n>  --text (print only; safe for hooks)
+
+exit codes: 0 nothing new to prove · 1 unproven claims (or claim drift) · 2 precondition failed · 3 usage
+`;
+
+const COMMANDS = { probe: probeCommand, claims: claimsCommand, baseline: baselineCommand, brief: briefCommand };
+
+export async function main(argv, io = { out: (s) => process.stdout.write(s + '\n'), err: (s) => process.stderr.write(s + '\n') }) {
+  let parsed;
+  try {
+    parsed = parseArgs({
+      args: argv,
+      allowPositionals: true,
+      options: {
+        claims: { type: 'string' },
+        confirm: { type: 'string', default: '3' },
+        budget: { type: 'string', default: '120000' },
+        out: { type: 'string' },
+        evidence: { type: 'string' },
+        baseline: { type: 'string' },
+        severity: { type: 'string', default: 'low' },
+        max: { type: 'string', default: '20' },
+        ref: { type: 'string', default: 'HEAD' },
+        'in-place': { type: 'boolean', default: false },
+        'no-escalate': { type: 'boolean', default: false },
+        'no-reuse': { type: 'boolean', default: false },
+        quiet: { type: 'boolean', default: false },
+        json: { type: 'boolean', default: false },
+        text: { type: 'boolean', default: false },
+        help: { type: 'boolean', short: 'h', default: false },
+        version: { type: 'boolean', short: 'v', default: false },
+      },
+    });
+  } catch (e) {
+    io.err(e.message);
+    io.err(USAGE);
+    return 3;
+  }
+  const { values, positionals } = parsed;
+  if (values.version) {
+    io.out(VERSION);
+    return 0;
+  }
+  const [command, dirArg] = positionals;
+  if (values.help || !command) {
+    io.out(USAGE);
+    return command ? 0 : 3;
+  }
+  const handler = COMMANDS[command];
+  if (!handler) {
+    io.err(`unknown command: ${command}`);
+    io.err(USAGE);
+    return 3;
+  }
+  if (!['critical', 'high', 'medium', 'low'].includes(values.severity)) {
+    io.err('--severity must be one of critical, high, medium, low');
+    return 3;
+  }
+  try {
+    return await handler({ projectDir: resolve(dirArg ?? '.'), values, version: VERSION }, io);
+  } catch (e) {
+    if (e instanceof ClaimsError || e instanceof PreconditionError || e instanceof GitError || e instanceof SpecDocError) {
+      io.err(`error: ${e.message}`);
+      return 2;
+    }
+    throw e;
+  }
+}

package/src/commands/baseline.mjs

@@ -0,0 +1,21 @@
+import { existsSync } from 'node:fs';
+import { resolve } from 'node:path';
+import { readSpecDoc, writeSpecDoc } from '../evidence/writer.mjs';
+import { buildBaseline } from '../baseline/baseline.mjs';
+import { evidencePath, baselinePath } from './probe.mjs';
+
+export async function baselineCommand({ projectDir, values }, io) {
+  const evPath = values.evidence ? resolve(values.evidence) : evidencePath(projectDir);
+  if (!existsSync(evPath)) {
+    io.err(`no evidence at ${evPath}; run \`testguard probe\` first`);
+    return 2;
+  }
+  const evidence = readSpecDoc('evidence', evPath);
+  const baseline = buildBaseline(evidence);
+  const outPath = values.out ? resolve(values.out) : baselinePath(projectDir);
+  writeSpecDoc('baseline', outPath, baseline);
+  const n = Object.values(baseline.fingerprints).reduce((a, b) => a + b, 0);
+  io.out(`baseline: ${n} unproven finding${n === 1 ? '' : 's'} frozen at ${baseline.head.slice(0, 12)} → ${outPath}`);
+  io.out('Commit this file. From now on only new findings gate.');
+  return 0;
+}

package/src/commands/brief.mjs

@@ -0,0 +1,27 @@
+import { existsSync } from 'node:fs';
+import { join, resolve } from 'node:path';
+import { readSpecDoc, writeSpecDoc } from '../evidence/writer.mjs';
+import { buildBrief } from '../brief/brief.mjs';
+import { evidencePath, baselinePath } from './probe.mjs';
+
+export async function briefCommand({ projectDir, values }, io) {
+  const evPath = values.evidence ? resolve(values.evidence) : evidencePath(projectDir);
+  if (!existsSync(evPath)) {
+    // A missing brief must never break an agent's session start.
+    if (values.text) return 0;
+    io.err(`no evidence at ${evPath}; run \`testguard probe\` first`);
+    return 2;
+  }
+  const evidence = readSpecDoc('evidence', evPath);
+  const basePath = values.baseline ? resolve(values.baseline) : baselinePath(projectDir);
+  const baseline = existsSync(basePath) ? readSpecDoc('baseline', basePath) : undefined;
+  const max = Number(values.max);
+  if (!Number.isInteger(max) || max < 1 || max > 50) {
+    io.err('--max must be an integer from 1 to 50');
+    return 3;
+  }
+  const brief = buildBrief(evidence, baseline, { max });
+  if (!values.text) writeSpecDoc('brief', values.out ? resolve(values.out) : join(projectDir, '.testguard', 'brief.json'), brief);
+  io.out(brief.text.trimEnd());
+  return 0;
+}

package/src/commands/claims.mjs

@@ -0,0 +1,27 @@
+import { resolve } from 'node:path';
+import { loadClaims, defaultClaimsPath } from '../claims/load.mjs';
+import { scanAnnotations, reconcile } from '../claims/annotations.mjs';
+import { resolveDefenders } from '../probe/runner-vitest.mjs';
+
+export async function claimsCommand({ projectDir, values }, io) {
+  const path = values.claims ? resolve(values.claims) : defaultClaimsPath(projectDir);
+  const claims = loadClaims(path);
+  const annotations = scanAnnotations(projectDir);
+  const drift = reconcile(claims, annotations);
+
+  if (values.json) {
+    io.out(JSON.stringify({ path, claims, annotations, drift }, null, 2));
+  } else {
+    io.out(`${claims.claims.length} claims in ${path}`);
+    io.out('');
+    for (const c of claims.claims) {
+      const defenders = resolveDefenders(projectDir, c.defendedBy);
+      const cover = defenders.length ? `${defenders.length} defender${defenders.length === 1 ? '' : 's'}` : 'NO DEFENDER';
+      io.out(`${c.id.padEnd(14)} ${c.severity.padEnd(8)} ${c.source.kind.padEnd(10)} ${String(c.faults.length).padStart(2)} fault${c.faults.length === 1 ? ' ' : 's'}  ${cover.padEnd(12)}  ${c.statement}`);
+    }
+    if (drift.undeclared.length || drift.stale.length) io.out('');
+    for (const a of drift.undeclared) io.out(`UNDECLARED   @claim ${a.id} at ${a.file}:${a.line} has no entry in the claims file — a claim with no fault model`);
+    for (const c of drift.stale) io.out(`STALE        ${c.id} is annotation-sourced but no source file carries @claim ${c.id}`);
+  }
+  return drift.undeclared.length || drift.stale.length ? 1 : 0;
+}

package/src/commands/probe.mjs

@@ -0,0 +1,53 @@
+import { existsSync } from 'node:fs';
+import { join, resolve } from 'node:path';
+import { loadClaims, defaultClaimsPath } from '../claims/load.mjs';
+import { probe } from '../probe/probe.mjs';
+import { writeSpecDoc, readSpecDoc } from '../evidence/writer.mjs';
+import { gate } from '../baseline/baseline.mjs';
+import { renderRecord, renderSummary, sortForReport } from '../render.mjs';
+
+export const evidencePath = (projectDir) => join(projectDir, '.testguard', 'evidence.json');
+export const baselinePath = (projectDir) => join(projectDir, '.testguard', 'baseline.json');
+
+export async function probeCommand({ projectDir, values, version }, io) {
+  const confirmRuns = Number(values.confirm);
+  const budgetMs = Number(values.budget);
+  if (!Number.isInteger(confirmRuns) || confirmRuns < 1 || !Number.isInteger(budgetMs) || budgetMs < 1000) {
+    io.err('--confirm must be a positive integer and --budget at least 1000');
+    return 3;
+  }
+  const claims = loadClaims(values.claims ? resolve(values.claims) : defaultClaimsPath(projectDir));
+  if (claims.claims.length === 0) {
+    io.err('claims file declares no claims; nothing to verify');
+    return 2;
+  }
+  const outPath = values.out ? resolve(values.out) : evidencePath(projectDir);
+  const previous = !values['no-reuse'] && existsSync(outPath) ? readSpecDoc('evidence', outPath) : undefined;
+  const basePath = values.baseline ? resolve(values.baseline) : baselinePath(projectDir);
+  const baseline = existsSync(basePath) ? readSpecDoc('baseline', basePath) : undefined;
+
+  const evidence = await probe({
+    projectDir,
+    claims,
+    previous,
+    confirmRuns,
+    budgetMs,
+    mode: values['in-place'] ? 'in-place' : 'worktree',
+    ref: values.ref,
+    escalate: !values['no-escalate'],
+    toolVersion: version,
+    onProgress: values.quiet ? undefined : (r) => io.out(renderRecord(r) + (r.reusedFrom ? '  (reused)' : '')),
+  });
+  writeSpecDoc('evidence', outPath, evidence);
+
+  const g = gate(evidence.records, baseline, { severityFloor: values.severity });
+  if (!values.quiet) {
+    io.out('');
+    const tag = (r) => (g.new.includes(r) ? '[NEW]      ' : g.baselined.includes(r) ? '[baseline] ' : '[below floor] ');
+    for (const r of sortForReport(evidence.records).filter((x) => x.verdict !== 'killed')) io.out('  ' + tag(r) + renderRecord(r));
+  }
+  io.out('');
+  io.out(renderSummary(evidence.records) + (baseline ? ` ${g.new.length} new since baseline, ${g.baselined.length} baselined.` : ' No baseline.'));
+  io.out(`evidence: ${outPath}`);
+  return g.new.length > 0 ? 1 : 0;
+}

package/src/evidence/writer.mjs

@@ -0,0 +1,36 @@
+import { mkdirSync, writeFileSync, readFileSync } from 'node:fs';
+import { dirname } from 'node:path';
+import { validate } from '../../spec/lib/validate.mjs';
+
+export class SpecDocError extends Error {}
+
+function describe(kind, path, result) {
+  const lines = result.errors.map((e) => `  ${e.path}: ${e.message}`).join('\n');
+  return `${kind} document ${path} does not conform to the spec:\n${lines}`;
+}
+
+/**
+ * Write a spec document — but only if it conforms. The tool is held to its
+ * own spec at the moment of output, not by a separate test someone might skip.
+ */
+export function writeSpecDoc(kind, path, doc) {
+  const result = validate(kind, doc);
+  if (!result.ok) throw new SpecDocError('refusing to write: ' + describe(kind, path, result));
+  mkdirSync(dirname(path), { recursive: true });
+  writeFileSync(path, JSON.stringify(doc, null, 2) + '\n');
+}
+
+/** Read a spec document, refusing one that does not conform. */
+export function readSpecDoc(kind, path) {
+  let doc;
+  try {
+    doc = JSON.parse(readFileSync(path, 'utf8'));
+  } catch (e) {
+    throw new SpecDocError(`cannot read ${kind} document ${path}: ${e.message}`);
+  }
+  const result = validate(kind, doc);
+  if (!result.ok) throw new SpecDocError(describe(kind, path, result));
+  return doc;
+}
+
+export const writeEvidence = (path, doc) => writeSpecDoc('evidence', path, doc);

package/src/git.mjs

@@ -0,0 +1,32 @@
+import { spawnSync } from 'node:child_process';
+import { rmSync } from 'node:fs';
+
+export class GitError extends Error {}
+
+export function git(args, cwd) {
+  const r = spawnSync('git', args, { cwd, encoding: 'utf8' });
+  if (r.status !== 0) throw new GitError(`git ${args.join(' ')}: ${(r.stderr || r.stdout).trim()}`);
+  return r.stdout.trim();
+}
+
+export const repoRoot = (dir) => git(['rev-parse', '--show-toplevel'], dir);
+
+/** Full sha of `ref` (default HEAD), or null when it does not resolve. */
+export function headSha(dir, ref = 'HEAD') {
+  try {
+    return git(['rev-parse', '--verify', `${ref}^{commit}`], dir);
+  } catch {
+    return null;
+  }
+}
+
+/** True when any of `paths` (repo-relative; empty = whole tree) has uncommitted changes. */
+export const isDirty = (dir, paths = []) => git(['status', '--porcelain', '--', ...paths], dir).length > 0;
+
+export const addWorktree = (repo, dest, ref = 'HEAD') => git(['worktree', 'add', '--detach', dest, ref], repo);
+
+export function removeWorktree(repo, dest) {
+  spawnSync('git', ['worktree', 'remove', '--force', dest], { cwd: repo });
+  rmSync(dest, { recursive: true, force: true });
+  spawnSync('git', ['worktree', 'prune'], { cwd: repo });
+}

package/src/probe/classify.mjs

@@ -0,0 +1,37 @@
+/**
+ * The verdict function. Pure: everything it needs is passed in, so every
+ * branch is unit-testable without a runner.
+ *
+ * Order matters. Each check short-circuits the ones below it, and the order
+ * encodes GATE-SEMANTICS.md: nothing to run → nocover; nothing to apply →
+ * unverifiable; defenders not green → flaky-defender; then, and only then,
+ * does the fault's own result count.
+ */
+export function classify({ defenders, anchor, baselineRuns, probeRuns, confirmRuns }) {
+  if (defenders.length === 0) return { verdict: 'nocover' };
+  if (anchor && anchor.status !== 'ok') return { verdict: 'unverifiable', reason: anchor.status };
+  if (baselineRuns.length === 0 || baselineRuns.some((r) => r.outcome !== 'pass')) {
+    return { verdict: 'flaky-defender', reason: 'defenders-not-green' };
+  }
+  const loadError = probeRuns.find((r) => r.outcome === 'error');
+  if (loadError) {
+    const parseError = /syntax|parse/i.test(loadError.loadMessage ?? '');
+    return { verdict: 'fault-invalid', reason: parseError ? 'replacement-does-not-compile' : 'suite-failed-to-load' };
+  }
+  if (probeRuns.some((r) => r.outcome === 'timeout' || r.timeouts > 0)) return { verdict: 'timeout', reason: 'test-timed-out' };
+
+  const killedRuns = probeRuns.filter((r) => r.outcome === 'fail' && r.assertionFailures > 0);
+  const passedRuns = probeRuns.filter((r) => r.outcome === 'pass');
+  if (probeRuns.length === confirmRuns && killedRuns.length === confirmRuns) return { verdict: 'killed' };
+  if (probeRuns.length === confirmRuns && passedRuns.length === confirmRuns) return { verdict: 'survived' };
+  // Some runs killed, some passed: the defenders' response to this fault is
+  // nondeterministic. Reporting it as killed would be the optimistic bias
+  // the research warned about; reporting survived would be a lie the other way.
+  return { verdict: 'flaky-defender', reason: 'inconsistent-probe' };
+}
+
+/** Decide whether another probe run is worth doing, or the verdict is already forced. */
+export function shouldStopEarly(probeRuns) {
+  const last = probeRuns[probeRuns.length - 1];
+  return last.outcome === 'error' || last.outcome === 'timeout' || last.timeouts > 0;
+}

package/src/probe/inject.mjs

@@ -0,0 +1,86 @@
+import { readFileSync, writeFileSync } from 'node:fs';
+import { join } from 'node:path';
+
+// Every live mutation registers its restore here so a signal or crash can
+// undo all of them before the process dies. Worktree mode makes this
+// belt-and-braces; in-place mode depends on it.
+const live = new Set();
+let handlersInstalled = false;
+
+function restoreAll() {
+  for (const restore of live) {
+    try {
+      restore();
+    } catch {}
+  }
+  live.clear();
+}
+
+function installHandlers() {
+  if (handlersInstalled) return;
+  handlersInstalled = true;
+  for (const sig of ['SIGINT', 'SIGTERM', 'SIGHUP']) {
+    process.on(sig, () => {
+      restoreAll();
+      process.exit(130);
+    });
+  }
+  process.on('uncaughtException', (err) => {
+    restoreAll();
+    throw err;
+  });
+  process.on('exit', restoreAll);
+}
+
+function countOccurrences(haystack, needle) {
+  let n = 0;
+  let i = 0;
+  while ((i = haystack.indexOf(needle, i)) !== -1) {
+    n++;
+    i += needle.length;
+  }
+  return n;
+}
+
+/** Does the anchor hit exactly as declared? */
+export function locate(source, fault) {
+  const hits = countOccurrences(source, fault.find);
+  const expected = fault.expectHits ?? 1;
+  if (hits === 0) return { status: 'anchor-missing', hits, expected };
+  if (hits !== expected) return { status: 'anchor-ambiguous', hits, expected };
+  return { status: 'ok', hits, expected };
+}
+
+/** Replace the declared occurrence of `find`. Pure; assumes `locate` returned ok. */
+export function mutate(source, fault) {
+  const nth = fault.occurrence ?? 1;
+  let idx = -1;
+  for (let k = 0; k < nth; k++) {
+    idx = source.indexOf(fault.find, idx + 1);
+    if (idx === -1) throw new RangeError(`occurrence ${nth} of anchor not found`);
+  }
+  return source.slice(0, idx) + fault.replace + source.slice(idx + fault.find.length);
+}
+
+/**
+ * Apply a fault to a file on disk. Returns a handle whose `restore()` puts the
+ * original back; restore is idempotent and also runs on process exit/signal.
+ */
+export function applyFault(projectDir, fault) {
+  installHandlers();
+  const path = join(projectDir, fault.file);
+  const original = readFileSync(path, 'utf8');
+  const anchor = locate(original, fault);
+  if (anchor.status !== 'ok') return { anchor, applied: false, restore: () => {} };
+
+  let restored = false;
+  const restore = () => {
+    if (restored) return;
+    restored = true;
+    live.delete(restore);
+    writeFileSync(path, original);
+  };
+  live.add(restore);
+  writeFileSync(path, mutate(original, fault));
+  return { anchor, applied: true, restore };
+}