testguard-cli 0.5.0 → 0.6.0

package/src/mcp/server.mjs

@@ -0,0 +1,104 @@
+import { TOOLS } from './tools.mjs';
+
+/**
+ * A Model Context Protocol server over stdio, hand-rolled.
+ *
+ * Why not the official SDK: this tool has one exact-pinned runtime dependency
+ * and intends to keep it that way (AGENTS.md rule 8). The surface here is
+ * three methods and five read-only tools and will not grow, so the protocol
+ * is written out rather than imported. The version is pinned below and the
+ * conformance is covered by tests that speak the wire format.
+ *
+ * JSON-RPC 2.0 framing: one message per line (newline-delimited JSON), which
+ * is what stdio transports use. A notification (no `id`) is never answered.
+ */
+export const PROTOCOL_VERSION = '2025-06-18';
+
+const ERROR = { parse: -32700, invalidRequest: -32600, methodNotFound: -32601, invalidParams: -32602, internal: -32603 };
+
+const ok = (id, result) => ({ jsonrpc: '2.0', id, result });
+const err = (id, code, message, data) => ({ jsonrpc: '2.0', id, error: { code, message, ...(data ? { data } : {}) } });
+
+/** Everything an agent may ask for, with the version stamped onto each call. */
+export function handle(message, { version = '0.0.0' } = {}) {
+  if (message === null || typeof message !== 'object' || Array.isArray(message)) return err(null, ERROR.invalidRequest, 'a request must be a JSON object');
+  const { id, method, params } = message;
+  const isNotification = id === undefined || id === null;
+
+  switch (method) {
+    case 'initialize':
+      return isNotification ? null : ok(id, {
+        protocolVersion: PROTOCOL_VERSION,
+        // Read-only: no prompts, no resources, no sampling. Saying so is part
+        // of the contract — nothing here can change a claims file.
+        capabilities: { tools: { listChanged: false } },
+        serverInfo: { name: 'testguard', version },
+        instructions: 'Read testguard_status first: it carries the one next action. Run the command it names in your own terminal — these tools never run a probe, because a probe is long-running, budgeted, and the person should see it happen.',
+      });
+
+    case 'notifications/initialized':
+    case 'notifications/cancelled':
+      return null;
+
+    case 'ping':
+      return isNotification ? null : ok(id, {});
+
+    case 'tools/list':
+      return isNotification ? null : ok(id, { tools: TOOLS.map(({ name, description, inputSchema }) => ({ name, description, inputSchema, annotations: { readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: false } })) });
+
+    case 'tools/call': {
+      if (isNotification) return null;
+      const tool = TOOLS.find((t) => t.name === params?.name);
+      if (!tool) return err(id, ERROR.invalidParams, `unknown tool: ${params?.name}`);
+      try {
+        const result = tool.handler({ ...(params?.arguments ?? {}), __version: version });
+        return ok(id, { content: [{ type: 'text', text: JSON.stringify(result, null, 2) }], structuredContent: result, isError: false });
+      } catch (e) {
+        // A tool failure is a tool result, not a protocol error: the agent
+        // should read the reason and act, not lose the connection.
+        return ok(id, { content: [{ type: 'text', text: `${tool.name} failed: ${e.message}` }], isError: true });
+      }
+    }
+
+    default:
+      return isNotification ? null : err(id, ERROR.methodNotFound, `unknown method: ${method}`);
+  }
+}
+
+/**
+ * Speak the protocol on a pair of streams. Resolves when input ends, so the
+ * caller can await a clean shutdown.
+ */
+export function serve({ input = process.stdin, output = process.stdout, version = '0.0.0' } = {}) {
+  return new Promise((resolveDone) => {
+    let buffer = '';
+    const write = (msg) => {
+      if (msg) output.write(JSON.stringify(msg) + '\n');
+    };
+    input.setEncoding?.('utf8');
+    input.on('data', (chunk) => {
+      buffer += chunk;
+      let nl;
+      while ((nl = buffer.indexOf('\n')) !== -1) {
+        const line = buffer.slice(0, nl).trim();
+        buffer = buffer.slice(nl + 1);
+        if (!line) continue;
+        let message;
+        try {
+          message = JSON.parse(line);
+        } catch {
+          write(err(null, ERROR.parse, 'invalid JSON'));
+          continue;
+        }
+        try {
+          write(handle(message, { version }));
+        } catch (e) {
+          // The server outlives any single bad request.
+          write(err(message?.id ?? null, ERROR.internal, e.message));
+        }
+      }
+    });
+    input.on('end', () => resolveDone(0));
+    input.on('close', () => resolveDone(0));
+  });
+}

package/src/mcp/tools.mjs

@@ -0,0 +1,114 @@
+import { existsSync } from 'node:fs';
+import { join, resolve } from 'node:path';
+import { computeStatus } from '../status/status.mjs';
+import { buildBrief, buildUnclaimedBrief } from '../brief/brief.mjs';
+import { loadClaims, defaultClaimsPath } from '../claims/load.mjs';
+import { scanAnnotations, reconcile } from '../claims/annotations.mjs';
+import { computeRemovedClaims } from '../claims/removed.mjs';
+import { readSpecDoc } from '../evidence/writer.mjs';
+import { resolveChangedRef, withChangedRef } from '../gate/changed.mjs';
+
+/**
+ * The read-only surface an agent needs, in any harness.
+ *
+ * TestGuard's operating loop currently lives in a Claude Code skill and a
+ * session-start hook, and both vanish the moment the harness is something
+ * else. These tools expose the documents the tool already computes, so the
+ * loop survives that change.
+ *
+ * Every tool is read-only and none of them probes. A probe is long-running
+ * and budgeted, and the human should see it happen: `next_command` hands back
+ * the exact shell line for the agent to run in its own terminal instead.
+ */
+const projectOf = (args) => resolve(args?.dir ?? '.');
+const evidenceOf = (args) => (args?.evidence ? resolve(args.evidence) : undefined);
+
+const DIR = { type: 'string', description: 'Project directory (where testguard.claims.json lives). Defaults to the current directory.' };
+const EVIDENCE = { type: 'string', description: "Read this evidence document instead of the project's own — CI's, fetched as an artifact." };
+const CHANGED = { type: 'string', description: 'A git reference to measure the change against, so claim coverage of the change is included.' };
+
+const statusFor = (args) => {
+  const projectDir = projectOf(args);
+  return withChangedRef(resolveChangedRef({ explicit: args?.changed }), (changedRef) =>
+    computeStatus({ projectDir, toolVersion: args?.__version ?? '0.0.0', changedRef, evidence: evidenceOf(args) }));
+};
+
+export const TOOLS = [
+  {
+    name: 'testguard_status',
+    description: 'Where this project stands and the ONE next action, as the machine-readable status document: state, next {action, command, why}, counts, stale inputs, faults edited since they were probed, and the ranked findings. This is the single source of truth every other rendering derives from — read it before deciding anything.',
+    inputSchema: { type: 'object', properties: { dir: DIR, changed: CHANGED, evidence: EVIDENCE }, additionalProperties: false },
+    handler: (args) => statusFor(args),
+  },
+  {
+    name: 'testguard_brief',
+    description: 'Where the test suite is blind, ranked and capped, with one actionable hint per finding and unclaimed changed files first. Read this before writing or changing code, which is what the session-start hook does for harnesses that have one.',
+    inputSchema: { type: 'object', properties: { dir: DIR, changed: CHANGED, evidence: EVIDENCE, max: { type: 'integer', minimum: 1, maximum: 50 } }, additionalProperties: false },
+    handler: (args) => {
+      const projectDir = projectOf(args);
+      const evPath = evidenceOf(args) ?? join(projectDir, '.testguard', 'evidence.json');
+      const status = (() => {
+        try {
+          return statusFor(args);
+        } catch {
+          return undefined; // a brief must never fail because status could not be computed
+        }
+      })();
+      if (!existsSync(evPath)) {
+        if (status?.changes?.uncovered?.length) {
+          return buildUnclaimedBrief({ tool: { name: 'testguard', version: args?.__version ?? '0.0.0' }, next: status.next, changes: status.changes });
+        }
+        return { note: `no evidence at ${evPath}: nothing has been probed yet`, next: status?.next };
+      }
+      const evidence = readSpecDoc('evidence', evPath);
+      const basePath = join(projectDir, '.testguard', 'baseline.json');
+      const baseline = existsSync(basePath) ? readSpecDoc('baseline', basePath) : undefined;
+      return buildBrief(evidence, baseline, { max: args?.max ?? 20, next: status?.next, changes: status?.changes });
+    },
+  },
+  {
+    name: 'testguard_claims',
+    description: "What this project claims must be true, each with its faults and defenders, plus drift against `@claim` annotations in source and — when `since` is given — every claim or fault that existed at that reference and does not now. A claim that disappeared is invisible to every other check.",
+    inputSchema: { type: 'object', properties: { dir: DIR, since: { type: 'string', description: 'A git reference: report claims and faults removed since it.' } }, additionalProperties: false },
+    handler: (args) => {
+      const projectDir = projectOf(args);
+      const path = defaultClaimsPath(projectDir);
+      if (!existsSync(path)) return { path, claims: [], note: 'no claims file in this project yet' };
+      const claims = loadClaims(path);
+      const drift = reconcile(claims, scanAnnotations(projectDir));
+      const removed = args?.since
+        ? computeRemovedClaims({ projectDir, ref: args.since, claimsPath: path, current: claims, evidencePath: join(projectDir, '.testguard', 'evidence.json'), toolVersion: args?.__version ?? '0.0.0' })
+        : undefined;
+      return { path, claims, drift, ...(removed ? { removed } : {}) };
+    },
+  },
+  {
+    name: 'testguard_evidence',
+    description: 'One run\'s findings on disk: every verdict with the runs that produced it. Pass `claim` (and optionally `fault`) for a single record instead of the whole document, which can be large.',
+    inputSchema: {
+      type: 'object',
+      properties: { dir: DIR, evidence: EVIDENCE, claim: { type: 'string', description: 'Return only this claim\'s records.' }, fault: { type: 'string', description: 'With `claim`: only this fault\'s record.' } },
+      additionalProperties: false,
+    },
+    handler: (args) => {
+      const projectDir = projectOf(args);
+      const evPath = evidenceOf(args) ?? join(projectDir, '.testguard', 'evidence.json');
+      if (!existsSync(evPath)) return { note: `no evidence at ${evPath}: run a probe first`, path: evPath };
+      const doc = readSpecDoc('evidence', evPath);
+      if (!args?.claim) return doc;
+      const records = doc.records.filter((r) => r.claim.id === args.claim && (!args.fault || r.subject.id === args.fault));
+      return { ...doc, records, filtered: { claim: args.claim, ...(args.fault ? { fault: args.fault } : {}) } };
+    },
+  },
+  {
+    name: 'testguard_next_command',
+    description: 'The exact shell command to run next, with the reason. Run it yourself in a terminal: a probe is long-running and budgeted, and the person should see it happen. This tool never runs anything.',
+    inputSchema: { type: 'object', properties: { dir: DIR, changed: CHANGED, evidence: EVIDENCE }, additionalProperties: false },
+    handler: (args) => {
+      const doc = statusFor(args);
+      return { state: doc.state, action: doc.next.action, command: doc.next.command, why: doc.next.why, ...(doc.next.target ? { target: doc.next.target } : {}), ...(doc.next.file ? { file: doc.next.file } : {}) };
+    },
+  },
+];
+
+export const TOOL_NAMES = TOOLS.map((t) => t.name);

package/src/probe/contention.mjs

@@ -0,0 +1,60 @@
+import { spawnSync } from 'node:child_process';
+
+/**
+ * Concurrent test runners (issue #26).
+ *
+ * A probe runs the defenders 3× clean and 3× with each fault applied. Heavy
+ * suites time out under parallel CPU load, and a timeout is a verdict —
+ * `TIMEOUT` or `FLAKY-DEFENDER` — that says nothing about the claim. A field
+ * report serialised everything by hand for exactly this reason. We cannot
+ * prevent the contention, but we can name it: warn before the first run, and
+ * record it on the evidence so a later reader of a flaky verdict knows the
+ * run was contended.
+ *
+ * Best effort by design: the detector never fails a probe, and a process list
+ * it cannot read is simply no detection.
+ */
+
+const RUNNER_RE = /\b(vitest|jest|playwright|mocha|ava|karma|cypress)\b/i;
+// Our own child processes and this process are not contention.
+const SELF_RE = /testguard/i;
+
+/** Rows of the process table as `{ pid, command }`, or [] when it cannot be read. */
+export function processList({ platform = process.platform, run = spawnSync } = {}) {
+  try {
+    const r = platform === 'win32'
+      ? run('wmic', ['process', 'get', 'ProcessId,CommandLine', '/format:csv'], { encoding: 'utf8', timeout: 5000 })
+      : run('ps', ['-Ao', 'pid=,args='], { encoding: 'utf8', timeout: 5000 });
+    if (r.status !== 0 || !r.stdout) return [];
+    return r.stdout.split('\n').map((line) => {
+      const m = platform === 'win32' ? /^[^,]*,(.*),(\d+)\s*$/.exec(line) : /^\s*(\d+)\s+(.*)$/.exec(line);
+      if (!m) return null;
+      const [pid, command] = platform === 'win32' ? [m[2], m[1]] : [m[1], m[2]];
+      return { pid: Number(pid), command: command.trim() };
+    }).filter(Boolean);
+  } catch {
+    return [];
+  }
+}
+
+/**
+ * Test runners already running, other than ours. Returns
+ * `{ detected, runners: [{ pid, command }] }`; `detected` is false when
+ * nothing was found or the process list could not be read.
+ */
+export function detectContention({ self = process.pid, ...opts } = {}) {
+  const rows = processList(opts);
+  const runners = rows
+    .filter((p) => p.pid !== self && RUNNER_RE.test(p.command) && !SELF_RE.test(p.command))
+    // `ps` lists the grep/ps itself and any shell wrapper; a runner's command
+    // line always names its own binary, so require it to look like an exec.
+    .filter((p) => !/^\s*(ps|grep|wmic)\b/.test(p.command))
+    .map((p) => ({ pid: p.pid, command: p.command.length > 200 ? p.command.slice(0, 197) + '…' : p.command }));
+  return { detected: runners.length > 0, runners };
+}
+
+/** One line for the operator: what is running, and what it costs. */
+export const contentionWarning = (c) =>
+  `${c.runners.length} test runner${c.runners.length === 1 ? ' is' : 's are'} already running (${c.runners.map((r) => `pid ${r.pid}`).join(', ')}); ` +
+  'defenders run N times each here, and a suite that times out under parallel load yields TIMEOUT or FLAKY-DEFENDER — verdicts about the load, not the claim. ' +
+  'Wait for it to finish, or pass --serial to run one test file at a time.';

package/src/probe/discover.mjs

@@ -1,12 +1,21 @@
 import { join } from 'node:path';
 import { listTestFiles } from './runners/shared.mjs';
 import { fileImports } from './rank.mjs';
+import { classifyDefenders } from './mocks.mjs';
 
 /**
  * When a claim declares no defenders: the test files that import the fault's
- * target file, directly, by relative path or through a resolved alias. This
- * is what `nocover` measures against — no test file even imports the source.
+ * target file — directly, by relative path or through a resolved alias — and
+ * do NOT mock it. A file that mocks the target cannot detect any fault in it;
+ * counting it would make `nocover` under-report and waste probe runs on files
+ * that cannot fail. `nocover` therefore means exactly "no test file imports
+ * this source without mocking it".
  */
+export function discoverDefendersDetailed(projectDir, targetRel) {
+  const importing = listTestFiles(projectDir).filter((t) => fileImports(projectDir, join(projectDir, t), targetRel));
+  return { importing, ...classifyDefenders(projectDir, targetRel, importing) };
+}
+
 export function discoverDefenders(projectDir, targetRel) {
-  return listTestFiles(projectDir).filter((t) => fileImports(projectDir, join(projectDir, t), targetRel));
+  return discoverDefendersDetailed(projectDir, targetRel).canDetect;
 }

package/src/probe/independence.mjs

@@ -0,0 +1,59 @@
+import { git, GitError } from '../git.mjs';
+
+/**
+ * Maturity-ladder L3: independence.
+ *
+ * `probe` measures POWER — would this test notice if the code were wrong.
+ * It says nothing about who wrote the test. An agent that writes the code and
+ * its defender in one change encodes whatever it believed, and a kill by such
+ * a test is weaker evidence than a kill by a test written separately
+ * (SpecBench: agents saturate the tests they can see).
+ *
+ * This is a SIGNAL, never a verdict. A co-authored kill is legitimate — a bug
+ * fix *should* ship with its regression test — but a repository where every
+ * kill is co-authored has no independent verification, whatever its claim
+ * verification rate says. Ranking may read it; `classify()` never does.
+ */
+const CLASSES = ['co-authored', 'separate-change', 'unknown'];
+
+/** `git log -1` for one path at a commit → {commit, author} or null when there is no history for it. */
+function lastCommit(dir, ref, path) {
+  try {
+    const out = git(['log', '-1', '--format=%H%x09%ae', ref, '--', path], dir);
+    if (!out) return null;
+    const [commit, author] = out.split('\t');
+    return commit && author ? { commit, author } : null;
+  } catch (e) {
+    if (e instanceof GitError) return null;
+    throw e;
+  }
+}
+
+/**
+ * Was the killing defender last touched by the same change as the fault's
+ * target? `defenders` are the files whose tests failed on the fault.
+ *
+ * `co-authored`     — same commit, or the same author identity within the
+ *                     commits the caller considers one change (`range`).
+ * `separate-change` — different commits by different identities.
+ * `unknown`         — no history for one of them: a working-tree snapshot of
+ *                     untracked files, a shallow clone, or no commits.
+ */
+export function classifyIndependence({ dir, ref = 'HEAD', targetFile, defenders = [] }) {
+  const target = lastCommit(dir, ref, targetFile);
+  if (!target || defenders.length === 0) return { class: 'unknown' };
+
+  let sameAuthor = false;
+  let worst = null; // the most independent defender decides: one separate author is independence
+  for (const d of defenders) {
+    const def = lastCommit(dir, ref, d);
+    if (!def) return { class: 'unknown', targetCommit: target.commit };
+    const co = def.commit === target.commit || def.author === target.author;
+    if (def.author === target.author) sameAuthor = true;
+    if (!co) return { class: 'separate-change', defenderCommit: def.commit, targetCommit: target.commit, sameAuthor: false };
+    worst ??= def;
+  }
+  return { class: 'co-authored', defenderCommit: worst.commit, targetCommit: target.commit, sameAuthor };
+}
+
+export { CLASSES as INDEPENDENCE_CLASSES };

package/src/probe/mocks.mjs

@@ -0,0 +1,120 @@
+import { readFileSync } from 'node:fs';
+import { join } from 'node:path';
+import { specifierResolvesTo } from './rank.mjs';
+
+/**
+ * Mock-awareness for defenders (issues #20, #21).
+ *
+ * A test file that `vi.mock`s / `jest.mock`s the target cannot detect any
+ * fault in it, yet it imports the target, so the import rule would count it
+ * as a defender. Two consequences the field reports measured: `nocover`
+ * under-reports (a module with eighteen "defenders" that is effectively
+ * uncovered) and every probe wastes runs on files that cannot fail. The
+ * mocked count is itself the strongest blindness signal.
+ *
+ * And the cheapest static signal of all: a file that mocks the target and
+ * never `expect(...)`s anything from it. That single fact explained the
+ * seam behind an escaped bug without injecting anything.
+ */
+
+const MOCK_CALL = /\b(?:vi|jest)\.(?:mock|doMock|unstable_mockModule)\(\s*(['"])([^'"]+)\1/g;
+const IMPORT_STMT = /import\s+([^'";]+?)\s+from\s*(['"])([^'"]+)\2|import\s*(['"])([^'"]+)\4|(?:const|let|var)\s+([^=;]+?)\s*=\s*(?:await\s+)?(?:require|import)\(\s*(['"])([^'"]+)\7\s*\)/g;
+const UNASSERTED = /unasserted:\s*(.+?)\s*(?:\*\/)?\s*$/;
+
+/** The specifiers this source mocks, with their line numbers. */
+export function mockSpecifiers(source) {
+  const out = [];
+  const lines = source.split('\n');
+  lines.forEach((text, i) => {
+    for (const m of text.matchAll(MOCK_CALL)) out.push({ specifier: m[2], line: i + 1 });
+  });
+  return out;
+}
+
+/** Local names bound to a module specifier by import/require statements. `*` marks a namespace/default object binding. */
+function bindingsFor(source, matchesTarget) {
+  const names = [];
+  for (const m of source.matchAll(IMPORT_STMT)) {
+    const spec = m[3] ?? m[5] ?? m[8];
+    if (!spec || !matchesTarget(spec)) continue;
+    const clause = (m[1] ?? m[6] ?? '').trim();
+    if (!clause) continue; // side-effect import
+    const ns = /\*\s+as\s+([\w$]+)/.exec(clause);
+    if (ns) names.push(ns[1]);
+    const braces = /\{([^}]*)\}/.exec(clause);
+    if (braces) for (const part of braces[1].split(',')) { const local = part.trim().split(/\s+as\s+|\s*:\s*/).pop()?.trim(); if (local && /^[\w$]+$/.test(local)) names.push(local); }
+    const def = clause.replace(/\{[^}]*\}/, '').replace(/\*\s+as\s+[\w$]+/, '').replace(/,/g, '').trim();
+    if (def && /^[\w$]+$/.test(def)) names.push(def);
+  }
+  return [...new Set(names)];
+}
+
+/** Does any `expect(...)` argument mention one of the bindings (as an identifier or a member root)? */
+function expectMentions(source, bindings) {
+  if (bindings.length === 0) return false;
+  const re = /\bexpect\s*\(((?:[^()]|\((?:[^()]|\([^()]*\))*\))*)\)/g;
+  for (const m of source.matchAll(re)) {
+    const arg = m[1];
+    for (const b of bindings) if (new RegExp(`(^|[^\\w$.])${b.replace(/\$/g, '\\$')}(?![\\w$])`).test(arg)) return true;
+  }
+  return false;
+}
+
+/**
+ * How one test file relates to one target: does it import it, does it mock it,
+ * does it ever assert on what it imported from it, and did the author annotate
+ * an unasserted mock with a reason (`// unasserted: <why>` on or above the
+ * mock line).
+ */
+export function analyzeDefender(projectDir, testRel, targetRel) {
+  const abs = join(projectDir, testRel);
+  let source;
+  try {
+    source = readFileSync(abs, 'utf8');
+  } catch {
+    return { file: testRel, imports: false, mocks: false, asserted: false, annotation: null };
+  }
+  const matchesTarget = (spec) => specifierResolvesTo(projectDir, abs, spec, targetRel);
+  const mocks = mockSpecifiers(source).filter((m) => matchesTarget(m.specifier));
+  const bindings = bindingsFor(source, matchesTarget);
+  const asserted = expectMentions(source, bindings);
+  let annotation = null;
+  if (mocks.length) {
+    const lines = source.split('\n');
+    for (const m of mocks) {
+      for (let i = m.line - 1; i >= Math.max(0, m.line - 3) && !annotation; i--) {
+        const a = UNASSERTED.exec(lines[i]);
+        if (a) annotation = a[1];
+      }
+    }
+  }
+  return { file: testRel, imports: bindings.length > 0 || matchesTargetAnyImport(source, matchesTarget), mocks: mocks.length > 0, asserted, annotation };
+}
+
+function matchesTargetAnyImport(source, matchesTarget) {
+  for (const m of source.matchAll(/(?:from\s*|import\s*\(?\s*|require\s*\(\s*)['"]([^'"]+)['"]/g)) if (matchesTarget(m[1])) return true;
+  return false;
+}
+
+/**
+ * Split candidate test files for one target into the ones that can detect a
+ * fault in it and the ones that mock it, plus the static signals. A file that
+ * mocks the target is never a defender; a mocking file that never asserts on
+ * the target carries `mocked-never-asserted`, unless annotated, in which case
+ * the annotation is recorded (`unasserted-annotated`) — silenced, never hidden.
+ */
+export function classifyDefenders(projectDir, targetRel, candidates) {
+  const canDetect = [];
+  const mocking = [];
+  const signals = [];
+  for (const file of candidates) {
+    const a = analyzeDefender(projectDir, file, targetRel);
+    if (a.mocks) {
+      mocking.push(file);
+      if (!a.asserted) signals.push(a.annotation ? { file, signal: 'unasserted-annotated', reason: a.annotation } : { file, signal: 'mocked-never-asserted' });
+    } else {
+      canDetect.push(file);
+    }
+  }
+  return { canDetect, mocking, signals };
+}