testguard-cli 0.10.4 → 0.12.0

package/src/supply/select.mjs

@@ -0,0 +1,168 @@
+/**
+ * Which drafted faults are worth probing, and in what order.
+ *
+ * `scaffold` proposes mechanically and proposes a lot: eight source files of a
+ * real application yielded 238 proposals. Probing all of them is the cost model
+ * the tool already refuses — `(baseline + faults x N) x defender-set cost`,
+ * per claim — so a sweep that probes everything is a sweep nobody runs twice.
+ *
+ * Google's mutation service solved the same problem by never surfacing more
+ * than 7 x |files| mutants per change and by ordering candidates on the
+ * measured productivity of their operator in similar context, which took their
+ * productive rate from 15% to 89% ("Practical Mutation Testing at Scale",
+ * Petrovic et al., 2021). This module is that idea at this tool's scale.
+ *
+ * TWO THINGS IT MUST NOT DO. It must not drop a candidate silently — a cap
+ * that hides work is a coverage claim nobody made, so everything not selected
+ * is returned as `deferred` and the caller is expected to say so. And it must
+ * not be nondeterministic: the same drafts in the same repository must select
+ * the same faults in the same order, or a sweep's output stops being evidence.
+ * Every tie therefore breaks on file, then line, then id.
+ *
+ * Pure: drafts in, a selection out. Nothing here reads a file or runs anything.
+ */
+
+/**
+ * P(a fault of this class survives a green suite), measured — not guessed.
+ *
+ * From 56 probed faults across 12 modules of a real AI-authored Next.js
+ * application (2026-09-21). Survival is the right prior for ordering because a
+ * fault that dies everywhere teaches nobody anything: the whole output of a
+ * sweep is the faults that lived.
+ *
+ * `n` is carried so a reader can see which of these is one observation and
+ * which is twelve, and so a later feedback pass can weight them honestly
+ * instead of treating 100%-of-one as certainty.
+ */
+export const MEASURED_PRODUCTIVITY = Object.freeze({
+  'element-removed': { p: 1.0, n: 1 },
+  'call-removed': { p: 0.67, n: 3 },
+  'field-dropped': { p: 0.5, n: 12 },
+  'statement-deleted': { p: 0.33, n: 12 },
+  'literal-changed': { p: 0.33, n: 3 },
+  'condition-forced': { p: 0.1, n: 10 },
+  'argument-swapped': { p: 0.09, n: 11 },
+  'return-altered': { p: 0.0, n: 4 },
+});
+
+/**
+ * A class nobody measured is worth more attention than the worst measured one
+ * and less than the best. Ranking it last would make every new producer
+ * invisible until someone hand-edited this table; ranking it first would let a
+ * new producer crowd out everything known to be productive.
+ */
+const UNMEASURED = 0.4;
+
+/**
+ * Shrink a small sample toward the unmeasured prior. One observation of
+ * `element-removed` is not evidence that it always survives, and ordering the
+ * whole sweep on it would be exactly the overconfidence this tool exists to
+ * refuse. m is the pseudo-count: at n = m the measurement and the prior weigh
+ * the same.
+ */
+const M = 4;
+export function priorFor(faultClass, table = MEASURED_PRODUCTIVITY) {
+  const hit = table[faultClass];
+  if (!hit) return UNMEASURED;
+  return (hit.p * hit.n + UNMEASURED * M) / (hit.n + M);
+}
+
+/**
+ * The write payload of a persisted record is where this project's own
+ * measurement found every survivor: 12 of 12 were a field dropped from a
+ * `data` / `where` / `update` object reaching a database call. A fault on such
+ * a line is worth more than its class alone says, because the assertion that
+ * would have to catch it is the one tests most often write loosely
+ * (`expect.objectContaining({...})` naming everything except the field that
+ * carried the data).
+ *
+ * Deliberately a line-shape test, not a type-aware one: the scaffold producers
+ * are line-oriented and this must not become the one part of the pipeline that
+ * needs an AST.
+ */
+const PERSISTENCE_CALL = /\b(?:prisma|db|tx|trx|knex|sequelize|mongoose|supabase|drizzle)\b\s*[.[]|\.(?:create|createMany|update|updateMany|upsert|insert|save|delete|deleteMany|destroy)\s*\(/;
+const PAYLOAD_KEY = /^\s*(?:data|where|update|create|select|set|values)\s*:/;
+
+/** Is this fault on the persistence path — a write call, or a field of its payload? */
+export function onWritePath(fault) {
+  const line = fault.find ?? '';
+  return PERSISTENCE_CALL.test(line) || PAYLOAD_KEY.test(line) || PERSISTENCE_CALL.test(fault.description ?? '');
+}
+
+/** The boost a write-path fault gets. Additive, not multiplicative: it must not be able to overtake a measured 1.0. */
+const WRITE_PATH_BONUS = 0.25;
+
+/** What one draft is worth probing, in [0, 1.25]. Exported so a test can pin the ordering rule rather than the order. */
+export function scoreOf(fault, { table } = {}) {
+  return priorFor(fault.faultClass, table) + (onWritePath(fault) ? WRITE_PATH_BONUS : 0);
+}
+
+/**
+ * Google surfaces at most 7 x |files| mutants per change. The number is theirs
+ * and the reason is general: past it the reader stops reading, and a finding
+ * nobody reads is worse than no finding, because it also spends the runner.
+ */
+export const capFor = (fileCount, perFile = 7) => Math.max(perFile, fileCount * perFile);
+
+/**
+ * Order every candidate and take the cap.
+ *
+ * `candidates` are `{ claim, fault }` pairs — the claim is carried so the
+ * caller can rebuild a claims document from the selection without a second
+ * lookup. Returns the selection, everything deferred, and the per-class tally
+ * a report needs to explain itself.
+ */
+export function selectFaults(candidates, { cap, table } = {}) {
+  const limit = Number.isInteger(cap) && cap > 0 ? cap : capFor(new Set(candidates.map((c) => c.fault.file)).size);
+  const ranked = candidates
+    .map((c) => ({ ...c, score: scoreOf(c.fault, { table }), writePath: onWritePath(c.fault) }))
+    // Deterministic to the last tie: score, then file, then line, then id.
+    // Two runs of the same sweep on the same tree are the same sweep.
+    .sort((a, b) =>
+      b.score - a.score
+      || (a.fault.file ?? '').localeCompare(b.fault.file ?? '')
+      || (a.fault.line ?? 0) - (b.fault.line ?? 0)
+      || `${a.claim.id}/${a.fault.id}`.localeCompare(`${b.claim.id}/${b.fault.id}`));
+
+  // ── Spread the cap across FILES, best-first within each. ──
+  // Taking the global top N looks right and is not: a file with many
+  // high-scoring lines takes the whole budget and every other changed file is
+  // reported as having no findings, which is indistinguishable from having
+  // none. Measured — the first sweep of a 14-file change spent all six slots
+  // on one page component and said nothing about the other twelve.
+  // The cap is already expressed per file (7 x |files|), so this is what that
+  // number was always supposed to mean.
+  const byFile = new Map();
+  for (const r of ranked) {
+    const key = r.fault.file ?? '';
+    if (!byFile.has(key)) byFile.set(key, []);
+    byFile.get(key).push(r);
+  }
+  // Files in the order of their best candidate, so a round that cannot serve
+  // everyone still serves the most promising files first. Ties on file name.
+  const queues = [...byFile.entries()]
+    .sort((a, b) => b[1][0].score - a[1][0].score || a[0].localeCompare(b[0]))
+    .map(([, q]) => q);
+
+  const selected = [];
+  for (let round = 0; selected.length < limit; round++) {
+    let served = false;
+    for (const q of queues) {
+      if (q.length <= round) continue;
+      selected.push(q[round]);
+      served = true;
+      if (selected.length >= limit) break;
+    }
+    if (!served) break;
+  }
+  const taken = new Set(selected);
+  const deferred = ranked.filter((r) => !taken.has(r));
+  const byClass = {};
+  for (const r of ranked) {
+    const e = (byClass[r.fault.faultClass] ??= { proposed: 0, selected: 0, writePath: 0 });
+    e.proposed += 1;
+    if (r.writePath) e.writePath += 1;
+  }
+  for (const r of selected) byClass[r.fault.faultClass].selected += 1;
+  return { selected, deferred, cap: limit, byClass };
+}

package/src/sweep/sweep.mjs

@@ -0,0 +1,302 @@
+/**
+ * The cold start: findings on a repository that has no claims yet.
+ *
+ * `gate --changed` already says which changed source files carry no claim, and
+ * stops there — correctly, because stating a claim is a human act. But a
+ * project adopting this tool reads that list, has nothing to compare it
+ * against, and closes the tab. Every escaped defect reported from the field so
+ * far was a claim gap, not a probe miss, so the bottleneck was never
+ * verification. It is oracle SUPPLY.
+ *
+ * Sweep is the lowest rung of that supply: no concern, no statement, nothing
+ * written by anybody. It takes the files the gate just called uncovered,
+ * proposes faults mechanically with the existing scaffold producers, probes a
+ * bounded selection of them, and reports what survived. A fault that survives
+ * needs no claim to be alarming: the code changed, something was deliberately
+ * broken, and not one test noticed.
+ *
+ * WHAT IT IS NOT. It does not write `testguard.claims.json`, and it never
+ * will — a claim is a sentence a human is willing to stand behind, and a
+ * sentence nobody wrote is not one. Its drafts land beside the evidence for a
+ * human to keep or drop, exactly as `scaffold` already works. It does not
+ * overwrite the canonical `.testguard/evidence.json` either: a sweep probes
+ * machine-proposed faults under TODO statements, and folding that into the
+ * document `status` and `baseline` read from would corrupt the record of what
+ * the project actually claims.
+ *
+ * The cap and the ordering come from Google's mutation service, which surfaces
+ * at most 7 x |files| mutants per change and orders candidates on the measured
+ * productivity of their operator, taking their productive rate from 15% to 89%
+ * ("Practical Mutation Testing at Scale", Petrovic et al., 2021). See
+ * `src/supply/select.mjs`.
+ */
+import { existsSync } from 'node:fs';
+import { loadClaims, defaultClaimsPath } from '../claims/load.mjs';
+import { computeChangedGate } from '../gate/changed.mjs';
+import { scaffoldFile } from '../scaffold/scaffold.mjs';
+import { selectFaults, onWritePath, capFor } from '../supply/select.mjs';
+import { saveSurface } from '../supply/savepath.mjs';
+import { persistenceSignals, persistenceHint } from '../supply/persistence.mjs';
+import { probe } from '../probe/probe.mjs';
+import { hintFor } from '../brief/brief.mjs';
+
+/**
+ * Findings first, in the order a reader should act on them. The same order
+ * `brief` uses, minus the verdicts a sweep does not gate on.
+ */
+const FINDING_ORDER = ['survived', 'nocover', 'timeout', 'flaky-defender', 'unverifiable', 'fault-invalid'];
+
+/**
+ * Which verdicts are a finding ABOUT THE PROJECT, and so decide the exit code.
+ *
+ * `survived` and `nocover` are statements about the test suite: something was
+ * broken and nothing failed, or nothing imports the file at all. The rest are
+ * statements about the PROPOSAL — an anchor that did not locate, a replacement
+ * that does not compile — and a sweep's proposals are machine-made and thrown
+ * away. Gating on them would make the tool fail because its own guess was bad,
+ * which is the fastest way to get a check switched off. They are still
+ * reported; they just do not gate.
+ */
+export const GATING = new Set(['survived', 'nocover']);
+
+/**
+ * Turn one evidence record into a finding, or null when it is not one.
+ *
+ * Pure, and exported, for the reason `classify.mjs` gives about itself: the
+ * only thing that could otherwise falsify these rules is a full fixture sweep,
+ * which is a minutes-long acceptance test standing in for a decision you can
+ * state in four lines. Self-probing this file found exactly that gap — the
+ * schema was guarded and the engine that fills it was not.
+ *
+ * `signalsFor` is injected so the rule about WHEN a signal applies can be
+ * tested without a filesystem; the default in `sweep()` reads the defenders.
+ */
+export function findingFrom(record, fault, signalsFor = () => []) {
+  if (record.verdict === 'killed') return null;
+  const writePath = fault ? onWritePath(fault) : false;
+  // The persistence signal explains a survivor on the write path and nothing
+  // else. On a kill it explains nothing, and on any other line a loose mock
+  // assertion is not evidence — reporting it everywhere is the
+  // non-actionable noise that gets a check switched off.
+  const signals = record.verdict === 'survived' && writePath ? signalsFor(record) : [];
+  return {
+    file: record.subject.file,
+    ...(fault?.line ? { line: fault.line } : {}),
+    claimId: record.claim.id,
+    faultId: record.subject.id,
+    faultClass: record.subject.faultClass,
+    verdict: record.verdict,
+    ...(record.detail?.reason ? { reason: record.detail.reason } : {}),
+    description: record.subject.description,
+    writePath,
+    hint: signals.length ? persistenceHint(signals[0], fault?.find) : hintFor(record),
+    ...(signals.length ? { signals } : {}),
+  };
+}
+
+/** Findings in the order a reader should act on them. Pure; deterministic to the last tie. */
+export function sortFindings(findings) {
+  return [...findings].sort((a, b) =>
+    FINDING_ORDER.indexOf(a.verdict) - FINDING_ORDER.indexOf(b.verdict)
+    || Number(b.writePath) - Number(a.writePath)
+    || a.file.localeCompare(b.file)
+    || (a.line ?? 0) - (b.line ?? 0));
+}
+
+/** How many findings a sweep is willing to fail on. */
+export const gatingCount = (findings) => findings.filter((f) => GATING.has(f.verdict)).length;
+
+/** Rebuild claim documents from a flat selection, keeping each claim's own faults together. */
+function claimsFromSelection(selected) {
+  const byClaim = new Map();
+  for (const { claim, fault } of selected) {
+    if (!byClaim.has(claim.id)) byClaim.set(claim.id, { ...claim, faults: [] });
+    byClaim.get(claim.id).faults.push(fault);
+  }
+  // Deterministic: the probe's own ordering, and the evidence it writes, must
+  // not depend on the order a Map happened to be filled in.
+  return [...byClaim.values()]
+    .map((c) => ({ ...c, faults: [...c.faults].sort((a, b) => a.id.localeCompare(b.id)) }))
+    .sort((a, b) => a.id.localeCompare(b.id));
+}
+
+/**
+ * Run a sweep. Returns the sweep document; the caller owns writing and exit
+ * codes, as every other engine module in this tool does.
+ */
+export async function sweep({
+  projectDir,
+  ref,
+  mode = 'changed',
+  includeDirty = false,
+  exclude = [],
+  cap,
+  confirmRuns = 3,
+  budgetMs = 120_000,
+  runnerCommand,
+  runnerName,
+  nodeModules,
+  toolVersion = '0.0.0',
+  generatedAt = new Date().toISOString(),
+  claimsPath,
+  ignorePath,
+  onStage,
+  onWarn = () => {},
+}) {
+  const gate = computeChangedGate({ projectDir, ref, includeDirty, exclude, toolVersion, claimsPath, ignorePath });
+
+  // ── What the sweep is pointed at. ──
+  // `changed` asks "what did this change leave unclaimed". `save-paths` asks a
+  // question the diff cannot: "of every place this project writes to storage,
+  // how many would notice if the write stopped carrying a field". The second
+  // needs the whole surface, because a clean report over an unstated
+  // denominator is a sample of unknown size, not a guarantee.
+  //
+  // A test file with no claim is a different problem — it has nothing to
+  // falsify — and the gate already names it. Sweep proposes against source.
+  const surface = mode === 'save-paths' ? saveSurface(projectDir) : null;
+  const targets = surface
+    ? surface.files.map((f) => f.file)
+    : gate.uncovered.filter((u) => u.kind === 'source').map((u) => u.file);
+
+  const cPath = claimsPath ?? defaultClaimsPath(projectDir);
+  const existingClaims = existsSync(cPath) ? loadClaims(cPath) : { claims: [] };
+
+  const candidates = [];
+  const skipped = [];
+  const drafts = [];
+  for (const file of targets) {
+    let result;
+    try {
+      result = scaffoldFile({ projectDir, file, existingClaims, toolVersion });
+    } catch (e) {
+      // One unscaffoldable file must not take the sweep with it: the other
+      // files' findings are still true, and a reader needs to know which file
+      // produced nothing and why.
+      skipped.push({ file, reason: e.message.split('\n')[0].slice(0, 200) });
+      continue;
+    }
+    if (result.stats.proposals === 0) {
+      skipped.push({ file, reason: 'no line in this file matches a fault producer' });
+      continue;
+    }
+    drafts.push(result.doc);
+    for (const claim of result.doc.claims) for (const fault of claim.faults) candidates.push({ claim, fault });
+  }
+
+  const limit = cap ?? capFor(targets.length);
+  const selection = selectFaults(candidates, { cap: limit });
+  const claims = claimsFromSelection(selection.selected);
+
+  let evidence = { records: [] };
+  if (claims.length > 0) {
+    evidence = await probe({
+      projectDir,
+      claims: { schemaVersion: 1, claims },
+      confirmRuns,
+      budgetMs,
+      mode: 'worktree',
+      includeDirty,
+      // A proposed fault has no declared defenders to be missing from, so the
+      // question escalation answers — "did something UNDECLARED catch it?" —
+      // has no meaning here, and it is the most expensive thing the probe does.
+      escalate: false,
+      runnerCommand,
+      runnerName,
+      nodeModules,
+      toolVersion,
+      onStage,
+      onWarn,
+    });
+  }
+
+  // Index the selection so a record can find the fault it came from without a
+  // second scan; `line` and `find` live on the draft, not on the evidence.
+  const draftOf = new Map(selection.selected.map(({ claim, fault }) => [`${claim.id}/${fault.id}`, fault]));
+
+  const signalsFor = (r) => persistenceSignals(projectDir, r.defenders?.resolved ?? []);
+  const counts = {};
+  const findings = [];
+  for (const r of evidence.records) {
+    counts[r.verdict] = (counts[r.verdict] ?? 0) + 1;
+    const f = findingFrom(r, draftOf.get(`${r.claim.id}/${r.subject.id}`), signalsFor);
+    if (f) findings.push(f);
+  }
+  const ordered = sortFindings(findings);
+  const gating = gatingCount(ordered);
+  return {
+    schemaVersion: 1,
+    tool: { name: 'testguard', version: toolVersion },
+    generatedAt,
+    ref,
+    base: gate.base,
+    head: gate.head,
+    includeDirty,
+    scope: {
+      mode,
+      changed: gate.changed,
+      ...(surface ? { writeSites: surface.siteCount, payloadFields: surface.keyCount } : {}),
+      targets: targets.length,
+      swept: drafts.length,
+      ...(skipped.length ? { skipped } : {}),
+    },
+    selection: {
+      proposed: candidates.length,
+      selected: selection.selected.length,
+      deferred: selection.deferred.length,
+      cap: selection.cap,
+      byClass: selection.byClass,
+    },
+    counts,
+    findings: ordered,
+    exitCode: gating > 0 ? 1 : 0,
+    drafts,
+  };
+}
+
+/** The sweep as text, for someone who ran it in a terminal or reads it in a CI log. */
+export function renderSweep(doc, { limit = 20 } = {}) {
+  const out = [];
+  const { scope, selection } = doc;
+  const saves = scope.mode === 'save-paths';
+  if (scope.targets === 0) {
+    out.push(saves
+      ? 'sweep: no write to storage found in this project. Nothing to propose.'
+      : `sweep: no changed source file is without a claim against ${doc.ref}. Nothing to propose.`);
+    return out.join('\n');
+  }
+  // The denominator first. A report that lists findings without saying how much
+  // was looked at invites the reader to assume the rest is fine.
+  out.push(saves
+    ? `${scope.writeSites} write${scope.writeSites === 1 ? '' : 's'} to storage across ${scope.targets} file${scope.targets === 1 ? '' : 's'}, carrying ${scope.payloadFields} payload field${scope.payloadFields === 1 ? '' : 's'}. Swept ${scope.swept}.`
+    : `swept ${scope.swept} of ${scope.targets} unclaimed changed file${scope.targets === 1 ? '' : 's'} against ${doc.ref}.`);
+  out.push(`proposed ${selection.proposed} fault${selection.proposed === 1 ? '' : 's'}, probed ${selection.selected} (cap ${selection.cap})${selection.deferred ? `, deferred ${selection.deferred}` : ''}.`);
+  if (selection.deferred) out.push(`  The ${selection.deferred} deferred are not a verdict: raise --cap${saves ? '.' : ', or sweep a smaller change.'}`);
+  for (const s of scope.skipped ?? []) out.push(`  skipped ${s.file}: ${s.reason}`);
+  out.push('');
+
+  const gating = doc.findings.filter((f) => GATING.has(f.verdict));
+  if (gating.length === 0) {
+    out.push(`No fault survived. ${doc.counts.killed ?? 0} of ${selection.selected} probed faults were caught.`);
+  } else {
+    out.push(`${gating.length} finding${gating.length === 1 ? '' : 's'} — a deliberate break that no test noticed:`);
+    out.push('');
+    for (const f of doc.findings.slice(0, limit)) {
+      const where = f.line ? `${f.file}:${f.line}` : f.file;
+      out.push(`  ${f.verdict.toUpperCase().padEnd(13)} ${where}${f.writePath ? '  [write path]' : ''}`);
+      out.push(`    ${f.description}`);
+      out.push(`    ${f.hint}`);
+      out.push('');
+    }
+    if (doc.findings.length > limit) out.push(`  … ${doc.findings.length - limit} more in the sweep document`);
+  }
+  if (saves) {
+    out.push('');
+    out.push(`This is a sample of the write surface, not a verdict over it: ${selection.selected} of`);
+    out.push(`${selection.proposed} proposed faults were probed. The ${scope.writeSites} writes above are the`);
+    out.push('denominator — a clean sweep says nothing about the ones nobody pointed a fault at.');
+  }
+  out.push('These are PROPOSALS, not claims. Keep the ones worth defending: state the');
+  out.push('claim, copy its fault into testguard.claims.json, and probe it from then on.');
+  return out.join('\n');
+}

package/src/util/glob.mjs

@@ -5,7 +5,15 @@ import { join } from 'node:path';
 // virtualenv inside the project holds thousands of `test_*.py` files belonging
 // to installed packages, and without skipping it every one of them would be
 // collected as a test file of the project under probe.
+// `dist` and `coverage` were here from the start; the framework build outputs
+// were not, and the claim above them already promised that "a build's test
+// files" are not this project's. A Next.js `.next/` carries compiled copies of
+// the application — including its writes and its tests — so a surface scan of a
+// real app counted the same code twice and a defender glob could match a
+// compiled chunk. The promise was right and the list was short.
 const SKIP_DIRS = new Set(['node_modules', '.git', 'dist', 'coverage', '.testguard',
+  '.next', '.nuxt', '.svelte-kit', '.output', '.turbo', '.parcel-cache', '.angular', '.astro',
+  'build', 'out', '.cache', '.vercel', '.netlify',
   '__pycache__', '.venv', 'venv', 'site-packages', '.tox', '.nox',
   '.pytest_cache', '.mypy_cache', '.ruff_cache', '.eggs']);