testguard-cli 0.11.0 → 0.13.0

package/src/sweep/sweep.mjs

@@ -35,8 +35,12 @@ import { loadClaims, defaultClaimsPath } from '../claims/load.mjs';
 import { computeChangedGate } from '../gate/changed.mjs';
 import { scaffoldFile } from '../scaffold/scaffold.mjs';
 import { selectFaults, onWritePath, capFor } from '../supply/select.mjs';
-import { persistenceSignals, persistenceHint } from '../supply/persistence.mjs';
+import { learnedProductivity } from '../supply/feedback.mjs';
+import { loadConcerns, concernById, targetsFor, filterByConcern } from '../supply/concerns.mjs';
+import { saveSurface } from '../supply/savepath.mjs';
+import { persistenceSignals, persistenceHint, provabilitySummary } from '../supply/persistence.mjs';
 import { probe } from '../probe/probe.mjs';
+import { discoverDefenders } from '../probe/discover.mjs';
 import { hintFor } from '../brief/brief.mjs';
 
 /**
@@ -105,6 +109,9 @@ export function sortFindings(findings) {
 /** How many findings a sweep is willing to fail on. */
 export const gatingCount = (findings) => findings.filter((f) => GATING.has(f.verdict)).length;
 
+/** A concern named on the command line that this project does not declare. */
+export class ConcernError extends Error {}
+
 /** Rebuild claim documents from a flat selection, keeping each claim's own faults together. */
 function claimsFromSelection(selected) {
   const byClaim = new Map();
@@ -126,6 +133,9 @@ function claimsFromSelection(selected) {
 export async function sweep({
   projectDir,
   ref,
+  mode = 'changed',
+  concern: concernId,
+  concernsPath,
   includeDirty = false,
   exclude = [],
   cap,
@@ -142,9 +152,33 @@ export async function sweep({
   onWarn = () => {},
 }) {
   const gate = computeChangedGate({ projectDir, ref, includeDirty, exclude, toolVersion, claimsPath, ignorePath });
+
+  // ── What the sweep is pointed at. ──
+  // `changed` asks "what did this change leave unclaimed". `save-paths` asks a
+  // question the diff cannot: "of every place this project writes to storage,
+  // how many would notice if the write stopped carrying a field". The second
+  // needs the whole surface, because a clean report over an unstated
+  // denominator is a sample of unknown size, not a guarantee.
+  //
   // A test file with no claim is a different problem — it has nothing to
   // falsify — and the gate already names it. Sweep proposes against source.
-  const targets = gate.uncovered.filter((u) => u.kind === 'source').map((u) => u.file);
+  // `--save-paths` is sugar for the built-in SAVE-PERSISTS concern: the same
+  // targets and the same producer selection, named so a project can retune it
+  // for an ORM this tool does not recognise without forking anything.
+  const loaded = loadConcerns(projectDir, { path: concernsPath });
+  const concern = concernId
+    ? concernById(loaded.concerns, concernId)
+    : mode === 'save-paths' ? concernById(loaded.concerns, 'SAVE-PERSISTS') : null;
+  if (concernId && !concern) {
+    throw new ConcernError(`unknown concern ${concernId}. Declared: ${loaded.concerns.map((c) => c.id).join(', ')}`);
+  }
+  const changedTargets = gate.uncovered.filter((u) => u.kind === 'source').map((u) => u.file);
+  // A concern whose targets are the diff defers to the gate rather than
+  // re-deriving what it already computed.
+  const concernTargets = concern ? targetsFor(projectDir, concern, { changedFiles: changedTargets }) : null;
+  const usesSurface = concern?.targets?.kind === 'write-sites' || mode === 'save-paths';
+  const surface = usesSurface ? saveSurface(projectDir) : null;
+  const targets = concernTargets ?? changedTargets;
 
   const cPath = claimsPath ?? defaultClaimsPath(projectDir);
   const existingClaims = existsSync(cPath) ? loadClaims(cPath) : { claims: [] };
@@ -171,8 +205,27 @@ export async function sweep({
     for (const claim of result.doc.claims) for (const fault of claim.faults) candidates.push({ claim, fault });
   }
 
+  // What the defenders of this surface are CAPABLE of proving, before anything
+  // is probed. A test that replaced the database can prove the call shape and
+  // never that the row landed, and that limit is a property of the suite rather
+  // than of any verdict — so it is measured once, over the whole surface, and
+  // reported whether or not a single fault survives.
+  const provability = surface
+    ? provabilitySummary(projectDir, targets, (f) => discoverDefenders(projectDir, f))
+    : null;
+
+  // Order by what THIS project's own runs have shown, not only by what was
+  // measured elsewhere. A project that has probed nothing falls back to the
+  // shipped prior; one with its own records overrides it in proportion to how
+  // many it has. This is the feedback half of Google's 15%-to-89%, using data
+  // the project already produced rather than a new thing to collect.
+  const learned = learnedProductivity(projectDir);
+  // Narrowing is what makes one sentence useful across two hundred files: a
+  // persistence concern that also reported every altered return value would
+  // bury the payload findings it exists to surface.
+  const relevant = filterByConcern(candidates, concern);
   const limit = cap ?? capFor(targets.length);
-  const selection = selectFaults(candidates, { cap: limit });
+  const selection = selectFaults(relevant, { cap: limit, table: learned.table });
   const claims = claimsFromSelection(selection.selected);
 
   let evidence = { records: [] };
@@ -220,19 +273,29 @@ export async function sweep({
     head: gate.head,
     includeDirty,
     scope: {
+      mode,
+      ...(concern ? { concern: concern.id } : {}),
       changed: gate.changed,
+      ...(surface ? { writeSites: surface.siteCount, payloadFields: surface.keyCount, provability } : {}),
       targets: targets.length,
       swept: drafts.length,
       ...(skipped.length ? { skipped } : {}),
     },
     selection: {
-      proposed: candidates.length,
+      proposed: relevant.length,
+      ...(candidates.length !== relevant.length ? { outOfScope: candidates.length - relevant.length } : {}),
       selected: selection.selected.length,
       deferred: selection.deferred.length,
       cap: selection.cap,
       byClass: selection.byClass,
     },
     counts,
+    ordering: { observed: learned.observed, sources: learned.sources },
+    // Carried, not written here: the caller owns I/O. Persisting it is what
+    // closes the feedback loop — the next sweep learns from these verdicts,
+    // which are MACHINE-proposed faults and so the closest observation of the
+    // distribution the ranker orders.
+    evidence,
     findings: ordered,
     exitCode: gating > 0 ? 1 : 0,
     drafts,
@@ -243,13 +306,48 @@ export async function sweep({
 export function renderSweep(doc, { limit = 20 } = {}) {
   const out = [];
   const { scope, selection } = doc;
+  const saves = scope.mode === 'save-paths';
   if (scope.targets === 0) {
-    out.push(`sweep: no changed source file is without a claim against ${doc.ref}. Nothing to propose.`);
+    out.push(saves
+      ? 'sweep: no write to storage found in this project. Nothing to propose.'
+      : scope.concern
+        ? `sweep: concern ${scope.concern} matches no file in this project. Nothing to propose.`
+        : `sweep: no changed source file is without a claim against ${doc.ref}. Nothing to propose.`);
     return out.join('\n');
   }
-  out.push(`swept ${scope.swept} of ${scope.targets} unclaimed changed file${scope.targets === 1 ? '' : 's'} against ${doc.ref}.`);
+  // The denominator first. A report that lists findings without saying how much
+  // was looked at invites the reader to assume the rest is fine.
+  // Say how the sweep was aimed. A glob concern that reports "unclaimed changed
+  // files" is describing a diff it never looked at.
+  out.push(saves
+    ? `${scope.writeSites} write${scope.writeSites === 1 ? '' : 's'} to storage across ${scope.targets} file${scope.targets === 1 ? '' : 's'}, carrying ${scope.payloadFields} payload field${scope.payloadFields === 1 ? '' : 's'}. Swept ${scope.swept}.`
+    : scope.concern
+      ? `concern ${scope.concern} matches ${scope.targets} file${scope.targets === 1 ? '' : 's'}. Swept ${scope.swept}.`
+      : `swept ${scope.swept} of ${scope.targets} unclaimed changed file${scope.targets === 1 ? '' : 's'} against ${doc.ref}.`);
+  if (saves && scope.provability) {
+    // The limit, before any verdict. A test that replaced the database can
+    // prove the call shape and never that the row landed, so on a surface
+    // defended entirely by such tests a clean probe is not evidence of
+    // persistence — it is evidence that nothing could have measured it.
+    const { mocked, unmocked, none } = scope.provability;
+    out.push(`  of those ${scope.targets} file${scope.targets === 1 ? '' : 's'}: ${mocked} defended only by tests that mock the persistence layer, ${unmocked} with an unmocked defender, ${none} with no defender at all.`);
+    if (unmocked === 0 && mocked > 0) {
+      out.push('  Nothing in this suite can prove a write reached storage. A mocked test proves the');
+      out.push('  call shape; a where-clause that matches nothing, a rolled-back transaction and a');
+      out.push('  rejected constraint all pass against it.');
+    }
+  }
   out.push(`proposed ${selection.proposed} fault${selection.proposed === 1 ? '' : 's'}, probed ${selection.selected} (cap ${selection.cap})${selection.deferred ? `, deferred ${selection.deferred}` : ''}.`);
-  if (selection.deferred) out.push(`  The ${selection.deferred} deferred are not a verdict: raise --cap, or sweep a smaller change.`);
+  if (selection.deferred) out.push(`  The ${selection.deferred} deferred are not a verdict: raise --cap${saves || scope.concern ? '.' : ', or sweep a smaller change.'}`);
+  // A narrow concern must not look like a quiet one: the producers found these,
+  // and this concern is not about them.
+  if (selection.outOfScope) out.push(`  ${selection.outOfScope} further proposal${selection.outOfScope === 1 ? ' was' : 's were'} outside this concern's fault classes.`);
+  // An ordering nobody can trace is a number nobody should trust.
+  if (doc.ordering) {
+    out.push(doc.ordering.observed === 0
+      ? '  ordering: the shipped productivity prior — this project has no probed evidence yet.'
+      : `  ordering: ${doc.ordering.observed} probed fault${doc.ordering.observed === 1 ? '' : 's'} of this project's own (${doc.ordering.sources.join(', ')}), shrunk toward the shipped prior.`);
+  }
   for (const s of scope.skipped ?? []) out.push(`  skipped ${s.file}: ${s.reason}`);
   out.push('');
 
@@ -268,6 +366,12 @@ export function renderSweep(doc, { limit = 20 } = {}) {
     }
     if (doc.findings.length > limit) out.push(`  … ${doc.findings.length - limit} more in the sweep document`);
   }
+  if (saves) {
+    out.push('');
+    out.push(`This is a sample of the write surface, not a verdict over it: ${selection.selected} of`);
+    out.push(`${selection.proposed} proposed faults were probed. The ${scope.writeSites} writes above are the`);
+    out.push('denominator — a clean sweep says nothing about the ones nobody pointed a fault at.');
+  }
   out.push('These are PROPOSALS, not claims. Keep the ones worth defending: state the');
   out.push('claim, copy its fault into testguard.claims.json, and probe it from then on.');
   return out.join('\n');

package/src/util/glob.mjs

@@ -5,7 +5,15 @@ import { join } from 'node:path';
 // virtualenv inside the project holds thousands of `test_*.py` files belonging
 // to installed packages, and without skipping it every one of them would be
 // collected as a test file of the project under probe.
+// `dist` and `coverage` were here from the start; the framework build outputs
+// were not, and the claim above them already promised that "a build's test
+// files" are not this project's. A Next.js `.next/` carries compiled copies of
+// the application — including its writes and its tests — so a surface scan of a
+// real app counted the same code twice and a defender glob could match a
+// compiled chunk. The promise was right and the list was short.
 const SKIP_DIRS = new Set(['node_modules', '.git', 'dist', 'coverage', '.testguard',
+  '.next', '.nuxt', '.svelte-kit', '.output', '.turbo', '.parcel-cache', '.angular', '.astro',
+  'build', 'out', '.cache', '.vercel', '.netlify',
   '__pycache__', '.venv', 'venv', 'site-packages', '.tox', '.nox',
   '.pytest_cache', '.mypy_cache', '.ruff_cache', '.eggs']);