testguard-cli 0.12.0 → 0.14.0

package/src/supply/persistence.mjs

@@ -38,6 +38,7 @@
  */
 import { readFileSync } from 'node:fs';
 import { join } from 'node:path';
+import { onWritePath } from './select.mjs';
 
 /**
  * Module specifiers that mean "the database". A project-local path counts when
@@ -158,6 +159,25 @@ export function persistenceSignals(projectDir, defenders) {
   return defenders.map((d) => analyzePersistence(projectDir, d)).filter(Boolean);
 }
 
+/**
+ * The persistence signals ONE EVIDENCE RECORD may carry, or none.
+ *
+ * The rule the sweep document already enforces, now applied where the record
+ * is written: the signal explains a SURVIVOR whose fault sits on the write
+ * path, and nothing else. On a kill it explains nothing; off the write path a
+ * loose mock assertion is not evidence of anything; on `nocover` or
+ * `unverifiable` the question was never asked. Reporting it anywhere else is
+ * the non-actionable noise that gets a check switched off.
+ *
+ * `analyze` is injected so the rule is testable without a filesystem; the
+ * probe passes the scratch tree, where the defenders are the project's own.
+ */
+export function persistenceSignalsFor({ verdict, fault, defenders, projectDir }, analyze = persistenceSignals) {
+  if (verdict !== 'survived') return [];
+  if (!onWritePath(fault)) return [];
+  return analyze(projectDir, defenders);
+}
+
 /**
  * One line an agent can act on, in the register `hintFor()` uses.
  *
@@ -171,3 +191,56 @@ export function persistenceHint(signal, faultLine) {
   const field = /^\s*([A-Za-z_$][\w$]*)\s*(?::|,\s*$)/.exec(faultLine ?? '')?.[1];
   return `${signal.file} ${signal.reason} Assert the whole object written by the call this fault changes${field ? `, including \`${field}\`` : ''} — or read the record back and assert on what was stored.`;
 }
+
+/**
+ * What this claim's defenders are CAPABLE of proving about a write.
+ *
+ * The read-back question — "did the row actually land?" — cannot be answered by
+ * a test that replaced the database with a mock. Such a test can prove the call
+ * shape and nothing beyond it: the `where` may match no rows, the transaction
+ * may roll back, a constraint may reject the write, and every one of those
+ * passes against a mock that recorded the arguments and returned a plausible
+ * object.
+ *
+ * So rather than pretend to measure persistence, this states the LIMIT. It is
+ * the same discipline as `nocover` — which says "no test imports this" instead
+ * of guessing — one level finer: these tests exist, they run, and there is a
+ * class of failure they structurally cannot see.
+ *
+ *   `mocked`   every defender replaces the persistence layer. Call shape only.
+ *   `unmocked` at least one does not, so a real write is at least possible.
+ *              NOT a promise that one happens: a test may simply never reach
+ *              the database. Possible is the honest word.
+ *   `none`     no defender at all; `nocover` already covers this.
+ *
+ * Measured on a real AI-authored application: of 37 files that write to
+ * storage, 32 were defended only by mocking tests, 4 had no defender, and 1
+ * had an unmocked defender — a mailer, not a database write. Nothing in that
+ * suite could prove any save persisted.
+ */
+export function persistenceProvability(projectDir, defenders) {
+  if (!defenders || defenders.length === 0) return { class: 'none', mocking: [], unmocked: [] };
+  const mocking = [];
+  const unmocked = [];
+  for (const d of defenders) (analyzePersistence(projectDir, d) ? mocking : unmocked).push(d);
+  return { class: unmocked.length ? 'unmocked' : 'mocked', mocking, unmocked };
+}
+
+/**
+ * The provability of a whole surface, as counts. The headline a save-path
+ * report opens with, because "how many of these could be proven at all" is a
+ * different question from "how many were probed" and a far more alarming one.
+ */
+export function provabilitySummary(projectDir, targets, defendersOf) {
+  const counts = { mocked: 0, unmocked: 0, none: 0 };
+  for (const file of targets) {
+    let defenders = [];
+    try {
+      defenders = defendersOf(file);
+    } catch {
+      defenders = [];
+    }
+    counts[persistenceProvability(projectDir, defenders).class] += 1;
+  }
+  return counts;
+}

package/src/supply/select.mjs

@@ -89,6 +89,35 @@ export function onWritePath(fault) {
   return PERSISTENCE_CALL.test(line) || PAYLOAD_KEY.test(line) || PERSISTENCE_CALL.test(fault.description ?? '');
 }
 
+/**
+ * Is this proposal a purely presentational JSX element — the arid class this
+ * tool actually has?
+ *
+ * Google suppresses "arid" nodes nobody would write a test for (logging,
+ * timeouts, flags). Measured here on 2026-09-22, those account for under 1% of
+ * proposals: the producers are shape-targeted and rarely land on a log line.
+ * What IS material is this: on a 14-file UI diff, 150 of 510 proposals were an
+ * `element-removed` on an icon (`<X />`) or a static wrapper (`<div>`, `<p>`
+ * with no expression and no handler). The ranker spent 29 of 30 slots on that
+ * class, 7 of the 10 survivors were of this shape, and because the ordering
+ * learns from survival those survivors then teach it to rank the class higher.
+ *
+ * Line-oriented, like every producer. An icon is a self-closing PascalCase
+ * element whose props are only sizing and styling; a static wrapper is a
+ * layout or text tag whose own line carries no `{expression}` and no
+ * interactive or test-facing attribute. Anything else — a component, an
+ * `<img>`, a `<label>`, an element with a handler or dynamic children — is a
+ * real proposal and stays.
+ */
+const ICON_ELEMENT = /^\s*<[A-Z]\w*(?:\s+(?:size|className|strokeWidth|aria-hidden|color|width|height|fill)(?:=(?:"[^"]*"|\{[^}]*\}))?)*\s*\/>\s*$/;
+const WRAPPER_ELEMENT = /^\s*<(?:div|span|p|h[1-6]|section|header|footer|main|nav|ul|ol|li|small|strong|em|b|i|hr|br)\b/;
+const INTERACTIVE_ATTR = /\b(?:on[A-Z]\w*=|href=|role=|aria-(?!hidden)\w+=|data-testid=|htmlFor=|tabIndex=)/;
+export function presentational(fault) {
+  if (fault.faultClass !== 'element-removed') return false;
+  const line = fault.find ?? '';
+  return ICON_ELEMENT.test(line) || (WRAPPER_ELEMENT.test(line) && !line.includes('{') && !INTERACTIVE_ATTR.test(line));
+}
+
 /** The boost a write-path fault gets. Additive, not multiplicative: it must not be able to overtake a measured 1.0. */
 const WRITE_PATH_BONUS = 0.25;
 
@@ -109,13 +138,17 @@ export const capFor = (fileCount, perFile = 7) => Math.max(perFile, fileCount *
  *
  * `candidates` are `{ claim, fault }` pairs — the claim is carried so the
  * caller can rebuild a claims document from the selection without a second
- * lookup. Returns the selection, everything deferred, and the per-class tally
- * a report needs to explain itself.
+ * lookup. Returns the selection, everything deferred, everything set aside as
+ * presentational, and the per-class tally a report needs to explain itself.
+ * The three lists partition the candidates: nothing is dropped without a name.
  */
 export function selectFaults(candidates, { cap, table } = {}) {
   const limit = Number.isInteger(cap) && cap > 0 ? cap : capFor(new Set(candidates.map((c) => c.fault.file)).size);
-  const ranked = candidates
-    .map((c) => ({ ...c, score: scoreOf(c.fault, { table }), writePath: onWritePath(c.fault) }))
+  const all = candidates.map((c) => ({ ...c, score: scoreOf(c.fault, { table }), writePath: onWritePath(c.fault) }));
+  // Set aside, never silently: a presentational element is not worth a probe
+  // run, and a survivor on one would teach the ordering the wrong lesson.
+  const presentationalList = all.filter((c) => presentational(c.fault));
+  const ranked = all.filter((c) => !presentational(c.fault))
     // Deterministic to the last tie: score, then file, then line, then id.
     // Two runs of the same sweep on the same tree are the same sweep.
     .sort((a, b) =>
@@ -158,11 +191,11 @@ export function selectFaults(candidates, { cap, table } = {}) {
   const taken = new Set(selected);
   const deferred = ranked.filter((r) => !taken.has(r));
   const byClass = {};
-  for (const r of ranked) {
+  for (const r of all) {
     const e = (byClass[r.fault.faultClass] ??= { proposed: 0, selected: 0, writePath: 0 });
     e.proposed += 1;
     if (r.writePath) e.writePath += 1;
   }
   for (const r of selected) byClass[r.fault.faultClass].selected += 1;
-  return { selected, deferred, cap: limit, byClass };
+  return { selected, deferred, presentational: presentationalList, cap: limit, byClass };
 }

package/src/sweep/sweep.mjs

@@ -35,9 +35,12 @@ import { loadClaims, defaultClaimsPath } from '../claims/load.mjs';
 import { computeChangedGate } from '../gate/changed.mjs';
 import { scaffoldFile } from '../scaffold/scaffold.mjs';
 import { selectFaults, onWritePath, capFor } from '../supply/select.mjs';
+import { learnedProductivity } from '../supply/feedback.mjs';
+import { loadConcerns, concernById, targetsFor, filterByConcern } from '../supply/concerns.mjs';
 import { saveSurface } from '../supply/savepath.mjs';
-import { persistenceSignals, persistenceHint } from '../supply/persistence.mjs';
+import { persistenceHint, provabilitySummary } from '../supply/persistence.mjs';
 import { probe } from '../probe/probe.mjs';
+import { discoverDefenders } from '../probe/discover.mjs';
 import { hintFor } from '../brief/brief.mjs';
 
 /**
@@ -106,6 +109,30 @@ export function sortFindings(findings) {
 /** How many findings a sweep is willing to fail on. */
 export const gatingCount = (findings) => findings.filter((f) => GATING.has(f.verdict)).length;
 
+/**
+ * The `selection` block of the document: every proposal accounted for.
+ *
+ * Pure and exported for the same reason as `findingFrom`: the arithmetic that
+ * proves nothing was hidden — proposed = probed + deferred + set aside — is a
+ * rule you can state in one line, and the validator that checks it is not the
+ * engine that has to satisfy it.
+ */
+export function summarizeSelection(selection, { proposed, outOfScope = 0 }) {
+  const presentationalCount = selection.presentational?.length ?? 0;
+  return {
+    proposed,
+    ...(outOfScope ? { outOfScope } : {}),
+    selected: selection.selected.length,
+    deferred: selection.deferred.length,
+    ...(presentationalCount ? { presentational: presentationalCount } : {}),
+    cap: selection.cap,
+    byClass: selection.byClass,
+  };
+}
+
+/** A concern named on the command line that this project does not declare. */
+export class ConcernError extends Error {}
+
 /** Rebuild claim documents from a flat selection, keeping each claim's own faults together. */
 function claimsFromSelection(selected) {
   const byClaim = new Map();
@@ -128,6 +155,8 @@ export async function sweep({
   projectDir,
   ref,
   mode = 'changed',
+  concern: concernId,
+  concernsPath,
   includeDirty = false,
   exclude = [],
   cap,
@@ -154,10 +183,23 @@ export async function sweep({
   //
   // A test file with no claim is a different problem — it has nothing to
   // falsify — and the gate already names it. Sweep proposes against source.
-  const surface = mode === 'save-paths' ? saveSurface(projectDir) : null;
-  const targets = surface
-    ? surface.files.map((f) => f.file)
-    : gate.uncovered.filter((u) => u.kind === 'source').map((u) => u.file);
+  // `--save-paths` is sugar for the built-in SAVE-PERSISTS concern: the same
+  // targets and the same producer selection, named so a project can retune it
+  // for an ORM this tool does not recognise without forking anything.
+  const loaded = loadConcerns(projectDir, { path: concernsPath });
+  const concern = concernId
+    ? concernById(loaded.concerns, concernId)
+    : mode === 'save-paths' ? concernById(loaded.concerns, 'SAVE-PERSISTS') : null;
+  if (concernId && !concern) {
+    throw new ConcernError(`unknown concern ${concernId}. Declared: ${loaded.concerns.map((c) => c.id).join(', ')}`);
+  }
+  const changedTargets = gate.uncovered.filter((u) => u.kind === 'source').map((u) => u.file);
+  // A concern whose targets are the diff defers to the gate rather than
+  // re-deriving what it already computed.
+  const concernTargets = concern ? targetsFor(projectDir, concern, { changedFiles: changedTargets }) : null;
+  const usesSurface = concern?.targets?.kind === 'write-sites' || mode === 'save-paths';
+  const surface = usesSurface ? saveSurface(projectDir) : null;
+  const targets = concernTargets ?? changedTargets;
 
   const cPath = claimsPath ?? defaultClaimsPath(projectDir);
   const existingClaims = existsSync(cPath) ? loadClaims(cPath) : { claims: [] };
@@ -184,8 +226,27 @@ export async function sweep({
     for (const claim of result.doc.claims) for (const fault of claim.faults) candidates.push({ claim, fault });
   }
 
+  // What the defenders of this surface are CAPABLE of proving, before anything
+  // is probed. A test that replaced the database can prove the call shape and
+  // never that the row landed, and that limit is a property of the suite rather
+  // than of any verdict — so it is measured once, over the whole surface, and
+  // reported whether or not a single fault survives.
+  const provability = surface
+    ? provabilitySummary(projectDir, targets, (f) => discoverDefenders(projectDir, f))
+    : null;
+
+  // Order by what THIS project's own runs have shown, not only by what was
+  // measured elsewhere. A project that has probed nothing falls back to the
+  // shipped prior; one with its own records overrides it in proportion to how
+  // many it has. This is the feedback half of Google's 15%-to-89%, using data
+  // the project already produced rather than a new thing to collect.
+  const learned = learnedProductivity(projectDir);
+  // Narrowing is what makes one sentence useful across two hundred files: a
+  // persistence concern that also reported every altered return value would
+  // bury the payload findings it exists to surface.
+  const relevant = filterByConcern(candidates, concern);
   const limit = cap ?? capFor(targets.length);
-  const selection = selectFaults(candidates, { cap: limit });
+  const selection = selectFaults(relevant, { cap: limit, table: learned.table });
   const claims = claimsFromSelection(selection.selected);
 
   let evidence = { records: [] };
@@ -214,7 +275,10 @@ export async function sweep({
   // second scan; `line` and `find` live on the draft, not on the evidence.
   const draftOf = new Map(selection.selected.map(({ claim, fault }) => [`${claim.id}/${fault.id}`, fault]));
 
-  const signalsFor = (r) => persistenceSignals(projectDir, r.defenders?.resolved ?? []);
+  // The probe attaches the persistence signal to the record itself now
+  // (claimspec v1 evidence, `defenders.signals`); the finding repeats what the
+  // record says rather than re-deriving it, so the two can never disagree.
+  const signalsFor = (r) => (r.defenders?.signals ?? []).filter((s) => s.signal === 'persistence-payload-unasserted');
   const counts = {};
   const findings = [];
   for (const r of evidence.records) {
@@ -234,20 +298,21 @@ export async function sweep({
     includeDirty,
     scope: {
       mode,
+      ...(concern ? { concern: concern.id } : {}),
       changed: gate.changed,
-      ...(surface ? { writeSites: surface.siteCount, payloadFields: surface.keyCount } : {}),
+      ...(surface ? { writeSites: surface.siteCount, payloadFields: surface.keyCount, provability } : {}),
       targets: targets.length,
       swept: drafts.length,
       ...(skipped.length ? { skipped } : {}),
     },
-    selection: {
-      proposed: candidates.length,
-      selected: selection.selected.length,
-      deferred: selection.deferred.length,
-      cap: selection.cap,
-      byClass: selection.byClass,
-    },
+    selection: summarizeSelection(selection, { proposed: relevant.length, outOfScope: candidates.length - relevant.length }),
     counts,
+    ordering: { observed: learned.observed, sources: learned.sources },
+    // Carried, not written here: the caller owns I/O. Persisting it is what
+    // closes the feedback loop — the next sweep learns from these verdicts,
+    // which are MACHINE-proposed faults and so the closest observation of the
+    // distribution the ranker orders.
+    evidence,
     findings: ordered,
     exitCode: gating > 0 ? 1 : 0,
     drafts,
@@ -262,16 +327,47 @@ export function renderSweep(doc, { limit = 20 } = {}) {
   if (scope.targets === 0) {
     out.push(saves
       ? 'sweep: no write to storage found in this project. Nothing to propose.'
-      : `sweep: no changed source file is without a claim against ${doc.ref}. Nothing to propose.`);
+      : scope.concern
+        ? `sweep: concern ${scope.concern} matches no file in this project. Nothing to propose.`
+        : `sweep: no changed source file is without a claim against ${doc.ref}. Nothing to propose.`);
     return out.join('\n');
   }
   // The denominator first. A report that lists findings without saying how much
   // was looked at invites the reader to assume the rest is fine.
+  // Say how the sweep was aimed. A glob concern that reports "unclaimed changed
+  // files" is describing a diff it never looked at.
   out.push(saves
     ? `${scope.writeSites} write${scope.writeSites === 1 ? '' : 's'} to storage across ${scope.targets} file${scope.targets === 1 ? '' : 's'}, carrying ${scope.payloadFields} payload field${scope.payloadFields === 1 ? '' : 's'}. Swept ${scope.swept}.`
-    : `swept ${scope.swept} of ${scope.targets} unclaimed changed file${scope.targets === 1 ? '' : 's'} against ${doc.ref}.`);
+    : scope.concern
+      ? `concern ${scope.concern} matches ${scope.targets} file${scope.targets === 1 ? '' : 's'}. Swept ${scope.swept}.`
+      : `swept ${scope.swept} of ${scope.targets} unclaimed changed file${scope.targets === 1 ? '' : 's'} against ${doc.ref}.`);
+  if (saves && scope.provability) {
+    // The limit, before any verdict. A test that replaced the database can
+    // prove the call shape and never that the row landed, so on a surface
+    // defended entirely by such tests a clean probe is not evidence of
+    // persistence — it is evidence that nothing could have measured it.
+    const { mocked, unmocked, none } = scope.provability;
+    out.push(`  of those ${scope.targets} file${scope.targets === 1 ? '' : 's'}: ${mocked} defended only by tests that mock the persistence layer, ${unmocked} with an unmocked defender, ${none} with no defender at all.`);
+    if (unmocked === 0 && mocked > 0) {
+      out.push('  Nothing in this suite can prove a write reached storage. A mocked test proves the');
+      out.push('  call shape; a where-clause that matches nothing, a rolled-back transaction and a');
+      out.push('  rejected constraint all pass against it.');
+    }
+  }
   out.push(`proposed ${selection.proposed} fault${selection.proposed === 1 ? '' : 's'}, probed ${selection.selected} (cap ${selection.cap})${selection.deferred ? `, deferred ${selection.deferred}` : ''}.`);
-  if (selection.deferred) out.push(`  The ${selection.deferred} deferred are not a verdict: raise --cap${saves ? '.' : ', or sweep a smaller change.'}`);
+  if (selection.deferred) out.push(`  The ${selection.deferred} deferred are not a verdict: raise --cap${saves || scope.concern ? '.' : ', or sweep a smaller change.'}`);
+  // A narrow concern must not look like a quiet one: the producers found these,
+  // and this concern is not about them.
+  if (selection.outOfScope) out.push(`  ${selection.outOfScope} further proposal${selection.outOfScope === 1 ? ' was' : 's were'} outside this concern's fault classes.`);
+  // The arid class this tool has. Set aside and said so: a count nobody prints
+  // is a proposal nobody made.
+  if (selection.presentational) out.push(`  ${selection.presentational} presentational element${selection.presentational === 1 ? '' : 's'} (icons, static wrappers) set aside: nobody writes a test for those, and a survivor there would teach the ordering the wrong lesson.`);
+  // An ordering nobody can trace is a number nobody should trust.
+  if (doc.ordering) {
+    out.push(doc.ordering.observed === 0
+      ? '  ordering: the shipped productivity prior — this project has no probed evidence yet.'
+      : `  ordering: ${doc.ordering.observed} probed fault${doc.ordering.observed === 1 ? '' : 's'} of this project's own (${doc.ordering.sources.join(', ')}), shrunk toward the shipped prior.`);
+  }
   for (const s of scope.skipped ?? []) out.push(`  skipped ${s.file}: ${s.reason}`);
   out.push('');
 

package/src/util/glob.mjs

@@ -11,7 +11,11 @@ import { join } from 'node:path';
 // the application — including its writes and its tests — so a surface scan of a
 // real app counted the same code twice and a defender glob could match a
 // compiled chunk. The promise was right and the list was short.
-const SKIP_DIRS = new Set(['node_modules', '.git', 'dist', 'coverage', '.testguard',
+// `.claude` holds agent tooling and, under `.claude/worktrees/`, whole nested
+// checkouts of OTHER branches. Measured: discovery on a real repository returned
+// another branch's test file as a defender — a file the scratch tree cannot
+// even contain, because the directory is gitignored.
+const SKIP_DIRS = new Set(['node_modules', '.git', 'dist', 'coverage', '.testguard', '.claude',
   '.next', '.nuxt', '.svelte-kit', '.output', '.turbo', '.parcel-cache', '.angular', '.astro',
   'build', 'out', '.cache', '.vercel', '.netlify',
   '__pycache__', '.venv', 'venv', 'site-packages', '.tox', '.nox',