testguard-cli 0.6.0 → 0.8.0

package/src/probe/probe.mjs

@@ -1,22 +1,27 @@
 import { existsSync, readFileSync, realpathSync as fsRealpathSync } from 'node:fs';
 import { git } from '../git.mjs';
 import { createRequire } from 'node:module';
-import { isAbsolute, join, relative, resolve } from 'node:path';
+import { isAbsolute, join, relative, resolve, sep } from 'node:path';
 import { repoRoot as gitRoot, headSha, isDirty, snapshotWorkingTree } from '../git.mjs';
 import { discoverDefenders, discoverDefendersDetailed } from './discover.mjs';
 import { classifyDefenders } from './mocks.mjs';
 import { detectContention, contentionWarning } from './contention.mjs';
 import { createScratch, inPlace, PreconditionError } from './worktree.mjs';
-import { applyFault, locate } from './inject.mjs';
+import { applyContent, applyFault, locate } from './inject.mjs';
 import { resolveDefenders, parseCommandTemplate } from './runners/shared.mjs';
-import { selectRunner, RUNNERS, OWNED_RUNNERS, partitionByRunner, mergeRuns } from './runners/index.mjs';
+import { selectRunner, RUNNERS, OWNED_RUNNERS, partitionByRunner, mergeRuns, runnerLabel, runnerFor } from './runners/index.mjs';
 import { classify, shouldStopEarly } from './classify.mjs';
 import { blastRadius, rank } from './rank.mjs';
 import { classifyIndependence } from './independence.mjs';
+import { escalationStart, foldEscalationRun, escalationResult, flakeRate, killersFromRuns, subjectOf, isReusable } from './attribution.mjs';
 import { hashFile, sha256 } from '../util/hash.mjs';
 import { fingerprint } from '../../spec/lib/fingerprint.mjs';
 
-const isKill = (r) => r.outcome === 'fail' && r.assertionFailures > 0;
+
+/** Is `child` the same file as `root`, or under it? Both must already be real paths. */
+function isInside(root, child) {
+  return child === root || child.startsWith(root.endsWith(sep) ? root : root + sep);
+}
 
 /** realpath when the path still exists; the path itself otherwise. Never throws. */
 function realpathSync(p) {
@@ -53,6 +58,7 @@ export async function probe({
   runnerCommand,
   runnerName = 'auto',
   nodeModules,
+  python,
   only,
   includeDirty = false,
   onStage = () => {},
@@ -125,12 +131,17 @@ export async function probe({
   let runnerVersion;
   let runnerSource;
   let runner = RUNNERS[runnerName === 'auto' ? 'vitest' : runnerName];
+  // A runner with more than one engine (Python: pytest or unittest) is recorded
+  // by the engine that actually ran, never by the module's name. TestGuard does
+  // not say pytest ran when the stdlib runner did.
+  const engines = new Map();
+  const labelOf = (r) => runnerLabel(r, engines);
   const runnersUsed = new Map(); // every runner that ran defenders, beyond the project runner
   try {
     const commandTemplate = runnerCommand ? parseCommandTemplate(runnerCommand) : undefined;
     if (!commandTemplate) {
       // An unresolvable runner is a precondition failure, not a flaky defender.
-      const sel = await selectRunner({ projectDir: iso.projectDir, name: runnerName });
+      const sel = await selectRunner({ projectDir: iso.projectDir, sourceDir: projectDir, python, name: runnerName });
       if (sel.error) {
         throw new PreconditionError(`test runner is not resolvable in the ${mode === 'worktree' ? 'scratch worktree' : 'project'} (${sel.error}). ` +
           (mode === 'worktree' ? 'No usable node_modules was linked: pass --node-modules <path>, or run with --in-place.' : 'Install dependencies first.'));
@@ -138,30 +149,40 @@ export async function probe({
       runner = sel.runner;
       runnerVersion = sel.version;
       runnerSource = sel.source;
+      if (sel.engine) engines.set(sel.runner, sel.engine);
     }
-    runnersUsed.set(runner.name, runnerVersion ?? readRunnerVersion(projectDir, runner.name));
+    runnersUsed.set(labelOf(runner), runnerVersion ?? readRunnerVersion(projectDir, runner.name));
     // Files an owning runner (Playwright) claims run under it, whatever the
     // project runner is; it must resolve before the first such defender runs.
     const owned = OWNED_RUNNERS.filter((r) => r !== runner);
     const ownedChecked = new Map();
     const ensureOwned = async (r, file) => {
-      if (!ownedChecked.has(r)) ownedChecked.set(r, await r.check({ projectDir: iso.projectDir }));
+      if (!ownedChecked.has(r)) ownedChecked.set(r, await r.check({ projectDir: iso.projectDir, sourceDir: projectDir, python }));
       const c = ownedChecked.get(r);
-      if (!c.ok) throw new PreconditionError(`${file} is a ${r.name} test (it lives under ${r.name}'s testDir) but ${r.name} is not resolvable in the ${mode === 'worktree' ? 'scratch worktree' : 'project'} (${c.message}).`);
-      runnersUsed.set(r.name, c.version);
+      if (!c.ok) throw new PreconditionError(`${file} is a ${r.name} test (${r.name === 'python' ? 'it is a .py file' : `it lives under ${r.name}'s testDir`}) but ${r.name} is not resolvable for the ${mode === 'worktree' ? 'scratch worktree' : 'project'} (${c.message}).`);
+      if (c.engine) engines.set(r, c.engine);
+      runnersUsed.set(labelOf(r), c.version);
     };
     const allTests = [...new Set([...runner.tests(iso.projectDir).filter((t) => !owned.some((r) => r.owns(iso.projectDir, t))), ...owned.flatMap((r) => r.tests(iso.projectDir))])].sort();
     const baselineCache = new Map();
+    // The negative control is charged per (target file, defender set): two
+    // claims over the same file with the same defenders ask the same question.
+    const controlCache = new Map();
+    // The runner that will LOAD the subject, which is not always the project
+    // runner — a `.py` source file is Python's whatever the project runs.
+    const fatalEditFor = (file) => runnerFor(iso.projectDir, file, runner).fatalEdit?.();
     const prior = previous && previous.run.confirmRuns === confirmRuns
       ? new Map(previous.records.map((r) => [`${r.claim.id}/${r.subject.id}`, r]))
       : new Map();
-    const runDefenders = async (files) => {
-      if (commandTemplate) return runner.run({ projectDir: iso.projectDir, files, budgetMs, commandTemplate, serial });
+    // `targets` are the fault's files, passed so a runner that can tell which
+    // file the interpreter actually loaded (Python) reports it back.
+    const runDefenders = async (files, targets = []) => {
+      if (commandTemplate) return runner.run({ projectDir: iso.projectDir, sourceDir: projectDir, python, files, budgetMs, commandTemplate, serial, targets });
       const groups = partitionByRunner(iso.projectDir, files, runner);
       const parts = [];
       for (const [r, group] of groups) {
         if (r !== runner) await ensureOwned(r, group[0]);
-        parts.push(await r.run({ projectDir: iso.projectDir, files: group, budgetMs, serial }));
+        parts.push(await r.run({ projectDir: iso.projectDir, sourceDir: projectDir, python, files: group, budgetMs, serial, targets }));
       }
       return mergeRuns(parts);
     };
@@ -169,7 +190,7 @@ export async function probe({
     const byRunner = (files) => {
       const groups = partitionByRunner(iso.projectDir, files, runner);
       if (groups.size <= 1) return undefined; // one runner ran them all, whichever it was
-      return Object.fromEntries([...groups].map(([r, group]) => [r.name, group]));
+      return Object.fromEntries([...groups].map(([r, group]) => [labelOf(r), group]));
     };
 
     for (const claim of claims.claims) {
@@ -183,14 +204,26 @@ export async function probe({
         const mockInfo = declared ? classifyDefenders(iso.projectDir, fault.file, declared) : discoverDefendersDetailed(iso.projectDir, fault.file);
         const defenders = declared ?? mockInfo.canDetect;
         const stage = (name, i, n) => onStage({ claimId: claim.id, faultId: fault.id, stage: name, i, n });
-        const record = await probeOne({ claim, fault, defenders, discovered: declared === null, allTests, iso, confirmRuns, escalate, baselineCache, runDefenders, stage, prior: prior.get(`${claim.id}/${fault.id}`), priorRunId: previous?.run.id, byRunner: byRunner(defenders), mocking: mockInfo.mocking, signals: mockInfo.signals,
-          historyRef: snapshot ?? (mode === 'worktree' ? head : 'HEAD'), historyDir: root,
-          // A path from the runner is absolute inside the SCRATCH worktree, or
-          // project-relative. Either way history is read from the real repository,
-          // so both must land on a repo-root-relative path. The runner reports
-          // realpaths (/private/var on macOS) while the worktree path may not be
-          // one — realpath both sides or every comparison silently misses.
-          toRepoPath: (p) => relative(root, join(projectDir, isAbsolute(p) ? relative(isoReal, realpathSync(p)) : p)) });
+        const common = { claim, fault, defenders, discovered: declared === null, byRunner: byRunner(defenders), mocking: mockInfo.mocking, signals: mockInfo.signals };
+        let record;
+        try {
+          record = await probeOne({ ...common, allTests, iso, isoReal, onWarn, confirmRuns, escalate, baselineCache, controlCache, fatalEditFor, runDefenders, stage, prior: prior.get(`${claim.id}/${fault.id}`), priorRunId: previous?.run.id,
+            historyRef: snapshot ?? (mode === 'worktree' ? head : 'HEAD'), historyDir: root,
+            // A path from the runner is absolute inside the SCRATCH worktree, or
+            // project-relative. Either way history is read from the real repository,
+            // so both must land on a repo-root-relative path. The runner reports
+            // realpaths (/private/var on macOS) while the worktree path may not be
+            // one — realpath both sides or every comparison silently misses.
+            toRepoPath: (p) => relative(root, join(projectDir, isAbsolute(p) ? relative(isoReal, realpathSync(p)) : p)) });
+        } catch (err) {
+          // A precondition failure is a statement about the whole run — the
+          // interpreter is loading a different copy of the source, the runner
+          // is not resolvable — and continuing would produce verdicts that are
+          // all false. Everything else is this one fault's problem.
+          if (err instanceof PreconditionError) throw err;
+          record = errorRecord({ ...common, error: err, inputs: safeInputs(iso.projectDir, fault, defenders) });
+          onWarn(`${claim.id}/${fault.id}: ${record.detail.message} — reported as unverifiable so the rest of the run still produces evidence.`);
+        }
         records.push(record);
         onProgress(record);
       }
@@ -207,19 +240,111 @@ export async function probe({
       startedAt,
       finishedAt: new Date().toISOString(),
       repo: { head, dirty: isDirty(root), ...(snapshot ? { snapshot } : {}), ...(ignoredDirty.length ? { ignoredDirty } : {}) },
-      runner: { name: runner.name, ...((runnerVersion ?? readRunnerVersion(projectDir, runner.name)) ? { version: runnerVersion ?? readRunnerVersion(projectDir, runner.name) } : {}) },
+      runner: { name: labelOf(runner), ...((runnerVersion ?? readRunnerVersion(projectDir, runner.name)) ? { version: runnerVersion ?? readRunnerVersion(projectDir, runner.name) } : {}) },
       ...(runnersUsed.size > 1 ? { runners: [...runnersUsed].map(([n, v]) => ({ name: n, ...(v ? { version: v } : {}) })) } : {}),
       confirmRuns,
       ...(serial ? { serial: true } : {}),
       ...(contention.detected ? { contention } : {}),
       ...(confirmRuns < 3 ? { provisional: true } : {}),
-      mode,
+      // The isolation that was actually created, not the mode that was asked
+      // for. A record saying `worktree` while the probe edited the project in
+      // place would be a false statement about where the evidence came from,
+      // and nothing downstream could detect it.
+      mode: iso.mode,
     },
     records,
   };
 }
 
-async function probeOne({ claim, fault, defenders, discovered, allTests, iso, confirmRuns, escalate, baselineCache, runDefenders, stage, prior, priorRunId, byRunner, mocking = [], signals = [], historyRef, historyDir, toRepoPath }) {
+/**
+ * The record a fault gets when probing it threw.
+ *
+ * Before this existed, an unexpected error anywhere inside one fault's probe
+ * killed the process: no evidence file at all, every verdict already decided
+ * thrown away, and an exit code no caller can read against GATE-SEMANTICS. One
+ * bad fault should cost that fault and nothing else.
+ *
+ * `unverifiable` is the honest verdict — the claim could not be probed — and it
+ * gates, so the failure is loud rather than absorbed into a green run. Pure, so
+ * every branch is testable without provoking a crash.
+ */
+export function errorRecord({ claim, fault, defenders, discovered, byRunner, mocking = [], signals = [], error, inputs }) {
+  const message = String(error?.message ?? error).split('\n')[0].slice(0, 1024);
+  return {
+    fingerprint: fingerprint({ claimId: claim.id, subjectId: fault.id, file: fault.file, verdict: 'unverifiable' }),
+    claim: { id: claim.id, statement: claim.statement, severity: claim.severity, source: claim.source, producedBy: claim.producedBy },
+    subject: subjectOf(fault, sha256),
+    verdict: 'unverifiable',
+    detail: { baselineRuns: [], probeRuns: [], reason: 'probe-error', message },
+    defenders: { requested: claim.defendedBy ?? [], resolved: defenders, nocover: defenders.length === 0, ...(discovered ? { discovered: true } : {}), ...(byRunner ? { byRunner } : {}), ...(mocking.length ? { mocking } : {}), ...(signals.length ? { signals } : {}) },
+    inputs,
+  };
+}
+
+/**
+ * `inputs` for a record built after something threw — which is exactly when a
+ * file may no longer be readable. Hashing must not be the second failure, so an
+ * unreadable file hashes as empty: never equal to a real file's hash, so the
+ * record can only ever fail a reuse check, never pass one by accident.
+ */
+function safeInputs(projectDir, fault, defenders) {
+  const h = (f) => {
+    try {
+      return hashFile(join(projectDir, f));
+    } catch {
+      return sha256('');
+    }
+  };
+  return { targetHash: h(fault.file), defenderHashes: Object.fromEntries(defenders.map((f) => [f, h(f)])) };
+}
+
+/**
+ * The negative control the green baseline cannot provide.
+ *
+ * A green baseline proves the defenders can PASS on unmodified source, which
+ * makes "the harness is broken, so everything looks killed" structurally
+ * impossible. It says nothing about the opposite direction: if the fault is
+ * applied to code the test process never executes, the baseline is green,
+ * every probe run is green, and every claim is reported SURVIVED. The output
+ * reads as a devastating audit finding and is entirely false, and nothing in
+ * the run contradicts it, because every individual check passed.
+ *
+ * So ask the question directly: replace the subject with something its loader
+ * cannot parse, and see whether the defenders go red. If they stay green they
+ * are not executing that file, and every verdict about it is meaningless.
+ *
+ * The predicate is "the run did not stay green", not "a test failed" — an
+ * unparseable module usually fails to LOAD, so the runner reports zero tests
+ * rather than a failure. Measured, not assumed.
+ *
+ * One run, not N: the defenders were already green N/N on this exact set, and
+ * neither answer here is the optimistic one — `survived` and `unverifiable`
+ * both gate.
+ */
+async function negativeControl({ file, defenders, iso, runDefenders, fatalEditFor, cache, stage }) {
+  const key = `${file}\n${defenders.join('\n')}`;
+  if (cache.has(key)) return cache.get(key);
+  const content = fatalEditFor(file);
+  // A runner that cannot say what "this cannot compile" looks like for its
+  // language does not get to guess. No control, no signal, verdict unchanged.
+  if (content == null) {
+    cache.set(key, undefined);
+    return undefined;
+  }
+  const control = applyContent(iso.projectDir, file, content, { inPlace: iso.mode === 'in-place' });
+  let run;
+  try {
+    stage('negative-control', 1, 1);
+    ({ run } = await runDefenders(defenders, [file]));
+  } finally {
+    control.restore();
+  }
+  const reached = run.outcome !== 'pass';
+  cache.set(key, reached);
+  return reached;
+}
+
+async function probeOne({ claim, fault, defenders, discovered, allTests, iso, isoReal, onWarn = () => {}, confirmRuns, escalate, baselineCache, controlCache, fatalEditFor, runDefenders, stage, prior, priorRunId, byRunner, mocking = [], signals = [], historyRef, historyDir, toRepoPath }) {
   const targetPath = join(iso.projectDir, fault.file);
   const targetExists = existsSync(targetPath);
   const inputs = {
@@ -227,14 +352,17 @@ async function probeOne({ claim, fault, defenders, discovered, allTests, iso, co
     defenderHashes: Object.fromEntries(defenders.map((f) => [f, hashFile(join(iso.projectDir, f))])),
   };
 
-  // Same source, same defenders, same N: the verdict cannot have changed.
-  if (prior && sameInputs(prior, inputs, claim.defendedBy ?? [], defenders)) {
+  const subject = subjectOf(fault, sha256);
+
+  // Same source, same defenders, same N, same fault: the verdict cannot have changed.
+  if (isReusable(prior, { inputs, requested: claim.defendedBy ?? [], resolved: defenders, contentHash: subject.contentHash })) {
     return { ...prior, reusedFrom: prior.reusedFrom ?? priorRunId };
   }
 
   const detail = { baselineRuns: [], probeRuns: [] };
   const rawProbeRuns = []; // spec testRun + the runner's timeout count, which classify needs
   let anchor = null;
+  let subjectReached; // the negative control's answer; undefined until it is worth asking
 
   if (defenders.length > 0) {
     anchor = targetExists ? locate(readFileSync(targetPath, 'utf8'), fault) : { status: 'file-missing', hits: 0, expected: fault.expectHits ?? 1 };
@@ -272,8 +400,8 @@ async function probeOne({ claim, fault, defenders, discovered, allTests, iso, co
       // the tests running and disagreeing with themselves. A load error or a
       // timeout is not flakiness — nothing was measured about stability — so
       // neither is counted, on either side of the ratio.
-      const measured = baseline.runs.filter((r) => r.outcome === 'pass' || r.outcome === 'fail');
-      if (measured.length) detail.flakeRate = { runs: measured.length, failures: measured.filter((r) => r.outcome === 'fail').length };
+      const rate = flakeRate(baseline.runs);
+      if (rate) detail.flakeRate = rate;
       if (baseline.runs.some((r) => r.outcome === 'error')) {
         // The defenders did not load. That is not flakiness and not a verdict
         // about the fault; the claim cannot be probed until they do.
@@ -281,53 +409,65 @@ async function probeOne({ claim, fault, defenders, discovered, allTests, iso, co
       }
 
       if (detail.baselineRuns.every((r) => r.outcome === 'pass') && detail.baselineRuns.length === confirmRuns) {
-        const mutation = applyFault(iso.projectDir, fault);
+        const mutation = applyFault(iso.projectDir, fault, { inPlace: iso.mode === 'in-place' });
         try {
           const probeRuns = rawProbeRuns;
           for (let i = 0; i < confirmRuns; i++) {
             stage('probe', i + 1, confirmRuns);
-            const { run, timeouts, loadMessage, failedTests } = await runDefenders(defenders);
+            const { run, timeouts, loadMessage, failedTests, provenance } = await runDefenders(defenders, [fault.file]);
             probeRuns.push({ ...run, timeouts, loadMessage, failedTests });
+            // Only when the suite actually loaded. A run that failed to load
+            // explains an absent import by itself — saying the defenders never
+            // reached the file would be true and useless, and it would put the
+            // signal on every fault whose replacement does not parse.
+            if (i === 0 && run.outcome !== 'error') checkProvenance({ claim, fault, provenance, isoReal, detail, onWarn });
             if (shouldStopEarly(probeRuns)) break;
           }
           detail.probeRuns = probeRuns.map(({ timeouts, loadMessage, failedTests, ...run }) => run);
           const provisional = classify({ defenders, anchor, baselineRuns: detail.baselineRuns, probeRuns, confirmRuns });
+          // `provisional` is deliberately computed BEFORE the control: the control is what turns a survivor into an unverifiable, so asking it first would be circular.
+
+          // Charge the negative control only where a false answer is expensive:
+          // on a survivor. A kill already proves the defenders reached the code,
+          // and paying for it on every fault would be waste.
+          if (provisional.verdict === 'survived') {
+            subjectReached = await negativeControl({ file: fault.file, defenders, iso, runDefenders, fatalEditFor, cache: controlCache, stage });
+            if (subjectReached !== undefined) detail.negativeControl = subjectReached ? 'reached' : 'not-reached';
+            if (subjectReached === false) {
+              onWarn(`${claim.id}/${fault.id}: the defenders stayed green with ${fault.file} replaced by something that cannot compile, so they never execute it. Every verdict about this file would be about their reach, not their assertions — reported as unverifiable rather than as a survivor.`);
+            }
+          }
 
           // Escalation: does anything *undeclared* catch it? A single run cannot
           // say — a flaky test elsewhere in the suite would take the credit — so
           // a test is an undeclared killer only if it fails in all N runs.
+          // Skipped when the subject is not executed at all: nothing the wider
+          // suite does could be evidence about a file nobody loads.
           const broader = allTests.filter((t) => !defenders.includes(t));
-          if (provisional.verdict === 'survived' && escalate && broader.length > 0) {
-            const runs = [];
-            let killers = null;
-            let stoppedEarly;
+          if (provisional.verdict === 'survived' && subjectReached !== false && escalate && broader.length > 0) {
+            // The rule lives in attribution.mjs as a pure fold; this loop only
+            // decides when to stop asking the runner.
+            let state = escalationStart();
             for (let i = 0; i < confirmRuns; i++) {
               stage('escalation', i + 1, confirmRuns);
-              const { run, failedTests } = await runDefenders(allTests);
-              runs.push(run);
-              killers = killers === null ? new Set(failedTests) : new Set(failedTests.filter((t) => killers.has(t)));
-              // Attribution needs a test that failed in ALL N runs. Once the
-              // intersection is empty no further run can name a killer, so the
-              // remaining runs are wasted; stopping can only fail to NAME a
-              // killer, never upgrade a verdict, which is the pessimistic side.
-              if (killers.size === 0) { stoppedEarly = 'no-common-failure'; break; }
-            }
-            detail.escalated = true;
-            detail.escalationRuns = runs;
-            if (stoppedEarly) detail.escalationStoppedEarly = stoppedEarly;
-            if (killers.size > 0 && runs.length === confirmRuns && runs.every(isKill)) {
-              detail.reason = 'killed-by-undeclared-tests';
-              detail.undeclaredKillers = [...killers].sort();
+              state = foldEscalationRun(state, await runDefenders(allTests));
+              if (state.stoppedEarly) break;
             }
+            Object.assign(detail, escalationResult(state, confirmRuns));
           }
         } finally {
           mutation.restore();
+          // The restore found nothing to put back. In worktree mode that is a
+          // disturbed run, not a failed one (see applyFault) — but a reader
+          // has to be able to tell, because it means something outside this
+          // process was deleting the tree the verdicts came from.
+          if (mutation.restoreSkipped) detail.restoreSkipped = mutation.restoreSkipped;
         }
       }
     }
   }
 
-  const { verdict, reason } = classify({ defenders, anchor, baselineRuns: detail.baselineRuns, probeRuns: rawProbeRuns, confirmRuns });
+  const { verdict, reason } = classify({ defenders, anchor, baselineRuns: detail.baselineRuns, probeRuns: rawProbeRuns, confirmRuns, subjectReached });
   if (reason && !detail.reason) detail.reason = reason;
   if (anchor && anchor.status !== 'ok' && anchor.status !== 'file-missing' && anchor.status !== 'defenders-failed-to-load') detail.anchor = { hits: anchor.hits, expected: anchor.expected };
 
@@ -338,14 +478,14 @@ async function probeOne({ claim, fault, defenders, discovered, allTests, iso, co
   // already decided above, and ranking is the only consumer.
   if (verdict === 'killed' && historyRef) {
     // Only the tests that actually failed on the fault are its killers.
-    const killers = [...new Set(rawProbeRuns.flatMap((r) => r.failedTests ?? []).map((t) => t.split('::')[0]).filter(Boolean))];
+    const killers = killersFromRuns(rawProbeRuns);
     detail.independence = classifyIndependence({ dir: historyDir, ref: historyRef, targetFile: toRepoPath(fault.file), defenders: (killers.length ? killers : defenders).map(toRepoPath) });
   }
 
   return {
     fingerprint: fingerprint({ claimId: claim.id, subjectId: fault.id, file: fault.file, verdict }),
     claim: { id: claim.id, statement: claim.statement, severity: claim.severity, source: claim.source, producedBy: claim.producedBy },
-    subject: { kind: 'fault', id: fault.id, description: fault.description, file: fault.file, faultClass: fault.faultClass, producedBy: fault.producedBy, contentHash: sha256(`${fault.find}\n${fault.replace}`) },
+    subject,
     verdict,
     detail,
     defenders: { requested: claim.defendedBy ?? [], resolved: defenders, nocover: defenders.length === 0, ...(discovered ? { discovered: true } : {}), ...(byRunner ? { byRunner } : {}), ...(mocking.length ? { mocking } : {}), ...(signals.length ? { signals } : {}) },
@@ -354,6 +494,49 @@ async function probeOne({ claim, fault, defenders, discovered, allTests, iso, co
   };
 }
 
+/**
+ * The negative control the green baseline cannot provide.
+ *
+ * A clean baseline proves the harness is not reporting everything as broken —
+ * the failure mode where a misread runner flatters nothing. It says nothing
+ * about the opposite direction: a fault that the interpreter never executes.
+ * Then the baseline is green, every fault SURVIVES, and the report reads as a
+ * devastating audit finding while being entirely false.
+ *
+ * Node cannot reach that state from a scratch worktree, because Node loads
+ * files by path. Python can. `sys.path[0]` is the working directory, so an
+ * ordinary editable install — a `.pth` file appended to sys.path — loses to
+ * the tree being probed and is harmless. What wins is a **meta path finder**:
+ * a strict editable install registers one, and it is consulted before every
+ * path entry. Then the tests import the original file while TestGuard faults
+ * the copy, and the whole run is false in the one direction nothing else
+ * catches. Measured, not assumed: with the fault live the reporter is asked
+ * which file was actually loaded, and the answer is checked.
+ *
+ * Loaded from outside the tree being probed is never recoverable and never
+ * ambiguous: the whole run is refused. Never loaded at all is recorded and
+ * warned about but not refused — an import inside a branch the fault does not
+ * reach is legitimate, and refusing would be the tool substituting its
+ * judgement for the author's.
+ */
+export function checkProvenance({ claim, fault, provenance, isoReal, detail, onWarn }) {
+  if (!provenance || !(fault.file in provenance)) return;
+  const loaded = provenance[fault.file];
+  if (loaded === null) {
+    detail.targetNotImported = true;
+    onWarn(`${claim.id}/${fault.id}: the defenders never imported ${fault.file}, so nothing they do could detect a fault in it. Whatever the runs show is a statement about the defenders' reach, not about their assertions.`);
+    return;
+  }
+  if (isoReal && !isInside(isoReal, loaded)) {
+    throw new PreconditionError(
+      `${fault.file}: the fault was applied to ${join(isoReal, fault.file)}, but the interpreter imported ${loaded} instead. ` +
+      'Python loaded a different copy of this module, so nothing TestGuard changes can ever run and every claim would be reported as SURVIVED however good the tests are. ' +
+      'The usual cause is an install that registers an import hook ahead of sys.path — a strict editable install (`pip install -e . --config-settings editable_mode=strict`, and some build backends by default) — or a plain install that left a copy in site-packages. ' +
+      'Either reinstall the project against the tree being probed, or run with --in-place so the tree the install points at is the tree that is faulted.',
+    );
+  }
+}
+
 function sameInputs(prior, inputs, requested, resolved) {
   const same = (a, b) => JSON.stringify(a) === JSON.stringify(b);
   return prior.inputs.targetHash === inputs.targetHash

package/src/probe/progress.mjs

@@ -0,0 +1,87 @@
+/**
+ * Progress, for readers that are not a terminal (issue #68).
+ *
+ * The stage line used to be written only when `process.stderr.isTTY`, with a
+ * carriage return and an erase-line so it rewrote itself in place. That is the
+ * right shape for a human at a terminal and the wrong shape everywhere else,
+ * and "everywhere else" is where a probe actually runs: CI, a redirected log,
+ * an agent harness. In all of those the tool printed **nothing** for twenty
+ * minutes, which is indistinguishable from being hung — and a gate you cannot
+ * tell from a hang is a gate people start killing.
+ *
+ * So the rewriting is a *rendering* choice, not a reason to withhold the
+ * information:
+ *
+ *   tty     `\r`-rewritten single line — unchanged, for a human at a terminal
+ *   plain   one line per stage, append-only — a redirected log, CI, an agent
+ *   ndjson  one JSON object per line — for a machine that parses progress
+ *   none    silence, for `--quiet` and `--json`
+ *
+ * `auto` picks `tty` when stderr is a terminal and `plain` when it is not.
+ * Progress always goes to **stderr**, so `--json` on stdout stays parseable.
+ */
+
+/** Which rendering to use. `explicit` is `--progress`; it always wins. */
+export function progressMode({ explicit, isTTY = false, quiet = false, json = false } = {}) {
+  if (explicit && explicit !== 'auto') return explicit;
+  // A silent mode is a choice about output, and an explicit --progress
+  // overrides it above; without one, --quiet and --json mean silence.
+  if (quiet || json) return 'none';
+  return isTTY ? 'tty' : 'plain';
+}
+
+/**
+ * The stream progress is written to. **Always stderr, never stdout.**
+ *
+ * stdout carries the result: with `--json` it is one document and nothing
+ * else, and every consumer pipes it somewhere that parses it. A progress line
+ * on stdout does not degrade that output, it destroys it. The choice is a
+ * function so it can be falsified without running a probe.
+ */
+export const progressStream = (streams) => streams.stderr;
+
+export const PROGRESS_MODES = ['auto', 'tty', 'plain', 'ndjson', 'none'];
+export const isProgressMode = (m) => PROGRESS_MODES.includes(m);
+
+/**
+ * A stage reporter for the chosen mode, or `undefined` for `none` — the probe
+ * takes `undefined` to mean "do not report", so no mode needs a no-op branch
+ * inside the hot loop.
+ *
+ * `plain` prints every stage line, including each run of an N-run phase. That
+ * is the point: the thing a reader of a twenty-minute log wants to know is
+ * that run 2 of 3 started, which is exactly what the TTY line was already
+ * saying and throwing away.
+ */
+export function stageReporter(mode, write) {
+  if (mode === 'none') return undefined;
+  if (mode === 'ndjson') {
+    return ({ claimId, faultId, stage, i, n }) => write(JSON.stringify({ event: 'stage', claim: claimId, fault: faultId, stage, run: i, of: n, at: new Date().toISOString() }) + '\n');
+  }
+  if (mode === 'plain') {
+    return ({ claimId, faultId, stage, i, n }) => write(`  … ${claimId}/${faultId} ${stage} ${i}/${n}\n`);
+  }
+  return ({ claimId, faultId, stage, i, n }) => write(`\r\x1b[K  … ${claimId}/${faultId} ${stage} ${i}/${n}`);
+}
+
+/**
+ * What to write before a verdict line, so a rewritten TTY stage line does not
+ * end up with a verdict appended to its tail. Only `tty` needs it; every other
+ * mode is append-only and already on its own line.
+ */
+export const clearStageLine = (mode) => (mode === 'tty' ? '\r\x1b[K' : '');
+
+/**
+ * A per-record line for `ndjson`, so a machine watching the stream sees
+ * verdicts as they land rather than only at the end.
+ */
+export const recordEvent = (r) => JSON.stringify({
+  event: 'verdict',
+  claim: r.claim.id,
+  fault: r.subject.id,
+  verdict: r.verdict,
+  severity: r.claim.severity,
+  file: r.subject.file,
+  ...(r.reusedFrom ? { reused: true } : {}),
+  at: new Date().toISOString(),
+}) + '\n';