testguard-cli 0.2.0 → 0.2.1

package/CHANGELOG.md

@@ -7,6 +7,20 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [0.2.1] - 2026-09-17
+
+### Added
+
+- **Provisional verdicts** (#5). A run with `--confirm` below 3 declares
+  `run.provisional: true` — a spec field whose equivalence with
+  `confirmRuns < 3` the validator enforces both ways. Provisional verdicts
+  print with a `?`, the summary is prefixed `PROVISIONAL`, a warning is
+  printed on stderr, evidence is written to `.testguard/evidence-provisional.json`
+  so the canonical file only ever holds confirmed runs, `brief` warns at the
+  top, and `baseline` refuses provisional evidence unless
+  `--allow-provisional`. A confirmed prior verdict is never reused by a
+  provisional run, nor the reverse.
+
 ## [0.2.0] - 2026-09-17
 
 ### Added

package/README.md

@@ -52,7 +52,7 @@ tests were written against the survivors, 39/39 were killed.
 | npm | `npm i -D testguard-cli` then `npx testguard probe` |
 | pip | `pip install testguard-cli` then `testguard probe` (needs Node ≥ 20) |
 | Homebrew | `brew tap raccioly/tap && brew install testguard` |
-| GitHub Action | `uses: raccioly/testguard@v0.2.0` — see [`action.yml`](./action.yml) |
+| GitHub Action | `uses: raccioly/testguard@v0.2.1` — see [`action.yml`](./action.yml) |
 | pre-commit | `repo: https://github.com/raccioly/testguard`, hooks `testguard-claims`, `testguard-probe` |
 
 Projects that set `min-release-age` in `.npmrc` cannot see a version published
@@ -112,7 +112,10 @@ npx testguard-cli scaffold src/x.ts   # propose faults for a file, as a draft to
    Practical loop: first pass `--no-escalate` (escalation re-runs the whole
    suite N times per survivor); iterate on one claim with `--claim <ID>` and
    either `--include-dirty` or `--in-place` (only fault target files must be
-   clean there; test files may be dirty); final pass with defaults. By default
+   clean there; test files may be dirty), optionally `--confirm 1` for a fast
+   **provisional** signal — verdicts print with a `?`, evidence goes to
+   `evidence-provisional.json`, and `baseline` refuses it; final pass with
+   defaults. By default
    the stream shows only unproven faults plus a killed count — `--verbose`
    shows every fault. A custom
    runner (`pnpm --filter`, a specific config) goes in

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "testguard-cli",
-  "version": "0.2.0",
+  "version": "0.2.1",
   "description": "Proves a test suite defends the claims a project makes: injects the faults those claims forbid and reports every one the tests fail to detect.",
   "type": "module",
   "license": "MIT",

package/spec/GATE-SEMANTICS.md

@@ -22,7 +22,11 @@ Rules that follow from the table:
 1. **Green baseline first.** Defenders run N times unmodified before any fault
    is applied. Anything short of N/N pass is `flaky-defender` and stops there.
 2. **Confirm over N runs.** `killed` and `survived` both require exactly N
-   probe runs, all agreeing. Default N is 3.
+   probe runs, all agreeing. Default N is 3. **Fewer than three runs is
+   provisional**: the evidence declares `run.provisional: true`, every
+   rendering marks the verdicts as unconfirmed, and a provisional run is
+   never frozen into a baseline. Provisional runs exist for the fix loop —
+   a fast signal while writing a test — not for a gate.
 3. **Only a test body rejecting the behaviour kills.** A test that fails by
    assertion — or by an exception the fault provoked inside it — counts. A
    timeout does not, and a suite that fails to load does not: neither is

package/spec/lib/validate.mjs

@@ -45,6 +45,8 @@ const semantic = {
   evidence(doc) {
     const errors = [];
     const n = doc.run.confirmRuns;
+    if (n < 3 && doc.run.provisional !== true) errors.push({ path: '/run/provisional', message: `confirmRuns ${n} is below 3; the run must declare provisional: true` });
+    if (n >= 3 && doc.run.provisional === true) errors.push({ path: '/run/provisional', message: `confirmRuns ${n} is confirmed; provisional must be absent or false` });
     doc.records.forEach((r, i) => {
       const p = `/records/${i}`;
       const expected = fingerprint({ claimId: r.claim.id, subjectId: r.subject.id, file: r.subject.file ?? '', verdict: r.verdict });

package/spec/schemas/evidence.schema.json

@@ -43,6 +43,10 @@
           "type": "integer",
           "minimum": 1
         },
+        "provisional": {
+          "description": "True iff confirmRuns < 3. A provisional run's verdicts are not confirmed: they must not be frozen into a baseline and must be rendered as provisional wherever they appear. Validators enforce the equivalence in both directions.",
+          "type": "boolean"
+        },
         "mode": {
           "description": "`worktree` = faults were applied in a scratch git worktree (default). `in-place` = the user's working tree was mutated and restored.",
           "enum": ["worktree", "in-place"]

package/src/brief/brief.mjs

@@ -55,7 +55,7 @@ export function buildBrief(evidence, baseline, { max = 20, generatedAt = new Dat
     baselined: g.baselined.length,
   };
   const doc = { schemaVersion: 1, tool: evidence.tool, generatedAt, head: evidence.run.repo.head, heading: HEADING, summary, items, text: '' };
-  doc.text = renderBriefText(doc, { hasBaseline: Boolean(baseline), total: evidence.records.length });
+  doc.text = renderBriefText({ ...doc, provisional: Boolean(evidence.run.provisional) }, { hasBaseline: Boolean(baseline), total: evidence.records.length });
   return doc;
 }
 
@@ -64,6 +64,7 @@ export function renderBriefText(brief, { hasBaseline, total }) {
   const lines = [
     brief.heading,
     '',
+    ...(brief.provisional ? ['**PROVISIONAL** — this evidence came from fewer than three confirmation runs; treat every verdict below as unconfirmed and re-probe with --confirm 3 before acting on it.', ''] : []),
     `testguard ${brief.tool.version}${brief.head ? ` @ ${brief.head.slice(0, 12)}` : ''} — ${brief.summary.claims} claims, ${total} faults probed, ${unproven} unproven` +
       (hasBaseline ? ` (${brief.summary.new} new since baseline).` : ' (no baseline; everything is new).'),
   ];

package/src/cli.mjs

@@ -33,6 +33,7 @@ probe
   --claim <ID,ID>      probe only these claims; writes .testguard/evidence-partial.json
   --include-dirty      probe the working tree (a snapshot commit) instead of HEAD; uncommitted tests count
   --verbose            also print each killed fault (default: only unproven ones, plus a count)
+                       --confirm below 3 is PROVISIONAL: verdicts print with "?", evidence goes to .testguard/evidence-provisional.json
   --runner-cmd "<cmd>" custom runner; must contain {files} and {out}, e.g. "pnpm vitest run {files} --reporter=json --outputFile={out}"
   --node-modules <dir> node_modules to link into the scratch worktree (or TESTGUARD_NODE_MODULES)
   --in-place           mutate the working tree instead of a scratch worktree
@@ -44,7 +45,7 @@ scaffold   --claim <ID> (put every proposal under this claim; copies it if it ex
            shapes: if-guard → if (false) · single-line guard/mutation removed · return <check> → return true
                    · security flag/window/cost literal weakened · verify/validate/check call removed
 claims     --json
-baseline   --evidence <path>  --out <path>
+baseline   --evidence <path>  --out <path>  --allow-provisional (freeze unconfirmed evidence; normally refused)
 brief      --evidence <path>  --baseline <path>  --max <n>  --text (print only; safe for hooks)
 
 exit codes: 0 nothing new to prove · 1 unproven claims (or claim drift) · 2 precondition failed · 3 usage
@@ -70,6 +71,7 @@ export async function main(argv, io = { out: (s) => process.stdout.write(s + '\n
         ref: { type: 'string', default: 'HEAD' },
         claim: { type: 'string' },
         'include-dirty': { type: 'boolean', default: false },
+        'allow-provisional': { type: 'boolean', default: false },
         verbose: { type: 'boolean', default: false },
         'runner-cmd': { type: 'string' },
         'node-modules': { type: 'string' },

package/src/commands/baseline.mjs

@@ -11,11 +11,15 @@ export async function baselineCommand({ projectDir, values }, io) {
     return 2;
   }
   const evidence = readSpecDoc('evidence', evPath);
+  if (evidence.run.provisional && !values['allow-provisional']) {
+    io.err(`error: ${evPath} is provisional (confirmRuns ${evidence.run.confirmRuns} < 3); a baseline must be frozen from confirmed evidence. Re-run probe with --confirm 3, or pass --allow-provisional if you accept unconfirmed findings as the frozen contract.`);
+    return 2;
+  }
   const baseline = buildBaseline(evidence);
   const outPath = values.out ? resolve(values.out) : baselinePath(projectDir);
   writeSpecDoc('baseline', outPath, baseline);
   const n = Object.values(baseline.fingerprints).reduce((a, b) => a + b, 0);
-  io.out(`baseline: ${n} unproven finding${n === 1 ? '' : 's'} frozen at ${baseline.head.slice(0, 12)}${baseline.dirty ? ' (working tree was dirty)' : ''} → ${outPath}`);
+  io.out(`baseline: ${n} unproven finding${n === 1 ? '' : 's'} frozen at ${baseline.head.slice(0, 12)}${baseline.dirty ? ' (working tree was dirty)' : ''}${evidence.run.provisional ? ' — FROM PROVISIONAL EVIDENCE (--allow-provisional)' : ''} → ${outPath}`);
   io.out('Commit this file; from now on only new findings gate. Ignore the regenerated ones — add to .gitignore:');
   io.out('  .testguard/evidence.json');
   io.out('  .testguard/brief.json');

package/src/commands/probe.mjs

@@ -4,7 +4,8 @@ import { loadClaims, defaultClaimsPath } from '../claims/load.mjs';
 import { probe } from '../probe/probe.mjs';
 import { writeSpecDoc, readSpecDoc } from '../evidence/writer.mjs';
 import { gate } from '../baseline/baseline.mjs';
-import { renderRecord, renderSummary, sortForReport } from '../render.mjs';
+import { renderRecord, renderSummary, sortForReport, PROVISIONAL_WARNING } from '../render.mjs';
+export const provisionalEvidencePath = (projectDir) => join(projectDir, '.testguard', 'evidence-provisional.json');
 
 export const evidencePath = (projectDir) => join(projectDir, '.testguard', 'evidence.json');
 export const baselinePath = (projectDir) => join(projectDir, '.testguard', 'baseline.json');
@@ -23,7 +24,10 @@ export async function probeCommand({ projectDir, values, version }, io) {
   }
   const only = values.claim ? values.claim.split(',').map((s) => s.trim()).filter(Boolean) : undefined;
   // A --claim run is partial evidence; keep it away from the canonical file unless --out says otherwise.
-  const outPath = values.out ? resolve(values.out) : only ? join(projectDir, '.testguard', 'evidence-partial.json') : evidencePath(projectDir);
+  const provisional = confirmRuns < 3;
+  // Provisional and partial runs never overwrite the canonical evidence: only confirmed, complete runs may feed a baseline.
+  const outPath = values.out ? resolve(values.out) : only ? join(projectDir, '.testguard', 'evidence-partial.json') : provisional ? provisionalEvidencePath(projectDir) : evidencePath(projectDir);
+  if (provisional && !values.quiet) io.err(PROVISIONAL_WARNING(confirmRuns));
   const previous = !values['no-reuse'] && existsSync(outPath) ? readSpecDoc('evidence', outPath) : undefined;
   const basePath = values.baseline ? resolve(values.baseline) : baselinePath(projectDir);
   const baseline = existsSync(basePath) ? readSpecDoc('baseline', basePath) : undefined;
@@ -45,7 +49,7 @@ export async function probeCommand({ projectDir, values, version }, io) {
     onStage: !values.quiet && process.stderr.isTTY ? ({ claimId, faultId, stage, i, n }) => process.stderr.write(`\r\x1b[K  … ${claimId}/${faultId} ${stage} ${i}/${n}`) : undefined,
     onProgress: values.quiet ? undefined : (r) => {
       if (process.stderr.isTTY) process.stderr.write('\r\x1b[K');
-      if (values.verbose || r.verdict !== 'killed') io.out(renderRecord(r) + (r.reusedFrom ? '  (reused)' : ''));
+      if (values.verbose || r.verdict !== 'killed') io.out(renderRecord(r, { provisional }) + (r.reusedFrom ? '  (reused)' : ''));
     },
   });
   writeSpecDoc('evidence', outPath, evidence);
@@ -54,7 +58,7 @@ export async function probeCommand({ projectDir, values, version }, io) {
   if (!values.quiet && baseline) {
     io.out('');
     const tag = (r) => (g.new.includes(r) ? '[NEW]      ' : g.baselined.includes(r) ? '[baseline] ' : '[below floor] ');
-    for (const r of sortForReport(evidence.records).filter((x) => x.verdict !== 'killed')) io.out('  ' + tag(r) + renderRecord(r));
+    for (const r of sortForReport(evidence.records).filter((x) => x.verdict !== 'killed')) io.out('  ' + tag(r) + renderRecord(r, { provisional }));
   }
   io.out('');
   if (!values.quiet && !values.verbose) {
@@ -62,6 +66,7 @@ export async function probeCommand({ projectDir, values, version }, io) {
     if (killed) io.out(`  ${killed} killed (not listed; --verbose to see them)`);
   }
   io.out(renderSummary(evidence.records, evidence.run) + (baseline ? ` ${g.new.length} new since baseline, ${g.baselined.length} baselined.` : ' No baseline.'));
-  io.out(`evidence: ${outPath}${only ? ` (partial: --claim ${only.join(',')}; not the canonical evidence file)` : ''}`);
+  io.out(`evidence: ${outPath}${only ? ` (partial: --claim ${only.join(',')}; not the canonical evidence file)` : provisional ? ' (provisional; not the canonical evidence file)' : ''}`);
+  if (provisional && !values.quiet) io.err(PROVISIONAL_WARNING(confirmRuns));
   return g.new.length > 0 ? 1 : 0;
 }

package/src/probe/probe.mjs

@@ -132,6 +132,7 @@ export async function probe({
       repo: { head, dirty: isDirty(root), ...(snapshot ? { snapshot } : {}) },
       runner: { name: vitest.name, ...((runnerVersion ?? readRunnerVersion(projectDir)) ? { version: runnerVersion ?? readRunnerVersion(projectDir) } : {}) },
       confirmRuns,
+      ...(confirmRuns < 3 ? { provisional: true } : {}),
       mode,
     },
     records,

package/src/render.mjs

@@ -1,10 +1,12 @@
 const ORDER = ['survived', 'nocover', 'unverifiable', 'fault-invalid', 'timeout', 'flaky-defender', 'killed'];
 
 /** Non-passing verdicts shout; the one pass does not. */
-export const formatVerdict = (v) => (v === 'killed' ? 'killed' : v.toUpperCase());
+export const formatVerdict = (v, provisional = false) => (v === 'killed' ? 'killed' : v.toUpperCase()) + (provisional ? '?' : '');
 
-export function renderRecord(r) {
-  const head = `${formatVerdict(r.verdict).padEnd(15)} ${r.claim.id}/${r.subject.id}`.padEnd(38);
+export const PROVISIONAL_WARNING = (n) => `PROVISIONAL — confirmRuns ${n} (< 3): nothing below is confirmed. Verdicts carry a "?"; this evidence cannot be frozen into a baseline. Re-run with --confirm 3 before trusting it.`;
+
+export function renderRecord(r, { provisional = false } = {}) {
+  const head = `${formatVerdict(r.verdict, provisional).padEnd(15)} ${r.claim.id}/${r.subject.id}`.padEnd(38);
   let why = '';
   if (r.detail.reason === 'killed-by-undeclared-tests' && r.detail.undeclaredKillers?.length) {
     const files = [...new Set(r.detail.undeclaredKillers.map((k) => k.split('::')[0]))];
@@ -26,11 +28,11 @@ export function summarize(records) {
 
 export function renderSummary(records, run) {
   const byVerdict = summarize(records);
-  const parts = ORDER.filter((v) => byVerdict[v]).map((v) => `${byVerdict[v]} ${formatVerdict(v)}`);
+  const parts = ORDER.filter((v) => byVerdict[v]).map((v) => `${byVerdict[v]} ${formatVerdict(v, run?.provisional)}`);
   const unproven = records.filter((r) => r.verdict !== 'killed');
   const claims = new Set(unproven.map((r) => r.claim.id)).size;
   const where = run ? ` Probed ${run.repo.snapshot ? `working tree (snapshot ${run.repo.snapshot.slice(0, 7)} of ${run.repo.head.slice(0, 7)})` : run.mode === 'in-place' ? `in place at ${run.repo.head.slice(0, 7)}${run.repo.dirty ? ' (dirty)' : ''}` : run.repo.head.slice(0, 7)}.` : '';
-  return `${records.length} faults probed: ${parts.join(', ')}. ${unproven.length} unproven fault${unproven.length === 1 ? '' : 's'} across ${claims} claim${claims === 1 ? '' : 's'}.${where}`;
+  return `${run?.provisional ? 'PROVISIONAL: ' : ''}${records.length} faults probed: ${parts.join(', ')}. ${unproven.length} unproven fault${unproven.length === 1 ? '' : 's'} across ${claims} claim${claims === 1 ? '' : 's'}.${where}`;
 }
 
 /** Survivors first, then by rank score; killed last. */