testguard-cli 0.4.0 → 0.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (51) hide show
  1. package/CHANGELOG.md +291 -0
  2. package/README.md +321 -27
  3. package/package.json +1 -1
  4. package/spec/GATE-SEMANTICS.md +109 -1
  5. package/spec/README.md +4 -2
  6. package/spec/lib/validate.mjs +75 -1
  7. package/spec/schemas/baseline.schema.json +8 -0
  8. package/spec/schemas/brief.schema.json +11 -1
  9. package/spec/schemas/common.schema.json +32 -1
  10. package/spec/schemas/evidence.schema.json +374 -63
  11. package/spec/schemas/gate.schema.json +52 -0
  12. package/spec/schemas/replay.schema.json +85 -0
  13. package/spec/schemas/status.schema.json +269 -45
  14. package/src/admit/admit.mjs +17 -0
  15. package/src/baseline/baseline.mjs +31 -2
  16. package/src/brief/brief.mjs +96 -8
  17. package/src/claims/removed.mjs +161 -0
  18. package/src/cli.mjs +69 -12
  19. package/src/commands/admit.mjs +116 -0
  20. package/src/commands/baseline.mjs +17 -3
  21. package/src/commands/brief.mjs +26 -12
  22. package/src/commands/claims.mjs +35 -6
  23. package/src/commands/gate.mjs +47 -0
  24. package/src/commands/init.mjs +19 -6
  25. package/src/commands/mcp.mjs +12 -0
  26. package/src/commands/probe.mjs +12 -3
  27. package/src/commands/replay.mjs +63 -0
  28. package/src/commands/status.mjs +4 -1
  29. package/src/gate/changed.mjs +222 -0
  30. package/src/git.mjs +5 -0
  31. package/src/init/init.mjs +182 -29
  32. package/src/init/templates/SKILL.md +21 -3
  33. package/src/mcp/server.mjs +104 -0
  34. package/src/mcp/tools.mjs +114 -0
  35. package/src/probe/contention.mjs +60 -0
  36. package/src/probe/discover.mjs +12 -3
  37. package/src/probe/independence.mjs +59 -0
  38. package/src/probe/mocks.mjs +120 -0
  39. package/src/probe/probe.mjs +126 -18
  40. package/src/probe/rank.mjs +73 -15
  41. package/src/probe/runners/index.mjs +49 -5
  42. package/src/probe/runners/jest.mjs +3 -3
  43. package/src/probe/runners/playwright.mjs +133 -0
  44. package/src/probe/runners/shared.mjs +70 -13
  45. package/src/probe/runners/vitest.mjs +4 -3
  46. package/src/render.mjs +2 -2
  47. package/src/replay/label.mjs +64 -0
  48. package/src/replay/replay.mjs +309 -0
  49. package/src/scaffold/producers.mjs +217 -6
  50. package/src/scaffold/scaffold.mjs +9 -3
  51. package/src/status/status.mjs +88 -7
@@ -1,15 +1,32 @@
1
1
  import { existsSync } from 'node:fs';
2
2
  import { join, resolve } from 'node:path';
3
3
  import { readSpecDoc, writeSpecDoc } from '../evidence/writer.mjs';
4
- import { buildBrief } from '../brief/brief.mjs';
4
+ import { buildBrief, buildUnclaimedBrief, renderBriefMarkdown, renderUnclaimedMarkdown } from '../brief/brief.mjs';
5
5
  import { computeStatus } from '../status/status.mjs';
6
6
  import { evidencePath, baselinePath } from './probe.mjs';
7
+ import { resolveChangedRef, withChangedRef } from '../gate/changed.mjs';
7
8
 
8
9
  export async function briefCommand({ projectDir, values, version }, io) {
9
10
  const evPath = values.evidence ? resolve(values.evidence) : evidencePath(projectDir);
11
+ let status;
12
+ try {
13
+ // A detected base that does not resolve is silent here: the brief is a
14
+ // session-start hook's output and must never add noise; `status` and
15
+ // `gate` are where that warning is printed.
16
+ status = withChangedRef(resolveChangedRef({ explicit: values.changed }), (changedRef) => computeStatus({ projectDir, toolVersion: version, changedRef, includeDirty: values['include-dirty'], evidence: values.evidence ? evPath : undefined }));
17
+ } catch {
18
+ status = undefined; // a brief must never fail because status could not be computed
19
+ }
10
20
  if (!existsSync(evPath)) {
11
- // A missing brief must never break an agent's session start.
12
- if (values.text) return 0;
21
+ // A missing brief must never break an agent's session start — but unclaimed
22
+ // changes are still said, because the claim comes before the probe.
23
+ if (values.text || values.markdown) {
24
+ if (status?.changes?.uncovered?.length) {
25
+ const args = { tool: { name: 'testguard', version }, next: status.next, changes: status.changes };
26
+ io.out((values.markdown ? renderUnclaimedMarkdown(args) : buildUnclaimedBrief(args).text).trimEnd());
27
+ }
28
+ return 0;
29
+ }
13
30
  io.err(`no evidence at ${evPath}; run \`testguard probe\` first`);
14
31
  return 2;
15
32
  }
@@ -21,14 +38,11 @@ export async function briefCommand({ projectDir, values, version }, io) {
21
38
  io.err('--max must be an integer from 1 to 50');
22
39
  return 3;
23
40
  }
24
- let next;
25
- try {
26
- next = computeStatus({ projectDir, toolVersion: version }).next;
27
- } catch {
28
- next = undefined; // a brief must never fail because status could not be computed
29
- }
30
- const brief = buildBrief(evidence, baseline, { max, next });
31
- if (!values.text) writeSpecDoc('brief', values.out ? resolve(values.out) : join(projectDir, '.testguard', 'brief.json'), brief);
32
- io.out(values.json ? JSON.stringify(brief, null, 2) : brief.text.trimEnd());
41
+ // Where this binary came from, so a stale install is visible in the session-start context.
42
+ const install = /[\\/]node_modules[\\/]/.test(process.argv[1] ?? '') ? 'local install' : 'global';
43
+ const brief = buildBrief(evidence, baseline, { max, next: status?.next, changes: status?.changes, install });
44
+ if (!values.text && !values.markdown) writeSpecDoc('brief', values.out ? resolve(values.out) : join(projectDir, '.testguard', 'brief.json'), brief);
45
+ if (values.markdown) io.out(renderBriefMarkdown({ ...brief, provisional: Boolean(evidence.run.provisional) }, { hasBaseline: Boolean(baseline), total: evidence.records.length, install }).trimEnd());
46
+ else io.out(values.json ? JSON.stringify(brief, null, 2) : brief.text.trimEnd());
33
47
  return 0;
34
48
  }
@@ -2,29 +2,58 @@ import { resolve } from 'node:path';
2
2
  import { loadClaims, defaultClaimsPath } from '../claims/load.mjs';
3
3
  import { scanAnnotations, reconcile } from '../claims/annotations.mjs';
4
4
  import { resolveDefenders } from '../probe/runners/shared.mjs';
5
- import { discoverDefenders } from '../probe/discover.mjs';
5
+ import { discoverDefendersDetailed } from '../probe/discover.mjs';
6
+ import { classifyDefenders } from '../probe/mocks.mjs';
7
+ import { computeRemovedClaims, renderRemoved } from '../claims/removed.mjs';
8
+ import { evidencePath } from './probe.mjs';
6
9
 
7
- export async function claimsCommand({ projectDir, values }, io) {
10
+ export async function claimsCommand({ projectDir, values, version }, io) {
8
11
  const path = values.claims ? resolve(values.claims) : defaultClaimsPath(projectDir);
9
12
  const claims = loadClaims(path);
10
13
  const annotations = scanAnnotations(projectDir);
11
14
  const drift = reconcile(claims, annotations);
12
15
 
16
+ // A claim that disappeared is invisible to every other command; compare
17
+ // identities against a reference when one is given.
18
+ const removed = values.since
19
+ ? computeRemovedClaims({ projectDir, ref: values.since, claimsPath: path, current: claims, ignorePath: values.ignore ? resolve(values.ignore) : undefined, evidencePath: evidencePath(projectDir), toolVersion: version })
20
+ : undefined;
21
+
13
22
  if (values.json) {
14
- io.out(JSON.stringify({ path, claims, annotations, drift }, null, 2));
23
+ io.out(JSON.stringify({ path, claims, annotations, drift, ...(removed ? { removed } : {}) }, null, 2));
15
24
  } else {
16
25
  const annotated = new Set(drift.annotated);
17
26
  io.out(`${claims.claims.length} claims in ${path} — ${annotated.size} carry a @claim annotation in source (test files are not scanned)`);
18
27
  io.out('');
28
+ const signalLines = [];
19
29
  for (const c of claims.claims) {
20
30
  const declared = c.defendedBy?.length > 0;
21
- const defenders = declared ? resolveDefenders(projectDir, c.defendedBy) : [...new Set(c.faults.flatMap((f) => discoverDefenders(projectDir, f.file)))];
22
- const cover = defenders.length ? `${defenders.length} ${declared ? 'defender' : 'discovered'}${defenders.length === 1 ? '' : 's'}` : 'NO DEFENDER';
31
+ const targets = [...new Set(c.faults.map((f) => f.file))];
32
+ let cover;
33
+ if (declared) {
34
+ const defenders = resolveDefenders(projectDir, c.defendedBy);
35
+ const mocking = new Set();
36
+ for (const t of targets) { const m = classifyDefenders(projectDir, t, defenders); m.mocking.forEach((f) => mocking.add(f)); for (const s of m.signals) signalLines.push({ claim: c.id, target: t, ...s }); }
37
+ cover = defenders.length ? `${defenders.length} defender${defenders.length === 1 ? '' : 's'}${mocking.size ? ` (${mocking.size} mock the target)` : ''}` : 'NO DEFENDER';
38
+ } else {
39
+ const importing = new Set(); const mocking = new Set(); const can = new Set();
40
+ for (const t of targets) { const d = discoverDefendersDetailed(projectDir, t); d.importing.forEach((f) => importing.add(f)); d.mocking.forEach((f) => mocking.add(f)); d.canDetect.forEach((f) => can.add(f)); for (const s of d.signals) signalLines.push({ claim: c.id, target: t, ...s }); }
41
+ cover = can.size ? `${importing.size} import · ${mocking.size} mock · ${can.size} can detect` : importing.size ? `NO DEFENDER (${importing.size} import, all mock the target)` : 'NO DEFENDER';
42
+ }
23
43
  io.out(`${annotated.has(c.id) ? '@ ' : ' '}${c.id.padEnd(14)} ${c.severity.padEnd(8)} ${c.source.kind.padEnd(10)} ${String(c.faults.length).padStart(2)} fault${c.faults.length === 1 ? ' ' : 's'} ${cover.padEnd(12)} ${c.statement}`);
24
44
  }
45
+ if (signalLines.length) io.out('');
46
+ for (const s of signalLines) {
47
+ if (s.signal === 'mocked-never-asserted') io.out(`MOCKED-NEVER-ASSERTED ${s.file} mocks ${s.target} and never asserts on it (${s.claim}) — assert on the mocked call, drive the real module, or annotate the mock \`// unasserted: <why>\``);
48
+ else io.out(`unasserted (annotated) ${s.file} mocks ${s.target}: ${s.reason} (${s.claim})`);
49
+ }
25
50
  if (drift.undeclared.length || drift.stale.length) io.out('');
26
51
  for (const a of drift.undeclared) io.out(`UNDECLARED @claim ${a.id} at ${a.file}:${a.line} has no entry in the claims file — a claim with no fault model`);
27
52
  for (const c of drift.stale) io.out(`STALE ${c.id} is annotation-sourced but no source file carries @claim ${c.id}`);
53
+ if (removed) {
54
+ io.out('');
55
+ io.out(renderRemoved(removed));
56
+ }
28
57
  }
29
- return drift.undeclared.length || drift.stale.length ? 1 : 0;
58
+ return drift.undeclared.length || drift.stale.length || (removed?.removed.length ?? 0) > 0 ? 1 : 0;
30
59
  }
@@ -0,0 +1,47 @@
1
+ import { join, resolve } from 'node:path';
2
+ import { computeChangedGate, resolveChangedRef, renderGate, DEFAULT_EXCLUDES } from '../gate/changed.mjs';
3
+ import { writeSpecDoc } from '../evidence/writer.mjs';
4
+
5
+ export const gatePath = (projectDir) => join(projectDir, '.testguard', 'gate.json');
6
+
7
+ /**
8
+ * `testguard gate --changed <ref>`: fail when a changed source file carries
9
+ * no claim and no excusing ignore entry. Exit 0 covered · 1 uncovered (or
10
+ * --strict with nothing evaluated) · 2 cannot evaluate · 3 no reference.
11
+ */
12
+ export async function gateCommand({ projectDir, values, version }, io) {
13
+ if (values.explain) {
14
+ io.out('Files that are source by extension but never carry claims (excluded by default; add more with --exclude <glob>):');
15
+ for (const g of DEFAULT_EXCLUDES) io.out(` ${g}`);
16
+ io.out('Non-source files (anything but .js .mjs .cjs .ts .mts .cts .jsx .tsx) are excluded before these patterns apply.');
17
+ return 0;
18
+ }
19
+ const resolved = resolveChangedRef({ explicit: values.changed });
20
+ if (!resolved) {
21
+ io.err('gate needs a reference to measure the change against: --changed <ref> (e.g. origin/main), or set TESTGUARD_CHANGED_REF. In GitHub Actions and GitLab CI the base branch is detected automatically.');
22
+ return 3;
23
+ }
24
+ const ref = resolved.ref;
25
+ // The gate itself never degrades: a detected reference that does not resolve is exit 2 here, because the gate's whole job is the measurement.
26
+ if (!resolved.required && !values.json && !values.quiet) io.err(`--changed not given; using ${ref} from ${resolved.from}`);
27
+ const doc = computeChangedGate({
28
+ projectDir,
29
+ ref,
30
+ includeDirty: values['include-dirty'],
31
+ exclude: values.exclude ?? [],
32
+ strict: values.strict,
33
+ toolVersion: version,
34
+ claimsPath: values.claims ? resolve(values.claims) : undefined,
35
+ ignorePath: values.ignore ? resolve(values.ignore) : undefined,
36
+ });
37
+ const outPath = values.out ? resolve(values.out) : gatePath(projectDir);
38
+ writeSpecDoc('gate', outPath, doc);
39
+ if (values.json) {
40
+ io.out(JSON.stringify(doc, null, 2));
41
+ } else {
42
+ io.out(renderGate(doc));
43
+ if (doc.uncovered.length) io.out(`Next: state the claim for each UNCLAIMED file (scaffold proposes the faults), or add a testguard.ignore.json path entry with a reason that a reviewer will accept.`);
44
+ if (!values.quiet) io.out(`gate: ${outPath}`);
45
+ }
46
+ return doc.exitCode;
47
+ }
@@ -1,15 +1,28 @@
1
1
  import { initProject } from '../init/init.mjs';
2
2
 
3
3
  export async function initCommand({ projectDir, values }, io) {
4
- const { done, skipped } = initProject({ projectDir, force: values.force });
4
+ const r = initProject({ projectDir, force: values.force, here: values.here, ciEvidence: values['ci-evidence'], mcp: values.mcp });
5
+ const { done, skipped, warnings, agentRoot, dir, mcpConfig } = r;
5
6
  if (values.json) {
6
- io.out(JSON.stringify({ done, skipped }, null, 2));
7
- return 0;
7
+ io.out(JSON.stringify({ done, skipped, warnings, agentRoot, dir, ...(mcpConfig ? { mcpConfig } : {}) }, null, 2));
8
+ return warnings.length ? 1 : 0;
8
9
  }
9
10
  for (const d of done) io.out(`+ ${d}`);
10
11
  for (const s of skipped) io.out(`= ${s}`);
12
+ for (const w of warnings) io.out(`! ${w}`);
11
13
  io.out('');
12
- io.out('Agents in this project now start with the blind-spot brief and can run `testguard status --json` to learn what to do next.');
13
- io.out(done.length ? 'Commit these files.' : 'Nothing to change.');
14
- return 0;
14
+ if (dir !== '.') io.out(`Agent layer (skill, session-start hook, AGENTS.md section) installed at the git root ${agentRoot}, where agent sessions run; the project layer stays in ${dir}/. Use --here to keep everything in ${dir}/.`);
15
+ io.out(`Agents now start with the blind-spot brief and can run \`testguard status --json${dir === '.' ? '' : ` ${dir}`}\` to learn what to do next. The hook prefers a local install and never fetches from the network.`);
16
+ io.out(warnings.length ? 'Some written files are ignored by git — fix that before committing.' : done.length ? 'Commit these files.' : 'Nothing to change.');
17
+ if (mcpConfig) {
18
+ io.out('');
19
+ io.out('MCP — five READ-ONLY tools (status, brief, claims, evidence, next_command), so the loop survives a change of harness. Nothing there runs a probe. Add ONE of these:');
20
+ for (const [where, snippet] of Object.entries(mcpConfig)) {
21
+ io.out('');
22
+ io.out(` ${where}`);
23
+ const text = typeof snippet === 'string' ? snippet : JSON.stringify(snippet, null, 2);
24
+ for (const line of text.split('\n')) io.out(` ${line}`);
25
+ }
26
+ }
27
+ return warnings.length ? 1 : 0;
15
28
  }
@@ -0,0 +1,12 @@
1
+ import { serve } from '../mcp/server.mjs';
2
+
3
+ export async function mcpCommand({ values, version }, io) {
4
+ if (values.json) {
5
+ // Not a JSON command: it is a long-lived server on stdio, and printing
6
+ // anything to stdout would corrupt the protocol stream.
7
+ io.err('mcp speaks JSON-RPC on stdin/stdout; there is no --json mode');
8
+ return 3;
9
+ }
10
+ await serve({ version });
11
+ return 0;
12
+ }
@@ -6,6 +6,7 @@ import { writeSpecDoc, readSpecDoc } from '../evidence/writer.mjs';
6
6
  import { gate } from '../baseline/baseline.mjs';
7
7
  import { renderRecord, renderSummary, sortForReport, PROVISIONAL_WARNING } from '../render.mjs';
8
8
  import { computeStatus } from '../status/status.mjs';
9
+ import { resolveChangedRef, withChangedRef } from '../gate/changed.mjs';
9
10
  export const provisionalEvidencePath = (projectDir) => join(projectDir, '.testguard', 'evidence-provisional.json');
10
11
 
11
12
  export const evidencePath = (projectDir) => join(projectDir, '.testguard', 'evidence.json');
@@ -29,6 +30,7 @@ export async function probeCommand({ projectDir, values, version }, io) {
29
30
  // Provisional and partial runs never overwrite the canonical evidence: only confirmed, complete runs may feed a baseline.
30
31
  const outPath = values.out ? resolve(values.out) : only ? join(projectDir, '.testguard', 'evidence-partial.json') : provisional ? provisionalEvidencePath(projectDir) : evidencePath(projectDir);
31
32
  if (provisional && !values.quiet) io.err(PROVISIONAL_WARNING(confirmRuns));
33
+ const discoveryNoted = new Set();
32
34
  const previous = !values['no-reuse'] && existsSync(outPath) ? readSpecDoc('evidence', outPath) : undefined;
33
35
  const basePath = values.baseline ? resolve(values.baseline) : baselinePath(projectDir);
34
36
  const baseline = existsSync(basePath) ? readSpecDoc('baseline', basePath) : undefined;
@@ -40,7 +42,11 @@ export async function probeCommand({ projectDir, values, version }, io) {
40
42
  confirmRuns,
41
43
  budgetMs,
42
44
  mode: values['in-place'] ? 'in-place' : 'worktree',
43
- ref: values.ref,
45
+ ref: values.ref ?? 'HEAD',
46
+ refExplicit: values.ref !== undefined,
47
+ ignoreDirty: values['ignore-dirty'],
48
+ serial: values.serial,
49
+ onWarn: (m) => io.err(`warning: ${m}`),
44
50
  runnerCommand: values['runner-cmd'],
45
51
  runnerName: values.runner,
46
52
  nodeModules: values['node-modules'] ? resolve(values['node-modules']) : process.env.TESTGUARD_NODE_MODULES,
@@ -51,14 +57,17 @@ export async function probeCommand({ projectDir, values, version }, io) {
51
57
  onStage: !values.quiet && !values.json && process.stderr.isTTY ? ({ claimId, faultId, stage, i, n }) => process.stderr.write(`\r\x1b[K … ${claimId}/${faultId} ${stage} ${i}/${n}`) : undefined,
52
58
  onProgress: values.quiet || values.json ? undefined : (r) => {
53
59
  if (process.stderr.isTTY) process.stderr.write('\r\x1b[K');
54
- if (values.verbose || r.verdict !== 'killed') io.out(renderRecord(r, { provisional }) + (r.reusedFrom ? ' (reused)' : ''));
60
+ // The discovery note is a property of the claim, not of each fault: say it once.
61
+ const firstOfClaim = !discoveryNoted.has(r.claim.id);
62
+ if (r.defenders.discovered) discoveryNoted.add(r.claim.id);
63
+ if (values.verbose || r.verdict !== 'killed') io.out(renderRecord(r, { provisional, showDiscovered: firstOfClaim }) + (r.reusedFrom ? ' (reused)' : ''));
55
64
  },
56
65
  });
57
66
  writeSpecDoc('evidence', outPath, evidence);
58
67
 
59
68
  const g = gate(evidence.records, baseline, { severityFloor: values.severity });
60
69
  if (values.json) {
61
- const status = computeStatus({ projectDir, toolVersion: version });
70
+ const status = withChangedRef(resolveChangedRef({ explicit: values.changed }), (changedRef) => computeStatus({ projectDir, toolVersion: version, changedRef, includeDirty: values['include-dirty'], evidence: outPath }), io.err);
62
71
  io.out(JSON.stringify({ ...status, run: { id: evidence.run.id, evidence: outPath, provisional, records: evidence.records.length, newSinceBaseline: g.new.length, exitCode: g.new.length > 0 ? 1 : 0 } }, null, 2));
63
72
  return g.new.length > 0 ? 1 : 0;
64
73
  }
@@ -0,0 +1,63 @@
1
+ import { dirname, join, resolve } from 'node:path';
2
+ import { replay, calibrationFrom, renderReplay } from '../replay/replay.mjs';
3
+ import { writeSpecDoc } from '../evidence/writer.mjs';
4
+
5
+ export const replayPath = (projectDir) => join(projectDir, '.testguard', 'replay.json');
6
+ export const calibrationPath = (projectDir) => join(projectDir, '.testguard', 'calibration.json');
7
+
8
+ export async function replayCommand({ projectDir, values, version }, io) {
9
+ const range = values.since ?? values.changed;
10
+ if (!range) {
11
+ io.err('usage: testguard replay [dir] --since <range> (e.g. --since HEAD~50..HEAD, or a tag)');
12
+ return 3;
13
+ }
14
+ const confirmRuns = Number(values.confirm);
15
+ const budgetMs = Number(values.budget);
16
+ const limit = values.max ? Number(values.max) : undefined;
17
+ if (!Number.isInteger(confirmRuns) || confirmRuns < 1 || !Number.isInteger(budgetMs) || budgetMs < 1000) {
18
+ io.err('--confirm must be a positive integer and --budget at least 1000');
19
+ return 3;
20
+ }
21
+
22
+ const doc = await replay({
23
+ projectDir,
24
+ range,
25
+ confirmRuns,
26
+ budgetMs,
27
+ runnerCommand: values['runner-cmd'],
28
+ runnerName: values.runner,
29
+ nodeModules: values['node-modules'] ? resolve(values['node-modules']) : process.env.TESTGUARD_NODE_MODULES,
30
+ limit,
31
+ toolVersion: version,
32
+ onStage: !values.quiet && !values.json && process.stderr.isTTY ? ({ commit, i, n, stage }) => process.stderr.write(`\r\x1b[K … ${commit.slice(0, 9)} (${i}/${n}) ${stage}`) : undefined,
33
+ onProgress: values.quiet || values.json ? undefined : () => {
34
+ if (process.stderr.isTTY) process.stderr.write('\r\x1b[K');
35
+ },
36
+ });
37
+
38
+ const outPath = values.out ? resolve(values.out) : replayPath(projectDir);
39
+ writeSpecDoc('replay', outPath, doc);
40
+ const calibration = calibrationFrom(doc, { toolVersion: version });
41
+ // Beside the replay document, whatever --out says: the two are one result,
42
+ // and splitting them across directories loses the pairing — and leaves a
43
+ // file behind in a repository the run is only meant to read.
44
+ const calPath = values.baseline ? resolve(values.baseline) : values.out ? join(dirname(outPath), 'calibration.json') : calibrationPath(projectDir);
45
+ writeSpecDoc('calibration', calPath, calibration);
46
+
47
+ if (values.json) {
48
+ io.out(JSON.stringify({ replay: doc, calibration, paths: { replay: outPath, calibration: calPath } }, null, 2));
49
+ } else {
50
+ io.out(renderReplay(doc));
51
+ io.out('');
52
+ for (const [cls, b] of Object.entries(calibration.buckets).sort((a, b2) => b2[1].n - a[1].n)) {
53
+ io.out(` ${cls.padEnd(22)} blind ${String(b.positives).padStart(3)}/${String(b.n).padEnd(3)} p=${b.p.toFixed(2)} ci [${b.ci[0].toFixed(2)}, ${b.ci[1].toFixed(2)}]`);
54
+ }
55
+ if (Object.keys(calibration.buckets).length === 0) io.out(' no measurable bug in this range: nothing to calibrate from yet.');
56
+ io.out('');
57
+ io.out(`replay: ${outPath}`);
58
+ io.out(`calibration: ${calPath}`);
59
+ }
60
+ // Replay reports; it never gates. A bug that escaped is history, not a
61
+ // regression in this change, and a gate on history is a gate nobody can pass.
62
+ return 0;
63
+ }
@@ -1,8 +1,11 @@
1
+ import { resolve } from 'node:path';
1
2
  import { computeStatus, renderStatus } from '../status/status.mjs';
2
3
  import { validate } from '../../spec/lib/validate.mjs';
4
+ import { resolveChangedRef, withChangedRef } from '../gate/changed.mjs';
3
5
 
4
6
  export async function statusCommand({ projectDir, values, version }, io) {
5
- const doc = computeStatus({ projectDir, toolVersion: version });
7
+ const evidence = values.evidence ? resolve(values.evidence) : undefined;
8
+ const doc = withChangedRef(resolveChangedRef({ explicit: values.changed }), (changedRef) => computeStatus({ projectDir, toolVersion: version, changedRef, includeDirty: values['include-dirty'], evidence }), io.err);
6
9
  const result = validate('status', doc);
7
10
  if (!result.ok) throw new Error(`status document does not conform: ${result.errors.map((e) => `${e.path}: ${e.message}`).join('; ')}`);
8
11
  io.out(values.json ? JSON.stringify(doc, null, 2) : renderStatus(doc));
@@ -0,0 +1,222 @@
1
+ import { existsSync, realpathSync } from 'node:fs';
2
+ import { extname, join, relative, resolve, dirname } from 'node:path';
3
+ import { git, repoRoot, headSha, GitError } from '../git.mjs';
4
+ import { loadClaims, defaultClaimsPath } from '../claims/load.mjs';
5
+ import { readSpecDoc } from '../evidence/writer.mjs';
6
+ import { resolveDefenders } from '../probe/runners/shared.mjs';
7
+ import { discoverDefenders } from '../probe/discover.mjs';
8
+ import { globToRegExp } from '../util/glob.mjs';
9
+
10
+ /**
11
+ * Claim coverage of a change.
12
+ *
13
+ * `probe` answers "do the tests defend the claims that exist?". This answers
14
+ * the question every field report so far turned on: "does the code I just
15
+ * changed have any claim at all?". A repository with ten old claims and one
16
+ * new, unclaimed feature exits 0 forever under `probe`; under `gate` the
17
+ * delta is measured, file by file, and an uncovered file is a finding.
18
+ *
19
+ * The unit is the file, on purpose: a fault anchors to a file, so the file is
20
+ * the smallest unit the rest of the contract already understands.
21
+ */
22
+
23
+ export const SOURCE_EXT = new Set(['.js', '.mjs', '.cjs', '.ts', '.mts', '.cts', '.jsx', '.tsx']);
24
+ const TEST_RE = /\.(test|spec)\.[cm]?[jt]sx?$/;
25
+
26
+ /** Files that are source by extension but never carry claims. Printed by --explain. */
27
+ export const DEFAULT_EXCLUDES = Object.freeze([
28
+ '**/*.d.ts',
29
+ '**/*.config.*',
30
+ '**/*.stories.*',
31
+ '**/*.generated.*',
32
+ '**/fixtures/**',
33
+ '**/__fixtures__/**',
34
+ '**/__mocks__/**',
35
+ '**/__snapshots__/**',
36
+ '.testguard/**',
37
+ ]);
38
+
39
+ export const defaultIgnorePath = (projectDir) => join(projectDir, 'testguard.ignore.json');
40
+
41
+ /**
42
+ * The reference the change is measured against. Explicit flag first, then the
43
+ * environments CI sets, then nothing: a wrong default would make the gate pass
44
+ * trivially (an upstream that already contains the change has an empty diff),
45
+ * so no ref is an error the caller reports, never a silent guess.
46
+ */
47
+ export function detectChangedRef(env = process.env) {
48
+ if (env.TESTGUARD_CHANGED_REF) return { ref: env.TESTGUARD_CHANGED_REF, from: 'TESTGUARD_CHANGED_REF' };
49
+ // GitHub Actions, pull_request events.
50
+ if (env.GITHUB_BASE_REF) return { ref: `origin/${env.GITHUB_BASE_REF}`, from: 'GITHUB_BASE_REF' };
51
+ // GitLab merge request pipelines. The diff base sha is exact and needs no
52
+ // remote-tracking ref; the target branch name is the fallback (it needs
53
+ // `git fetch origin <branch>` or GIT_DEPTH: 0 on the job).
54
+ const sha = env.CI_MERGE_REQUEST_DIFF_BASE_SHA;
55
+ if (sha && /^[0-9a-f]{7,40}$/i.test(sha) && !/^0+$/.test(sha)) return { ref: sha, from: 'CI_MERGE_REQUEST_DIFF_BASE_SHA' };
56
+ if (env.CI_MERGE_REQUEST_TARGET_BRANCH_NAME) return { ref: `origin/${env.CI_MERGE_REQUEST_TARGET_BRANCH_NAME}`, from: 'CI_MERGE_REQUEST_TARGET_BRANCH_NAME' };
57
+ return null;
58
+ }
59
+
60
+ /**
61
+ * Resolve the reference a command should measure against, and whether it may
62
+ * be dropped on failure. An explicit `--changed` is the user's word: if it
63
+ * does not resolve, that is an error. A reference detected from CI variables
64
+ * is a convenience: if it does not resolve (shallow clone, temp directory,
65
+ * no remote) the command says so on stderr and continues without a change
66
+ * measurement, because `status` and `brief` must keep working everywhere.
67
+ */
68
+ export function resolveChangedRef({ explicit, env = process.env } = {}) {
69
+ if (explicit) return { ref: explicit, from: '--changed', required: true };
70
+ const d = detectChangedRef(env);
71
+ return d ? { ...d, required: false } : null;
72
+ }
73
+
74
+ /**
75
+ * Run `compute(ref)` for a resolved reference. A required reference propagates
76
+ * its error; a detected one that fails yields `undefined` after `warn(message)`.
77
+ */
78
+ export function withChangedRef(resolved, compute, warn = () => {}) {
79
+ if (!resolved) return compute(undefined);
80
+ try {
81
+ return compute(resolved.ref);
82
+ } catch (e) {
83
+ if (resolved.required || !(e instanceof GitError)) throw e;
84
+ warn(`change coverage skipped: ${resolved.ref} (from ${resolved.from}) does not resolve here — ${e.message.split('. ')[0]}. Pass --changed <ref> to make this an error.`);
85
+ return compute(undefined);
86
+ }
87
+ }
88
+
89
+ /**
90
+ * Files changed since the merge-base of `ref` and HEAD, repo-root-relative,
91
+ * deletions dropped. With `includeDirty` the working tree (tracked edits,
92
+ * staged or not, plus untracked non-ignored files) is compared instead of HEAD.
93
+ */
94
+ export function changedFiles({ root, ref, includeDirty = false }) {
95
+ let base;
96
+ try {
97
+ base = git(['merge-base', ref, 'HEAD'], root);
98
+ } catch (e) {
99
+ throw new GitError(`cannot resolve --changed ${ref}: ${e.message.split('\n').pop()}. In CI the base must be present locally: GitHub Actions — actions/checkout with fetch-depth: 0; GitLab — GIT_DEPTH: 0 or \`git fetch origin <target branch>\` before the job (merge request pipelines also provide CI_MERGE_REQUEST_DIFF_BASE_SHA, which needs no fetch).`);
100
+ }
101
+ const head = headSha(root);
102
+ const out = new Set(git(['diff', '--name-only', '--diff-filter=d', '-z', base, ...(includeDirty ? [] : ['HEAD'])], root).split('\0').filter(Boolean));
103
+ if (includeDirty) for (const f of git(['ls-files', '--others', '--exclude-standard', '-z'], root).split('\0').filter(Boolean)) out.add(f);
104
+ return { base, head, files: [...out].sort() };
105
+ }
106
+
107
+ const isExpired = (entry, now) => Boolean(entry.expires) && new Date(entry.expires).getTime() <= now.getTime();
108
+
109
+ /**
110
+ * The claim to attach to, if any: one whose fault files live in the same
111
+ * directory as the changed file. A claim that merely shares `src/` is not
112
+ * "nearest", it is noise, and a wrong `--claim` hint is worse than none.
113
+ */
114
+ function nearestClaim(file, faultFiles) {
115
+ const dir = dirname(file);
116
+ for (const [ff, ids] of faultFiles) if (dirname(ff) === dir) return ids[0];
117
+ return null;
118
+ }
119
+
120
+ /**
121
+ * Compute the gate for one change. Pure apart from reading the repository,
122
+ * the claims file and the ignore file; the caller decides what to print and
123
+ * what to write.
124
+ */
125
+ export function computeChangedGate({ projectDir, ref, includeDirty = false, exclude = [], strict = false, now = new Date(), toolVersion = '0.0.0', generatedAt = now.toISOString(), claimsPath, ignorePath }) {
126
+ projectDir = realpathSync(resolve(projectDir));
127
+ const root = repoRoot(projectDir);
128
+ const relProject = relative(root, projectDir).split('\\').join('/');
129
+ const { base, head, files } = changedFiles({ root, ref, includeDirty });
130
+
131
+ // Repo-root paths → project paths; a change outside the project directory is not this project's change.
132
+ const changed = [];
133
+ for (const f of files) {
134
+ if (!relProject) { changed.push(f); continue; }
135
+ if (f.startsWith(relProject + '/')) changed.push(f.slice(relProject.length + 1));
136
+ }
137
+
138
+ const cPath = claimsPath ?? defaultClaimsPath(projectDir);
139
+ const claims = existsSync(cPath) ? loadClaims(cPath) : { claims: [] };
140
+ const iPath = ignorePath ?? defaultIgnorePath(projectDir);
141
+ const ignore = existsSync(iPath) ? readSpecDoc('ignore', iPath) : { entries: [] };
142
+
143
+ // Coverage index: which files carry a fault, which test files defend a claim.
144
+ const faultFiles = new Map(); // file → claimIds
145
+ const defenderFiles = new Map(); // test file → claimIds
146
+ for (const c of claims.claims) {
147
+ for (const f of c.faults) faultFiles.set(f.file, [...new Set([...(faultFiles.get(f.file) ?? []), c.id])]);
148
+ const defenders = c.defendedBy?.length ? resolveDefenders(projectDir, c.defendedBy) : [...new Set(c.faults.flatMap((f) => discoverDefenders(projectDir, f.file)))];
149
+ for (const d of defenders) defenderFiles.set(d, [...new Set([...(defenderFiles.get(d) ?? []), c.id])]);
150
+ }
151
+
152
+ const pathEntries = ignore.entries.filter((e) => e.kind === 'path').map((e) => ({ entry: e, re: globToRegExp(e.pattern), expired: isExpired(e, now), files: [] }));
153
+ const userExcludes = exclude.map((g) => ({ glob: g, re: globToRegExp(g) }));
154
+ const defaultExcludes = DEFAULT_EXCLUDES.map((g) => ({ glob: g, re: globToRegExp(g) }));
155
+
156
+ const excluded = [];
157
+ const covered = [];
158
+ const uncovered = [];
159
+ for (const file of changed) {
160
+ if (!SOURCE_EXT.has(extname(file))) { excluded.push({ file, by: 'non-source' }); continue; }
161
+ const ux = userExcludes.find((x) => x.re.test(file));
162
+ if (ux) { excluded.push({ file, by: `exclude:${ux.glob}` }); continue; }
163
+ const dx = defaultExcludes.find((x) => x.re.test(file));
164
+ if (dx) { excluded.push({ file, by: `default:${dx.glob}` }); continue; }
165
+
166
+ const isTest = TEST_RE.test(file);
167
+ if (faultFiles.has(file)) { covered.push({ file, by: 'fault', claimIds: faultFiles.get(file) }); continue; }
168
+ if (isTest && defenderFiles.has(file)) { covered.push({ file, by: 'defender', claimIds: defenderFiles.get(file) }); continue; }
169
+ const active = pathEntries.find((p) => !p.expired && p.re.test(file));
170
+ if (active) { active.files.push(file); covered.push({ file, by: 'ignore', pattern: active.entry.pattern }); continue; }
171
+ for (const p of pathEntries) if (p.expired && p.re.test(file)) p.files.push(file);
172
+
173
+ // Uncovered. Point at a same-directory claim so the author knows where to attach.
174
+ const nearest = nearestClaim(file, faultFiles);
175
+ const item = {
176
+ file,
177
+ kind: isTest ? 'test' : 'source',
178
+ ...(nearest ? { nearestClaimId: nearest } : {}),
179
+ suggestion: isTest
180
+ ? `add ${file} to the defendedBy of the claim it defends${nearest ? ` (nearest: ${nearest})` : ''}, or state a new claim for what it asserts`
181
+ : `testguard scaffold ${file}${nearest ? ` --claim ${nearest}` : ''}`,
182
+ };
183
+ uncovered.push(item);
184
+ }
185
+
186
+ const evaluated = covered.length + uncovered.length;
187
+ const reliedOn = pathEntries.filter((p) => !p.expired && p.files.length).map((p) => ({ pattern: p.entry.pattern, reason: p.entry.reason, ...(p.entry.expires ? { expires: p.entry.expires } : {}), files: p.files }));
188
+ const expired = pathEntries.filter((p) => p.expired && p.files.length).map((p) => ({ pattern: p.entry.pattern, reason: p.entry.reason, expires: p.entry.expires, files: p.files }));
189
+ const exitCode = uncovered.length > 0 ? 1 : strict && changed.length > 0 && evaluated === 0 ? 1 : 0;
190
+
191
+ return {
192
+ schemaVersion: 1,
193
+ tool: { name: 'testguard', version: toolVersion },
194
+ generatedAt,
195
+ ref,
196
+ base,
197
+ head,
198
+ includeDirty,
199
+ strict,
200
+ changed: changed.length,
201
+ evaluated,
202
+ excluded,
203
+ covered,
204
+ uncovered,
205
+ reliedOn,
206
+ expired,
207
+ exitCode,
208
+ };
209
+ }
210
+
211
+ export function renderGate(doc) {
212
+ const lines = [];
213
+ const where = doc.includeDirty ? 'working tree' : `HEAD ${doc.head.slice(0, 7)}`;
214
+ lines.push(`gate: ${doc.changed} file${doc.changed === 1 ? '' : 's'} changed since ${doc.ref} (merge-base ${doc.base.slice(0, 7)}) in the ${where}; ${doc.evaluated} evaluated, ${doc.excluded.length} excluded, ${doc.covered.length} covered, ${doc.uncovered.length} uncovered`);
215
+ for (const u of doc.uncovered) lines.push(`UNCLAIMED ${u.file} (${u.kind}${u.nearestClaimId ? `, nearest claim ${u.nearestClaimId}` : ''})\n → ${u.suggestion}`);
216
+ for (const r of doc.reliedOn) lines.push(`excused ${r.files.join(', ')} by ignore "${r.pattern}": ${r.reason}${r.expires ? ` (expires ${r.expires})` : ''}`);
217
+ for (const e of doc.expired) lines.push(`EXPIRED ignore "${e.pattern}" (expired ${e.expires}) no longer excuses ${e.files.join(', ')}`);
218
+ if (doc.changed > 0 && doc.evaluated === 0) lines.push(`note: every changed file was excluded (${[...new Set(doc.excluded.map((x) => x.by))].join(', ')}); 0 evaluated${doc.strict ? ' — --strict makes this a failure' : ''}`);
219
+ if (doc.changed === 0) lines.push(`no changes since ${doc.ref}; nothing to claim`);
220
+ if (doc.uncovered.length === 0 && doc.exitCode === 0 && doc.evaluated > 0) lines.push('every changed source file carries a claim or an excusing ignore entry');
221
+ return lines.join('\n');
222
+ }
package/src/git.mjs CHANGED
@@ -23,6 +23,11 @@ export function headSha(dir, ref = 'HEAD') {
23
23
  }
24
24
  }
25
25
 
26
+ /** Is `ancestor` reachable from `ref`? False when either does not resolve. */
27
+ export function isAncestor(dir, ancestor, ref = 'HEAD') {
28
+ return spawnSync('git', ['merge-base', '--is-ancestor', ancestor, ref], { cwd: dir }).status === 0;
29
+ }
30
+
26
31
  /** True when any of `paths` (repo-relative; empty = whole tree) has uncommitted changes. */
27
32
  export const isDirty = (dir, paths = []) => git(['status', '--porcelain', '--', ...paths], dir).length > 0;
28
33