testguard-cli 0.5.0 → 0.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (65) hide show
  1. package/CHANGELOG.md +628 -0
  2. package/README.md +536 -42
  3. package/package.json +2 -2
  4. package/spec/GATE-SEMANTICS.md +166 -2
  5. package/spec/README.md +29 -10
  6. package/spec/lib/validate.mjs +140 -8
  7. package/spec/schemas/baseline.schema.json +13 -5
  8. package/spec/schemas/brief.schema.json +12 -12
  9. package/spec/schemas/calibration.schema.json +5 -5
  10. package/spec/schemas/claims.schema.json +122 -25
  11. package/spec/schemas/common.schema.json +131 -29
  12. package/spec/schemas/evidence.schema.json +476 -69
  13. package/spec/schemas/gate.schema.json +12 -12
  14. package/spec/schemas/ignore.schema.json +4 -4
  15. package/spec/schemas/replay.schema.json +85 -0
  16. package/spec/schemas/status.schema.json +258 -56
  17. package/src/admit/admit.mjs +17 -0
  18. package/src/baseline/baseline.mjs +31 -2
  19. package/src/brief/brief.mjs +71 -7
  20. package/src/claims/removed.mjs +161 -0
  21. package/src/cli.mjs +68 -10
  22. package/src/commands/admit.mjs +116 -0
  23. package/src/commands/baseline.mjs +17 -3
  24. package/src/commands/brief.mjs +13 -7
  25. package/src/commands/claims.mjs +55 -6
  26. package/src/commands/init.mjs +19 -6
  27. package/src/commands/mcp.mjs +12 -0
  28. package/src/commands/probe.mjs +35 -7
  29. package/src/commands/replay.mjs +63 -0
  30. package/src/commands/status.mjs +3 -1
  31. package/src/gate/changed.mjs +9 -2
  32. package/src/git.mjs +5 -0
  33. package/src/init/init.mjs +182 -29
  34. package/src/init/templates/SKILL.md +3 -3
  35. package/src/mcp/server.mjs +104 -0
  36. package/src/mcp/tools.mjs +114 -0
  37. package/src/probe/attribution.mjs +123 -0
  38. package/src/probe/classify.mjs +18 -2
  39. package/src/probe/contention.mjs +60 -0
  40. package/src/probe/cost.mjs +145 -0
  41. package/src/probe/discover.mjs +20 -3
  42. package/src/probe/independence.mjs +59 -0
  43. package/src/probe/inject.mjs +41 -9
  44. package/src/probe/mocks.mjs +122 -0
  45. package/src/probe/probe.mjs +331 -40
  46. package/src/probe/progress.mjs +87 -0
  47. package/src/probe/pyimports.mjs +287 -0
  48. package/src/probe/rank.mjs +75 -15
  49. package/src/probe/runners/index.mjs +79 -8
  50. package/src/probe/runners/jest.mjs +15 -3
  51. package/src/probe/runners/playwright.mjs +142 -0
  52. package/src/probe/runners/python/_testguard_pytest_plugin.py +132 -0
  53. package/src/probe/runners/python/_testguard_report.py +141 -0
  54. package/src/probe/runners/python/_testguard_unittest_main.py +168 -0
  55. package/src/probe/runners/python.mjs +316 -0
  56. package/src/probe/runners/shared.mjs +80 -14
  57. package/src/probe/runners/vitest.mjs +16 -3
  58. package/src/render.mjs +2 -2
  59. package/src/replay/label.mjs +64 -0
  60. package/src/replay/replay.mjs +310 -0
  61. package/src/scaffold/producers.mjs +217 -6
  62. package/src/scaffold/producers.python.mjs +209 -0
  63. package/src/scaffold/scaffold.mjs +90 -30
  64. package/src/status/status.mjs +47 -6
  65. package/src/util/glob.mjs +7 -1
@@ -1,7 +1,7 @@
1
1
  import { existsSync } from 'node:fs';
2
2
  import { join, resolve } from 'node:path';
3
3
  import { readSpecDoc, writeSpecDoc } from '../evidence/writer.mjs';
4
- import { buildBrief, buildUnclaimedBrief } from '../brief/brief.mjs';
4
+ import { buildBrief, buildUnclaimedBrief, renderBriefMarkdown, renderUnclaimedMarkdown } from '../brief/brief.mjs';
5
5
  import { computeStatus } from '../status/status.mjs';
6
6
  import { evidencePath, baselinePath } from './probe.mjs';
7
7
  import { resolveChangedRef, withChangedRef } from '../gate/changed.mjs';
@@ -13,15 +13,18 @@ export async function briefCommand({ projectDir, values, version }, io) {
13
13
  // A detected base that does not resolve is silent here: the brief is a
14
14
  // session-start hook's output and must never add noise; `status` and
15
15
  // `gate` are where that warning is printed.
16
- status = withChangedRef(resolveChangedRef({ explicit: values.changed }), (changedRef) => computeStatus({ projectDir, toolVersion: version, changedRef, includeDirty: values['include-dirty'] }));
16
+ status = withChangedRef(resolveChangedRef({ explicit: values.changed }), (changedRef) => computeStatus({ projectDir, toolVersion: version, changedRef, includeDirty: values['include-dirty'], evidence: values.evidence ? evPath : undefined }));
17
17
  } catch {
18
18
  status = undefined; // a brief must never fail because status could not be computed
19
19
  }
20
20
  if (!existsSync(evPath)) {
21
21
  // A missing brief must never break an agent's session start — but unclaimed
22
22
  // changes are still said, because the claim comes before the probe.
23
- if (values.text) {
24
- if (status?.changes?.uncovered?.length) io.out(buildUnclaimedBrief({ tool: { name: 'testguard', version }, next: status.next, changes: status.changes }).text.trimEnd());
23
+ if (values.text || values.markdown) {
24
+ if (status?.changes?.uncovered?.length) {
25
+ const args = { tool: { name: 'testguard', version }, next: status.next, changes: status.changes };
26
+ io.out((values.markdown ? renderUnclaimedMarkdown(args) : buildUnclaimedBrief(args).text).trimEnd());
27
+ }
25
28
  return 0;
26
29
  }
27
30
  io.err(`no evidence at ${evPath}; run \`testguard probe\` first`);
@@ -35,8 +38,11 @@ export async function briefCommand({ projectDir, values, version }, io) {
35
38
  io.err('--max must be an integer from 1 to 50');
36
39
  return 3;
37
40
  }
38
- const brief = buildBrief(evidence, baseline, { max, next: status?.next, changes: status?.changes });
39
- if (!values.text) writeSpecDoc('brief', values.out ? resolve(values.out) : join(projectDir, '.testguard', 'brief.json'), brief);
40
- io.out(values.json ? JSON.stringify(brief, null, 2) : brief.text.trimEnd());
41
+ // Where this binary came from, so a stale install is visible in the session-start context.
42
+ const install = /[\\/]node_modules[\\/]/.test(process.argv[1] ?? '') ? 'local install' : 'global';
43
+ const brief = buildBrief(evidence, baseline, { max, next: status?.next, changes: status?.changes, install });
44
+ if (!values.text && !values.markdown) writeSpecDoc('brief', values.out ? resolve(values.out) : join(projectDir, '.testguard', 'brief.json'), brief);
45
+ if (values.markdown) io.out(renderBriefMarkdown({ ...brief, provisional: Boolean(evidence.run.provisional) }, { hasBaseline: Boolean(baseline), total: evidence.records.length, install }).trimEnd());
46
+ else io.out(values.json ? JSON.stringify(brief, null, 2) : brief.text.trimEnd());
41
47
  return 0;
42
48
  }
@@ -2,29 +2,78 @@ import { resolve } from 'node:path';
2
2
  import { loadClaims, defaultClaimsPath } from '../claims/load.mjs';
3
3
  import { scanAnnotations, reconcile } from '../claims/annotations.mjs';
4
4
  import { resolveDefenders } from '../probe/runners/shared.mjs';
5
- import { discoverDefenders } from '../probe/discover.mjs';
5
+ import { discoverDefendersDetailed } from '../probe/discover.mjs';
6
+ import { classifyDefenders } from '../probe/mocks.mjs';
7
+ import { computeRemovedClaims, renderRemoved } from '../claims/removed.mjs';
8
+ import { evidencePath } from './probe.mjs';
9
+ import { costReport, renderCost } from '../probe/cost.mjs';
10
+ import { readSpecDoc } from '../evidence/writer.mjs';
11
+ import { existsSync } from 'node:fs';
6
12
 
7
- export async function claimsCommand({ projectDir, values }, io) {
13
+ export async function claimsCommand({ projectDir, values, version }, io) {
8
14
  const path = values.claims ? resolve(values.claims) : defaultClaimsPath(projectDir);
9
15
  const claims = loadClaims(path);
10
16
  const annotations = scanAnnotations(projectDir);
11
17
  const drift = reconcile(claims, annotations);
12
18
 
19
+ // A claim that disappeared is invisible to every other command; compare
20
+ // identities against a reference when one is given.
21
+ const removed = values.since
22
+ ? computeRemovedClaims({ projectDir, ref: values.since, claimsPath: path, current: claims, ignorePath: values.ignore ? resolve(values.ignore) : undefined, evidencePath: evidencePath(projectDir), toolVersion: version })
23
+ : undefined;
24
+
25
+ // Cost is read back out of evidence the probe already wrote; it never runs a test.
26
+ let cost;
27
+ if (values.cost) {
28
+ const ev = values.evidence ? resolve(values.evidence) : evidencePath(projectDir);
29
+ cost = existsSync(ev) ? costReport(readSpecDoc('evidence', ev).records) : undefined;
30
+ if (!cost && !values.json) io.err(`no evidence at ${ev} — run \`testguard probe\` first; cost is derived from the run durations it records`);
31
+ }
32
+
13
33
  if (values.json) {
14
- io.out(JSON.stringify({ path, claims, annotations, drift }, null, 2));
34
+ io.out(JSON.stringify({ path, claims, annotations, drift, ...(removed ? { removed } : {}), ...(cost ? { cost } : {}) }, null, 2));
15
35
  } else {
16
36
  const annotated = new Set(drift.annotated);
17
37
  io.out(`${claims.claims.length} claims in ${path} — ${annotated.size} carry a @claim annotation in source (test files are not scanned)`);
18
38
  io.out('');
39
+ const signalLines = [];
19
40
  for (const c of claims.claims) {
20
41
  const declared = c.defendedBy?.length > 0;
21
- const defenders = declared ? resolveDefenders(projectDir, c.defendedBy) : [...new Set(c.faults.flatMap((f) => discoverDefenders(projectDir, f.file)))];
22
- const cover = defenders.length ? `${defenders.length} ${declared ? 'defender' : 'discovered'}${defenders.length === 1 ? '' : 's'}` : 'NO DEFENDER';
42
+ const targets = [...new Set(c.faults.map((f) => f.file))];
43
+ let cover;
44
+ if (declared) {
45
+ const defenders = resolveDefenders(projectDir, c.defendedBy);
46
+ const mocking = new Set();
47
+ for (const t of targets) { const m = classifyDefenders(projectDir, t, defenders); m.mocking.forEach((f) => mocking.add(f)); for (const s of m.signals) signalLines.push({ claim: c.id, target: t, ...s }); }
48
+ cover = defenders.length ? `${defenders.length} defender${defenders.length === 1 ? '' : 's'}${mocking.size ? ` (${mocking.size} mock the target)` : ''}` : 'NO DEFENDER';
49
+ } else {
50
+ const importing = new Set(); const mocking = new Set(); const can = new Set();
51
+ for (const t of targets) { const d = discoverDefendersDetailed(projectDir, t); d.importing.forEach((f) => importing.add(f)); d.mocking.forEach((f) => mocking.add(f)); d.canDetect.forEach((f) => can.add(f)); for (const s of d.signals) signalLines.push({ claim: c.id, target: t, ...s }); }
52
+ cover = can.size ? `${importing.size} import · ${mocking.size} mock · ${can.size} can detect` : importing.size ? `NO DEFENDER (${importing.size} import, all mock the target)` : 'NO DEFENDER';
53
+ }
23
54
  io.out(`${annotated.has(c.id) ? '@ ' : ' '}${c.id.padEnd(14)} ${c.severity.padEnd(8)} ${c.source.kind.padEnd(10)} ${String(c.faults.length).padStart(2)} fault${c.faults.length === 1 ? ' ' : 's'} ${cover.padEnd(12)} ${c.statement}`);
24
55
  }
56
+ if (signalLines.length) io.out('');
57
+ for (const s of signalLines) {
58
+ // The annotation's comment marker is the probed file's, not JavaScript's.
59
+ const marker = s.file.endsWith('.py') ? '#' : '//';
60
+ if (s.signal === 'mocked-never-asserted') io.out(`MOCKED-NEVER-ASSERTED ${s.file} mocks ${s.target} and never asserts on it (${s.claim}) — assert on the mocked call, drive the real module, or annotate the mock \`${marker} unasserted: <why>\``);
61
+ // An attribute patch does not remove a defender; saying it "mocks" the
62
+ // module would tell the author the opposite of what happened.
63
+ else if (s.signal === 'target-attribute-patched') io.out(`attribute-patched ${s.file} defends ${s.target} but replaces ${s.reason} (${s.claim}) — a fault in those attributes is where it is least likely to notice`);
64
+ else io.out(`unasserted (annotated) ${s.file} mocks ${s.target}: ${s.reason} (${s.claim})`);
65
+ }
25
66
  if (drift.undeclared.length || drift.stale.length) io.out('');
26
67
  for (const a of drift.undeclared) io.out(`UNDECLARED @claim ${a.id} at ${a.file}:${a.line} has no entry in the claims file — a claim with no fault model`);
27
68
  for (const c of drift.stale) io.out(`STALE ${c.id} is annotation-sourced but no source file carries @claim ${c.id}`);
69
+ if (removed) {
70
+ io.out('');
71
+ io.out(renderRemoved(removed));
72
+ }
73
+ if (cost) {
74
+ io.out('');
75
+ io.out(renderCost(cost));
76
+ }
28
77
  }
29
- return drift.undeclared.length || drift.stale.length ? 1 : 0;
78
+ return drift.undeclared.length || drift.stale.length || (removed?.removed.length ?? 0) > 0 ? 1 : 0;
30
79
  }
@@ -1,15 +1,28 @@
1
1
  import { initProject } from '../init/init.mjs';
2
2
 
3
3
  export async function initCommand({ projectDir, values }, io) {
4
- const { done, skipped } = initProject({ projectDir, force: values.force });
4
+ const r = initProject({ projectDir, force: values.force, here: values.here, ciEvidence: values['ci-evidence'], mcp: values.mcp });
5
+ const { done, skipped, warnings, agentRoot, dir, mcpConfig } = r;
5
6
  if (values.json) {
6
- io.out(JSON.stringify({ done, skipped }, null, 2));
7
- return 0;
7
+ io.out(JSON.stringify({ done, skipped, warnings, agentRoot, dir, ...(mcpConfig ? { mcpConfig } : {}) }, null, 2));
8
+ return warnings.length ? 1 : 0;
8
9
  }
9
10
  for (const d of done) io.out(`+ ${d}`);
10
11
  for (const s of skipped) io.out(`= ${s}`);
12
+ for (const w of warnings) io.out(`! ${w}`);
11
13
  io.out('');
12
- io.out('Agents in this project now start with the blind-spot brief and can run `testguard status --json` to learn what to do next.');
13
- io.out(done.length ? 'Commit these files.' : 'Nothing to change.');
14
- return 0;
14
+ if (dir !== '.') io.out(`Agent layer (skill, session-start hook, AGENTS.md section) installed at the git root ${agentRoot}, where agent sessions run; the project layer stays in ${dir}/. Use --here to keep everything in ${dir}/.`);
15
+ io.out(`Agents now start with the blind-spot brief and can run \`testguard status --json${dir === '.' ? '' : ` ${dir}`}\` to learn what to do next. The hook prefers a local install and never fetches from the network.`);
16
+ io.out(warnings.length ? 'Some written files are ignored by git — fix that before committing.' : done.length ? 'Commit these files.' : 'Nothing to change.');
17
+ if (mcpConfig) {
18
+ io.out('');
19
+ io.out('MCP — five READ-ONLY tools (status, brief, claims, evidence, next_command), so the loop survives a change of harness. Nothing there runs a probe. Add ONE of these:');
20
+ for (const [where, snippet] of Object.entries(mcpConfig)) {
21
+ io.out('');
22
+ io.out(` ${where}`);
23
+ const text = typeof snippet === 'string' ? snippet : JSON.stringify(snippet, null, 2);
24
+ for (const line of text.split('\n')) io.out(` ${line}`);
25
+ }
26
+ }
27
+ return warnings.length ? 1 : 0;
15
28
  }
@@ -0,0 +1,12 @@
1
+ import { serve } from '../mcp/server.mjs';
2
+
3
+ export async function mcpCommand({ values, version }, io) {
4
+ if (values.json) {
5
+ // Not a JSON command: it is a long-lived server on stdio, and printing
6
+ // anything to stdout would corrupt the protocol stream.
7
+ io.err('mcp speaks JSON-RPC on stdin/stdout; there is no --json mode');
8
+ return 3;
9
+ }
10
+ await serve({ version });
11
+ return 0;
12
+ }
@@ -7,6 +7,8 @@ import { gate } from '../baseline/baseline.mjs';
7
7
  import { renderRecord, renderSummary, sortForReport, PROVISIONAL_WARNING } from '../render.mjs';
8
8
  import { computeStatus } from '../status/status.mjs';
9
9
  import { resolveChangedRef, withChangedRef } from '../gate/changed.mjs';
10
+ import { costReport, renderCost } from '../probe/cost.mjs';
11
+ import { progressMode, stageReporter, clearStageLine, recordEvent, isProgressMode, progressStream } from '../probe/progress.mjs';
10
12
  export const provisionalEvidencePath = (projectDir) => join(projectDir, '.testguard', 'evidence-provisional.json');
11
13
 
12
14
  export const evidencePath = (projectDir) => join(projectDir, '.testguard', 'evidence.json');
@@ -19,6 +21,14 @@ export async function probeCommand({ projectDir, values, version }, io) {
19
21
  io.err('--confirm must be a positive integer and --budget at least 1000');
20
22
  return 3;
21
23
  }
24
+ if (values.progress && !isProgressMode(values.progress)) {
25
+ io.err(`--progress must be one of auto, tty, plain, ndjson, none (got ${values.progress})`);
26
+ return 3;
27
+ }
28
+ // Progress always goes to stderr, so --json keeps stdout parseable.
29
+ const progress = progressMode({ explicit: values.progress, isTTY: process.stderr.isTTY, quiet: values.quiet, json: values.json });
30
+ const progressTo = progressStream(process);
31
+ const stageOut = stageReporter(progress, (s) => progressTo.write(s));
22
32
  const claims = loadClaims(values.claims ? resolve(values.claims) : defaultClaimsPath(projectDir));
23
33
  if (claims.claims.length === 0) {
24
34
  io.err('claims file declares no claims; nothing to verify');
@@ -30,6 +40,7 @@ export async function probeCommand({ projectDir, values, version }, io) {
30
40
  // Provisional and partial runs never overwrite the canonical evidence: only confirmed, complete runs may feed a baseline.
31
41
  const outPath = values.out ? resolve(values.out) : only ? join(projectDir, '.testguard', 'evidence-partial.json') : provisional ? provisionalEvidencePath(projectDir) : evidencePath(projectDir);
32
42
  if (provisional && !values.quiet) io.err(PROVISIONAL_WARNING(confirmRuns));
43
+ const discoveryNoted = new Set();
33
44
  const previous = !values['no-reuse'] && existsSync(outPath) ? readSpecDoc('evidence', outPath) : undefined;
34
45
  const basePath = values.baseline ? resolve(values.baseline) : baselinePath(projectDir);
35
46
  const baseline = existsSync(basePath) ? readSpecDoc('baseline', basePath) : undefined;
@@ -41,26 +52,38 @@ export async function probeCommand({ projectDir, values, version }, io) {
41
52
  confirmRuns,
42
53
  budgetMs,
43
54
  mode: values['in-place'] ? 'in-place' : 'worktree',
44
- ref: values.ref,
55
+ ref: values.ref ?? 'HEAD',
56
+ refExplicit: values.ref !== undefined,
57
+ ignoreDirty: values['ignore-dirty'],
58
+ serial: values.serial,
59
+ onWarn: (m) => io.err(`warning: ${m}`),
45
60
  runnerCommand: values['runner-cmd'],
46
61
  runnerName: values.runner,
47
62
  nodeModules: values['node-modules'] ? resolve(values['node-modules']) : process.env.TESTGUARD_NODE_MODULES,
63
+ python: values.python ? resolve(values.python) : undefined,
48
64
  only,
49
65
  escalate: !values['no-escalate'],
50
66
  toolVersion: version,
51
67
  includeDirty: values['include-dirty'],
52
- onStage: !values.quiet && !values.json && process.stderr.isTTY ? ({ claimId, faultId, stage, i, n }) => process.stderr.write(`\r\x1b[K … ${claimId}/${faultId} ${stage} ${i}/${n}`) : undefined,
53
- onProgress: values.quiet || values.json ? undefined : (r) => {
54
- if (process.stderr.isTTY) process.stderr.write('\r\x1b[K');
55
- if (values.verbose || r.verdict !== 'killed') io.out(renderRecord(r, { provisional }) + (r.reusedFrom ? ' (reused)' : ''));
68
+ onStage: stageOut,
69
+ onProgress: (r) => {
70
+ // ndjson streams verdicts as they land; a watcher sees them without
71
+ // waiting for the document at the end.
72
+ if (progress === 'ndjson') progressTo.write(recordEvent(r));
73
+ if (values.quiet || values.json) return;
74
+ progressTo.write(clearStageLine(progress));
75
+ // The discovery note is a property of the claim, not of each fault: say it once.
76
+ const firstOfClaim = !discoveryNoted.has(r.claim.id);
77
+ if (r.defenders.discovered) discoveryNoted.add(r.claim.id);
78
+ if (values.verbose || r.verdict !== 'killed') io.out(renderRecord(r, { provisional, showDiscovered: firstOfClaim }) + (r.reusedFrom ? ' (reused)' : ''));
56
79
  },
57
80
  });
58
81
  writeSpecDoc('evidence', outPath, evidence);
59
82
 
60
83
  const g = gate(evidence.records, baseline, { severityFloor: values.severity });
61
84
  if (values.json) {
62
- const status = withChangedRef(resolveChangedRef({ explicit: values.changed }), (changedRef) => computeStatus({ projectDir, toolVersion: version, changedRef, includeDirty: values['include-dirty'] }), io.err);
63
- io.out(JSON.stringify({ ...status, run: { id: evidence.run.id, evidence: outPath, provisional, records: evidence.records.length, newSinceBaseline: g.new.length, exitCode: g.new.length > 0 ? 1 : 0 } }, null, 2));
85
+ const status = withChangedRef(resolveChangedRef({ explicit: values.changed }), (changedRef) => computeStatus({ projectDir, toolVersion: version, changedRef, includeDirty: values['include-dirty'], evidence: outPath }), io.err);
86
+ io.out(JSON.stringify({ ...status, run: { id: evidence.run.id, evidence: outPath, provisional, records: evidence.records.length, newSinceBaseline: g.new.length, exitCode: g.new.length > 0 ? 1 : 0 }, ...(values.cost ? { cost: costReport(evidence.records) } : {}) }, null, 2));
64
87
  return g.new.length > 0 ? 1 : 0;
65
88
  }
66
89
  if (!values.quiet && baseline) {
@@ -74,6 +97,11 @@ export async function probeCommand({ projectDir, values, version }, io) {
74
97
  if (killed) io.out(` ${killed} killed (not listed; --verbose to see them)`);
75
98
  }
76
99
  io.out(renderSummary(evidence.records, evidence.run) + (baseline ? ` ${g.new.length} new since baseline, ${g.baselined.length} baselined.` : ' No baseline.'));
100
+ if (values.cost) {
101
+ io.out('');
102
+ io.out(renderCost(costReport(evidence.records)));
103
+ io.out('');
104
+ }
77
105
  io.out(`evidence: ${outPath}${only ? ` (partial: --claim ${only.join(',')}; not the canonical evidence file)` : provisional ? ' (provisional; not the canonical evidence file)' : ''}`);
78
106
  if (provisional && !values.quiet) io.err(PROVISIONAL_WARNING(confirmRuns));
79
107
  return g.new.length > 0 ? 1 : 0;
@@ -0,0 +1,63 @@
1
+ import { dirname, join, resolve } from 'node:path';
2
+ import { replay, calibrationFrom, renderReplay } from '../replay/replay.mjs';
3
+ import { writeSpecDoc } from '../evidence/writer.mjs';
4
+
5
+ export const replayPath = (projectDir) => join(projectDir, '.testguard', 'replay.json');
6
+ export const calibrationPath = (projectDir) => join(projectDir, '.testguard', 'calibration.json');
7
+
8
+ export async function replayCommand({ projectDir, values, version }, io) {
9
+ const range = values.since ?? values.changed;
10
+ if (!range) {
11
+ io.err('usage: testguard replay [dir] --since <range> (e.g. --since HEAD~50..HEAD, or a tag)');
12
+ return 3;
13
+ }
14
+ const confirmRuns = Number(values.confirm);
15
+ const budgetMs = Number(values.budget);
16
+ const limit = values.max ? Number(values.max) : undefined;
17
+ if (!Number.isInteger(confirmRuns) || confirmRuns < 1 || !Number.isInteger(budgetMs) || budgetMs < 1000) {
18
+ io.err('--confirm must be a positive integer and --budget at least 1000');
19
+ return 3;
20
+ }
21
+
22
+ const doc = await replay({
23
+ projectDir,
24
+ range,
25
+ confirmRuns,
26
+ budgetMs,
27
+ runnerCommand: values['runner-cmd'],
28
+ runnerName: values.runner,
29
+ nodeModules: values['node-modules'] ? resolve(values['node-modules']) : process.env.TESTGUARD_NODE_MODULES,
30
+ limit,
31
+ toolVersion: version,
32
+ onStage: !values.quiet && !values.json && process.stderr.isTTY ? ({ commit, i, n, stage }) => process.stderr.write(`\r\x1b[K … ${commit.slice(0, 9)} (${i}/${n}) ${stage}`) : undefined,
33
+ onProgress: values.quiet || values.json ? undefined : () => {
34
+ if (process.stderr.isTTY) process.stderr.write('\r\x1b[K');
35
+ },
36
+ });
37
+
38
+ const outPath = values.out ? resolve(values.out) : replayPath(projectDir);
39
+ writeSpecDoc('replay', outPath, doc);
40
+ const calibration = calibrationFrom(doc, { toolVersion: version });
41
+ // Beside the replay document, whatever --out says: the two are one result,
42
+ // and splitting them across directories loses the pairing — and leaves a
43
+ // file behind in a repository the run is only meant to read.
44
+ const calPath = values.baseline ? resolve(values.baseline) : values.out ? join(dirname(outPath), 'calibration.json') : calibrationPath(projectDir);
45
+ writeSpecDoc('calibration', calPath, calibration);
46
+
47
+ if (values.json) {
48
+ io.out(JSON.stringify({ replay: doc, calibration, paths: { replay: outPath, calibration: calPath } }, null, 2));
49
+ } else {
50
+ io.out(renderReplay(doc));
51
+ io.out('');
52
+ for (const [cls, b] of Object.entries(calibration.buckets).sort((a, b2) => b2[1].n - a[1].n)) {
53
+ io.out(` ${cls.padEnd(22)} blind ${String(b.positives).padStart(3)}/${String(b.n).padEnd(3)} p=${b.p.toFixed(2)} ci [${b.ci[0].toFixed(2)}, ${b.ci[1].toFixed(2)}]`);
54
+ }
55
+ if (Object.keys(calibration.buckets).length === 0) io.out(' no measurable bug in this range: nothing to calibrate from yet.');
56
+ io.out('');
57
+ io.out(`replay: ${outPath}`);
58
+ io.out(`calibration: ${calPath}`);
59
+ }
60
+ // Replay reports; it never gates. A bug that escaped is history, not a
61
+ // regression in this change, and a gate on history is a gate nobody can pass.
62
+ return 0;
63
+ }
@@ -1,9 +1,11 @@
1
+ import { resolve } from 'node:path';
1
2
  import { computeStatus, renderStatus } from '../status/status.mjs';
2
3
  import { validate } from '../../spec/lib/validate.mjs';
3
4
  import { resolveChangedRef, withChangedRef } from '../gate/changed.mjs';
4
5
 
5
6
  export async function statusCommand({ projectDir, values, version }, io) {
6
- const doc = withChangedRef(resolveChangedRef({ explicit: values.changed }), (changedRef) => computeStatus({ projectDir, toolVersion: version, changedRef, includeDirty: values['include-dirty'] }), io.err);
7
+ const evidence = values.evidence ? resolve(values.evidence) : undefined;
8
+ const doc = withChangedRef(resolveChangedRef({ explicit: values.changed }), (changedRef) => computeStatus({ projectDir, toolVersion: version, changedRef, includeDirty: values['include-dirty'], evidence }), io.err);
7
9
  const result = validate('status', doc);
8
10
  if (!result.ok) throw new Error(`status document does not conform: ${result.errors.map((e) => `${e.path}: ${e.message}`).join('; ')}`);
9
11
  io.out(values.json ? JSON.stringify(doc, null, 2) : renderStatus(doc));
@@ -5,6 +5,7 @@ import { loadClaims, defaultClaimsPath } from '../claims/load.mjs';
5
5
  import { readSpecDoc } from '../evidence/writer.mjs';
6
6
  import { resolveDefenders } from '../probe/runners/shared.mjs';
7
7
  import { discoverDefenders } from '../probe/discover.mjs';
8
+ import { IS_PY_TEST } from '../probe/pyimports.mjs';
8
9
  import { globToRegExp } from '../util/glob.mjs';
9
10
 
10
11
  /**
@@ -20,8 +21,14 @@ import { globToRegExp } from '../util/glob.mjs';
20
21
  * the smallest unit the rest of the contract already understands.
21
22
  */
22
23
 
23
- export const SOURCE_EXT = new Set(['.js', '.mjs', '.cjs', '.ts', '.mts', '.cts', '.jsx', '.tsx']);
24
+ // `.py` belongs here for the same reason every other extension does: a file
25
+ // the gate does not recognise as source is excluded as `non-source`, which
26
+ // means a changed Python file could never be reported as uncovered and the
27
+ // gate would pass while saying nothing.
28
+ export const SOURCE_EXT = new Set(['.js', '.mjs', '.cjs', '.ts', '.mts', '.cts', '.jsx', '.tsx', '.py']);
24
29
  const TEST_RE = /\.(test|spec)\.[cm]?[jt]sx?$/;
30
+ // Python names tests by file, not by a dotted suffix: `test_foo.py`, `foo_test.py`.
31
+ const isTestFile = (f) => TEST_RE.test(f) || IS_PY_TEST.test(f);
25
32
 
26
33
  /** Files that are source by extension but never carry claims. Printed by --explain. */
27
34
  export const DEFAULT_EXCLUDES = Object.freeze([
@@ -163,7 +170,7 @@ export function computeChangedGate({ projectDir, ref, includeDirty = false, excl
163
170
  const dx = defaultExcludes.find((x) => x.re.test(file));
164
171
  if (dx) { excluded.push({ file, by: `default:${dx.glob}` }); continue; }
165
172
 
166
- const isTest = TEST_RE.test(file);
173
+ const isTest = isTestFile(file);
167
174
  if (faultFiles.has(file)) { covered.push({ file, by: 'fault', claimIds: faultFiles.get(file) }); continue; }
168
175
  if (isTest && defenderFiles.has(file)) { covered.push({ file, by: 'defender', claimIds: defenderFiles.get(file) }); continue; }
169
176
  const active = pathEntries.find((p) => !p.expired && p.re.test(file));
package/src/git.mjs CHANGED
@@ -23,6 +23,11 @@ export function headSha(dir, ref = 'HEAD') {
23
23
  }
24
24
  }
25
25
 
26
+ /** Is `ancestor` reachable from `ref`? False when either does not resolve. */
27
+ export function isAncestor(dir, ancestor, ref = 'HEAD') {
28
+ return spawnSync('git', ['merge-base', '--is-ancestor', ancestor, ref], { cwd: dir }).status === 0;
29
+ }
30
+
26
31
  /** True when any of `paths` (repo-relative; empty = whole tree) has uncommitted changes. */
27
32
  export const isDirty = (dir, paths = []) => git(['status', '--porcelain', '--', ...paths], dir).length > 0;
28
33