testguard-cli 0.4.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +291 -0
- package/README.md +321 -27
- package/package.json +1 -1
- package/spec/GATE-SEMANTICS.md +109 -1
- package/spec/README.md +4 -2
- package/spec/lib/validate.mjs +75 -1
- package/spec/schemas/baseline.schema.json +8 -0
- package/spec/schemas/brief.schema.json +11 -1
- package/spec/schemas/common.schema.json +32 -1
- package/spec/schemas/evidence.schema.json +374 -63
- package/spec/schemas/gate.schema.json +52 -0
- package/spec/schemas/replay.schema.json +85 -0
- package/spec/schemas/status.schema.json +269 -45
- package/src/admit/admit.mjs +17 -0
- package/src/baseline/baseline.mjs +31 -2
- package/src/brief/brief.mjs +96 -8
- package/src/claims/removed.mjs +161 -0
- package/src/cli.mjs +69 -12
- package/src/commands/admit.mjs +116 -0
- package/src/commands/baseline.mjs +17 -3
- package/src/commands/brief.mjs +26 -12
- package/src/commands/claims.mjs +35 -6
- package/src/commands/gate.mjs +47 -0
- package/src/commands/init.mjs +19 -6
- package/src/commands/mcp.mjs +12 -0
- package/src/commands/probe.mjs +12 -3
- package/src/commands/replay.mjs +63 -0
- package/src/commands/status.mjs +4 -1
- package/src/gate/changed.mjs +222 -0
- package/src/git.mjs +5 -0
- package/src/init/init.mjs +182 -29
- package/src/init/templates/SKILL.md +21 -3
- package/src/mcp/server.mjs +104 -0
- package/src/mcp/tools.mjs +114 -0
- package/src/probe/contention.mjs +60 -0
- package/src/probe/discover.mjs +12 -3
- package/src/probe/independence.mjs +59 -0
- package/src/probe/mocks.mjs +120 -0
- package/src/probe/probe.mjs +126 -18
- package/src/probe/rank.mjs +73 -15
- package/src/probe/runners/index.mjs +49 -5
- package/src/probe/runners/jest.mjs +3 -3
- package/src/probe/runners/playwright.mjs +133 -0
- package/src/probe/runners/shared.mjs +70 -13
- package/src/probe/runners/vitest.mjs +4 -3
- package/src/render.mjs +2 -2
- package/src/replay/label.mjs +64 -0
- package/src/replay/replay.mjs +309 -0
- package/src/scaffold/producers.mjs +217 -6
- package/src/scaffold/scaffold.mjs +9 -3
- package/src/status/status.mjs +88 -7
package/src/commands/brief.mjs
CHANGED
|
@@ -1,15 +1,32 @@
|
|
|
1
1
|
import { existsSync } from 'node:fs';
|
|
2
2
|
import { join, resolve } from 'node:path';
|
|
3
3
|
import { readSpecDoc, writeSpecDoc } from '../evidence/writer.mjs';
|
|
4
|
-
import { buildBrief } from '../brief/brief.mjs';
|
|
4
|
+
import { buildBrief, buildUnclaimedBrief, renderBriefMarkdown, renderUnclaimedMarkdown } from '../brief/brief.mjs';
|
|
5
5
|
import { computeStatus } from '../status/status.mjs';
|
|
6
6
|
import { evidencePath, baselinePath } from './probe.mjs';
|
|
7
|
+
import { resolveChangedRef, withChangedRef } from '../gate/changed.mjs';
|
|
7
8
|
|
|
8
9
|
export async function briefCommand({ projectDir, values, version }, io) {
|
|
9
10
|
const evPath = values.evidence ? resolve(values.evidence) : evidencePath(projectDir);
|
|
11
|
+
let status;
|
|
12
|
+
try {
|
|
13
|
+
// A detected base that does not resolve is silent here: the brief is a
|
|
14
|
+
// session-start hook's output and must never add noise; `status` and
|
|
15
|
+
// `gate` are where that warning is printed.
|
|
16
|
+
status = withChangedRef(resolveChangedRef({ explicit: values.changed }), (changedRef) => computeStatus({ projectDir, toolVersion: version, changedRef, includeDirty: values['include-dirty'], evidence: values.evidence ? evPath : undefined }));
|
|
17
|
+
} catch {
|
|
18
|
+
status = undefined; // a brief must never fail because status could not be computed
|
|
19
|
+
}
|
|
10
20
|
if (!existsSync(evPath)) {
|
|
11
|
-
// A missing brief must never break an agent's session start
|
|
12
|
-
|
|
21
|
+
// A missing brief must never break an agent's session start — but unclaimed
|
|
22
|
+
// changes are still said, because the claim comes before the probe.
|
|
23
|
+
if (values.text || values.markdown) {
|
|
24
|
+
if (status?.changes?.uncovered?.length) {
|
|
25
|
+
const args = { tool: { name: 'testguard', version }, next: status.next, changes: status.changes };
|
|
26
|
+
io.out((values.markdown ? renderUnclaimedMarkdown(args) : buildUnclaimedBrief(args).text).trimEnd());
|
|
27
|
+
}
|
|
28
|
+
return 0;
|
|
29
|
+
}
|
|
13
30
|
io.err(`no evidence at ${evPath}; run \`testguard probe\` first`);
|
|
14
31
|
return 2;
|
|
15
32
|
}
|
|
@@ -21,14 +38,11 @@ export async function briefCommand({ projectDir, values, version }, io) {
|
|
|
21
38
|
io.err('--max must be an integer from 1 to 50');
|
|
22
39
|
return 3;
|
|
23
40
|
}
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
const brief = buildBrief(evidence, baseline, { max, next });
|
|
31
|
-
if (!values.text) writeSpecDoc('brief', values.out ? resolve(values.out) : join(projectDir, '.testguard', 'brief.json'), brief);
|
|
32
|
-
io.out(values.json ? JSON.stringify(brief, null, 2) : brief.text.trimEnd());
|
|
41
|
+
// Where this binary came from, so a stale install is visible in the session-start context.
|
|
42
|
+
const install = /[\\/]node_modules[\\/]/.test(process.argv[1] ?? '') ? 'local install' : 'global';
|
|
43
|
+
const brief = buildBrief(evidence, baseline, { max, next: status?.next, changes: status?.changes, install });
|
|
44
|
+
if (!values.text && !values.markdown) writeSpecDoc('brief', values.out ? resolve(values.out) : join(projectDir, '.testguard', 'brief.json'), brief);
|
|
45
|
+
if (values.markdown) io.out(renderBriefMarkdown({ ...brief, provisional: Boolean(evidence.run.provisional) }, { hasBaseline: Boolean(baseline), total: evidence.records.length, install }).trimEnd());
|
|
46
|
+
else io.out(values.json ? JSON.stringify(brief, null, 2) : brief.text.trimEnd());
|
|
33
47
|
return 0;
|
|
34
48
|
}
|
package/src/commands/claims.mjs
CHANGED
|
@@ -2,29 +2,58 @@ import { resolve } from 'node:path';
|
|
|
2
2
|
import { loadClaims, defaultClaimsPath } from '../claims/load.mjs';
|
|
3
3
|
import { scanAnnotations, reconcile } from '../claims/annotations.mjs';
|
|
4
4
|
import { resolveDefenders } from '../probe/runners/shared.mjs';
|
|
5
|
-
import {
|
|
5
|
+
import { discoverDefendersDetailed } from '../probe/discover.mjs';
|
|
6
|
+
import { classifyDefenders } from '../probe/mocks.mjs';
|
|
7
|
+
import { computeRemovedClaims, renderRemoved } from '../claims/removed.mjs';
|
|
8
|
+
import { evidencePath } from './probe.mjs';
|
|
6
9
|
|
|
7
|
-
export async function claimsCommand({ projectDir, values }, io) {
|
|
10
|
+
export async function claimsCommand({ projectDir, values, version }, io) {
|
|
8
11
|
const path = values.claims ? resolve(values.claims) : defaultClaimsPath(projectDir);
|
|
9
12
|
const claims = loadClaims(path);
|
|
10
13
|
const annotations = scanAnnotations(projectDir);
|
|
11
14
|
const drift = reconcile(claims, annotations);
|
|
12
15
|
|
|
16
|
+
// A claim that disappeared is invisible to every other command; compare
|
|
17
|
+
// identities against a reference when one is given.
|
|
18
|
+
const removed = values.since
|
|
19
|
+
? computeRemovedClaims({ projectDir, ref: values.since, claimsPath: path, current: claims, ignorePath: values.ignore ? resolve(values.ignore) : undefined, evidencePath: evidencePath(projectDir), toolVersion: version })
|
|
20
|
+
: undefined;
|
|
21
|
+
|
|
13
22
|
if (values.json) {
|
|
14
|
-
io.out(JSON.stringify({ path, claims, annotations, drift }, null, 2));
|
|
23
|
+
io.out(JSON.stringify({ path, claims, annotations, drift, ...(removed ? { removed } : {}) }, null, 2));
|
|
15
24
|
} else {
|
|
16
25
|
const annotated = new Set(drift.annotated);
|
|
17
26
|
io.out(`${claims.claims.length} claims in ${path} — ${annotated.size} carry a @claim annotation in source (test files are not scanned)`);
|
|
18
27
|
io.out('');
|
|
28
|
+
const signalLines = [];
|
|
19
29
|
for (const c of claims.claims) {
|
|
20
30
|
const declared = c.defendedBy?.length > 0;
|
|
21
|
-
const
|
|
22
|
-
|
|
31
|
+
const targets = [...new Set(c.faults.map((f) => f.file))];
|
|
32
|
+
let cover;
|
|
33
|
+
if (declared) {
|
|
34
|
+
const defenders = resolveDefenders(projectDir, c.defendedBy);
|
|
35
|
+
const mocking = new Set();
|
|
36
|
+
for (const t of targets) { const m = classifyDefenders(projectDir, t, defenders); m.mocking.forEach((f) => mocking.add(f)); for (const s of m.signals) signalLines.push({ claim: c.id, target: t, ...s }); }
|
|
37
|
+
cover = defenders.length ? `${defenders.length} defender${defenders.length === 1 ? '' : 's'}${mocking.size ? ` (${mocking.size} mock the target)` : ''}` : 'NO DEFENDER';
|
|
38
|
+
} else {
|
|
39
|
+
const importing = new Set(); const mocking = new Set(); const can = new Set();
|
|
40
|
+
for (const t of targets) { const d = discoverDefendersDetailed(projectDir, t); d.importing.forEach((f) => importing.add(f)); d.mocking.forEach((f) => mocking.add(f)); d.canDetect.forEach((f) => can.add(f)); for (const s of d.signals) signalLines.push({ claim: c.id, target: t, ...s }); }
|
|
41
|
+
cover = can.size ? `${importing.size} import · ${mocking.size} mock · ${can.size} can detect` : importing.size ? `NO DEFENDER (${importing.size} import, all mock the target)` : 'NO DEFENDER';
|
|
42
|
+
}
|
|
23
43
|
io.out(`${annotated.has(c.id) ? '@ ' : ' '}${c.id.padEnd(14)} ${c.severity.padEnd(8)} ${c.source.kind.padEnd(10)} ${String(c.faults.length).padStart(2)} fault${c.faults.length === 1 ? ' ' : 's'} ${cover.padEnd(12)} ${c.statement}`);
|
|
24
44
|
}
|
|
45
|
+
if (signalLines.length) io.out('');
|
|
46
|
+
for (const s of signalLines) {
|
|
47
|
+
if (s.signal === 'mocked-never-asserted') io.out(`MOCKED-NEVER-ASSERTED ${s.file} mocks ${s.target} and never asserts on it (${s.claim}) — assert on the mocked call, drive the real module, or annotate the mock \`// unasserted: <why>\``);
|
|
48
|
+
else io.out(`unasserted (annotated) ${s.file} mocks ${s.target}: ${s.reason} (${s.claim})`);
|
|
49
|
+
}
|
|
25
50
|
if (drift.undeclared.length || drift.stale.length) io.out('');
|
|
26
51
|
for (const a of drift.undeclared) io.out(`UNDECLARED @claim ${a.id} at ${a.file}:${a.line} has no entry in the claims file — a claim with no fault model`);
|
|
27
52
|
for (const c of drift.stale) io.out(`STALE ${c.id} is annotation-sourced but no source file carries @claim ${c.id}`);
|
|
53
|
+
if (removed) {
|
|
54
|
+
io.out('');
|
|
55
|
+
io.out(renderRemoved(removed));
|
|
56
|
+
}
|
|
28
57
|
}
|
|
29
|
-
return drift.undeclared.length || drift.stale.length ? 1 : 0;
|
|
58
|
+
return drift.undeclared.length || drift.stale.length || (removed?.removed.length ?? 0) > 0 ? 1 : 0;
|
|
30
59
|
}
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
import { join, resolve } from 'node:path';
|
|
2
|
+
import { computeChangedGate, resolveChangedRef, renderGate, DEFAULT_EXCLUDES } from '../gate/changed.mjs';
|
|
3
|
+
import { writeSpecDoc } from '../evidence/writer.mjs';
|
|
4
|
+
|
|
5
|
+
export const gatePath = (projectDir) => join(projectDir, '.testguard', 'gate.json');
|
|
6
|
+
|
|
7
|
+
/**
|
|
8
|
+
* `testguard gate --changed <ref>`: fail when a changed source file carries
|
|
9
|
+
* no claim and no excusing ignore entry. Exit 0 covered · 1 uncovered (or
|
|
10
|
+
* --strict with nothing evaluated) · 2 cannot evaluate · 3 no reference.
|
|
11
|
+
*/
|
|
12
|
+
export async function gateCommand({ projectDir, values, version }, io) {
|
|
13
|
+
if (values.explain) {
|
|
14
|
+
io.out('Files that are source by extension but never carry claims (excluded by default; add more with --exclude <glob>):');
|
|
15
|
+
for (const g of DEFAULT_EXCLUDES) io.out(` ${g}`);
|
|
16
|
+
io.out('Non-source files (anything but .js .mjs .cjs .ts .mts .cts .jsx .tsx) are excluded before these patterns apply.');
|
|
17
|
+
return 0;
|
|
18
|
+
}
|
|
19
|
+
const resolved = resolveChangedRef({ explicit: values.changed });
|
|
20
|
+
if (!resolved) {
|
|
21
|
+
io.err('gate needs a reference to measure the change against: --changed <ref> (e.g. origin/main), or set TESTGUARD_CHANGED_REF. In GitHub Actions and GitLab CI the base branch is detected automatically.');
|
|
22
|
+
return 3;
|
|
23
|
+
}
|
|
24
|
+
const ref = resolved.ref;
|
|
25
|
+
// The gate itself never degrades: a detected reference that does not resolve is exit 2 here, because the gate's whole job is the measurement.
|
|
26
|
+
if (!resolved.required && !values.json && !values.quiet) io.err(`--changed not given; using ${ref} from ${resolved.from}`);
|
|
27
|
+
const doc = computeChangedGate({
|
|
28
|
+
projectDir,
|
|
29
|
+
ref,
|
|
30
|
+
includeDirty: values['include-dirty'],
|
|
31
|
+
exclude: values.exclude ?? [],
|
|
32
|
+
strict: values.strict,
|
|
33
|
+
toolVersion: version,
|
|
34
|
+
claimsPath: values.claims ? resolve(values.claims) : undefined,
|
|
35
|
+
ignorePath: values.ignore ? resolve(values.ignore) : undefined,
|
|
36
|
+
});
|
|
37
|
+
const outPath = values.out ? resolve(values.out) : gatePath(projectDir);
|
|
38
|
+
writeSpecDoc('gate', outPath, doc);
|
|
39
|
+
if (values.json) {
|
|
40
|
+
io.out(JSON.stringify(doc, null, 2));
|
|
41
|
+
} else {
|
|
42
|
+
io.out(renderGate(doc));
|
|
43
|
+
if (doc.uncovered.length) io.out(`Next: state the claim for each UNCLAIMED file (scaffold proposes the faults), or add a testguard.ignore.json path entry with a reason that a reviewer will accept.`);
|
|
44
|
+
if (!values.quiet) io.out(`gate: ${outPath}`);
|
|
45
|
+
}
|
|
46
|
+
return doc.exitCode;
|
|
47
|
+
}
|
package/src/commands/init.mjs
CHANGED
|
@@ -1,15 +1,28 @@
|
|
|
1
1
|
import { initProject } from '../init/init.mjs';
|
|
2
2
|
|
|
3
3
|
export async function initCommand({ projectDir, values }, io) {
|
|
4
|
-
const
|
|
4
|
+
const r = initProject({ projectDir, force: values.force, here: values.here, ciEvidence: values['ci-evidence'], mcp: values.mcp });
|
|
5
|
+
const { done, skipped, warnings, agentRoot, dir, mcpConfig } = r;
|
|
5
6
|
if (values.json) {
|
|
6
|
-
io.out(JSON.stringify({ done, skipped }, null, 2));
|
|
7
|
-
return 0;
|
|
7
|
+
io.out(JSON.stringify({ done, skipped, warnings, agentRoot, dir, ...(mcpConfig ? { mcpConfig } : {}) }, null, 2));
|
|
8
|
+
return warnings.length ? 1 : 0;
|
|
8
9
|
}
|
|
9
10
|
for (const d of done) io.out(`+ ${d}`);
|
|
10
11
|
for (const s of skipped) io.out(`= ${s}`);
|
|
12
|
+
for (const w of warnings) io.out(`! ${w}`);
|
|
11
13
|
io.out('');
|
|
12
|
-
io.out(
|
|
13
|
-
io.out(
|
|
14
|
-
|
|
14
|
+
if (dir !== '.') io.out(`Agent layer (skill, session-start hook, AGENTS.md section) installed at the git root ${agentRoot}, where agent sessions run; the project layer stays in ${dir}/. Use --here to keep everything in ${dir}/.`);
|
|
15
|
+
io.out(`Agents now start with the blind-spot brief and can run \`testguard status --json${dir === '.' ? '' : ` ${dir}`}\` to learn what to do next. The hook prefers a local install and never fetches from the network.`);
|
|
16
|
+
io.out(warnings.length ? 'Some written files are ignored by git — fix that before committing.' : done.length ? 'Commit these files.' : 'Nothing to change.');
|
|
17
|
+
if (mcpConfig) {
|
|
18
|
+
io.out('');
|
|
19
|
+
io.out('MCP — five READ-ONLY tools (status, brief, claims, evidence, next_command), so the loop survives a change of harness. Nothing there runs a probe. Add ONE of these:');
|
|
20
|
+
for (const [where, snippet] of Object.entries(mcpConfig)) {
|
|
21
|
+
io.out('');
|
|
22
|
+
io.out(` ${where}`);
|
|
23
|
+
const text = typeof snippet === 'string' ? snippet : JSON.stringify(snippet, null, 2);
|
|
24
|
+
for (const line of text.split('\n')) io.out(` ${line}`);
|
|
25
|
+
}
|
|
26
|
+
}
|
|
27
|
+
return warnings.length ? 1 : 0;
|
|
15
28
|
}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import { serve } from '../mcp/server.mjs';
|
|
2
|
+
|
|
3
|
+
export async function mcpCommand({ values, version }, io) {
|
|
4
|
+
if (values.json) {
|
|
5
|
+
// Not a JSON command: it is a long-lived server on stdio, and printing
|
|
6
|
+
// anything to stdout would corrupt the protocol stream.
|
|
7
|
+
io.err('mcp speaks JSON-RPC on stdin/stdout; there is no --json mode');
|
|
8
|
+
return 3;
|
|
9
|
+
}
|
|
10
|
+
await serve({ version });
|
|
11
|
+
return 0;
|
|
12
|
+
}
|
package/src/commands/probe.mjs
CHANGED
|
@@ -6,6 +6,7 @@ import { writeSpecDoc, readSpecDoc } from '../evidence/writer.mjs';
|
|
|
6
6
|
import { gate } from '../baseline/baseline.mjs';
|
|
7
7
|
import { renderRecord, renderSummary, sortForReport, PROVISIONAL_WARNING } from '../render.mjs';
|
|
8
8
|
import { computeStatus } from '../status/status.mjs';
|
|
9
|
+
import { resolveChangedRef, withChangedRef } from '../gate/changed.mjs';
|
|
9
10
|
export const provisionalEvidencePath = (projectDir) => join(projectDir, '.testguard', 'evidence-provisional.json');
|
|
10
11
|
|
|
11
12
|
export const evidencePath = (projectDir) => join(projectDir, '.testguard', 'evidence.json');
|
|
@@ -29,6 +30,7 @@ export async function probeCommand({ projectDir, values, version }, io) {
|
|
|
29
30
|
// Provisional and partial runs never overwrite the canonical evidence: only confirmed, complete runs may feed a baseline.
|
|
30
31
|
const outPath = values.out ? resolve(values.out) : only ? join(projectDir, '.testguard', 'evidence-partial.json') : provisional ? provisionalEvidencePath(projectDir) : evidencePath(projectDir);
|
|
31
32
|
if (provisional && !values.quiet) io.err(PROVISIONAL_WARNING(confirmRuns));
|
|
33
|
+
const discoveryNoted = new Set();
|
|
32
34
|
const previous = !values['no-reuse'] && existsSync(outPath) ? readSpecDoc('evidence', outPath) : undefined;
|
|
33
35
|
const basePath = values.baseline ? resolve(values.baseline) : baselinePath(projectDir);
|
|
34
36
|
const baseline = existsSync(basePath) ? readSpecDoc('baseline', basePath) : undefined;
|
|
@@ -40,7 +42,11 @@ export async function probeCommand({ projectDir, values, version }, io) {
|
|
|
40
42
|
confirmRuns,
|
|
41
43
|
budgetMs,
|
|
42
44
|
mode: values['in-place'] ? 'in-place' : 'worktree',
|
|
43
|
-
ref: values.ref,
|
|
45
|
+
ref: values.ref ?? 'HEAD',
|
|
46
|
+
refExplicit: values.ref !== undefined,
|
|
47
|
+
ignoreDirty: values['ignore-dirty'],
|
|
48
|
+
serial: values.serial,
|
|
49
|
+
onWarn: (m) => io.err(`warning: ${m}`),
|
|
44
50
|
runnerCommand: values['runner-cmd'],
|
|
45
51
|
runnerName: values.runner,
|
|
46
52
|
nodeModules: values['node-modules'] ? resolve(values['node-modules']) : process.env.TESTGUARD_NODE_MODULES,
|
|
@@ -51,14 +57,17 @@ export async function probeCommand({ projectDir, values, version }, io) {
|
|
|
51
57
|
onStage: !values.quiet && !values.json && process.stderr.isTTY ? ({ claimId, faultId, stage, i, n }) => process.stderr.write(`\r\x1b[K … ${claimId}/${faultId} ${stage} ${i}/${n}`) : undefined,
|
|
52
58
|
onProgress: values.quiet || values.json ? undefined : (r) => {
|
|
53
59
|
if (process.stderr.isTTY) process.stderr.write('\r\x1b[K');
|
|
54
|
-
|
|
60
|
+
// The discovery note is a property of the claim, not of each fault: say it once.
|
|
61
|
+
const firstOfClaim = !discoveryNoted.has(r.claim.id);
|
|
62
|
+
if (r.defenders.discovered) discoveryNoted.add(r.claim.id);
|
|
63
|
+
if (values.verbose || r.verdict !== 'killed') io.out(renderRecord(r, { provisional, showDiscovered: firstOfClaim }) + (r.reusedFrom ? ' (reused)' : ''));
|
|
55
64
|
},
|
|
56
65
|
});
|
|
57
66
|
writeSpecDoc('evidence', outPath, evidence);
|
|
58
67
|
|
|
59
68
|
const g = gate(evidence.records, baseline, { severityFloor: values.severity });
|
|
60
69
|
if (values.json) {
|
|
61
|
-
const status = computeStatus({ projectDir, toolVersion: version });
|
|
70
|
+
const status = withChangedRef(resolveChangedRef({ explicit: values.changed }), (changedRef) => computeStatus({ projectDir, toolVersion: version, changedRef, includeDirty: values['include-dirty'], evidence: outPath }), io.err);
|
|
62
71
|
io.out(JSON.stringify({ ...status, run: { id: evidence.run.id, evidence: outPath, provisional, records: evidence.records.length, newSinceBaseline: g.new.length, exitCode: g.new.length > 0 ? 1 : 0 } }, null, 2));
|
|
63
72
|
return g.new.length > 0 ? 1 : 0;
|
|
64
73
|
}
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
import { dirname, join, resolve } from 'node:path';
|
|
2
|
+
import { replay, calibrationFrom, renderReplay } from '../replay/replay.mjs';
|
|
3
|
+
import { writeSpecDoc } from '../evidence/writer.mjs';
|
|
4
|
+
|
|
5
|
+
export const replayPath = (projectDir) => join(projectDir, '.testguard', 'replay.json');
|
|
6
|
+
export const calibrationPath = (projectDir) => join(projectDir, '.testguard', 'calibration.json');
|
|
7
|
+
|
|
8
|
+
export async function replayCommand({ projectDir, values, version }, io) {
|
|
9
|
+
const range = values.since ?? values.changed;
|
|
10
|
+
if (!range) {
|
|
11
|
+
io.err('usage: testguard replay [dir] --since <range> (e.g. --since HEAD~50..HEAD, or a tag)');
|
|
12
|
+
return 3;
|
|
13
|
+
}
|
|
14
|
+
const confirmRuns = Number(values.confirm);
|
|
15
|
+
const budgetMs = Number(values.budget);
|
|
16
|
+
const limit = values.max ? Number(values.max) : undefined;
|
|
17
|
+
if (!Number.isInteger(confirmRuns) || confirmRuns < 1 || !Number.isInteger(budgetMs) || budgetMs < 1000) {
|
|
18
|
+
io.err('--confirm must be a positive integer and --budget at least 1000');
|
|
19
|
+
return 3;
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
const doc = await replay({
|
|
23
|
+
projectDir,
|
|
24
|
+
range,
|
|
25
|
+
confirmRuns,
|
|
26
|
+
budgetMs,
|
|
27
|
+
runnerCommand: values['runner-cmd'],
|
|
28
|
+
runnerName: values.runner,
|
|
29
|
+
nodeModules: values['node-modules'] ? resolve(values['node-modules']) : process.env.TESTGUARD_NODE_MODULES,
|
|
30
|
+
limit,
|
|
31
|
+
toolVersion: version,
|
|
32
|
+
onStage: !values.quiet && !values.json && process.stderr.isTTY ? ({ commit, i, n, stage }) => process.stderr.write(`\r\x1b[K … ${commit.slice(0, 9)} (${i}/${n}) ${stage}`) : undefined,
|
|
33
|
+
onProgress: values.quiet || values.json ? undefined : () => {
|
|
34
|
+
if (process.stderr.isTTY) process.stderr.write('\r\x1b[K');
|
|
35
|
+
},
|
|
36
|
+
});
|
|
37
|
+
|
|
38
|
+
const outPath = values.out ? resolve(values.out) : replayPath(projectDir);
|
|
39
|
+
writeSpecDoc('replay', outPath, doc);
|
|
40
|
+
const calibration = calibrationFrom(doc, { toolVersion: version });
|
|
41
|
+
// Beside the replay document, whatever --out says: the two are one result,
|
|
42
|
+
// and splitting them across directories loses the pairing — and leaves a
|
|
43
|
+
// file behind in a repository the run is only meant to read.
|
|
44
|
+
const calPath = values.baseline ? resolve(values.baseline) : values.out ? join(dirname(outPath), 'calibration.json') : calibrationPath(projectDir);
|
|
45
|
+
writeSpecDoc('calibration', calPath, calibration);
|
|
46
|
+
|
|
47
|
+
if (values.json) {
|
|
48
|
+
io.out(JSON.stringify({ replay: doc, calibration, paths: { replay: outPath, calibration: calPath } }, null, 2));
|
|
49
|
+
} else {
|
|
50
|
+
io.out(renderReplay(doc));
|
|
51
|
+
io.out('');
|
|
52
|
+
for (const [cls, b] of Object.entries(calibration.buckets).sort((a, b2) => b2[1].n - a[1].n)) {
|
|
53
|
+
io.out(` ${cls.padEnd(22)} blind ${String(b.positives).padStart(3)}/${String(b.n).padEnd(3)} p=${b.p.toFixed(2)} ci [${b.ci[0].toFixed(2)}, ${b.ci[1].toFixed(2)}]`);
|
|
54
|
+
}
|
|
55
|
+
if (Object.keys(calibration.buckets).length === 0) io.out(' no measurable bug in this range: nothing to calibrate from yet.');
|
|
56
|
+
io.out('');
|
|
57
|
+
io.out(`replay: ${outPath}`);
|
|
58
|
+
io.out(`calibration: ${calPath}`);
|
|
59
|
+
}
|
|
60
|
+
// Replay reports; it never gates. A bug that escaped is history, not a
|
|
61
|
+
// regression in this change, and a gate on history is a gate nobody can pass.
|
|
62
|
+
return 0;
|
|
63
|
+
}
|
package/src/commands/status.mjs
CHANGED
|
@@ -1,8 +1,11 @@
|
|
|
1
|
+
import { resolve } from 'node:path';
|
|
1
2
|
import { computeStatus, renderStatus } from '../status/status.mjs';
|
|
2
3
|
import { validate } from '../../spec/lib/validate.mjs';
|
|
4
|
+
import { resolveChangedRef, withChangedRef } from '../gate/changed.mjs';
|
|
3
5
|
|
|
4
6
|
export async function statusCommand({ projectDir, values, version }, io) {
|
|
5
|
-
const
|
|
7
|
+
const evidence = values.evidence ? resolve(values.evidence) : undefined;
|
|
8
|
+
const doc = withChangedRef(resolveChangedRef({ explicit: values.changed }), (changedRef) => computeStatus({ projectDir, toolVersion: version, changedRef, includeDirty: values['include-dirty'], evidence }), io.err);
|
|
6
9
|
const result = validate('status', doc);
|
|
7
10
|
if (!result.ok) throw new Error(`status document does not conform: ${result.errors.map((e) => `${e.path}: ${e.message}`).join('; ')}`);
|
|
8
11
|
io.out(values.json ? JSON.stringify(doc, null, 2) : renderStatus(doc));
|
|
@@ -0,0 +1,222 @@
|
|
|
1
|
+
import { existsSync, realpathSync } from 'node:fs';
|
|
2
|
+
import { extname, join, relative, resolve, dirname } from 'node:path';
|
|
3
|
+
import { git, repoRoot, headSha, GitError } from '../git.mjs';
|
|
4
|
+
import { loadClaims, defaultClaimsPath } from '../claims/load.mjs';
|
|
5
|
+
import { readSpecDoc } from '../evidence/writer.mjs';
|
|
6
|
+
import { resolveDefenders } from '../probe/runners/shared.mjs';
|
|
7
|
+
import { discoverDefenders } from '../probe/discover.mjs';
|
|
8
|
+
import { globToRegExp } from '../util/glob.mjs';
|
|
9
|
+
|
|
10
|
+
/**
|
|
11
|
+
* Claim coverage of a change.
|
|
12
|
+
*
|
|
13
|
+
* `probe` answers "do the tests defend the claims that exist?". This answers
|
|
14
|
+
* the question every field report so far turned on: "does the code I just
|
|
15
|
+
* changed have any claim at all?". A repository with ten old claims and one
|
|
16
|
+
* new, unclaimed feature exits 0 forever under `probe`; under `gate` the
|
|
17
|
+
* delta is measured, file by file, and an uncovered file is a finding.
|
|
18
|
+
*
|
|
19
|
+
* The unit is the file, on purpose: a fault anchors to a file, so the file is
|
|
20
|
+
* the smallest unit the rest of the contract already understands.
|
|
21
|
+
*/
|
|
22
|
+
|
|
23
|
+
export const SOURCE_EXT = new Set(['.js', '.mjs', '.cjs', '.ts', '.mts', '.cts', '.jsx', '.tsx']);
|
|
24
|
+
const TEST_RE = /\.(test|spec)\.[cm]?[jt]sx?$/;
|
|
25
|
+
|
|
26
|
+
/** Files that are source by extension but never carry claims. Printed by --explain. */
|
|
27
|
+
export const DEFAULT_EXCLUDES = Object.freeze([
|
|
28
|
+
'**/*.d.ts',
|
|
29
|
+
'**/*.config.*',
|
|
30
|
+
'**/*.stories.*',
|
|
31
|
+
'**/*.generated.*',
|
|
32
|
+
'**/fixtures/**',
|
|
33
|
+
'**/__fixtures__/**',
|
|
34
|
+
'**/__mocks__/**',
|
|
35
|
+
'**/__snapshots__/**',
|
|
36
|
+
'.testguard/**',
|
|
37
|
+
]);
|
|
38
|
+
|
|
39
|
+
export const defaultIgnorePath = (projectDir) => join(projectDir, 'testguard.ignore.json');
|
|
40
|
+
|
|
41
|
+
/**
|
|
42
|
+
* The reference the change is measured against. Explicit flag first, then the
|
|
43
|
+
* environments CI sets, then nothing: a wrong default would make the gate pass
|
|
44
|
+
* trivially (an upstream that already contains the change has an empty diff),
|
|
45
|
+
* so no ref is an error the caller reports, never a silent guess.
|
|
46
|
+
*/
|
|
47
|
+
export function detectChangedRef(env = process.env) {
|
|
48
|
+
if (env.TESTGUARD_CHANGED_REF) return { ref: env.TESTGUARD_CHANGED_REF, from: 'TESTGUARD_CHANGED_REF' };
|
|
49
|
+
// GitHub Actions, pull_request events.
|
|
50
|
+
if (env.GITHUB_BASE_REF) return { ref: `origin/${env.GITHUB_BASE_REF}`, from: 'GITHUB_BASE_REF' };
|
|
51
|
+
// GitLab merge request pipelines. The diff base sha is exact and needs no
|
|
52
|
+
// remote-tracking ref; the target branch name is the fallback (it needs
|
|
53
|
+
// `git fetch origin <branch>` or GIT_DEPTH: 0 on the job).
|
|
54
|
+
const sha = env.CI_MERGE_REQUEST_DIFF_BASE_SHA;
|
|
55
|
+
if (sha && /^[0-9a-f]{7,40}$/i.test(sha) && !/^0+$/.test(sha)) return { ref: sha, from: 'CI_MERGE_REQUEST_DIFF_BASE_SHA' };
|
|
56
|
+
if (env.CI_MERGE_REQUEST_TARGET_BRANCH_NAME) return { ref: `origin/${env.CI_MERGE_REQUEST_TARGET_BRANCH_NAME}`, from: 'CI_MERGE_REQUEST_TARGET_BRANCH_NAME' };
|
|
57
|
+
return null;
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
/**
|
|
61
|
+
* Resolve the reference a command should measure against, and whether it may
|
|
62
|
+
* be dropped on failure. An explicit `--changed` is the user's word: if it
|
|
63
|
+
* does not resolve, that is an error. A reference detected from CI variables
|
|
64
|
+
* is a convenience: if it does not resolve (shallow clone, temp directory,
|
|
65
|
+
* no remote) the command says so on stderr and continues without a change
|
|
66
|
+
* measurement, because `status` and `brief` must keep working everywhere.
|
|
67
|
+
*/
|
|
68
|
+
export function resolveChangedRef({ explicit, env = process.env } = {}) {
|
|
69
|
+
if (explicit) return { ref: explicit, from: '--changed', required: true };
|
|
70
|
+
const d = detectChangedRef(env);
|
|
71
|
+
return d ? { ...d, required: false } : null;
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
/**
|
|
75
|
+
* Run `compute(ref)` for a resolved reference. A required reference propagates
|
|
76
|
+
* its error; a detected one that fails yields `undefined` after `warn(message)`.
|
|
77
|
+
*/
|
|
78
|
+
export function withChangedRef(resolved, compute, warn = () => {}) {
|
|
79
|
+
if (!resolved) return compute(undefined);
|
|
80
|
+
try {
|
|
81
|
+
return compute(resolved.ref);
|
|
82
|
+
} catch (e) {
|
|
83
|
+
if (resolved.required || !(e instanceof GitError)) throw e;
|
|
84
|
+
warn(`change coverage skipped: ${resolved.ref} (from ${resolved.from}) does not resolve here — ${e.message.split('. ')[0]}. Pass --changed <ref> to make this an error.`);
|
|
85
|
+
return compute(undefined);
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
/**
|
|
90
|
+
* Files changed since the merge-base of `ref` and HEAD, repo-root-relative,
|
|
91
|
+
* deletions dropped. With `includeDirty` the working tree (tracked edits,
|
|
92
|
+
* staged or not, plus untracked non-ignored files) is compared instead of HEAD.
|
|
93
|
+
*/
|
|
94
|
+
export function changedFiles({ root, ref, includeDirty = false }) {
|
|
95
|
+
let base;
|
|
96
|
+
try {
|
|
97
|
+
base = git(['merge-base', ref, 'HEAD'], root);
|
|
98
|
+
} catch (e) {
|
|
99
|
+
throw new GitError(`cannot resolve --changed ${ref}: ${e.message.split('\n').pop()}. In CI the base must be present locally: GitHub Actions — actions/checkout with fetch-depth: 0; GitLab — GIT_DEPTH: 0 or \`git fetch origin <target branch>\` before the job (merge request pipelines also provide CI_MERGE_REQUEST_DIFF_BASE_SHA, which needs no fetch).`);
|
|
100
|
+
}
|
|
101
|
+
const head = headSha(root);
|
|
102
|
+
const out = new Set(git(['diff', '--name-only', '--diff-filter=d', '-z', base, ...(includeDirty ? [] : ['HEAD'])], root).split('\0').filter(Boolean));
|
|
103
|
+
if (includeDirty) for (const f of git(['ls-files', '--others', '--exclude-standard', '-z'], root).split('\0').filter(Boolean)) out.add(f);
|
|
104
|
+
return { base, head, files: [...out].sort() };
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
const isExpired = (entry, now) => Boolean(entry.expires) && new Date(entry.expires).getTime() <= now.getTime();
|
|
108
|
+
|
|
109
|
+
/**
|
|
110
|
+
* The claim to attach to, if any: one whose fault files live in the same
|
|
111
|
+
* directory as the changed file. A claim that merely shares `src/` is not
|
|
112
|
+
* "nearest", it is noise, and a wrong `--claim` hint is worse than none.
|
|
113
|
+
*/
|
|
114
|
+
function nearestClaim(file, faultFiles) {
|
|
115
|
+
const dir = dirname(file);
|
|
116
|
+
for (const [ff, ids] of faultFiles) if (dirname(ff) === dir) return ids[0];
|
|
117
|
+
return null;
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
/**
|
|
121
|
+
* Compute the gate for one change. Pure apart from reading the repository,
|
|
122
|
+
* the claims file and the ignore file; the caller decides what to print and
|
|
123
|
+
* what to write.
|
|
124
|
+
*/
|
|
125
|
+
export function computeChangedGate({ projectDir, ref, includeDirty = false, exclude = [], strict = false, now = new Date(), toolVersion = '0.0.0', generatedAt = now.toISOString(), claimsPath, ignorePath }) {
|
|
126
|
+
projectDir = realpathSync(resolve(projectDir));
|
|
127
|
+
const root = repoRoot(projectDir);
|
|
128
|
+
const relProject = relative(root, projectDir).split('\\').join('/');
|
|
129
|
+
const { base, head, files } = changedFiles({ root, ref, includeDirty });
|
|
130
|
+
|
|
131
|
+
// Repo-root paths → project paths; a change outside the project directory is not this project's change.
|
|
132
|
+
const changed = [];
|
|
133
|
+
for (const f of files) {
|
|
134
|
+
if (!relProject) { changed.push(f); continue; }
|
|
135
|
+
if (f.startsWith(relProject + '/')) changed.push(f.slice(relProject.length + 1));
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
const cPath = claimsPath ?? defaultClaimsPath(projectDir);
|
|
139
|
+
const claims = existsSync(cPath) ? loadClaims(cPath) : { claims: [] };
|
|
140
|
+
const iPath = ignorePath ?? defaultIgnorePath(projectDir);
|
|
141
|
+
const ignore = existsSync(iPath) ? readSpecDoc('ignore', iPath) : { entries: [] };
|
|
142
|
+
|
|
143
|
+
// Coverage index: which files carry a fault, which test files defend a claim.
|
|
144
|
+
const faultFiles = new Map(); // file → claimIds
|
|
145
|
+
const defenderFiles = new Map(); // test file → claimIds
|
|
146
|
+
for (const c of claims.claims) {
|
|
147
|
+
for (const f of c.faults) faultFiles.set(f.file, [...new Set([...(faultFiles.get(f.file) ?? []), c.id])]);
|
|
148
|
+
const defenders = c.defendedBy?.length ? resolveDefenders(projectDir, c.defendedBy) : [...new Set(c.faults.flatMap((f) => discoverDefenders(projectDir, f.file)))];
|
|
149
|
+
for (const d of defenders) defenderFiles.set(d, [...new Set([...(defenderFiles.get(d) ?? []), c.id])]);
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
const pathEntries = ignore.entries.filter((e) => e.kind === 'path').map((e) => ({ entry: e, re: globToRegExp(e.pattern), expired: isExpired(e, now), files: [] }));
|
|
153
|
+
const userExcludes = exclude.map((g) => ({ glob: g, re: globToRegExp(g) }));
|
|
154
|
+
const defaultExcludes = DEFAULT_EXCLUDES.map((g) => ({ glob: g, re: globToRegExp(g) }));
|
|
155
|
+
|
|
156
|
+
const excluded = [];
|
|
157
|
+
const covered = [];
|
|
158
|
+
const uncovered = [];
|
|
159
|
+
for (const file of changed) {
|
|
160
|
+
if (!SOURCE_EXT.has(extname(file))) { excluded.push({ file, by: 'non-source' }); continue; }
|
|
161
|
+
const ux = userExcludes.find((x) => x.re.test(file));
|
|
162
|
+
if (ux) { excluded.push({ file, by: `exclude:${ux.glob}` }); continue; }
|
|
163
|
+
const dx = defaultExcludes.find((x) => x.re.test(file));
|
|
164
|
+
if (dx) { excluded.push({ file, by: `default:${dx.glob}` }); continue; }
|
|
165
|
+
|
|
166
|
+
const isTest = TEST_RE.test(file);
|
|
167
|
+
if (faultFiles.has(file)) { covered.push({ file, by: 'fault', claimIds: faultFiles.get(file) }); continue; }
|
|
168
|
+
if (isTest && defenderFiles.has(file)) { covered.push({ file, by: 'defender', claimIds: defenderFiles.get(file) }); continue; }
|
|
169
|
+
const active = pathEntries.find((p) => !p.expired && p.re.test(file));
|
|
170
|
+
if (active) { active.files.push(file); covered.push({ file, by: 'ignore', pattern: active.entry.pattern }); continue; }
|
|
171
|
+
for (const p of pathEntries) if (p.expired && p.re.test(file)) p.files.push(file);
|
|
172
|
+
|
|
173
|
+
// Uncovered. Point at a same-directory claim so the author knows where to attach.
|
|
174
|
+
const nearest = nearestClaim(file, faultFiles);
|
|
175
|
+
const item = {
|
|
176
|
+
file,
|
|
177
|
+
kind: isTest ? 'test' : 'source',
|
|
178
|
+
...(nearest ? { nearestClaimId: nearest } : {}),
|
|
179
|
+
suggestion: isTest
|
|
180
|
+
? `add ${file} to the defendedBy of the claim it defends${nearest ? ` (nearest: ${nearest})` : ''}, or state a new claim for what it asserts`
|
|
181
|
+
: `testguard scaffold ${file}${nearest ? ` --claim ${nearest}` : ''}`,
|
|
182
|
+
};
|
|
183
|
+
uncovered.push(item);
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
const evaluated = covered.length + uncovered.length;
|
|
187
|
+
const reliedOn = pathEntries.filter((p) => !p.expired && p.files.length).map((p) => ({ pattern: p.entry.pattern, reason: p.entry.reason, ...(p.entry.expires ? { expires: p.entry.expires } : {}), files: p.files }));
|
|
188
|
+
const expired = pathEntries.filter((p) => p.expired && p.files.length).map((p) => ({ pattern: p.entry.pattern, reason: p.entry.reason, expires: p.entry.expires, files: p.files }));
|
|
189
|
+
const exitCode = uncovered.length > 0 ? 1 : strict && changed.length > 0 && evaluated === 0 ? 1 : 0;
|
|
190
|
+
|
|
191
|
+
return {
|
|
192
|
+
schemaVersion: 1,
|
|
193
|
+
tool: { name: 'testguard', version: toolVersion },
|
|
194
|
+
generatedAt,
|
|
195
|
+
ref,
|
|
196
|
+
base,
|
|
197
|
+
head,
|
|
198
|
+
includeDirty,
|
|
199
|
+
strict,
|
|
200
|
+
changed: changed.length,
|
|
201
|
+
evaluated,
|
|
202
|
+
excluded,
|
|
203
|
+
covered,
|
|
204
|
+
uncovered,
|
|
205
|
+
reliedOn,
|
|
206
|
+
expired,
|
|
207
|
+
exitCode,
|
|
208
|
+
};
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
export function renderGate(doc) {
|
|
212
|
+
const lines = [];
|
|
213
|
+
const where = doc.includeDirty ? 'working tree' : `HEAD ${doc.head.slice(0, 7)}`;
|
|
214
|
+
lines.push(`gate: ${doc.changed} file${doc.changed === 1 ? '' : 's'} changed since ${doc.ref} (merge-base ${doc.base.slice(0, 7)}) in the ${where}; ${doc.evaluated} evaluated, ${doc.excluded.length} excluded, ${doc.covered.length} covered, ${doc.uncovered.length} uncovered`);
|
|
215
|
+
for (const u of doc.uncovered) lines.push(`UNCLAIMED ${u.file} (${u.kind}${u.nearestClaimId ? `, nearest claim ${u.nearestClaimId}` : ''})\n → ${u.suggestion}`);
|
|
216
|
+
for (const r of doc.reliedOn) lines.push(`excused ${r.files.join(', ')} by ignore "${r.pattern}": ${r.reason}${r.expires ? ` (expires ${r.expires})` : ''}`);
|
|
217
|
+
for (const e of doc.expired) lines.push(`EXPIRED ignore "${e.pattern}" (expired ${e.expires}) no longer excuses ${e.files.join(', ')}`);
|
|
218
|
+
if (doc.changed > 0 && doc.evaluated === 0) lines.push(`note: every changed file was excluded (${[...new Set(doc.excluded.map((x) => x.by))].join(', ')}); 0 evaluated${doc.strict ? ' — --strict makes this a failure' : ''}`);
|
|
219
|
+
if (doc.changed === 0) lines.push(`no changes since ${doc.ref}; nothing to claim`);
|
|
220
|
+
if (doc.uncovered.length === 0 && doc.exitCode === 0 && doc.evaluated > 0) lines.push('every changed source file carries a claim or an excusing ignore entry');
|
|
221
|
+
return lines.join('\n');
|
|
222
|
+
}
|
package/src/git.mjs
CHANGED
|
@@ -23,6 +23,11 @@ export function headSha(dir, ref = 'HEAD') {
|
|
|
23
23
|
}
|
|
24
24
|
}
|
|
25
25
|
|
|
26
|
+
/** Is `ancestor` reachable from `ref`? False when either does not resolve. */
|
|
27
|
+
export function isAncestor(dir, ancestor, ref = 'HEAD') {
|
|
28
|
+
return spawnSync('git', ['merge-base', '--is-ancestor', ancestor, ref], { cwd: dir }).status === 0;
|
|
29
|
+
}
|
|
30
|
+
|
|
26
31
|
/** True when any of `paths` (repo-relative; empty = whole tree) has uncommitted changes. */
|
|
27
32
|
export const isDirty = (dir, paths = []) => git(['status', '--porcelain', '--', ...paths], dir).length > 0;
|
|
28
33
|
|