testguard-cli 0.5.0 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +628 -0
- package/README.md +536 -42
- package/package.json +2 -2
- package/spec/GATE-SEMANTICS.md +166 -2
- package/spec/README.md +29 -10
- package/spec/lib/validate.mjs +140 -8
- package/spec/schemas/baseline.schema.json +13 -5
- package/spec/schemas/brief.schema.json +12 -12
- package/spec/schemas/calibration.schema.json +5 -5
- package/spec/schemas/claims.schema.json +122 -25
- package/spec/schemas/common.schema.json +131 -29
- package/spec/schemas/evidence.schema.json +476 -69
- package/spec/schemas/gate.schema.json +12 -12
- package/spec/schemas/ignore.schema.json +4 -4
- package/spec/schemas/replay.schema.json +85 -0
- package/spec/schemas/status.schema.json +258 -56
- package/src/admit/admit.mjs +17 -0
- package/src/baseline/baseline.mjs +31 -2
- package/src/brief/brief.mjs +71 -7
- package/src/claims/removed.mjs +161 -0
- package/src/cli.mjs +68 -10
- package/src/commands/admit.mjs +116 -0
- package/src/commands/baseline.mjs +17 -3
- package/src/commands/brief.mjs +13 -7
- package/src/commands/claims.mjs +55 -6
- package/src/commands/init.mjs +19 -6
- package/src/commands/mcp.mjs +12 -0
- package/src/commands/probe.mjs +35 -7
- package/src/commands/replay.mjs +63 -0
- package/src/commands/status.mjs +3 -1
- package/src/gate/changed.mjs +9 -2
- package/src/git.mjs +5 -0
- package/src/init/init.mjs +182 -29
- package/src/init/templates/SKILL.md +3 -3
- package/src/mcp/server.mjs +104 -0
- package/src/mcp/tools.mjs +114 -0
- package/src/probe/attribution.mjs +123 -0
- package/src/probe/classify.mjs +18 -2
- package/src/probe/contention.mjs +60 -0
- package/src/probe/cost.mjs +145 -0
- package/src/probe/discover.mjs +20 -3
- package/src/probe/independence.mjs +59 -0
- package/src/probe/inject.mjs +41 -9
- package/src/probe/mocks.mjs +122 -0
- package/src/probe/probe.mjs +331 -40
- package/src/probe/progress.mjs +87 -0
- package/src/probe/pyimports.mjs +287 -0
- package/src/probe/rank.mjs +75 -15
- package/src/probe/runners/index.mjs +79 -8
- package/src/probe/runners/jest.mjs +15 -3
- package/src/probe/runners/playwright.mjs +142 -0
- package/src/probe/runners/python/_testguard_pytest_plugin.py +132 -0
- package/src/probe/runners/python/_testguard_report.py +141 -0
- package/src/probe/runners/python/_testguard_unittest_main.py +168 -0
- package/src/probe/runners/python.mjs +316 -0
- package/src/probe/runners/shared.mjs +80 -14
- package/src/probe/runners/vitest.mjs +16 -3
- package/src/render.mjs +2 -2
- package/src/replay/label.mjs +64 -0
- package/src/replay/replay.mjs +310 -0
- package/src/scaffold/producers.mjs +217 -6
- package/src/scaffold/producers.python.mjs +209 -0
- package/src/scaffold/scaffold.mjs +90 -30
- package/src/status/status.mjs +47 -6
- package/src/util/glob.mjs +7 -1
package/src/commands/brief.mjs
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { existsSync } from 'node:fs';
|
|
2
2
|
import { join, resolve } from 'node:path';
|
|
3
3
|
import { readSpecDoc, writeSpecDoc } from '../evidence/writer.mjs';
|
|
4
|
-
import { buildBrief, buildUnclaimedBrief } from '../brief/brief.mjs';
|
|
4
|
+
import { buildBrief, buildUnclaimedBrief, renderBriefMarkdown, renderUnclaimedMarkdown } from '../brief/brief.mjs';
|
|
5
5
|
import { computeStatus } from '../status/status.mjs';
|
|
6
6
|
import { evidencePath, baselinePath } from './probe.mjs';
|
|
7
7
|
import { resolveChangedRef, withChangedRef } from '../gate/changed.mjs';
|
|
@@ -13,15 +13,18 @@ export async function briefCommand({ projectDir, values, version }, io) {
|
|
|
13
13
|
// A detected base that does not resolve is silent here: the brief is a
|
|
14
14
|
// session-start hook's output and must never add noise; `status` and
|
|
15
15
|
// `gate` are where that warning is printed.
|
|
16
|
-
status = withChangedRef(resolveChangedRef({ explicit: values.changed }), (changedRef) => computeStatus({ projectDir, toolVersion: version, changedRef, includeDirty: values['include-dirty'] }));
|
|
16
|
+
status = withChangedRef(resolveChangedRef({ explicit: values.changed }), (changedRef) => computeStatus({ projectDir, toolVersion: version, changedRef, includeDirty: values['include-dirty'], evidence: values.evidence ? evPath : undefined }));
|
|
17
17
|
} catch {
|
|
18
18
|
status = undefined; // a brief must never fail because status could not be computed
|
|
19
19
|
}
|
|
20
20
|
if (!existsSync(evPath)) {
|
|
21
21
|
// A missing brief must never break an agent's session start — but unclaimed
|
|
22
22
|
// changes are still said, because the claim comes before the probe.
|
|
23
|
-
if (values.text) {
|
|
24
|
-
if (status?.changes?.uncovered?.length)
|
|
23
|
+
if (values.text || values.markdown) {
|
|
24
|
+
if (status?.changes?.uncovered?.length) {
|
|
25
|
+
const args = { tool: { name: 'testguard', version }, next: status.next, changes: status.changes };
|
|
26
|
+
io.out((values.markdown ? renderUnclaimedMarkdown(args) : buildUnclaimedBrief(args).text).trimEnd());
|
|
27
|
+
}
|
|
25
28
|
return 0;
|
|
26
29
|
}
|
|
27
30
|
io.err(`no evidence at ${evPath}; run \`testguard probe\` first`);
|
|
@@ -35,8 +38,11 @@ export async function briefCommand({ projectDir, values, version }, io) {
|
|
|
35
38
|
io.err('--max must be an integer from 1 to 50');
|
|
36
39
|
return 3;
|
|
37
40
|
}
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
+
// Where this binary came from, so a stale install is visible in the session-start context.
|
|
42
|
+
const install = /[\\/]node_modules[\\/]/.test(process.argv[1] ?? '') ? 'local install' : 'global';
|
|
43
|
+
const brief = buildBrief(evidence, baseline, { max, next: status?.next, changes: status?.changes, install });
|
|
44
|
+
if (!values.text && !values.markdown) writeSpecDoc('brief', values.out ? resolve(values.out) : join(projectDir, '.testguard', 'brief.json'), brief);
|
|
45
|
+
if (values.markdown) io.out(renderBriefMarkdown({ ...brief, provisional: Boolean(evidence.run.provisional) }, { hasBaseline: Boolean(baseline), total: evidence.records.length, install }).trimEnd());
|
|
46
|
+
else io.out(values.json ? JSON.stringify(brief, null, 2) : brief.text.trimEnd());
|
|
41
47
|
return 0;
|
|
42
48
|
}
|
package/src/commands/claims.mjs
CHANGED
|
@@ -2,29 +2,78 @@ import { resolve } from 'node:path';
|
|
|
2
2
|
import { loadClaims, defaultClaimsPath } from '../claims/load.mjs';
|
|
3
3
|
import { scanAnnotations, reconcile } from '../claims/annotations.mjs';
|
|
4
4
|
import { resolveDefenders } from '../probe/runners/shared.mjs';
|
|
5
|
-
import {
|
|
5
|
+
import { discoverDefendersDetailed } from '../probe/discover.mjs';
|
|
6
|
+
import { classifyDefenders } from '../probe/mocks.mjs';
|
|
7
|
+
import { computeRemovedClaims, renderRemoved } from '../claims/removed.mjs';
|
|
8
|
+
import { evidencePath } from './probe.mjs';
|
|
9
|
+
import { costReport, renderCost } from '../probe/cost.mjs';
|
|
10
|
+
import { readSpecDoc } from '../evidence/writer.mjs';
|
|
11
|
+
import { existsSync } from 'node:fs';
|
|
6
12
|
|
|
7
|
-
export async function claimsCommand({ projectDir, values }, io) {
|
|
13
|
+
export async function claimsCommand({ projectDir, values, version }, io) {
|
|
8
14
|
const path = values.claims ? resolve(values.claims) : defaultClaimsPath(projectDir);
|
|
9
15
|
const claims = loadClaims(path);
|
|
10
16
|
const annotations = scanAnnotations(projectDir);
|
|
11
17
|
const drift = reconcile(claims, annotations);
|
|
12
18
|
|
|
19
|
+
// A claim that disappeared is invisible to every other command; compare
|
|
20
|
+
// identities against a reference when one is given.
|
|
21
|
+
const removed = values.since
|
|
22
|
+
? computeRemovedClaims({ projectDir, ref: values.since, claimsPath: path, current: claims, ignorePath: values.ignore ? resolve(values.ignore) : undefined, evidencePath: evidencePath(projectDir), toolVersion: version })
|
|
23
|
+
: undefined;
|
|
24
|
+
|
|
25
|
+
// Cost is read back out of evidence the probe already wrote; it never runs a test.
|
|
26
|
+
let cost;
|
|
27
|
+
if (values.cost) {
|
|
28
|
+
const ev = values.evidence ? resolve(values.evidence) : evidencePath(projectDir);
|
|
29
|
+
cost = existsSync(ev) ? costReport(readSpecDoc('evidence', ev).records) : undefined;
|
|
30
|
+
if (!cost && !values.json) io.err(`no evidence at ${ev} — run \`testguard probe\` first; cost is derived from the run durations it records`);
|
|
31
|
+
}
|
|
32
|
+
|
|
13
33
|
if (values.json) {
|
|
14
|
-
io.out(JSON.stringify({ path, claims, annotations, drift }, null, 2));
|
|
34
|
+
io.out(JSON.stringify({ path, claims, annotations, drift, ...(removed ? { removed } : {}), ...(cost ? { cost } : {}) }, null, 2));
|
|
15
35
|
} else {
|
|
16
36
|
const annotated = new Set(drift.annotated);
|
|
17
37
|
io.out(`${claims.claims.length} claims in ${path} — ${annotated.size} carry a @claim annotation in source (test files are not scanned)`);
|
|
18
38
|
io.out('');
|
|
39
|
+
const signalLines = [];
|
|
19
40
|
for (const c of claims.claims) {
|
|
20
41
|
const declared = c.defendedBy?.length > 0;
|
|
21
|
-
const
|
|
22
|
-
|
|
42
|
+
const targets = [...new Set(c.faults.map((f) => f.file))];
|
|
43
|
+
let cover;
|
|
44
|
+
if (declared) {
|
|
45
|
+
const defenders = resolveDefenders(projectDir, c.defendedBy);
|
|
46
|
+
const mocking = new Set();
|
|
47
|
+
for (const t of targets) { const m = classifyDefenders(projectDir, t, defenders); m.mocking.forEach((f) => mocking.add(f)); for (const s of m.signals) signalLines.push({ claim: c.id, target: t, ...s }); }
|
|
48
|
+
cover = defenders.length ? `${defenders.length} defender${defenders.length === 1 ? '' : 's'}${mocking.size ? ` (${mocking.size} mock the target)` : ''}` : 'NO DEFENDER';
|
|
49
|
+
} else {
|
|
50
|
+
const importing = new Set(); const mocking = new Set(); const can = new Set();
|
|
51
|
+
for (const t of targets) { const d = discoverDefendersDetailed(projectDir, t); d.importing.forEach((f) => importing.add(f)); d.mocking.forEach((f) => mocking.add(f)); d.canDetect.forEach((f) => can.add(f)); for (const s of d.signals) signalLines.push({ claim: c.id, target: t, ...s }); }
|
|
52
|
+
cover = can.size ? `${importing.size} import · ${mocking.size} mock · ${can.size} can detect` : importing.size ? `NO DEFENDER (${importing.size} import, all mock the target)` : 'NO DEFENDER';
|
|
53
|
+
}
|
|
23
54
|
io.out(`${annotated.has(c.id) ? '@ ' : ' '}${c.id.padEnd(14)} ${c.severity.padEnd(8)} ${c.source.kind.padEnd(10)} ${String(c.faults.length).padStart(2)} fault${c.faults.length === 1 ? ' ' : 's'} ${cover.padEnd(12)} ${c.statement}`);
|
|
24
55
|
}
|
|
56
|
+
if (signalLines.length) io.out('');
|
|
57
|
+
for (const s of signalLines) {
|
|
58
|
+
// The annotation's comment marker is the probed file's, not JavaScript's.
|
|
59
|
+
const marker = s.file.endsWith('.py') ? '#' : '//';
|
|
60
|
+
if (s.signal === 'mocked-never-asserted') io.out(`MOCKED-NEVER-ASSERTED ${s.file} mocks ${s.target} and never asserts on it (${s.claim}) — assert on the mocked call, drive the real module, or annotate the mock \`${marker} unasserted: <why>\``);
|
|
61
|
+
// An attribute patch does not remove a defender; saying it "mocks" the
|
|
62
|
+
// module would tell the author the opposite of what happened.
|
|
63
|
+
else if (s.signal === 'target-attribute-patched') io.out(`attribute-patched ${s.file} defends ${s.target} but replaces ${s.reason} (${s.claim}) — a fault in those attributes is where it is least likely to notice`);
|
|
64
|
+
else io.out(`unasserted (annotated) ${s.file} mocks ${s.target}: ${s.reason} (${s.claim})`);
|
|
65
|
+
}
|
|
25
66
|
if (drift.undeclared.length || drift.stale.length) io.out('');
|
|
26
67
|
for (const a of drift.undeclared) io.out(`UNDECLARED @claim ${a.id} at ${a.file}:${a.line} has no entry in the claims file — a claim with no fault model`);
|
|
27
68
|
for (const c of drift.stale) io.out(`STALE ${c.id} is annotation-sourced but no source file carries @claim ${c.id}`);
|
|
69
|
+
if (removed) {
|
|
70
|
+
io.out('');
|
|
71
|
+
io.out(renderRemoved(removed));
|
|
72
|
+
}
|
|
73
|
+
if (cost) {
|
|
74
|
+
io.out('');
|
|
75
|
+
io.out(renderCost(cost));
|
|
76
|
+
}
|
|
28
77
|
}
|
|
29
|
-
return drift.undeclared.length || drift.stale.length ? 1 : 0;
|
|
78
|
+
return drift.undeclared.length || drift.stale.length || (removed?.removed.length ?? 0) > 0 ? 1 : 0;
|
|
30
79
|
}
|
package/src/commands/init.mjs
CHANGED
|
@@ -1,15 +1,28 @@
|
|
|
1
1
|
import { initProject } from '../init/init.mjs';
|
|
2
2
|
|
|
3
3
|
export async function initCommand({ projectDir, values }, io) {
|
|
4
|
-
const
|
|
4
|
+
const r = initProject({ projectDir, force: values.force, here: values.here, ciEvidence: values['ci-evidence'], mcp: values.mcp });
|
|
5
|
+
const { done, skipped, warnings, agentRoot, dir, mcpConfig } = r;
|
|
5
6
|
if (values.json) {
|
|
6
|
-
io.out(JSON.stringify({ done, skipped }, null, 2));
|
|
7
|
-
return 0;
|
|
7
|
+
io.out(JSON.stringify({ done, skipped, warnings, agentRoot, dir, ...(mcpConfig ? { mcpConfig } : {}) }, null, 2));
|
|
8
|
+
return warnings.length ? 1 : 0;
|
|
8
9
|
}
|
|
9
10
|
for (const d of done) io.out(`+ ${d}`);
|
|
10
11
|
for (const s of skipped) io.out(`= ${s}`);
|
|
12
|
+
for (const w of warnings) io.out(`! ${w}`);
|
|
11
13
|
io.out('');
|
|
12
|
-
io.out(
|
|
13
|
-
io.out(
|
|
14
|
-
|
|
14
|
+
if (dir !== '.') io.out(`Agent layer (skill, session-start hook, AGENTS.md section) installed at the git root ${agentRoot}, where agent sessions run; the project layer stays in ${dir}/. Use --here to keep everything in ${dir}/.`);
|
|
15
|
+
io.out(`Agents now start with the blind-spot brief and can run \`testguard status --json${dir === '.' ? '' : ` ${dir}`}\` to learn what to do next. The hook prefers a local install and never fetches from the network.`);
|
|
16
|
+
io.out(warnings.length ? 'Some written files are ignored by git — fix that before committing.' : done.length ? 'Commit these files.' : 'Nothing to change.');
|
|
17
|
+
if (mcpConfig) {
|
|
18
|
+
io.out('');
|
|
19
|
+
io.out('MCP — five READ-ONLY tools (status, brief, claims, evidence, next_command), so the loop survives a change of harness. Nothing there runs a probe. Add ONE of these:');
|
|
20
|
+
for (const [where, snippet] of Object.entries(mcpConfig)) {
|
|
21
|
+
io.out('');
|
|
22
|
+
io.out(` ${where}`);
|
|
23
|
+
const text = typeof snippet === 'string' ? snippet : JSON.stringify(snippet, null, 2);
|
|
24
|
+
for (const line of text.split('\n')) io.out(` ${line}`);
|
|
25
|
+
}
|
|
26
|
+
}
|
|
27
|
+
return warnings.length ? 1 : 0;
|
|
15
28
|
}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import { serve } from '../mcp/server.mjs';
|
|
2
|
+
|
|
3
|
+
export async function mcpCommand({ values, version }, io) {
|
|
4
|
+
if (values.json) {
|
|
5
|
+
// Not a JSON command: it is a long-lived server on stdio, and printing
|
|
6
|
+
// anything to stdout would corrupt the protocol stream.
|
|
7
|
+
io.err('mcp speaks JSON-RPC on stdin/stdout; there is no --json mode');
|
|
8
|
+
return 3;
|
|
9
|
+
}
|
|
10
|
+
await serve({ version });
|
|
11
|
+
return 0;
|
|
12
|
+
}
|
package/src/commands/probe.mjs
CHANGED
|
@@ -7,6 +7,8 @@ import { gate } from '../baseline/baseline.mjs';
|
|
|
7
7
|
import { renderRecord, renderSummary, sortForReport, PROVISIONAL_WARNING } from '../render.mjs';
|
|
8
8
|
import { computeStatus } from '../status/status.mjs';
|
|
9
9
|
import { resolveChangedRef, withChangedRef } from '../gate/changed.mjs';
|
|
10
|
+
import { costReport, renderCost } from '../probe/cost.mjs';
|
|
11
|
+
import { progressMode, stageReporter, clearStageLine, recordEvent, isProgressMode, progressStream } from '../probe/progress.mjs';
|
|
10
12
|
export const provisionalEvidencePath = (projectDir) => join(projectDir, '.testguard', 'evidence-provisional.json');
|
|
11
13
|
|
|
12
14
|
export const evidencePath = (projectDir) => join(projectDir, '.testguard', 'evidence.json');
|
|
@@ -19,6 +21,14 @@ export async function probeCommand({ projectDir, values, version }, io) {
|
|
|
19
21
|
io.err('--confirm must be a positive integer and --budget at least 1000');
|
|
20
22
|
return 3;
|
|
21
23
|
}
|
|
24
|
+
if (values.progress && !isProgressMode(values.progress)) {
|
|
25
|
+
io.err(`--progress must be one of auto, tty, plain, ndjson, none (got ${values.progress})`);
|
|
26
|
+
return 3;
|
|
27
|
+
}
|
|
28
|
+
// Progress always goes to stderr, so --json keeps stdout parseable.
|
|
29
|
+
const progress = progressMode({ explicit: values.progress, isTTY: process.stderr.isTTY, quiet: values.quiet, json: values.json });
|
|
30
|
+
const progressTo = progressStream(process);
|
|
31
|
+
const stageOut = stageReporter(progress, (s) => progressTo.write(s));
|
|
22
32
|
const claims = loadClaims(values.claims ? resolve(values.claims) : defaultClaimsPath(projectDir));
|
|
23
33
|
if (claims.claims.length === 0) {
|
|
24
34
|
io.err('claims file declares no claims; nothing to verify');
|
|
@@ -30,6 +40,7 @@ export async function probeCommand({ projectDir, values, version }, io) {
|
|
|
30
40
|
// Provisional and partial runs never overwrite the canonical evidence: only confirmed, complete runs may feed a baseline.
|
|
31
41
|
const outPath = values.out ? resolve(values.out) : only ? join(projectDir, '.testguard', 'evidence-partial.json') : provisional ? provisionalEvidencePath(projectDir) : evidencePath(projectDir);
|
|
32
42
|
if (provisional && !values.quiet) io.err(PROVISIONAL_WARNING(confirmRuns));
|
|
43
|
+
const discoveryNoted = new Set();
|
|
33
44
|
const previous = !values['no-reuse'] && existsSync(outPath) ? readSpecDoc('evidence', outPath) : undefined;
|
|
34
45
|
const basePath = values.baseline ? resolve(values.baseline) : baselinePath(projectDir);
|
|
35
46
|
const baseline = existsSync(basePath) ? readSpecDoc('baseline', basePath) : undefined;
|
|
@@ -41,26 +52,38 @@ export async function probeCommand({ projectDir, values, version }, io) {
|
|
|
41
52
|
confirmRuns,
|
|
42
53
|
budgetMs,
|
|
43
54
|
mode: values['in-place'] ? 'in-place' : 'worktree',
|
|
44
|
-
ref: values.ref,
|
|
55
|
+
ref: values.ref ?? 'HEAD',
|
|
56
|
+
refExplicit: values.ref !== undefined,
|
|
57
|
+
ignoreDirty: values['ignore-dirty'],
|
|
58
|
+
serial: values.serial,
|
|
59
|
+
onWarn: (m) => io.err(`warning: ${m}`),
|
|
45
60
|
runnerCommand: values['runner-cmd'],
|
|
46
61
|
runnerName: values.runner,
|
|
47
62
|
nodeModules: values['node-modules'] ? resolve(values['node-modules']) : process.env.TESTGUARD_NODE_MODULES,
|
|
63
|
+
python: values.python ? resolve(values.python) : undefined,
|
|
48
64
|
only,
|
|
49
65
|
escalate: !values['no-escalate'],
|
|
50
66
|
toolVersion: version,
|
|
51
67
|
includeDirty: values['include-dirty'],
|
|
52
|
-
onStage:
|
|
53
|
-
onProgress:
|
|
54
|
-
|
|
55
|
-
|
|
68
|
+
onStage: stageOut,
|
|
69
|
+
onProgress: (r) => {
|
|
70
|
+
// ndjson streams verdicts as they land; a watcher sees them without
|
|
71
|
+
// waiting for the document at the end.
|
|
72
|
+
if (progress === 'ndjson') progressTo.write(recordEvent(r));
|
|
73
|
+
if (values.quiet || values.json) return;
|
|
74
|
+
progressTo.write(clearStageLine(progress));
|
|
75
|
+
// The discovery note is a property of the claim, not of each fault: say it once.
|
|
76
|
+
const firstOfClaim = !discoveryNoted.has(r.claim.id);
|
|
77
|
+
if (r.defenders.discovered) discoveryNoted.add(r.claim.id);
|
|
78
|
+
if (values.verbose || r.verdict !== 'killed') io.out(renderRecord(r, { provisional, showDiscovered: firstOfClaim }) + (r.reusedFrom ? ' (reused)' : ''));
|
|
56
79
|
},
|
|
57
80
|
});
|
|
58
81
|
writeSpecDoc('evidence', outPath, evidence);
|
|
59
82
|
|
|
60
83
|
const g = gate(evidence.records, baseline, { severityFloor: values.severity });
|
|
61
84
|
if (values.json) {
|
|
62
|
-
const status = withChangedRef(resolveChangedRef({ explicit: values.changed }), (changedRef) => computeStatus({ projectDir, toolVersion: version, changedRef, includeDirty: values['include-dirty'] }), io.err);
|
|
63
|
-
io.out(JSON.stringify({ ...status, run: { id: evidence.run.id, evidence: outPath, provisional, records: evidence.records.length, newSinceBaseline: g.new.length, exitCode: g.new.length > 0 ? 1 : 0 } }, null, 2));
|
|
85
|
+
const status = withChangedRef(resolveChangedRef({ explicit: values.changed }), (changedRef) => computeStatus({ projectDir, toolVersion: version, changedRef, includeDirty: values['include-dirty'], evidence: outPath }), io.err);
|
|
86
|
+
io.out(JSON.stringify({ ...status, run: { id: evidence.run.id, evidence: outPath, provisional, records: evidence.records.length, newSinceBaseline: g.new.length, exitCode: g.new.length > 0 ? 1 : 0 }, ...(values.cost ? { cost: costReport(evidence.records) } : {}) }, null, 2));
|
|
64
87
|
return g.new.length > 0 ? 1 : 0;
|
|
65
88
|
}
|
|
66
89
|
if (!values.quiet && baseline) {
|
|
@@ -74,6 +97,11 @@ export async function probeCommand({ projectDir, values, version }, io) {
|
|
|
74
97
|
if (killed) io.out(` ${killed} killed (not listed; --verbose to see them)`);
|
|
75
98
|
}
|
|
76
99
|
io.out(renderSummary(evidence.records, evidence.run) + (baseline ? ` ${g.new.length} new since baseline, ${g.baselined.length} baselined.` : ' No baseline.'));
|
|
100
|
+
if (values.cost) {
|
|
101
|
+
io.out('');
|
|
102
|
+
io.out(renderCost(costReport(evidence.records)));
|
|
103
|
+
io.out('');
|
|
104
|
+
}
|
|
77
105
|
io.out(`evidence: ${outPath}${only ? ` (partial: --claim ${only.join(',')}; not the canonical evidence file)` : provisional ? ' (provisional; not the canonical evidence file)' : ''}`);
|
|
78
106
|
if (provisional && !values.quiet) io.err(PROVISIONAL_WARNING(confirmRuns));
|
|
79
107
|
return g.new.length > 0 ? 1 : 0;
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
import { dirname, join, resolve } from 'node:path';
|
|
2
|
+
import { replay, calibrationFrom, renderReplay } from '../replay/replay.mjs';
|
|
3
|
+
import { writeSpecDoc } from '../evidence/writer.mjs';
|
|
4
|
+
|
|
5
|
+
export const replayPath = (projectDir) => join(projectDir, '.testguard', 'replay.json');
|
|
6
|
+
export const calibrationPath = (projectDir) => join(projectDir, '.testguard', 'calibration.json');
|
|
7
|
+
|
|
8
|
+
export async function replayCommand({ projectDir, values, version }, io) {
|
|
9
|
+
const range = values.since ?? values.changed;
|
|
10
|
+
if (!range) {
|
|
11
|
+
io.err('usage: testguard replay [dir] --since <range> (e.g. --since HEAD~50..HEAD, or a tag)');
|
|
12
|
+
return 3;
|
|
13
|
+
}
|
|
14
|
+
const confirmRuns = Number(values.confirm);
|
|
15
|
+
const budgetMs = Number(values.budget);
|
|
16
|
+
const limit = values.max ? Number(values.max) : undefined;
|
|
17
|
+
if (!Number.isInteger(confirmRuns) || confirmRuns < 1 || !Number.isInteger(budgetMs) || budgetMs < 1000) {
|
|
18
|
+
io.err('--confirm must be a positive integer and --budget at least 1000');
|
|
19
|
+
return 3;
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
const doc = await replay({
|
|
23
|
+
projectDir,
|
|
24
|
+
range,
|
|
25
|
+
confirmRuns,
|
|
26
|
+
budgetMs,
|
|
27
|
+
runnerCommand: values['runner-cmd'],
|
|
28
|
+
runnerName: values.runner,
|
|
29
|
+
nodeModules: values['node-modules'] ? resolve(values['node-modules']) : process.env.TESTGUARD_NODE_MODULES,
|
|
30
|
+
limit,
|
|
31
|
+
toolVersion: version,
|
|
32
|
+
onStage: !values.quiet && !values.json && process.stderr.isTTY ? ({ commit, i, n, stage }) => process.stderr.write(`\r\x1b[K … ${commit.slice(0, 9)} (${i}/${n}) ${stage}`) : undefined,
|
|
33
|
+
onProgress: values.quiet || values.json ? undefined : () => {
|
|
34
|
+
if (process.stderr.isTTY) process.stderr.write('\r\x1b[K');
|
|
35
|
+
},
|
|
36
|
+
});
|
|
37
|
+
|
|
38
|
+
const outPath = values.out ? resolve(values.out) : replayPath(projectDir);
|
|
39
|
+
writeSpecDoc('replay', outPath, doc);
|
|
40
|
+
const calibration = calibrationFrom(doc, { toolVersion: version });
|
|
41
|
+
// Beside the replay document, whatever --out says: the two are one result,
|
|
42
|
+
// and splitting them across directories loses the pairing — and leaves a
|
|
43
|
+
// file behind in a repository the run is only meant to read.
|
|
44
|
+
const calPath = values.baseline ? resolve(values.baseline) : values.out ? join(dirname(outPath), 'calibration.json') : calibrationPath(projectDir);
|
|
45
|
+
writeSpecDoc('calibration', calPath, calibration);
|
|
46
|
+
|
|
47
|
+
if (values.json) {
|
|
48
|
+
io.out(JSON.stringify({ replay: doc, calibration, paths: { replay: outPath, calibration: calPath } }, null, 2));
|
|
49
|
+
} else {
|
|
50
|
+
io.out(renderReplay(doc));
|
|
51
|
+
io.out('');
|
|
52
|
+
for (const [cls, b] of Object.entries(calibration.buckets).sort((a, b2) => b2[1].n - a[1].n)) {
|
|
53
|
+
io.out(` ${cls.padEnd(22)} blind ${String(b.positives).padStart(3)}/${String(b.n).padEnd(3)} p=${b.p.toFixed(2)} ci [${b.ci[0].toFixed(2)}, ${b.ci[1].toFixed(2)}]`);
|
|
54
|
+
}
|
|
55
|
+
if (Object.keys(calibration.buckets).length === 0) io.out(' no measurable bug in this range: nothing to calibrate from yet.');
|
|
56
|
+
io.out('');
|
|
57
|
+
io.out(`replay: ${outPath}`);
|
|
58
|
+
io.out(`calibration: ${calPath}`);
|
|
59
|
+
}
|
|
60
|
+
// Replay reports; it never gates. A bug that escaped is history, not a
|
|
61
|
+
// regression in this change, and a gate on history is a gate nobody can pass.
|
|
62
|
+
return 0;
|
|
63
|
+
}
|
package/src/commands/status.mjs
CHANGED
|
@@ -1,9 +1,11 @@
|
|
|
1
|
+
import { resolve } from 'node:path';
|
|
1
2
|
import { computeStatus, renderStatus } from '../status/status.mjs';
|
|
2
3
|
import { validate } from '../../spec/lib/validate.mjs';
|
|
3
4
|
import { resolveChangedRef, withChangedRef } from '../gate/changed.mjs';
|
|
4
5
|
|
|
5
6
|
export async function statusCommand({ projectDir, values, version }, io) {
|
|
6
|
-
const
|
|
7
|
+
const evidence = values.evidence ? resolve(values.evidence) : undefined;
|
|
8
|
+
const doc = withChangedRef(resolveChangedRef({ explicit: values.changed }), (changedRef) => computeStatus({ projectDir, toolVersion: version, changedRef, includeDirty: values['include-dirty'], evidence }), io.err);
|
|
7
9
|
const result = validate('status', doc);
|
|
8
10
|
if (!result.ok) throw new Error(`status document does not conform: ${result.errors.map((e) => `${e.path}: ${e.message}`).join('; ')}`);
|
|
9
11
|
io.out(values.json ? JSON.stringify(doc, null, 2) : renderStatus(doc));
|
package/src/gate/changed.mjs
CHANGED
|
@@ -5,6 +5,7 @@ import { loadClaims, defaultClaimsPath } from '../claims/load.mjs';
|
|
|
5
5
|
import { readSpecDoc } from '../evidence/writer.mjs';
|
|
6
6
|
import { resolveDefenders } from '../probe/runners/shared.mjs';
|
|
7
7
|
import { discoverDefenders } from '../probe/discover.mjs';
|
|
8
|
+
import { IS_PY_TEST } from '../probe/pyimports.mjs';
|
|
8
9
|
import { globToRegExp } from '../util/glob.mjs';
|
|
9
10
|
|
|
10
11
|
/**
|
|
@@ -20,8 +21,14 @@ import { globToRegExp } from '../util/glob.mjs';
|
|
|
20
21
|
* the smallest unit the rest of the contract already understands.
|
|
21
22
|
*/
|
|
22
23
|
|
|
23
|
-
|
|
24
|
+
// `.py` belongs here for the same reason every other extension does: a file
|
|
25
|
+
// the gate does not recognise as source is excluded as `non-source`, which
|
|
26
|
+
// means a changed Python file could never be reported as uncovered and the
|
|
27
|
+
// gate would pass while saying nothing.
|
|
28
|
+
export const SOURCE_EXT = new Set(['.js', '.mjs', '.cjs', '.ts', '.mts', '.cts', '.jsx', '.tsx', '.py']);
|
|
24
29
|
const TEST_RE = /\.(test|spec)\.[cm]?[jt]sx?$/;
|
|
30
|
+
// Python names tests by file, not by a dotted suffix: `test_foo.py`, `foo_test.py`.
|
|
31
|
+
const isTestFile = (f) => TEST_RE.test(f) || IS_PY_TEST.test(f);
|
|
25
32
|
|
|
26
33
|
/** Files that are source by extension but never carry claims. Printed by --explain. */
|
|
27
34
|
export const DEFAULT_EXCLUDES = Object.freeze([
|
|
@@ -163,7 +170,7 @@ export function computeChangedGate({ projectDir, ref, includeDirty = false, excl
|
|
|
163
170
|
const dx = defaultExcludes.find((x) => x.re.test(file));
|
|
164
171
|
if (dx) { excluded.push({ file, by: `default:${dx.glob}` }); continue; }
|
|
165
172
|
|
|
166
|
-
const isTest =
|
|
173
|
+
const isTest = isTestFile(file);
|
|
167
174
|
if (faultFiles.has(file)) { covered.push({ file, by: 'fault', claimIds: faultFiles.get(file) }); continue; }
|
|
168
175
|
if (isTest && defenderFiles.has(file)) { covered.push({ file, by: 'defender', claimIds: defenderFiles.get(file) }); continue; }
|
|
169
176
|
const active = pathEntries.find((p) => !p.expired && p.re.test(file));
|
package/src/git.mjs
CHANGED
|
@@ -23,6 +23,11 @@ export function headSha(dir, ref = 'HEAD') {
|
|
|
23
23
|
}
|
|
24
24
|
}
|
|
25
25
|
|
|
26
|
+
/** Is `ancestor` reachable from `ref`? False when either does not resolve. */
|
|
27
|
+
export function isAncestor(dir, ancestor, ref = 'HEAD') {
|
|
28
|
+
return spawnSync('git', ['merge-base', '--is-ancestor', ancestor, ref], { cwd: dir }).status === 0;
|
|
29
|
+
}
|
|
30
|
+
|
|
26
31
|
/** True when any of `paths` (repo-relative; empty = whole tree) has uncommitted changes. */
|
|
27
32
|
export const isDirty = (dir, paths = []) => git(['status', '--porcelain', '--', ...paths], dir).length > 0;
|
|
28
33
|
|