testguard-cli 0.5.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +255 -0
- package/README.md +275 -31
- package/package.json +1 -1
- package/spec/GATE-SEMANTICS.md +58 -1
- package/spec/README.md +1 -0
- package/spec/lib/validate.mjs +57 -1
- package/spec/schemas/baseline.schema.json +8 -0
- package/spec/schemas/common.schema.json +5 -1
- package/spec/schemas/evidence.schema.json +374 -63
- package/spec/schemas/replay.schema.json +85 -0
- package/spec/schemas/status.schema.json +256 -54
- package/src/admit/admit.mjs +17 -0
- package/src/baseline/baseline.mjs +31 -2
- package/src/brief/brief.mjs +71 -7
- package/src/claims/removed.mjs +161 -0
- package/src/cli.mjs +53 -10
- package/src/commands/admit.mjs +116 -0
- package/src/commands/baseline.mjs +17 -3
- package/src/commands/brief.mjs +13 -7
- package/src/commands/claims.mjs +35 -6
- package/src/commands/init.mjs +19 -6
- package/src/commands/mcp.mjs +12 -0
- package/src/commands/probe.mjs +11 -3
- package/src/commands/replay.mjs +63 -0
- package/src/commands/status.mjs +3 -1
- package/src/git.mjs +5 -0
- package/src/init/init.mjs +182 -29
- package/src/init/templates/SKILL.md +3 -3
- package/src/mcp/server.mjs +104 -0
- package/src/mcp/tools.mjs +114 -0
- package/src/probe/contention.mjs +60 -0
- package/src/probe/discover.mjs +12 -3
- package/src/probe/independence.mjs +59 -0
- package/src/probe/mocks.mjs +120 -0
- package/src/probe/probe.mjs +126 -18
- package/src/probe/rank.mjs +73 -15
- package/src/probe/runners/index.mjs +49 -5
- package/src/probe/runners/jest.mjs +3 -3
- package/src/probe/runners/playwright.mjs +133 -0
- package/src/probe/runners/shared.mjs +70 -13
- package/src/probe/runners/vitest.mjs +4 -3
- package/src/render.mjs +2 -2
- package/src/replay/label.mjs +64 -0
- package/src/replay/replay.mjs +309 -0
- package/src/scaffold/producers.mjs +217 -6
- package/src/scaffold/scaffold.mjs +9 -3
- package/src/status/status.mjs +47 -6
package/src/brief/brief.mjs
CHANGED
|
@@ -2,26 +2,44 @@ import { gate } from '../baseline/baseline.mjs';
|
|
|
2
2
|
import { summarize, formatVerdict } from '../render.mjs';
|
|
3
3
|
|
|
4
4
|
export const HEADING = '## TEST BLINDSPOT CONTEXT';
|
|
5
|
+
/** First line of every markdown brief, so a poster can find and update its own note instead of adding another. */
|
|
6
|
+
export const MARKDOWN_MARKER = '<!-- testguard:brief -->';
|
|
5
7
|
const ORDER = ['survived', 'nocover', 'unverifiable', 'fault-invalid', 'timeout', 'flaky-defender'];
|
|
6
8
|
|
|
7
9
|
/** One line the agent can act on. Names the mechanism, never just the verdict. */
|
|
8
10
|
export function hintFor(r) {
|
|
9
11
|
const defenders = r.defenders.resolved.join(', ');
|
|
12
|
+
const base = hintBase(r, defenders);
|
|
13
|
+
const mocked = (r.defenders.mocking ?? []);
|
|
14
|
+
const never = (r.defenders.signals ?? []).filter((s) => s.signal === 'mocked-never-asserted').map((s) => s.file);
|
|
15
|
+
const extra = [
|
|
16
|
+
mocked.length ? `${mocked.length} importing test${mocked.length === 1 ? '' : 's'} mock${mocked.length === 1 ? 's' : ''} the target and cannot detect this (${mocked.join(', ')})` : '',
|
|
17
|
+
never.length ? `${never.join(', ')} mock${never.length === 1 ? 's' : ''} it and never assert${never.length === 1 ? 's' : ''} on it` : '',
|
|
18
|
+
].filter(Boolean);
|
|
19
|
+
return extra.length ? `${base} ${extra.join('; ')}.` : base;
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
function hintBase(r, defenders) {
|
|
10
23
|
switch (r.verdict) {
|
|
11
24
|
case 'survived':
|
|
12
25
|
return r.detail.reason === 'killed-by-undeclared-tests'
|
|
13
26
|
? `Only tests outside its declared defenders (${defenders}) catch this; fix the claim's defendedBy or move the assertion.`
|
|
14
27
|
: `${defenders} stayed green with this fault applied; add an assertion that fails on it and passes on HEAD. If no test's outcome can change, first check the fault is observable at all.`;
|
|
15
28
|
case 'nocover':
|
|
16
|
-
return
|
|
29
|
+
return r.defenders.discovered
|
|
30
|
+
? 'No test file imports the target without mocking it; nothing defends this claim.'
|
|
31
|
+
: `No test file matches ${r.defenders.requested.join(', ') || '(no defenders declared)'}; nothing defends this claim.`;
|
|
17
32
|
case 'unverifiable':
|
|
18
33
|
return `Anchor ${r.detail.reason}; re-author fault ${r.subject.id} in the claims file before trusting this claim.`;
|
|
19
34
|
case 'fault-invalid':
|
|
20
35
|
return `Replacement does not load (${r.detail.reason}); fix the fault definition, not the code.`;
|
|
21
36
|
case 'timeout':
|
|
22
37
|
return 'Defenders time out with this fault applied; a hang is not a detection.';
|
|
23
|
-
case 'flaky-defender':
|
|
24
|
-
|
|
38
|
+
case 'flaky-defender': {
|
|
39
|
+
const fr = r.detail.flakeRate;
|
|
40
|
+
const rate = fr && fr.failures > 0 ? ` — failed ${fr.failures} of ${fr.runs} runs on unmodified source` : '';
|
|
41
|
+
return `${defenders} not reliably green (${r.detail.reason})${rate}; fix the flake before trusting any verdict here.`;
|
|
42
|
+
}
|
|
25
43
|
default:
|
|
26
44
|
return '';
|
|
27
45
|
}
|
|
@@ -31,7 +49,7 @@ function orderItems(a, b) {
|
|
|
31
49
|
return (b.isNew - a.isNew) || (ORDER.indexOf(a.verdict) - ORDER.indexOf(b.verdict)) || ((b.rank ?? 0) - (a.rank ?? 0));
|
|
32
50
|
}
|
|
33
51
|
|
|
34
|
-
export function buildBrief(evidence, baseline, { max = 20, generatedAt = new Date().toISOString(), next, changes } = {}) {
|
|
52
|
+
export function buildBrief(evidence, baseline, { max = 20, generatedAt = new Date().toISOString(), next, changes, install } = {}) {
|
|
35
53
|
const g = gate(evidence.records, baseline);
|
|
36
54
|
const toItem = (r, isNew) => ({
|
|
37
55
|
fingerprint: r.fingerprint,
|
|
@@ -54,19 +72,21 @@ export function buildBrief(evidence, baseline, { max = 20, generatedAt = new Dat
|
|
|
54
72
|
new: g.new.length + g.belowFloor.length,
|
|
55
73
|
baselined: g.baselined.length,
|
|
56
74
|
};
|
|
75
|
+
const kills = evidence.records.filter((r) => r.verdict === 'killed');
|
|
76
|
+
const coAuthored = kills.filter((r) => r.detail.independence?.class === 'co-authored').length;
|
|
57
77
|
const unclaimed = changes?.uncovered?.length ? { ref: changes.ref, files: changes.uncovered } : undefined;
|
|
58
78
|
const doc = { schemaVersion: 1, tool: evidence.tool, generatedAt, head: evidence.run.repo.head, heading: HEADING, summary, ...(next ? { next: { action: next.action, command: next.command, why: next.why } } : {}), ...(unclaimed ? { unclaimed } : {}), items, text: '' };
|
|
59
|
-
doc.text = renderBriefText({ ...doc, provisional: Boolean(evidence.run.provisional) }, { hasBaseline: Boolean(baseline), total: evidence.records.length });
|
|
79
|
+
doc.text = renderBriefText({ ...doc, provisional: Boolean(evidence.run.provisional), install }, { hasBaseline: Boolean(baseline), total: evidence.records.length, independence: kills.length ? { coAuthored, kills: kills.length } : undefined });
|
|
60
80
|
return doc;
|
|
61
81
|
}
|
|
62
82
|
|
|
63
|
-
export function renderBriefText(brief, { hasBaseline, total }) {
|
|
83
|
+
export function renderBriefText(brief, { hasBaseline, total, independence }) {
|
|
64
84
|
const unproven = total - (brief.summary.byVerdict.killed ?? 0);
|
|
65
85
|
const lines = [
|
|
66
86
|
brief.heading,
|
|
67
87
|
'',
|
|
68
88
|
...(brief.provisional ? ['**PROVISIONAL** — this evidence came from fewer than three confirmation runs; treat every verdict below as unconfirmed and re-probe with --confirm 3 before acting on it.', ''] : []),
|
|
69
|
-
`testguard ${brief.tool.version}${brief.head ? ` @ ${brief.head.slice(0, 12)}` : ''} — ${brief.summary.claims} claims, ${total} faults probed, ${unproven} unproven` +
|
|
89
|
+
`testguard ${brief.tool.version}${brief.install ? ` (${brief.install})` : ''}${brief.head ? ` @ ${brief.head.slice(0, 12)}` : ''} — ${brief.summary.claims} claims, ${total} faults probed, ${unproven} unproven` +
|
|
70
90
|
(unproven === 0 ? '.' : hasBaseline ? ` (${brief.summary.new} new since baseline).` : ' (no baseline; everything is new).'),
|
|
71
91
|
];
|
|
72
92
|
// Unclaimed changes come before everything else: the claim is written
|
|
@@ -76,6 +96,9 @@ export function renderBriefText(brief, { hasBaseline, total }) {
|
|
|
76
96
|
lines.push('', `UNCLAIMED CHANGES since ${brief.unclaimed.ref}: ${n} changed file${n === 1 ? '' : 's'} carr${n === 1 ? 'ies' : 'y'} no claim. State the claim first; nothing below can see this code.`);
|
|
77
97
|
for (const u of brief.unclaimed.files) lines.push(` - ${u.file} (${u.kind}) → ${u.suggestion}`);
|
|
78
98
|
}
|
|
99
|
+
// L3: a kill written in the same change as the code it guards is not
|
|
100
|
+
// independent evidence. One line, no per-item noise.
|
|
101
|
+
if (independence?.coAuthored) lines.push('', `${independence.coAuthored} of ${independence.kills} kills are co-authored with the code they defend — the test and the code were written in the same change, so those kills are not independent evidence.`);
|
|
79
102
|
if (brief.next) lines.push('', `NEXT [${brief.next.action}]: ${brief.next.command}`, ` why: ${brief.next.why}`);
|
|
80
103
|
if (brief.items.length === 0) {
|
|
81
104
|
lines.push('', 'Every probed claim is defended. Keep it that way: new claims need a fault and a test that fails on it.');
|
|
@@ -111,3 +134,44 @@ export function buildUnclaimedBrief({ tool, next, changes, generatedAt = new Dat
|
|
|
111
134
|
doc.text = lines.join('\n') + '\n';
|
|
112
135
|
return doc;
|
|
113
136
|
}
|
|
137
|
+
|
|
138
|
+
const cell = (v) => String(v ?? '').replace(/\|/g, '\\|').replace(/\r?\n/g, ' ');
|
|
139
|
+
|
|
140
|
+
/**
|
|
141
|
+
* The brief as a merge-request note: same content, same order as the text —
|
|
142
|
+
* unclaimed changes first, then `next`, then the ranked findings, capped at
|
|
143
|
+
* `--max` — in GitLab/GitHub-flavoured markdown. Rendering only; the document
|
|
144
|
+
* it renders is the validated one.
|
|
145
|
+
*/
|
|
146
|
+
export function renderBriefMarkdown(brief, { hasBaseline, total, install } = {}) {
|
|
147
|
+
const unproven = total - (brief.summary.byVerdict.killed ?? 0);
|
|
148
|
+
const lines = [MARKDOWN_MARKER, brief.heading, ''];
|
|
149
|
+
if (brief.provisional) lines.push('**PROVISIONAL** — fewer than three confirmation runs; nothing below is confirmed. Re-probe with `--confirm 3`.', '');
|
|
150
|
+
lines.push(`\`testguard ${brief.tool.version}${install ? ` (${install})` : ''}\`${brief.head ? ` @ \`${brief.head.slice(0, 12)}\`` : ''} — **${brief.summary.claims}** claims, **${total}** faults probed, **${unproven}** unproven` +
|
|
151
|
+
(unproven === 0 ? '.' : hasBaseline ? ` (**${brief.summary.new}** new since baseline).` : ' (no baseline; everything is new).'));
|
|
152
|
+
if (brief.unclaimed) {
|
|
153
|
+
const n = brief.unclaimed.files.length;
|
|
154
|
+
lines.push('', `### Unclaimed changes since \`${brief.unclaimed.ref}\``, '', `${n} changed file${n === 1 ? '' : 's'} carr${n === 1 ? 'ies' : 'y'} no claim. State the claim first; nothing below can see this code.`, '');
|
|
155
|
+
for (const u of brief.unclaimed.files) lines.push(`- \`${u.file}\` (${u.kind}) → \`${u.suggestion}\``);
|
|
156
|
+
}
|
|
157
|
+
if (brief.next) lines.push('', `**Next** \`[${brief.next.action}]\`: \`${brief.next.command}\``, '', `> ${brief.next.why}`);
|
|
158
|
+
if (brief.items.length === 0) {
|
|
159
|
+
lines.push('', 'Every probed claim is defended. Keep it that way: new claims need a fault and a test that fails on it.');
|
|
160
|
+
return lines.join('\n') + '\n';
|
|
161
|
+
}
|
|
162
|
+
lines.push('', 'Where the test suite is blind, ranked. A **SURVIVED** fault means its defenders stayed green while the claim was false. Do not close these by asserting current behaviour; write a test that fails on the described fault and passes on HEAD.', '');
|
|
163
|
+
lines.push('| # | verdict | claim / fault | severity | file | what to do |', '|---|---|---|---|---|---|');
|
|
164
|
+
brief.items.forEach((it, i) => {
|
|
165
|
+
lines.push(`| ${i + 1} | ${it.isNew ? '**NEW** ' : ''}${formatVerdict(it.verdict)} | \`${cell(it.claimId)}/${cell(it.subjectId ?? '?')}\`<br>${cell(it.statement)} | ${it.severity} | ${it.file ? `\`${cell(it.file)}\`` : ''} | ${cell(it.hint)} |`);
|
|
166
|
+
});
|
|
167
|
+
if (unproven > brief.items.length) lines.push('', `… and ${unproven - brief.items.length} more in the evidence file.`);
|
|
168
|
+
return lines.join('\n') + '\n';
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
/** Markdown for the no-evidence-yet, unclaimed-changes-only brief. */
|
|
172
|
+
export function renderUnclaimedMarkdown({ tool, next, changes, install }) {
|
|
173
|
+
const lines = [MARKDOWN_MARKER, HEADING, '', `\`testguard ${tool.version}${install ? ` (${install})` : ''}\` — no evidence yet; **${changes.uncovered.length}** unclaimed changed file${changes.uncovered.length === 1 ? '' : 's'} since \`${changes.ref}\`.`, '', `### Unclaimed changes since \`${changes.ref}\``, '', 'State the claim first; nothing can be probed for this code until it has one.', ''];
|
|
174
|
+
for (const u of changes.uncovered) lines.push(`- \`${u.file}\` (${u.kind}) → \`${u.suggestion}\``);
|
|
175
|
+
if (next) lines.push('', `**Next** \`[${next.action}]\`: \`${next.command}\``, '', `> ${next.why}`);
|
|
176
|
+
return lines.join('\n') + '\n';
|
|
177
|
+
}
|
|
@@ -0,0 +1,161 @@
|
|
|
1
|
+
import { existsSync, realpathSync } from 'node:fs';
|
|
2
|
+
import { basename, dirname, join, relative, resolve } from 'node:path';
|
|
3
|
+
import { git, GitError, repoRoot } from '../git.mjs';
|
|
4
|
+
import { readSpecDoc } from '../evidence/writer.mjs';
|
|
5
|
+
import { defaultIgnorePath } from '../gate/changed.mjs';
|
|
6
|
+
import { PreconditionError } from '../probe/worktree.mjs';
|
|
7
|
+
|
|
8
|
+
/**
|
|
9
|
+
* A claim that no longer exists is invisible by construction: `probe` verifies
|
|
10
|
+
* what is there, `gate` sees the source file covered by some other claim, and
|
|
11
|
+
* `status` says clean. Deleting a claim is therefore cheaper than weakening
|
|
12
|
+
* its fault — and `changedFaults` exists precisely because weakening was the
|
|
13
|
+
* cheap escape. This closes the other half: removal is allowed, and recorded.
|
|
14
|
+
*
|
|
15
|
+
* Identities only. A fault whose text changed is `changedFaults`' business.
|
|
16
|
+
*/
|
|
17
|
+
const isExpired = (entry, now) => Boolean(entry.expires) && new Date(entry.expires).getTime() <= now.getTime();
|
|
18
|
+
|
|
19
|
+
/** The claims document as of `ref`, or null when the file did not exist there. */
|
|
20
|
+
export function claimsAt({ projectDir, ref, claimsPath }) {
|
|
21
|
+
// git reports the repository root by its real path (/private/var on macOS);
|
|
22
|
+
// a relative() that does not start from the same place yields `../../..`
|
|
23
|
+
// and `git show` then fails as if the file had never existed.
|
|
24
|
+
const root = repoRoot(realpathSync(resolve(projectDir)));
|
|
25
|
+
// The file may not exist at this ref (or at all); realpath its DIRECTORY so
|
|
26
|
+
// a missing file still resolves to a repository-relative path.
|
|
27
|
+
const p0 = resolve(claimsPath);
|
|
28
|
+
const abs = existsSync(p0) ? realpathSync(p0) : join(realpathSync(dirname(p0)), basename(p0));
|
|
29
|
+
const rel = relative(root, abs).split('\\').join('/');
|
|
30
|
+
if (rel.startsWith('..')) throw new PreconditionError(`the claims file ${claimsPath} is outside the repository at ${root}`);
|
|
31
|
+
let text;
|
|
32
|
+
try {
|
|
33
|
+
text = git(['show', `${ref}:${rel}`], root);
|
|
34
|
+
} catch (e) {
|
|
35
|
+
if (e instanceof GitError) {
|
|
36
|
+
// Distinguish "that ref is not a thing" (the caller's mistake) from
|
|
37
|
+
// "the file did not exist yet" (nothing to compare, not an error).
|
|
38
|
+
try {
|
|
39
|
+
git(['rev-parse', '--verify', `${ref}^{commit}`], root);
|
|
40
|
+
} catch {
|
|
41
|
+
throw new PreconditionError(`cannot read ${ref}: no such commit`);
|
|
42
|
+
}
|
|
43
|
+
return null;
|
|
44
|
+
}
|
|
45
|
+
throw e;
|
|
46
|
+
}
|
|
47
|
+
try {
|
|
48
|
+
return JSON.parse(text);
|
|
49
|
+
} catch (e) {
|
|
50
|
+
throw new PreconditionError(`the claims file at ${ref} is not valid JSON (${e.message})`);
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
const index = (doc) => {
|
|
55
|
+
const claims = new Map();
|
|
56
|
+
for (const c of doc?.claims ?? []) claims.set(c.id, { statement: c.statement, severity: c.severity, faults: new Set((c.faults ?? []).map((f) => f.id)) });
|
|
57
|
+
return claims;
|
|
58
|
+
};
|
|
59
|
+
|
|
60
|
+
/**
|
|
61
|
+
* Compare claim and fault identities at `ref` with the working file.
|
|
62
|
+
*
|
|
63
|
+
* A removed id whose statement reappears verbatim under a new id is a
|
|
64
|
+
* `renamed-claim`, not a removal: an ordinary rename must not read as an
|
|
65
|
+
* escape, or the check becomes noise nobody acts on.
|
|
66
|
+
*/
|
|
67
|
+
export function computeRemovedClaims({ projectDir, ref, claimsPath, current, ignorePath, evidencePath, now = new Date(), toolVersion = '0.0.0' }) {
|
|
68
|
+
const before = claimsAt({ projectDir, ref, claimsPath });
|
|
69
|
+
const beforeIdx = index(before);
|
|
70
|
+
const afterIdx = index(current);
|
|
71
|
+
|
|
72
|
+
const iPath = ignorePath ?? defaultIgnorePath(projectDir);
|
|
73
|
+
const ignore = existsSync(iPath) ? readSpecDoc('ignore', iPath) : { entries: [] };
|
|
74
|
+
const excuses = ignore.entries
|
|
75
|
+
.filter((e) => e.kind === 'claim' || e.kind === 'fault')
|
|
76
|
+
.map((e) => ({ entry: e, expired: isExpired(e, now), used: [] }));
|
|
77
|
+
|
|
78
|
+
const lastVerdict = new Map();
|
|
79
|
+
if (evidencePath && existsSync(evidencePath)) {
|
|
80
|
+
try {
|
|
81
|
+
for (const r of readSpecDoc('evidence', evidencePath).records) lastVerdict.set(`${r.claim.id}/${r.subject.id}`, r.verdict);
|
|
82
|
+
} catch {
|
|
83
|
+
// Evidence that cannot be read must not stop the check; the verdict is
|
|
84
|
+
// extra context, never the finding itself.
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
const statementsAfter = new Map();
|
|
89
|
+
for (const [id, c] of afterIdx) if (!beforeIdx.has(id)) statementsAfter.set(c.statement, id);
|
|
90
|
+
|
|
91
|
+
const findings = [];
|
|
92
|
+
for (const [id, c] of beforeIdx) {
|
|
93
|
+
if (!afterIdx.has(id)) {
|
|
94
|
+
const renamedTo = statementsAfter.get(c.statement);
|
|
95
|
+
if (renamedTo) {
|
|
96
|
+
findings.push({ kind: 'renamed-claim', claimId: id, renamedTo, severity: c.severity });
|
|
97
|
+
continue;
|
|
98
|
+
}
|
|
99
|
+
const verdicts = [...c.faults].map((f) => lastVerdict.get(`${id}/${f}`)).filter(Boolean);
|
|
100
|
+
findings.push({ kind: 'removed-claim', claimId: id, severity: c.severity, faults: [...c.faults].sort(), ...(verdicts.length ? { lastVerdicts: [...new Set(verdicts)].sort() } : {}) });
|
|
101
|
+
continue;
|
|
102
|
+
}
|
|
103
|
+
for (const f of c.faults) {
|
|
104
|
+
if (afterIdx.get(id).faults.has(f)) continue;
|
|
105
|
+
const v = lastVerdict.get(`${id}/${f}`);
|
|
106
|
+
findings.push({ kind: 'removed-fault', claimId: id, subjectId: f, severity: c.severity, ...(v ? { lastVerdicts: [v] } : {}) });
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
// An excuse applies to a removal, never to a rename: a rename is already not a finding.
|
|
111
|
+
const gating = [];
|
|
112
|
+
for (const f of findings) {
|
|
113
|
+
if (f.kind === 'renamed-claim') continue;
|
|
114
|
+
const target = f.kind === 'removed-fault' ? `${f.claimId}/${f.subjectId}` : f.claimId;
|
|
115
|
+
const active = excuses.find((e) => !e.expired && (e.entry.pattern === target || (f.kind === 'removed-fault' && e.entry.kind === 'claim' && e.entry.pattern === f.claimId)));
|
|
116
|
+
if (active) {
|
|
117
|
+
active.used.push(target);
|
|
118
|
+
f.excusedBy = active.entry.pattern;
|
|
119
|
+
continue;
|
|
120
|
+
}
|
|
121
|
+
const expired = excuses.find((e) => e.expired && (e.entry.pattern === target || (f.kind === 'removed-fault' && e.entry.kind === 'claim' && e.entry.pattern === f.claimId)));
|
|
122
|
+
if (expired) {
|
|
123
|
+
expired.used.push(target);
|
|
124
|
+
f.expiredExcuse = expired.entry.pattern;
|
|
125
|
+
}
|
|
126
|
+
gating.push(f);
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
const order = { 'removed-claim': 0, 'removed-fault': 1, 'renamed-claim': 2 };
|
|
130
|
+
findings.sort((a, b) => order[a.kind] - order[b.kind] || a.claimId.localeCompare(b.claimId) || (a.subjectId ?? '').localeCompare(b.subjectId ?? ''));
|
|
131
|
+
|
|
132
|
+
return {
|
|
133
|
+
ref,
|
|
134
|
+
comparedAgainst: before ? 'claims-file' : 'nothing',
|
|
135
|
+
findings,
|
|
136
|
+
removed: gating,
|
|
137
|
+
reliedOn: excuses.filter((e) => !e.expired && e.used.length).map((e) => ({ pattern: e.entry.pattern, reason: e.entry.reason, ...(e.entry.expires ? { expires: e.entry.expires } : {}), ids: e.used })),
|
|
138
|
+
expired: excuses.filter((e) => e.expired && e.used.length).map((e) => ({ pattern: e.entry.pattern, reason: e.entry.reason, expires: e.entry.expires, ids: e.used })),
|
|
139
|
+
toolVersion,
|
|
140
|
+
};
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
export function renderRemoved(doc) {
|
|
144
|
+
if (doc.comparedAgainst === 'nothing') return `no claims file at ${doc.ref}; nothing to compare`;
|
|
145
|
+
const lines = [];
|
|
146
|
+
for (const f of doc.findings) {
|
|
147
|
+
const verdicts = f.lastVerdicts?.length ? ` — last ${f.lastVerdicts.length === 1 ? 'verdict' : 'verdicts'} ${f.lastVerdicts.join(', ')}` : '';
|
|
148
|
+
if (f.kind === 'removed-claim') lines.push(`REMOVED ${f.claimId} (${f.severity}, ${f.faults.length} fault${f.faults.length === 1 ? '' : 's'})${verdicts}${f.excusedBy ? ` [excused by ignore "${f.excusedBy}"]` : ''}${f.expiredExcuse ? ` [ignore "${f.expiredExcuse}" has EXPIRED]` : ''}`);
|
|
149
|
+
else if (f.kind === 'removed-fault') lines.push(`REMOVED ${f.claimId}/${f.subjectId} (${f.severity})${verdicts}${f.excusedBy ? ` [excused by ignore "${f.excusedBy}"]` : ''}${f.expiredExcuse ? ` [ignore "${f.expiredExcuse}" has EXPIRED]` : ''}`);
|
|
150
|
+
else lines.push(`renamed ${f.claimId} → ${f.renamedTo} (same statement; not a removal)`);
|
|
151
|
+
}
|
|
152
|
+
for (const r of doc.reliedOn) lines.push(`excused ${r.ids.join(', ')} by ignore "${r.pattern}": ${r.reason}${r.expires ? ` (expires ${r.expires})` : ''}`);
|
|
153
|
+
for (const e of doc.expired) lines.push(`EXPIRED ignore "${e.pattern}" (expired ${e.expires}) no longer excuses ${e.ids.join(', ')}`);
|
|
154
|
+
if (doc.removed.length) {
|
|
155
|
+
lines.push('', `${doc.removed.length} claim${doc.removed.length === 1 ? '' : 's'}/fault${doc.removed.length === 1 ? '' : 's'} present at ${doc.ref} no longer exist. Removing a claim is allowed — claims can be wrong, superseded or split — but it is the cheapest way to make a finding disappear, so it is never silent.`);
|
|
156
|
+
lines.push('Either restore them, or add a `claim` entry to testguard.ignore.json with a reason a reviewer will accept.');
|
|
157
|
+
} else if (doc.findings.length === 0) {
|
|
158
|
+
lines.push(`every claim and fault present at ${doc.ref} still exists`);
|
|
159
|
+
}
|
|
160
|
+
return lines.join('\n');
|
|
161
|
+
}
|
package/src/cli.mjs
CHANGED
|
@@ -14,19 +14,25 @@ import { scaffoldCommand } from './commands/scaffold.mjs';
|
|
|
14
14
|
import { statusCommand } from './commands/status.mjs';
|
|
15
15
|
import { initCommand } from './commands/init.mjs';
|
|
16
16
|
import { gateCommand } from './commands/gate.mjs';
|
|
17
|
+
import { replayCommand } from './commands/replay.mjs';
|
|
18
|
+
import { mcpCommand } from './commands/mcp.mjs';
|
|
19
|
+
import { admitCommand } from './commands/admit.mjs';
|
|
17
20
|
|
|
18
21
|
const VERSION = JSON.parse(readFileSync(join(fileURLToPath(import.meta.url), '..', '..', 'package.json'), 'utf8')).version;
|
|
19
22
|
|
|
20
23
|
const USAGE = `testguard ${VERSION} — proves a test suite defends the claims a project makes
|
|
21
24
|
|
|
22
25
|
testguard status [dir] where the project is and the ONE next action; --json is the machine entry point
|
|
23
|
-
testguard init [dir] install the agent layer
|
|
26
|
+
testguard init [dir] install the agent layer at the git root (skill, session-start hook, AGENTS.md section) and the .gitignore lines in [dir]
|
|
24
27
|
testguard claims [dir] list the claims file and report drift against @claim annotations in code
|
|
25
28
|
testguard probe [dir] inject each claim's faults, run its defenders, report what survived
|
|
26
29
|
testguard baseline [dir] freeze today's unproven findings so only new ones gate
|
|
27
30
|
testguard brief [dir] emit the blind-spot block for an agent's session-start context
|
|
28
31
|
testguard gate [dir] fail when a changed source file carries no claim and no excusing ignore entry
|
|
29
32
|
testguard scaffold <file> propose faults mechanically for one source file, as a draft claims document
|
|
33
|
+
testguard mcp serve the read-only status/brief/claims/evidence tools over MCP on stdio, so the loop works in any agent harness
|
|
34
|
+
testguard replay [dir] would this suite have caught the bugs that already escaped? Replays real fix commits and calibrates fault classes against them
|
|
35
|
+
testguard admit <test> --claim <ID> the two-gate rule as one verb: is this test green on HEAD and does it fail on every fault of the claim?
|
|
30
36
|
|
|
31
37
|
probe
|
|
32
38
|
--claims <path> claims file (default: <dir>/testguard.claims.json)
|
|
@@ -35,22 +41,42 @@ probe
|
|
|
35
41
|
--out <path> evidence file (default: <dir>/.testguard/evidence.json)
|
|
36
42
|
--baseline <path> baseline to gate against (default: <dir>/.testguard/baseline.json if present)
|
|
37
43
|
--severity <level> gate only at or above (default: low)
|
|
38
|
-
--ref <commit> probe this commit in the scratch worktree (default: HEAD)
|
|
44
|
+
--ref <commit> probe this commit in the scratch worktree (default: HEAD). An explicit --ref is honoured even when
|
|
45
|
+
defender/target files are dirty: a warning names them, the evidence records them (repo.ignoredDirty)
|
|
46
|
+
--ignore-dirty probe HEAD as committed although defender/target files are dirty (same warning and record)
|
|
39
47
|
--claim <ID,ID> probe only these claims; writes .testguard/evidence-partial.json
|
|
40
48
|
--include-dirty probe the working tree (a snapshot commit) instead of HEAD; uncommitted tests count
|
|
49
|
+
(without it, a dirty defender/target with the implicit HEAD is refused: the silent-mismatch trap)
|
|
41
50
|
--verbose also print each killed fault (default: only unproven ones, plus a count)
|
|
42
51
|
--confirm below 3 is PROVISIONAL: verdicts print with "?", evidence goes to .testguard/evidence-provisional.json
|
|
43
|
-
--runner <name> vitest | jest | auto (default: auto — first of vitest, jest that resolves
|
|
52
|
+
--runner <name> vitest | jest | playwright | auto (default: auto — first of vitest, jest that resolves; a defender under
|
|
53
|
+
playwright's testDir always runs under playwright, whatever the project runner)
|
|
44
54
|
--runner-cmd "<cmd>" custom runner; must contain {files} and {out}, e.g. "pnpm vitest run {files} --reporter=json --outputFile={out}"
|
|
45
55
|
--node-modules <dir> node_modules to link into the scratch worktree (or TESTGUARD_NODE_MODULES)
|
|
56
|
+
--serial run one test file at a time (vitest --no-file-parallelism, jest --runInBand, playwright --workers=1);
|
|
57
|
+
use it when another test runner is already running — probe warns and records the contention either way
|
|
46
58
|
--in-place mutate the working tree instead of a scratch worktree
|
|
47
59
|
--no-escalate do not re-run survivors against the whole suite
|
|
48
60
|
--no-reuse re-probe claims whose inputs have not changed
|
|
49
61
|
--quiet suppress the per-fault stream and ranked block; print only the summary and evidence path
|
|
62
|
+
--json the status document plus this run's result (records, newSinceBaseline, exitCode)
|
|
50
63
|
|
|
51
64
|
scaffold --claim <ID> (put every proposal under this claim; copies it if it exists) --out <path> --json
|
|
52
65
|
shapes: if-guard → if (false) · single-line guard/mutation removed · return <check> → return true
|
|
53
66
|
· security flag/window/cost literal weakened · verify/validate/check call removed
|
|
67
|
+
· field dropped from a payload/allow-list/schema/merge · parameter-derived argument swapped for undefined/{}
|
|
68
|
+
mcp no options. JSON-RPC 2.0 over stdio; five READ-ONLY tools (status, brief, claims, evidence, next_command).
|
|
69
|
+
Nothing here runs a probe: next_command hands back the shell line for you to run where the person can see it.
|
|
70
|
+
Register it with your harness — testguard init --mcp prints the config for Claude Code, Cursor and Codex.
|
|
71
|
+
replay --since <range> commit range to search for fix commits (HEAD~50..HEAD, a tag, origin/main..HEAD)
|
|
72
|
+
--max <n> replay at most n fixes (they are slow: one worktree and N runs each)
|
|
73
|
+
--confirm <n> runs per verdict (default 3); a flaky failure reads as "the suite caught it", so mixed runs are never caught
|
|
74
|
+
--out <path> replay document (default: <dir>/.testguard/replay.json); the calibration goes beside it
|
|
75
|
+
reports, never gates: a bug that escaped is history, not a regression in this change
|
|
76
|
+
· one-line JSX element removed · on<Event> handler prop dropped
|
|
77
|
+
admit --claim <ID> (required) --fault <FID> (one fault only) --confirm <n> --json
|
|
78
|
+
ADMITTED (exit 0) only when every fault of the claim is killed N/N by defenders that are green N/N; anything else is NOT ADMITTED (exit 1) and names the first blocking fault
|
|
79
|
+
the test must be a declared or discovered defender of the claim (exit 3 otherwise); evidence goes to .testguard/evidence-partial.json
|
|
54
80
|
gate --changed <ref> measure the change since merge-base(ref, HEAD); auto-detected in GitHub Actions / GitLab CI
|
|
55
81
|
--include-dirty compare the working tree (staged, unstaged and untracked) instead of HEAD — the pre-commit shape
|
|
56
82
|
--exclude <glob> (repeatable) more files that never carry claims; --explain lists the defaults
|
|
@@ -58,17 +84,25 @@ gate --changed <ref> measure the change since merge-base(ref, HEAD); aut
|
|
|
58
84
|
--ignore <path> ignore file (default: <dir>/testguard.ignore.json; kind=path entries excuse files, with a reason)
|
|
59
85
|
exit 0 every changed source file is claimed or excused · 1 unclaimed file · 2 cannot evaluate · 3 no reference
|
|
60
86
|
status --json (exit 0 clean · 1 unproven/stale/unclaimed · 2 nothing to probe yet)
|
|
87
|
+
--evidence <path> read this evidence instead of .testguard/evidence.json (e.g. CI's, fetched as an artifact); staleness is still computed from the recorded input hashes, and both commits are named
|
|
61
88
|
--changed <ref> also compute claim coverage of the change; unclaimed-changes then precedes every evidence state
|
|
62
|
-
init --force (replace an existing skill file) --json
|
|
89
|
+
init --force (replace an existing skill file) --here (keep the agent layer in [dir] instead of the git root) --json
|
|
90
|
+
--ci-evidence github|gitlab write .testguard/fetch-ci-evidence.sh, an ON-DEMAND helper that downloads CI's evidence artifact and briefs from it; the session-start hook stays offline
|
|
91
|
+
agent layer (skill, SessionStart hook, AGENTS.md section) → git root; project layer (.gitignore lines) → [dir]
|
|
92
|
+
the hook prefers a local install, falls back to npx --no-install, never fetches; exit 1 if a written file is gitignored
|
|
93
|
+
--mcp print the MCP server config for Claude Code, Cursor and Codex (printed, never written: a harness config is yours)
|
|
63
94
|
every command accepts --json; probe/baseline emit the status document plus their own result
|
|
64
95
|
claims --json
|
|
96
|
+
--since <ref> report every claim and fault that existed at <ref> and does not now; a claim entry in testguard.ignore.json (with a reason) excuses one. exit 1 on any unexcused removal
|
|
65
97
|
baseline --evidence <path> --out <path> --allow-provisional (freeze unconfirmed evidence; normally refused)
|
|
66
|
-
|
|
98
|
+
--restamp move head to the commit of a later CLEAN probe that reproduced the same fingerprints
|
|
99
|
+
(a baseline frozen from a snapshot points at the parent of the commit that carries its tests)
|
|
100
|
+
brief --evidence <path> --baseline <path> --max <n> --text (print only; safe for hooks) --markdown (print only, as a merge-request note)
|
|
67
101
|
|
|
68
102
|
exit codes: 0 nothing new to prove · 1 unproven claims (or claim drift, or unclaimed changes) · 2 precondition failed · 3 usage
|
|
69
103
|
`;
|
|
70
104
|
|
|
71
|
-
const COMMANDS = { probe: probeCommand, claims: claimsCommand, baseline: baselineCommand, brief: briefCommand, scaffold: scaffoldCommand, status: statusCommand, init: initCommand, gate: gateCommand };
|
|
105
|
+
const COMMANDS = { mcp: mcpCommand, replay: replayCommand, probe: probeCommand, claims: claimsCommand, baseline: baselineCommand, brief: briefCommand, scaffold: scaffoldCommand, status: statusCommand, init: initCommand, gate: gateCommand, admit: admitCommand };
|
|
72
106
|
|
|
73
107
|
export async function main(argv, io = { out: (s) => process.stdout.write(s + '\n'), err: (s) => process.stderr.write(s + '\n') }) {
|
|
74
108
|
let parsed;
|
|
@@ -85,16 +119,24 @@ export async function main(argv, io = { out: (s) => process.stdout.write(s + '\n
|
|
|
85
119
|
baseline: { type: 'string' },
|
|
86
120
|
severity: { type: 'string', default: 'low' },
|
|
87
121
|
max: { type: 'string', default: '20' },
|
|
88
|
-
ref: { type: 'string'
|
|
122
|
+
ref: { type: 'string' },
|
|
123
|
+
'ignore-dirty': { type: 'boolean', default: false },
|
|
124
|
+
serial: { type: 'boolean', default: false },
|
|
89
125
|
claim: { type: 'string' },
|
|
126
|
+
fault: { type: 'string' },
|
|
90
127
|
'include-dirty': { type: 'boolean', default: false },
|
|
91
128
|
changed: { type: 'string' },
|
|
129
|
+
since: { type: 'string' },
|
|
92
130
|
exclude: { type: 'string', multiple: true },
|
|
93
131
|
strict: { type: 'boolean', default: false },
|
|
94
132
|
explain: { type: 'boolean', default: false },
|
|
95
133
|
ignore: { type: 'string' },
|
|
96
134
|
'allow-provisional': { type: 'boolean', default: false },
|
|
135
|
+
restamp: { type: 'boolean', default: false },
|
|
97
136
|
force: { type: 'boolean', default: false },
|
|
137
|
+
mcp: { type: 'boolean', default: false },
|
|
138
|
+
'ci-evidence': { type: 'string' },
|
|
139
|
+
here: { type: 'boolean', default: false },
|
|
98
140
|
verbose: { type: 'boolean', default: false },
|
|
99
141
|
'runner-cmd': { type: 'string' },
|
|
100
142
|
runner: { type: 'string', default: 'auto' },
|
|
@@ -105,6 +147,7 @@ export async function main(argv, io = { out: (s) => process.stdout.write(s + '\n
|
|
|
105
147
|
quiet: { type: 'boolean', default: false },
|
|
106
148
|
json: { type: 'boolean', default: false },
|
|
107
149
|
text: { type: 'boolean', default: false },
|
|
150
|
+
markdown: { type: 'boolean', default: false },
|
|
108
151
|
help: { type: 'boolean', short: 'h', default: false },
|
|
109
152
|
version: { type: 'boolean', short: 'v', default: false },
|
|
110
153
|
},
|
|
@@ -130,8 +173,8 @@ export async function main(argv, io = { out: (s) => process.stdout.write(s + '\n
|
|
|
130
173
|
io.err(USAGE);
|
|
131
174
|
return 3;
|
|
132
175
|
}
|
|
133
|
-
if (!['vitest', 'jest', 'auto'].includes(values.runner)) {
|
|
134
|
-
io.err('--runner must be vitest, jest or auto');
|
|
176
|
+
if (!['vitest', 'jest', 'playwright', 'auto'].includes(values.runner)) {
|
|
177
|
+
io.err('--runner must be vitest, jest, playwright or auto');
|
|
135
178
|
return 3;
|
|
136
179
|
}
|
|
137
180
|
if (!['critical', 'high', 'medium', 'low'].includes(values.severity)) {
|
|
@@ -139,7 +182,7 @@ export async function main(argv, io = { out: (s) => process.stdout.write(s + '\n
|
|
|
139
182
|
return 3;
|
|
140
183
|
}
|
|
141
184
|
try {
|
|
142
|
-
const projectDir = command === 'scaffold' ? resolve('.') : resolve(dirArg ?? '.');
|
|
185
|
+
const projectDir = command === 'scaffold' || command === 'admit' ? resolve('.') : resolve(dirArg ?? '.');
|
|
143
186
|
return await handler({ projectDir, file: dirArg, values, version: VERSION }, io);
|
|
144
187
|
} catch (e) {
|
|
145
188
|
if (e instanceof ClaimsError || e instanceof PreconditionError || e instanceof GitError || e instanceof SpecDocError) {
|
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
import { existsSync } from 'node:fs';
|
|
2
|
+
import { dirname, join, relative, resolve } from 'node:path';
|
|
3
|
+
import { loadClaims, defaultClaimsPath } from '../claims/load.mjs';
|
|
4
|
+
import { probe } from '../probe/probe.mjs';
|
|
5
|
+
import { writeSpecDoc } from '../evidence/writer.mjs';
|
|
6
|
+
import { resolveDefenders } from '../probe/runners/shared.mjs';
|
|
7
|
+
import { discoverDefenders } from '../probe/discover.mjs';
|
|
8
|
+
import { PreconditionError } from '../probe/worktree.mjs';
|
|
9
|
+
import { hintFor } from '../brief/brief.mjs';
|
|
10
|
+
import { formatVerdict, PROVISIONAL_WARNING } from '../render.mjs';
|
|
11
|
+
import { decideAdmission } from '../admit/admit.mjs';
|
|
12
|
+
|
|
13
|
+
/** The project is wherever the claims file is, walking up from the test file; else cwd. */
|
|
14
|
+
function findProjectDir(testAbs, claimsFlag) {
|
|
15
|
+
if (claimsFlag) return dirname(resolve(claimsFlag));
|
|
16
|
+
let dir = dirname(testAbs);
|
|
17
|
+
for (let hop = 0; hop < 20; hop++) {
|
|
18
|
+
if (existsSync(defaultClaimsPath(dir))) return dir;
|
|
19
|
+
const up = dirname(dir);
|
|
20
|
+
if (up === dir) break;
|
|
21
|
+
dir = up;
|
|
22
|
+
}
|
|
23
|
+
return resolve('.');
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
/**
|
|
27
|
+
* `testguard admit <test-file> --claim <ID>`: the two-gate rule as one verb.
|
|
28
|
+
* Sugar over `probe --claim <ID> --include-dirty`; no second engine, so the
|
|
29
|
+
* decision here can never disagree with the gate.
|
|
30
|
+
*/
|
|
31
|
+
export async function admitCommand({ file, values, version }, io) {
|
|
32
|
+
if (!file || !values.claim) {
|
|
33
|
+
io.err('usage: testguard admit <test-file> --claim <ID> [--fault <FID>] [--confirm <n>] [--json]');
|
|
34
|
+
return 3;
|
|
35
|
+
}
|
|
36
|
+
const confirmRuns = Number(values.confirm);
|
|
37
|
+
const budgetMs = Number(values.budget);
|
|
38
|
+
if (!Number.isInteger(confirmRuns) || confirmRuns < 1 || !Number.isInteger(budgetMs) || budgetMs < 1000) {
|
|
39
|
+
io.err('--confirm must be a positive integer and --budget at least 1000');
|
|
40
|
+
return 3;
|
|
41
|
+
}
|
|
42
|
+
const testAbs = resolve(file);
|
|
43
|
+
if (!existsSync(testAbs)) throw new PreconditionError(`no such file: ${file}`);
|
|
44
|
+
const projectDir = findProjectDir(testAbs, values.claims);
|
|
45
|
+
const testRel = relative(projectDir, testAbs);
|
|
46
|
+
if (testRel.startsWith('..')) throw new PreconditionError(`${file} is outside the project directory ${projectDir}`);
|
|
47
|
+
|
|
48
|
+
const all = loadClaims(values.claims ? resolve(values.claims) : defaultClaimsPath(projectDir));
|
|
49
|
+
const claimId = values.claim.trim();
|
|
50
|
+
const claim = all.claims.find((c) => c.id === claimId);
|
|
51
|
+
if (!claim) throw new PreconditionError(`--claim: unknown claim id ${claimId}`);
|
|
52
|
+
const faults = values.fault ? claim.faults.filter((f) => f.id === values.fault) : claim.faults;
|
|
53
|
+
if (faults.length === 0) throw new PreconditionError(`--fault: ${claimId} has no fault ${values.fault}`);
|
|
54
|
+
|
|
55
|
+
// The named test must be a defender of the claim, declared or discovered;
|
|
56
|
+
// otherwise `killed` below would be someone else's test doing the work.
|
|
57
|
+
const declared = claim.defendedBy?.length ? resolveDefenders(projectDir, claim.defendedBy) : null;
|
|
58
|
+
const isDefender = declared ? declared.includes(testRel) : faults.some((f) => discoverDefenders(projectDir, f.file).includes(testRel));
|
|
59
|
+
if (!isDefender) {
|
|
60
|
+
io.err(declared
|
|
61
|
+
? `${testRel} is not a defender of ${claimId}. Add it: "defendedBy": [${[...claim.defendedBy, testRel].map((d) => JSON.stringify(d)).join(', ')}] in testguard.claims.json, then run admit again.`
|
|
62
|
+
: `${testRel} does not import ${[...new Set(faults.map((f) => f.file))].join(' or ')}, so it cannot be discovered as a defender of ${claimId}. Import the module under test, or declare "defendedBy": ["${testRel}"] on the claim.`);
|
|
63
|
+
return 3;
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
const provisional = confirmRuns < 3;
|
|
67
|
+
if (provisional && !values.quiet) io.err(PROVISIONAL_WARNING(confirmRuns));
|
|
68
|
+
const command = `testguard probe --claim ${claimId} --include-dirty --confirm ${confirmRuns} --no-escalate`;
|
|
69
|
+
const evidence = await probe({
|
|
70
|
+
projectDir,
|
|
71
|
+
claims: { ...all, claims: [{ ...claim, faults }] },
|
|
72
|
+
confirmRuns,
|
|
73
|
+
budgetMs,
|
|
74
|
+
mode: 'worktree',
|
|
75
|
+
includeDirty: true,
|
|
76
|
+
only: [claimId],
|
|
77
|
+
escalate: false,
|
|
78
|
+
runnerCommand: values['runner-cmd'],
|
|
79
|
+
runnerName: values.runner,
|
|
80
|
+
nodeModules: values['node-modules'] ? resolve(values['node-modules']) : process.env.TESTGUARD_NODE_MODULES,
|
|
81
|
+
toolVersion: version,
|
|
82
|
+
onStage: !values.quiet && !values.json && process.stderr.isTTY ? ({ claimId: c, faultId, stage, i, n }) => process.stderr.write(`\r\x1b[K … ${c}/${faultId} ${stage} ${i}/${n}`) : undefined,
|
|
83
|
+
});
|
|
84
|
+
if (process.stderr.isTTY && !values.quiet && !values.json) process.stderr.write('\r\x1b[K');
|
|
85
|
+
const outPath = values.out ? resolve(values.out) : join(projectDir, '.testguard', 'evidence-partial.json');
|
|
86
|
+
writeSpecDoc('evidence', outPath, evidence);
|
|
87
|
+
|
|
88
|
+
const decision = decideAdmission(evidence.records);
|
|
89
|
+
const result = {
|
|
90
|
+
admitted: decision.admitted,
|
|
91
|
+
provisional,
|
|
92
|
+
claim: claimId,
|
|
93
|
+
test: testRel,
|
|
94
|
+
faults: evidence.records.map((r) => ({ id: r.subject.id, verdict: r.verdict, ...(r.detail.reason ? { reason: r.detail.reason } : {}), hint: hintFor(r) })),
|
|
95
|
+
evidence: outPath,
|
|
96
|
+
command,
|
|
97
|
+
};
|
|
98
|
+
if (values.json) {
|
|
99
|
+
io.out(JSON.stringify(result, null, 2));
|
|
100
|
+
} else {
|
|
101
|
+
for (const r of evidence.records) io.out(` ${formatVerdict(r.verdict, provisional).padEnd(15)} ${claimId}/${r.subject.id} ${r.subject.description}${r.detail.reason ? ` [${r.detail.reason}]` : ''}`);
|
|
102
|
+
io.out('');
|
|
103
|
+
if (decision.admitted) {
|
|
104
|
+
io.out(provisional
|
|
105
|
+
? `ADMITTED? — ${testRel} kills ${evidence.records.length === 1 ? 'the fault' : `all ${evidence.records.length} faults`} of ${claimId} at --confirm ${confirmRuns}. Provisional: confirm with --confirm 3 before committing.`
|
|
106
|
+
: `ADMITTED — ${testRel} passes on HEAD and fails on ${evidence.records.length === 1 ? 'the fault' : `all ${evidence.records.length} faults`} of ${claimId}, ${confirmRuns}/${confirmRuns}. Commit it.`);
|
|
107
|
+
} else {
|
|
108
|
+
const b = decision.blocking;
|
|
109
|
+
io.out(`NOT ADMITTED: ${b.verdict}${b.detail.reason ? ` (${b.detail.reason})` : ''} on ${claimId}/${b.subject.id}`);
|
|
110
|
+
io.out(` ${hintFor(b)}`);
|
|
111
|
+
}
|
|
112
|
+
io.out(`evidence: ${outPath} (partial; not the canonical evidence file)`);
|
|
113
|
+
}
|
|
114
|
+
if (provisional && !values.quiet && !values.json) io.err(PROVISIONAL_WARNING(confirmRuns));
|
|
115
|
+
return decision.admitted ? 0 : 1;
|
|
116
|
+
}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { existsSync } from 'node:fs';
|
|
2
2
|
import { resolve } from 'node:path';
|
|
3
3
|
import { readSpecDoc, writeSpecDoc } from '../evidence/writer.mjs';
|
|
4
|
-
import { buildBaseline } from '../baseline/baseline.mjs';
|
|
4
|
+
import { buildBaseline, restampBaseline } from '../baseline/baseline.mjs';
|
|
5
5
|
import { evidencePath, baselinePath } from './probe.mjs';
|
|
6
6
|
import { computeStatus } from '../status/status.mjs';
|
|
7
7
|
|
|
@@ -16,15 +16,29 @@ export async function baselineCommand({ projectDir, values, version }, io) {
|
|
|
16
16
|
io.err(`error: ${evPath} is provisional (confirmRuns ${evidence.run.confirmRuns} < 3); a baseline must be frozen from confirmed evidence. Re-run probe with --confirm 3, or pass --allow-provisional if you accept unconfirmed findings as the frozen contract.`);
|
|
17
17
|
return 2;
|
|
18
18
|
}
|
|
19
|
-
const baseline = buildBaseline(evidence);
|
|
20
19
|
const outPath = values.out ? resolve(values.out) : baselinePath(projectDir);
|
|
20
|
+
if (values.restamp) {
|
|
21
|
+
if (!existsSync(outPath)) {
|
|
22
|
+
io.err(`error: no baseline at ${outPath} to re-stamp; run \`testguard baseline\` first`);
|
|
23
|
+
return 2;
|
|
24
|
+
}
|
|
25
|
+
const r = restampBaseline(readSpecDoc('baseline', outPath), evidence);
|
|
26
|
+
if (!r.ok) {
|
|
27
|
+
io.err(`error: cannot re-stamp: ${r.reason}`);
|
|
28
|
+
return 2;
|
|
29
|
+
}
|
|
30
|
+
writeSpecDoc('baseline', outPath, r.baseline);
|
|
31
|
+
io.out(values.json ? JSON.stringify({ ...computeStatus({ projectDir, toolVersion: version }), baseline: { path: outPath, restamped: r.baseline.head } }, null, 2) : `baseline: re-stamped to ${r.baseline.head.slice(0, 12)} (same fingerprints, clean tree) → ${outPath}`);
|
|
32
|
+
return 0;
|
|
33
|
+
}
|
|
34
|
+
const baseline = buildBaseline(evidence);
|
|
21
35
|
writeSpecDoc('baseline', outPath, baseline);
|
|
22
36
|
const n = Object.values(baseline.fingerprints).reduce((a, b) => a + b, 0);
|
|
23
37
|
if (values.json) {
|
|
24
38
|
io.out(JSON.stringify({ ...computeStatus({ projectDir, toolVersion: version }), baseline: { path: outPath, frozen: n } }, null, 2));
|
|
25
39
|
return 0;
|
|
26
40
|
}
|
|
27
|
-
io.out(`baseline: ${n} unproven finding${n === 1 ? '' : 's'} frozen at ${baseline.head.slice(0, 12)}${baseline.dirty ? ' (working tree was dirty)' : ''}${evidence.run.provisional ? ' — FROM PROVISIONAL EVIDENCE (--allow-provisional)' : ''} → ${outPath}`);
|
|
41
|
+
io.out(`baseline: ${n} unproven finding${n === 1 ? '' : 's'} frozen at ${baseline.head.slice(0, 12)}${baseline.snapshot ? ` (working-tree snapshot ${baseline.snapshot.slice(0, 7)}; after you commit, a clean probe + \`baseline --restamp\` moves head to that commit)` : baseline.dirty ? ' (working tree was dirty)' : ''}${evidence.run.provisional ? ' — FROM PROVISIONAL EVIDENCE (--allow-provisional)' : ''} → ${outPath}`);
|
|
28
42
|
io.out('Commit this file; from now on only new findings gate. Ignore the regenerated ones — add to .gitignore:');
|
|
29
43
|
io.out(' .testguard/evidence.json');
|
|
30
44
|
io.out(' .testguard/brief.json');
|