testguard-cli 0.5.0 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +628 -0
- package/README.md +536 -42
- package/package.json +2 -2
- package/spec/GATE-SEMANTICS.md +166 -2
- package/spec/README.md +29 -10
- package/spec/lib/validate.mjs +140 -8
- package/spec/schemas/baseline.schema.json +13 -5
- package/spec/schemas/brief.schema.json +12 -12
- package/spec/schemas/calibration.schema.json +5 -5
- package/spec/schemas/claims.schema.json +122 -25
- package/spec/schemas/common.schema.json +131 -29
- package/spec/schemas/evidence.schema.json +476 -69
- package/spec/schemas/gate.schema.json +12 -12
- package/spec/schemas/ignore.schema.json +4 -4
- package/spec/schemas/replay.schema.json +85 -0
- package/spec/schemas/status.schema.json +258 -56
- package/src/admit/admit.mjs +17 -0
- package/src/baseline/baseline.mjs +31 -2
- package/src/brief/brief.mjs +71 -7
- package/src/claims/removed.mjs +161 -0
- package/src/cli.mjs +68 -10
- package/src/commands/admit.mjs +116 -0
- package/src/commands/baseline.mjs +17 -3
- package/src/commands/brief.mjs +13 -7
- package/src/commands/claims.mjs +55 -6
- package/src/commands/init.mjs +19 -6
- package/src/commands/mcp.mjs +12 -0
- package/src/commands/probe.mjs +35 -7
- package/src/commands/replay.mjs +63 -0
- package/src/commands/status.mjs +3 -1
- package/src/gate/changed.mjs +9 -2
- package/src/git.mjs +5 -0
- package/src/init/init.mjs +182 -29
- package/src/init/templates/SKILL.md +3 -3
- package/src/mcp/server.mjs +104 -0
- package/src/mcp/tools.mjs +114 -0
- package/src/probe/attribution.mjs +123 -0
- package/src/probe/classify.mjs +18 -2
- package/src/probe/contention.mjs +60 -0
- package/src/probe/cost.mjs +145 -0
- package/src/probe/discover.mjs +20 -3
- package/src/probe/independence.mjs +59 -0
- package/src/probe/inject.mjs +41 -9
- package/src/probe/mocks.mjs +122 -0
- package/src/probe/probe.mjs +331 -40
- package/src/probe/progress.mjs +87 -0
- package/src/probe/pyimports.mjs +287 -0
- package/src/probe/rank.mjs +75 -15
- package/src/probe/runners/index.mjs +79 -8
- package/src/probe/runners/jest.mjs +15 -3
- package/src/probe/runners/playwright.mjs +142 -0
- package/src/probe/runners/python/_testguard_pytest_plugin.py +132 -0
- package/src/probe/runners/python/_testguard_report.py +141 -0
- package/src/probe/runners/python/_testguard_unittest_main.py +168 -0
- package/src/probe/runners/python.mjs +316 -0
- package/src/probe/runners/shared.mjs +80 -14
- package/src/probe/runners/vitest.mjs +16 -3
- package/src/render.mjs +2 -2
- package/src/replay/label.mjs +64 -0
- package/src/replay/replay.mjs +310 -0
- package/src/scaffold/producers.mjs +217 -6
- package/src/scaffold/producers.python.mjs +209 -0
- package/src/scaffold/scaffold.mjs +90 -30
- package/src/status/status.mjs +47 -6
- package/src/util/glob.mjs +7 -1
package/src/init/init.mjs
CHANGED
|
@@ -1,68 +1,198 @@
|
|
|
1
|
-
import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
|
|
2
|
-
import { join, dirname } from 'node:path';
|
|
1
|
+
import { existsSync, mkdirSync, readFileSync, writeFileSync, realpathSync } from 'node:fs';
|
|
2
|
+
import { join, dirname, relative, resolve, sep } from 'node:path';
|
|
3
3
|
import { fileURLToPath } from 'node:url';
|
|
4
|
+
import { spawnSync } from 'node:child_process';
|
|
4
5
|
|
|
5
6
|
const TEMPLATES = join(dirname(fileURLToPath(import.meta.url)), 'templates');
|
|
6
|
-
const HOOK_CMD = 'npx -y testguard-cli brief --text';
|
|
7
7
|
const AGENTS_BEGIN = '<!-- testguard:begin -->';
|
|
8
8
|
const AGENTS_END = '<!-- testguard:end -->';
|
|
9
|
-
const
|
|
9
|
+
const GITIGNORE_LINES = ['.testguard/evidence.json', '.testguard/evidence-provisional.json', '.testguard/evidence-partial.json', '.testguard/brief.json', '.testguard/gate.json', '.testguard/scaffold-*.json'];
|
|
10
|
+
|
|
11
|
+
/**
|
|
12
|
+
* The session-start hook command. It runs from the git root (where the agent
|
|
13
|
+
* session lives) and must never fetch from the network: a local install of
|
|
14
|
+
* the project (or of the root) is preferred, then a `testguard` already on
|
|
15
|
+
* PATH, then nothing — the hook exits 0 with no output rather than break a
|
|
16
|
+
* session.
|
|
17
|
+
*
|
|
18
|
+
* No form of `npx` appears here, and `--no-install` is not an exception.
|
|
19
|
+
* Measured: in a project with nothing installed,
|
|
20
|
+
* `npx --no-install --loglevel=http testguard-cli --version` logs
|
|
21
|
+
* `npm http fetch GET 200 https://registry.npmjs.org/testguard-cli`, and
|
|
22
|
+
* against an unreachable registry it exits non-zero. npm resolves the
|
|
23
|
+
* packument before it decides not to install, so `--no-install` is quiet, not
|
|
24
|
+
* offline. A `command -v` lookup covers the global-install case with no
|
|
25
|
+
* network at all, which is the whole point in a repository that pins and
|
|
26
|
+
* audits its dependencies.
|
|
27
|
+
*/
|
|
28
|
+
export function hookCommand(dir) {
|
|
29
|
+
const arg = dir === '.' ? '' : ` ${dir}`;
|
|
30
|
+
const candidates = dir === '.' ? ['node_modules/.bin/testguard'] : [`${dir}/node_modules/.bin/testguard`, 'node_modules/.bin/testguard'];
|
|
31
|
+
// The PATH branch is braced: `a || b && c` binds as `(a || b) && c` in sh,
|
|
32
|
+
// so an unbraced `&&` would run the brief a second time whenever the local
|
|
33
|
+
// binary succeeded.
|
|
34
|
+
const onPath = `{ command -v testguard >/dev/null 2>&1 && testguard brief --text${arg} 2>/dev/null; }`;
|
|
35
|
+
return [...candidates.map((c) => `${c} brief --text${arg} 2>/dev/null`), onPath, 'true'].join(' || ');
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
function agentsBlock(entries) {
|
|
39
|
+
const lines = entries.map((dir) => (dir === '.'
|
|
40
|
+
? '- This project: claims in `testguard.claims.json`; run `testguard status --json` and follow `next`.'
|
|
41
|
+
: `- \`${dir}/\`: claims in \`${dir}/testguard.claims.json\`; run \`testguard status --json ${dir}\` and follow \`next\`.`));
|
|
42
|
+
return `${AGENTS_BEGIN}
|
|
10
43
|
## TestGuard
|
|
11
44
|
|
|
12
|
-
This
|
|
13
|
-
Before writing or changing tests, run \`testguard status --json\` and follow \`next\`.
|
|
45
|
+
This repository's tests are verified by [TestGuard](https://github.com/raccioly/testguard).
|
|
46
|
+
Before writing or changing tests, run \`testguard status --json [dir]\` and follow \`next\`.
|
|
14
47
|
The full operating loop and the verdict table are in \`.claude/skills/testguard/SKILL.md\`.
|
|
15
48
|
Never make a fault die by editing \`testguard.claims.json\`; write the test. Claim edits are recorded in the evidence.
|
|
49
|
+
|
|
50
|
+
${lines.join('\n')}
|
|
16
51
|
${AGENTS_END}`;
|
|
17
|
-
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
/** The git root of `dir`, or null outside a repository. */
|
|
55
|
+
export function gitRoot(dir) {
|
|
56
|
+
const r = spawnSync('git', ['rev-parse', '--show-toplevel'], { cwd: dir, encoding: 'utf8' });
|
|
57
|
+
return r.status === 0 ? r.stdout.trim() : null;
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
/** Is `file` (absolute) ignored by git in `root`? False outside a repository. */
|
|
61
|
+
export function isGitIgnored(root, file) {
|
|
62
|
+
if (!root) return false;
|
|
63
|
+
return spawnSync('git', ['check-ignore', '-q', '--', file], { cwd: root }).status === 0;
|
|
64
|
+
}
|
|
18
65
|
|
|
19
66
|
/**
|
|
20
|
-
* Install the
|
|
21
|
-
*
|
|
22
|
-
*
|
|
67
|
+
* Install the operating layer.
|
|
68
|
+
*
|
|
69
|
+
* Two layers, two places. The AGENT layer — the skill, the session-start
|
|
70
|
+
* hook, the AGENTS.md section — goes to the git root, because that is where
|
|
71
|
+
* an agent session runs; installed in a subdirectory it never fires and is
|
|
72
|
+
* never read (a field report installed it in a backend/ folder and got
|
|
73
|
+
* nothing). The PROJECT layer — the `.gitignore` lines beside the claims
|
|
74
|
+
* file — stays in the project directory. `here: true` keeps everything in
|
|
75
|
+
* the project directory for a subdirectory that is its own agent root.
|
|
76
|
+
*
|
|
77
|
+
* Idempotent per project: a second project in the same repository adds a
|
|
78
|
+
* hook line and an AGENTS.md bullet, never duplicates. Every written file is
|
|
79
|
+
* checked against .gitignore; an ignored file is reported as ignored, never
|
|
80
|
+
* as something to commit.
|
|
23
81
|
*/
|
|
24
|
-
|
|
82
|
+
const CI_EVIDENCE_HELPERS = {
|
|
83
|
+
github: `#!/bin/sh
|
|
84
|
+
# Fetch the evidence CI wrote on a branch, then brief from it. Run it when you
|
|
85
|
+
# want it: the session-start hook never touches the network, and this script
|
|
86
|
+
# exits 0 with a message whenever anything is missing.
|
|
87
|
+
BRANCH="\${1:-main}"
|
|
88
|
+
OUT=.testguard/ci
|
|
89
|
+
command -v gh >/dev/null 2>&1 || { echo "gh is not installed; see README > Read CI's evidence locally" >&2; exit 0; }
|
|
90
|
+
command -v testguard >/dev/null 2>&1 || { echo "testguard is not on PATH; install it or run npx testguard-cli brief --evidence <file>" >&2; exit 0; }
|
|
91
|
+
mkdir -p "$OUT" || exit 0
|
|
92
|
+
gh run download --name "testguard-evidence-$BRANCH" --dir "$OUT" >/dev/null 2>&1 || { echo "no testguard-evidence-$BRANCH artifact to download" >&2; exit 0; }
|
|
93
|
+
exec testguard brief . --text --evidence "$OUT/ci-self-evidence.json"
|
|
94
|
+
`,
|
|
95
|
+
gitlab: `#!/bin/sh
|
|
96
|
+
# Fetch the evidence CI wrote on a branch, then brief from it. Run it when you
|
|
97
|
+
# want it: the session-start hook never touches the network, and this script
|
|
98
|
+
# exits 0 with a message whenever anything is missing.
|
|
99
|
+
BRANCH="\${1:-main}"
|
|
100
|
+
OUT=.testguard/ci
|
|
101
|
+
command -v glab >/dev/null 2>&1 || { echo "glab is not installed; see README > Read CI's evidence locally" >&2; exit 0; }
|
|
102
|
+
command -v testguard >/dev/null 2>&1 || { echo "testguard is not on PATH; install it or run npx testguard-cli brief --evidence <file>" >&2; exit 0; }
|
|
103
|
+
mkdir -p "$OUT" || exit 0
|
|
104
|
+
glab ci artifact "$BRANCH" testguard:probe --path "$OUT/" >/dev/null 2>&1 || { echo "no testguard:probe artifact to download for $BRANCH" >&2; exit 0; }
|
|
105
|
+
exec testguard brief . --text --evidence "$OUT/.testguard/evidence.json"
|
|
106
|
+
`,
|
|
107
|
+
};
|
|
108
|
+
|
|
109
|
+
export function initProject({ projectDir, force = false, here = false, ciEvidence, mcp = false }) {
|
|
110
|
+
projectDir = realpathSync(resolve(projectDir));
|
|
111
|
+
const root = here ? null : gitRoot(projectDir);
|
|
112
|
+
const agentRoot = root ?? projectDir;
|
|
113
|
+
const ignoreRoot = root ?? gitRoot(projectDir);
|
|
114
|
+
const dir = (relative(agentRoot, projectDir).split(sep).join('/')) || '.';
|
|
25
115
|
const done = [];
|
|
116
|
+
const notes = [];
|
|
117
|
+
let mcpConfig;
|
|
26
118
|
const skipped = [];
|
|
119
|
+
const warnings = [];
|
|
120
|
+
const rel = (p) => relative(agentRoot, p).split(sep).join('/');
|
|
121
|
+
const note = (p) => { if (isGitIgnored(ignoreRoot, p)) warnings.push(`${rel(p)} is ignored by .gitignore — it was written, but git will not track it and agents in a fresh clone will never see it; unignore it or move it`); };
|
|
27
122
|
|
|
28
|
-
const skillDir = join(
|
|
123
|
+
const skillDir = join(agentRoot, '.claude', 'skills', 'testguard');
|
|
29
124
|
const skillPath = join(skillDir, 'SKILL.md');
|
|
30
125
|
if (!existsSync(skillPath) || force) {
|
|
126
|
+
const replaced = force && existsSync(skillPath);
|
|
31
127
|
mkdirSync(skillDir, { recursive: true });
|
|
32
128
|
writeFileSync(skillPath, readFileSync(join(TEMPLATES, 'SKILL.md'), 'utf8'));
|
|
33
|
-
done.push(
|
|
129
|
+
done.push(`${rel(skillPath)}${replaced ? ' (replaced)' : ''}`);
|
|
130
|
+
note(skillPath);
|
|
34
131
|
} else {
|
|
35
|
-
skipped.push(
|
|
132
|
+
skipped.push(`${rel(skillPath)} exists (use --force to replace)`);
|
|
36
133
|
}
|
|
37
134
|
|
|
38
|
-
const settingsPath = join(
|
|
135
|
+
const settingsPath = join(agentRoot, '.claude', 'settings.json');
|
|
39
136
|
let settings = {};
|
|
40
137
|
if (existsSync(settingsPath)) {
|
|
41
138
|
try {
|
|
42
139
|
settings = JSON.parse(readFileSync(settingsPath, 'utf8'));
|
|
43
140
|
} catch (e) {
|
|
44
|
-
throw new Error(
|
|
141
|
+
throw new Error(`${rel(settingsPath)} is not valid JSON (${e.message}); fix it before init can add the hook`);
|
|
45
142
|
}
|
|
46
143
|
}
|
|
47
144
|
settings.hooks ??= {};
|
|
48
145
|
settings.hooks.SessionStart ??= [];
|
|
49
|
-
const
|
|
50
|
-
|
|
51
|
-
|
|
146
|
+
const cmd = hookCommand(dir);
|
|
147
|
+
const hasCmd = settings.hooks.SessionStart.some((e) => JSON.stringify(e).includes(cmd));
|
|
148
|
+
// Any earlier hook that reached for npx — `-y` fetched outright, and
|
|
149
|
+
// `--no-install` still resolves the packument from the registry — is replaced.
|
|
150
|
+
// Which project a hook is for is the token right after `brief --text`, if
|
|
151
|
+
// that token is an argument rather than a redirect or a shell operator.
|
|
152
|
+
// Testing for any non-space there mistook `brief --text 2>/dev/null` for a
|
|
153
|
+
// hook belonging to a directory, so a legacy root hook was never replaced.
|
|
154
|
+
const hookDir = (text) => {
|
|
155
|
+
const m = /brief --text(?:\s+([^\s"|;&]+))?/.exec(text);
|
|
156
|
+
const token = m?.[1];
|
|
157
|
+
return !token || token.startsWith('2>') || token.startsWith('>') ? '.' : token;
|
|
158
|
+
};
|
|
159
|
+
const legacy = settings.hooks.SessionStart.findIndex((e) => {
|
|
160
|
+
const text = JSON.stringify(e);
|
|
161
|
+
return /npx\s+(-y|--no-install)[^"]*brief --text/.test(text) && hookDir(text) === dir;
|
|
162
|
+
});
|
|
163
|
+
if (hasCmd) {
|
|
164
|
+
skipped.push(`SessionStart hook for ${dir === '.' ? 'this project' : dir} already present in ${rel(settingsPath)}`);
|
|
52
165
|
} else {
|
|
53
|
-
|
|
166
|
+
if (legacy !== -1) {
|
|
167
|
+
settings.hooks.SessionStart.splice(legacy, 1);
|
|
168
|
+
done.push(`${rel(settingsPath)}: replaced the network-reaching npx hook with a local-then-PATH one`);
|
|
169
|
+
}
|
|
170
|
+
settings.hooks.SessionStart.push({ hooks: [{ type: 'command', command: cmd }] });
|
|
54
171
|
mkdirSync(dirname(settingsPath), { recursive: true });
|
|
55
172
|
writeFileSync(settingsPath, JSON.stringify(settings, null, 2) + '\n');
|
|
56
|
-
done.push(
|
|
173
|
+
done.push(`${rel(settingsPath)}: SessionStart hook → brief --text${dir === '.' ? '' : ` ${dir}`} (local install first, npx --no-install fallback, never a fetch)`);
|
|
174
|
+
note(settingsPath);
|
|
57
175
|
}
|
|
58
176
|
|
|
59
|
-
const agentsPath = join(
|
|
177
|
+
const agentsPath = join(agentRoot, 'AGENTS.md');
|
|
60
178
|
const agents = existsSync(agentsPath) ? readFileSync(agentsPath, 'utf8') : '';
|
|
61
|
-
|
|
62
|
-
|
|
179
|
+
const begin = agents.indexOf(AGENTS_BEGIN);
|
|
180
|
+
const end = agents.indexOf(AGENTS_END);
|
|
181
|
+
if (begin !== -1 && end !== -1) {
|
|
182
|
+
const block = agents.slice(begin, end + AGENTS_END.length);
|
|
183
|
+
const entries = [...block.matchAll(/^- (?:This project|`([^`]+)\/`)/gm)].map((m) => m[1] ?? '.');
|
|
184
|
+
const legacyBlock = entries.length === 0; // a pre-0.6 section with no project list
|
|
185
|
+
if (entries.includes(dir)) {
|
|
186
|
+
skipped.push(`AGENTS.md already has the TestGuard section for ${dir === '.' ? 'this project' : dir}`);
|
|
187
|
+
} else {
|
|
188
|
+
writeFileSync(agentsPath, agents.slice(0, begin) + agentsBlock([...entries, dir]) + agents.slice(end + AGENTS_END.length));
|
|
189
|
+
done.push(legacyBlock ? `AGENTS.md: TestGuard section rewritten to list ${dir}` : `AGENTS.md: TestGuard section now lists ${dir}`);
|
|
190
|
+
note(agentsPath);
|
|
191
|
+
}
|
|
63
192
|
} else {
|
|
64
|
-
writeFileSync(agentsPath, (agents ? agents.replace(/\s*$/, '\n\n') : '# Agent instructions\n\n') +
|
|
193
|
+
writeFileSync(agentsPath, (agents ? agents.replace(/\s*$/, '\n\n') : '# Agent instructions\n\n') + agentsBlock([dir]) + '\n');
|
|
65
194
|
done.push(agents ? 'AGENTS.md: TestGuard section appended' : 'AGENTS.md created with the TestGuard section');
|
|
195
|
+
note(agentsPath);
|
|
66
196
|
}
|
|
67
197
|
|
|
68
198
|
const giPath = join(projectDir, '.gitignore');
|
|
@@ -71,12 +201,35 @@ export function initProject({ projectDir, force = false }) {
|
|
|
71
201
|
const dirIgnored = entries.some((x) => /^\/?\.testguard\/?$/.test(x));
|
|
72
202
|
const missing = dirIgnored ? [] : GITIGNORE_LINES.filter((l) => !entries.includes(l));
|
|
73
203
|
if (dirIgnored) {
|
|
74
|
-
skipped.push(
|
|
204
|
+
skipped.push(`${rel(giPath)} ignores .testguard/ entirely — note that baseline.json should be committed; ignore only the regenerated files if you adopt a baseline`);
|
|
75
205
|
} else if (missing.length) {
|
|
76
206
|
writeFileSync(giPath, (gi ? gi.replace(/\s*$/, '\n') : '') + '# TestGuard: regenerated per run (baseline.json IS committed)\n' + missing.join('\n') + '\n');
|
|
77
|
-
done.push(
|
|
207
|
+
done.push(`${rel(giPath)}: ${missing.length} line${missing.length === 1 ? '' : 's'} added`);
|
|
78
208
|
} else {
|
|
79
|
-
skipped.push(
|
|
209
|
+
skipped.push(`${rel(giPath)} already ignores the regenerated files`);
|
|
210
|
+
}
|
|
211
|
+
if (ciEvidence) {
|
|
212
|
+
const helperPath = join(projectDir, '.testguard', 'fetch-ci-evidence.sh');
|
|
213
|
+
const body = CI_EVIDENCE_HELPERS[ciEvidence];
|
|
214
|
+
if (!body) throw new Error(`--ci-evidence must be github or gitlab, not ${ciEvidence}`);
|
|
215
|
+
if (!existsSync(helperPath) || force) {
|
|
216
|
+
mkdirSync(dirname(helperPath), { recursive: true });
|
|
217
|
+
writeFileSync(helperPath, body, { mode: 0o755 });
|
|
218
|
+
done.push(`.testguard/fetch-ci-evidence.sh (${ciEvidence}) — run it on demand; the session-start hook stays offline`);
|
|
219
|
+
} else {
|
|
220
|
+
skipped.push('.testguard/fetch-ci-evidence.sh exists (use --force to replace)');
|
|
221
|
+
}
|
|
222
|
+
}
|
|
223
|
+
if (mcp) {
|
|
224
|
+
// Printed, never written: a harness's own config is the person's file
|
|
225
|
+
// (and often global), so init shows the snippet and lets them place it.
|
|
226
|
+
const cmd = { command: 'npx', args: ['-y', 'testguard-cli', 'mcp'] };
|
|
227
|
+
mcpConfig = {
|
|
228
|
+
'Claude Code — .mcp.json in the project root, or `claude mcp add`': { mcpServers: { testguard: cmd } },
|
|
229
|
+
'Cursor — .cursor/mcp.json': { mcpServers: { testguard: cmd } },
|
|
230
|
+
'Codex CLI — ~/.codex/config.toml': '[mcp_servers.testguard]\ncommand = "npx"\nargs = ["-y", "testguard-cli", "mcp"]',
|
|
231
|
+
};
|
|
232
|
+
notes.push('MCP: five read-only tools (status, brief, claims, evidence, next_command). Nothing there runs a probe.');
|
|
80
233
|
}
|
|
81
|
-
return { done, skipped };
|
|
234
|
+
return { done, skipped, warnings, agentRoot, projectDir, dir, ...(mcpConfig ? { mcpConfig } : {}) };
|
|
82
235
|
}
|
|
@@ -45,7 +45,7 @@ code by construction. So, for every source file you create or change:
|
|
|
45
45
|
thin wrapper whose logic is claimed elsewhere): a `path` entry in
|
|
46
46
|
`testguard.ignore.json` with a reason and, where possible, an `expires`.
|
|
47
47
|
The gate prints every entry it relied on; a reviewer reads them.
|
|
48
|
-
4. Then `testguard probe --claim <ID> --include-dirty` for the
|
|
48
|
+
4. Then `testguard admit <test-file> --claim <ID>` for the new claim's test (or `testguard probe --claim <ID> --include-dirty` for the full report).
|
|
49
49
|
|
|
50
50
|
## Verdict → action
|
|
51
51
|
|
|
@@ -63,8 +63,8 @@ code by construction. So, for every source file you create or change:
|
|
|
63
63
|
|
|
64
64
|
1. `testguard status --json` → take `next.target`.
|
|
65
65
|
2. Write the test. It must **fail when the fault is applied and pass on HEAD**. To check the first half by hand: apply `find` → `replace` in the target file, run the defender, restore.
|
|
66
|
-
3. `testguard
|
|
67
|
-
4. When it is `
|
|
66
|
+
3. `testguard admit <test-file> --claim <ID>` — the two gates as one verb, on your uncommitted test, without touching the tree: `ADMITTED` (exit 0) means the test is green on HEAD and fails on every fault of the claim, 3/3; `NOT ADMITTED` (exit 1) names the first blocking fault and what to do. `--fault <FID>` judges one fault; `--confirm 1` gives a fast **provisional** `ADMITTED?` that must be confirmed at 3 before you commit. (`testguard probe --claim <ID> --include-dirty` is the same run with the full report.)
|
|
67
|
+
4. When it is `ADMITTED` at `--confirm 3`, commit the test.
|
|
68
68
|
5. When everything is killed or baselined, `testguard baseline` if `next` says so, and commit `.testguard/baseline.json`.
|
|
69
69
|
|
|
70
70
|
## Rules that are not yours to bend
|
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
import { TOOLS } from './tools.mjs';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* A Model Context Protocol server over stdio, hand-rolled.
|
|
5
|
+
*
|
|
6
|
+
* Why not the official SDK: this tool has one exact-pinned runtime dependency
|
|
7
|
+
* and intends to keep it that way (AGENTS.md rule 8). The surface here is
|
|
8
|
+
* three methods and five read-only tools and will not grow, so the protocol
|
|
9
|
+
* is written out rather than imported. The version is pinned below and the
|
|
10
|
+
* conformance is covered by tests that speak the wire format.
|
|
11
|
+
*
|
|
12
|
+
* JSON-RPC 2.0 framing: one message per line (newline-delimited JSON), which
|
|
13
|
+
* is what stdio transports use. A notification (no `id`) is never answered.
|
|
14
|
+
*/
|
|
15
|
+
export const PROTOCOL_VERSION = '2025-06-18';
|
|
16
|
+
|
|
17
|
+
const ERROR = { parse: -32700, invalidRequest: -32600, methodNotFound: -32601, invalidParams: -32602, internal: -32603 };
|
|
18
|
+
|
|
19
|
+
const ok = (id, result) => ({ jsonrpc: '2.0', id, result });
|
|
20
|
+
const err = (id, code, message, data) => ({ jsonrpc: '2.0', id, error: { code, message, ...(data ? { data } : {}) } });
|
|
21
|
+
|
|
22
|
+
/** Everything an agent may ask for, with the version stamped onto each call. */
|
|
23
|
+
export function handle(message, { version = '0.0.0' } = {}) {
|
|
24
|
+
if (message === null || typeof message !== 'object' || Array.isArray(message)) return err(null, ERROR.invalidRequest, 'a request must be a JSON object');
|
|
25
|
+
const { id, method, params } = message;
|
|
26
|
+
const isNotification = id === undefined || id === null;
|
|
27
|
+
|
|
28
|
+
switch (method) {
|
|
29
|
+
case 'initialize':
|
|
30
|
+
return isNotification ? null : ok(id, {
|
|
31
|
+
protocolVersion: PROTOCOL_VERSION,
|
|
32
|
+
// Read-only: no prompts, no resources, no sampling. Saying so is part
|
|
33
|
+
// of the contract — nothing here can change a claims file.
|
|
34
|
+
capabilities: { tools: { listChanged: false } },
|
|
35
|
+
serverInfo: { name: 'testguard', version },
|
|
36
|
+
instructions: 'Read testguard_status first: it carries the one next action. Run the command it names in your own terminal — these tools never run a probe, because a probe is long-running, budgeted, and the person should see it happen.',
|
|
37
|
+
});
|
|
38
|
+
|
|
39
|
+
case 'notifications/initialized':
|
|
40
|
+
case 'notifications/cancelled':
|
|
41
|
+
return null;
|
|
42
|
+
|
|
43
|
+
case 'ping':
|
|
44
|
+
return isNotification ? null : ok(id, {});
|
|
45
|
+
|
|
46
|
+
case 'tools/list':
|
|
47
|
+
return isNotification ? null : ok(id, { tools: TOOLS.map(({ name, description, inputSchema }) => ({ name, description, inputSchema, annotations: { readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: false } })) });
|
|
48
|
+
|
|
49
|
+
case 'tools/call': {
|
|
50
|
+
if (isNotification) return null;
|
|
51
|
+
const tool = TOOLS.find((t) => t.name === params?.name);
|
|
52
|
+
if (!tool) return err(id, ERROR.invalidParams, `unknown tool: ${params?.name}`);
|
|
53
|
+
try {
|
|
54
|
+
const result = tool.handler({ ...(params?.arguments ?? {}), __version: version });
|
|
55
|
+
return ok(id, { content: [{ type: 'text', text: JSON.stringify(result, null, 2) }], structuredContent: result, isError: false });
|
|
56
|
+
} catch (e) {
|
|
57
|
+
// A tool failure is a tool result, not a protocol error: the agent
|
|
58
|
+
// should read the reason and act, not lose the connection.
|
|
59
|
+
return ok(id, { content: [{ type: 'text', text: `${tool.name} failed: ${e.message}` }], isError: true });
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
default:
|
|
64
|
+
return isNotification ? null : err(id, ERROR.methodNotFound, `unknown method: ${method}`);
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
/**
|
|
69
|
+
* Speak the protocol on a pair of streams. Resolves when input ends, so the
|
|
70
|
+
* caller can await a clean shutdown.
|
|
71
|
+
*/
|
|
72
|
+
export function serve({ input = process.stdin, output = process.stdout, version = '0.0.0' } = {}) {
|
|
73
|
+
return new Promise((resolveDone) => {
|
|
74
|
+
let buffer = '';
|
|
75
|
+
const write = (msg) => {
|
|
76
|
+
if (msg) output.write(JSON.stringify(msg) + '\n');
|
|
77
|
+
};
|
|
78
|
+
input.setEncoding?.('utf8');
|
|
79
|
+
input.on('data', (chunk) => {
|
|
80
|
+
buffer += chunk;
|
|
81
|
+
let nl;
|
|
82
|
+
while ((nl = buffer.indexOf('\n')) !== -1) {
|
|
83
|
+
const line = buffer.slice(0, nl).trim();
|
|
84
|
+
buffer = buffer.slice(nl + 1);
|
|
85
|
+
if (!line) continue;
|
|
86
|
+
let message;
|
|
87
|
+
try {
|
|
88
|
+
message = JSON.parse(line);
|
|
89
|
+
} catch {
|
|
90
|
+
write(err(null, ERROR.parse, 'invalid JSON'));
|
|
91
|
+
continue;
|
|
92
|
+
}
|
|
93
|
+
try {
|
|
94
|
+
write(handle(message, { version }));
|
|
95
|
+
} catch (e) {
|
|
96
|
+
// The server outlives any single bad request.
|
|
97
|
+
write(err(message?.id ?? null, ERROR.internal, e.message));
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
});
|
|
101
|
+
input.on('end', () => resolveDone(0));
|
|
102
|
+
input.on('close', () => resolveDone(0));
|
|
103
|
+
});
|
|
104
|
+
}
|
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
import { existsSync } from 'node:fs';
|
|
2
|
+
import { join, resolve } from 'node:path';
|
|
3
|
+
import { computeStatus } from '../status/status.mjs';
|
|
4
|
+
import { buildBrief, buildUnclaimedBrief } from '../brief/brief.mjs';
|
|
5
|
+
import { loadClaims, defaultClaimsPath } from '../claims/load.mjs';
|
|
6
|
+
import { scanAnnotations, reconcile } from '../claims/annotations.mjs';
|
|
7
|
+
import { computeRemovedClaims } from '../claims/removed.mjs';
|
|
8
|
+
import { readSpecDoc } from '../evidence/writer.mjs';
|
|
9
|
+
import { resolveChangedRef, withChangedRef } from '../gate/changed.mjs';
|
|
10
|
+
|
|
11
|
+
/**
|
|
12
|
+
* The read-only surface an agent needs, in any harness.
|
|
13
|
+
*
|
|
14
|
+
* TestGuard's operating loop currently lives in a Claude Code skill and a
|
|
15
|
+
* session-start hook, and both vanish the moment the harness is something
|
|
16
|
+
* else. These tools expose the documents the tool already computes, so the
|
|
17
|
+
* loop survives that change.
|
|
18
|
+
*
|
|
19
|
+
* Every tool is read-only and none of them probes. A probe is long-running
|
|
20
|
+
* and budgeted, and the human should see it happen: `next_command` hands back
|
|
21
|
+
* the exact shell line for the agent to run in its own terminal instead.
|
|
22
|
+
*/
|
|
23
|
+
const projectOf = (args) => resolve(args?.dir ?? '.');
|
|
24
|
+
const evidenceOf = (args) => (args?.evidence ? resolve(args.evidence) : undefined);
|
|
25
|
+
|
|
26
|
+
const DIR = { type: 'string', description: 'Project directory (where testguard.claims.json lives). Defaults to the current directory.' };
|
|
27
|
+
const EVIDENCE = { type: 'string', description: "Read this evidence document instead of the project's own — CI's, fetched as an artifact." };
|
|
28
|
+
const CHANGED = { type: 'string', description: 'A git reference to measure the change against, so claim coverage of the change is included.' };
|
|
29
|
+
|
|
30
|
+
const statusFor = (args) => {
|
|
31
|
+
const projectDir = projectOf(args);
|
|
32
|
+
return withChangedRef(resolveChangedRef({ explicit: args?.changed }), (changedRef) =>
|
|
33
|
+
computeStatus({ projectDir, toolVersion: args?.__version ?? '0.0.0', changedRef, evidence: evidenceOf(args) }));
|
|
34
|
+
};
|
|
35
|
+
|
|
36
|
+
export const TOOLS = [
|
|
37
|
+
{
|
|
38
|
+
name: 'testguard_status',
|
|
39
|
+
description: 'Where this project stands and the ONE next action, as the machine-readable status document: state, next {action, command, why}, counts, stale inputs, faults edited since they were probed, and the ranked findings. This is the single source of truth every other rendering derives from — read it before deciding anything.',
|
|
40
|
+
inputSchema: { type: 'object', properties: { dir: DIR, changed: CHANGED, evidence: EVIDENCE }, additionalProperties: false },
|
|
41
|
+
handler: (args) => statusFor(args),
|
|
42
|
+
},
|
|
43
|
+
{
|
|
44
|
+
name: 'testguard_brief',
|
|
45
|
+
description: 'Where the test suite is blind, ranked and capped, with one actionable hint per finding and unclaimed changed files first. Read this before writing or changing code, which is what the session-start hook does for harnesses that have one.',
|
|
46
|
+
inputSchema: { type: 'object', properties: { dir: DIR, changed: CHANGED, evidence: EVIDENCE, max: { type: 'integer', minimum: 1, maximum: 50 } }, additionalProperties: false },
|
|
47
|
+
handler: (args) => {
|
|
48
|
+
const projectDir = projectOf(args);
|
|
49
|
+
const evPath = evidenceOf(args) ?? join(projectDir, '.testguard', 'evidence.json');
|
|
50
|
+
const status = (() => {
|
|
51
|
+
try {
|
|
52
|
+
return statusFor(args);
|
|
53
|
+
} catch {
|
|
54
|
+
return undefined; // a brief must never fail because status could not be computed
|
|
55
|
+
}
|
|
56
|
+
})();
|
|
57
|
+
if (!existsSync(evPath)) {
|
|
58
|
+
if (status?.changes?.uncovered?.length) {
|
|
59
|
+
return buildUnclaimedBrief({ tool: { name: 'testguard', version: args?.__version ?? '0.0.0' }, next: status.next, changes: status.changes });
|
|
60
|
+
}
|
|
61
|
+
return { note: `no evidence at ${evPath}: nothing has been probed yet`, next: status?.next };
|
|
62
|
+
}
|
|
63
|
+
const evidence = readSpecDoc('evidence', evPath);
|
|
64
|
+
const basePath = join(projectDir, '.testguard', 'baseline.json');
|
|
65
|
+
const baseline = existsSync(basePath) ? readSpecDoc('baseline', basePath) : undefined;
|
|
66
|
+
return buildBrief(evidence, baseline, { max: args?.max ?? 20, next: status?.next, changes: status?.changes });
|
|
67
|
+
},
|
|
68
|
+
},
|
|
69
|
+
{
|
|
70
|
+
name: 'testguard_claims',
|
|
71
|
+
description: "What this project claims must be true, each with its faults and defenders, plus drift against `@claim` annotations in source and — when `since` is given — every claim or fault that existed at that reference and does not now. A claim that disappeared is invisible to every other check.",
|
|
72
|
+
inputSchema: { type: 'object', properties: { dir: DIR, since: { type: 'string', description: 'A git reference: report claims and faults removed since it.' } }, additionalProperties: false },
|
|
73
|
+
handler: (args) => {
|
|
74
|
+
const projectDir = projectOf(args);
|
|
75
|
+
const path = defaultClaimsPath(projectDir);
|
|
76
|
+
if (!existsSync(path)) return { path, claims: [], note: 'no claims file in this project yet' };
|
|
77
|
+
const claims = loadClaims(path);
|
|
78
|
+
const drift = reconcile(claims, scanAnnotations(projectDir));
|
|
79
|
+
const removed = args?.since
|
|
80
|
+
? computeRemovedClaims({ projectDir, ref: args.since, claimsPath: path, current: claims, evidencePath: join(projectDir, '.testguard', 'evidence.json'), toolVersion: args?.__version ?? '0.0.0' })
|
|
81
|
+
: undefined;
|
|
82
|
+
return { path, claims, drift, ...(removed ? { removed } : {}) };
|
|
83
|
+
},
|
|
84
|
+
},
|
|
85
|
+
{
|
|
86
|
+
name: 'testguard_evidence',
|
|
87
|
+
description: 'One run\'s findings on disk: every verdict with the runs that produced it. Pass `claim` (and optionally `fault`) for a single record instead of the whole document, which can be large.',
|
|
88
|
+
inputSchema: {
|
|
89
|
+
type: 'object',
|
|
90
|
+
properties: { dir: DIR, evidence: EVIDENCE, claim: { type: 'string', description: 'Return only this claim\'s records.' }, fault: { type: 'string', description: 'With `claim`: only this fault\'s record.' } },
|
|
91
|
+
additionalProperties: false,
|
|
92
|
+
},
|
|
93
|
+
handler: (args) => {
|
|
94
|
+
const projectDir = projectOf(args);
|
|
95
|
+
const evPath = evidenceOf(args) ?? join(projectDir, '.testguard', 'evidence.json');
|
|
96
|
+
if (!existsSync(evPath)) return { note: `no evidence at ${evPath}: run a probe first`, path: evPath };
|
|
97
|
+
const doc = readSpecDoc('evidence', evPath);
|
|
98
|
+
if (!args?.claim) return doc;
|
|
99
|
+
const records = doc.records.filter((r) => r.claim.id === args.claim && (!args.fault || r.subject.id === args.fault));
|
|
100
|
+
return { ...doc, records, filtered: { claim: args.claim, ...(args.fault ? { fault: args.fault } : {}) } };
|
|
101
|
+
},
|
|
102
|
+
},
|
|
103
|
+
{
|
|
104
|
+
name: 'testguard_next_command',
|
|
105
|
+
description: 'The exact shell command to run next, with the reason. Run it yourself in a terminal: a probe is long-running and budgeted, and the person should see it happen. This tool never runs anything.',
|
|
106
|
+
inputSchema: { type: 'object', properties: { dir: DIR, changed: CHANGED, evidence: EVIDENCE }, additionalProperties: false },
|
|
107
|
+
handler: (args) => {
|
|
108
|
+
const doc = statusFor(args);
|
|
109
|
+
return { state: doc.state, action: doc.next.action, command: doc.next.command, why: doc.next.why, ...(doc.next.target ? { target: doc.next.target } : {}), ...(doc.next.file ? { file: doc.next.file } : {}) };
|
|
110
|
+
},
|
|
111
|
+
},
|
|
112
|
+
];
|
|
113
|
+
|
|
114
|
+
export const TOOL_NAMES = TOOLS.map((t) => t.name);
|
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The decisions escalation and flake measurement make, as pure functions.
|
|
3
|
+
*
|
|
4
|
+
* These used to live inline in `probeOne`, which meant the only thing that
|
|
5
|
+
* could falsify them was a full fixture probe — a 50-second acceptance test
|
|
6
|
+
* standing in for a rule you can state in three lines. `classify()` has always
|
|
7
|
+
* been pure for this reason: the order of its checks *is* the spec, and every
|
|
8
|
+
* branch is unit-testable without a runner. Same principle here.
|
|
9
|
+
*
|
|
10
|
+
* Nothing in this file runs a test, touches the filesystem, or knows what a
|
|
11
|
+
* worktree is. Everything it needs is passed in.
|
|
12
|
+
*/
|
|
13
|
+
|
|
14
|
+
const isKill = (r) => r.outcome === 'fail' && (r.assertionFailures ?? 0) > 0;
|
|
15
|
+
|
|
16
|
+
/**
|
|
17
|
+
* Fold one escalation run into the running attribution state.
|
|
18
|
+
*
|
|
19
|
+
* A test is an undeclared killer only if it fails in **every** run: a single
|
|
20
|
+
* run cannot say, because a test that is flaky elsewhere in the suite would
|
|
21
|
+
* take the credit. So the killer set is an intersection, seeded by the first
|
|
22
|
+
* run rather than by the empty set.
|
|
23
|
+
*
|
|
24
|
+
* Once the intersection is empty no further run can name a killer, so the
|
|
25
|
+
* caller may stop early. Stopping can only fail to *name* a killer, never
|
|
26
|
+
* upgrade a verdict, which is the pessimistic side and therefore the safe one.
|
|
27
|
+
*/
|
|
28
|
+
export function foldEscalationRun(state, { run, failedTests }) {
|
|
29
|
+
const runs = [...state.runs, run];
|
|
30
|
+
const killers = state.killers === null ? new Set(failedTests) : new Set([...state.killers].filter((t) => failedTests.includes(t)));
|
|
31
|
+
return { runs, killers, stoppedEarly: killers.size === 0 ? 'no-common-failure' : undefined };
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
/** The initial attribution state, before any escalation run. */
|
|
35
|
+
export const escalationStart = () => ({ runs: [], killers: null, stoppedEarly: undefined });
|
|
36
|
+
|
|
37
|
+
/**
|
|
38
|
+
* What escalation concluded. `undeclaredKillers` is named only when a test
|
|
39
|
+
* failed in all N runs AND every run was a genuine kill (an assertion
|
|
40
|
+
* failure). A suite that errored or timed out proves nothing about the fault,
|
|
41
|
+
* so it never earns an attribution.
|
|
42
|
+
*/
|
|
43
|
+
export function escalationResult(state, confirmRuns) {
|
|
44
|
+
const complete = state.runs.length === confirmRuns;
|
|
45
|
+
const attributable = state.killers !== null && state.killers.size > 0 && complete && state.runs.every(isKill);
|
|
46
|
+
return {
|
|
47
|
+
escalated: true,
|
|
48
|
+
escalationRuns: state.runs,
|
|
49
|
+
...(state.stoppedEarly ? { escalationStoppedEarly: state.stoppedEarly } : {}),
|
|
50
|
+
...(attributable ? { reason: 'killed-by-undeclared-tests', undeclaredKillers: [...state.killers].sort() } : {}),
|
|
51
|
+
};
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
/**
|
|
55
|
+
* The defenders' flake rate on unmodified source.
|
|
56
|
+
*
|
|
57
|
+
* Only runs that **actually ran** carry information. A load error or a timeout
|
|
58
|
+
* measures the environment, not the suite's stability, and counting either as
|
|
59
|
+
* a "failure" would invent flakiness that was never observed. Returns
|
|
60
|
+
* `undefined` when nothing was measurable, so the caller omits the field
|
|
61
|
+
* rather than recording a rate of zero it cannot support.
|
|
62
|
+
*/
|
|
63
|
+
export function flakeRate(runs) {
|
|
64
|
+
const measured = runs.filter((r) => r.outcome === 'pass' || r.outcome === 'fail');
|
|
65
|
+
if (!measured.length) return undefined;
|
|
66
|
+
return { runs: measured.length, failures: measured.filter((r) => r.outcome === 'fail').length };
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
/**
|
|
70
|
+
* The tests that actually failed on the fault, by file. Only these are its
|
|
71
|
+
* killers; everything else in a run is noise for attribution purposes.
|
|
72
|
+
*/
|
|
73
|
+
export function killersFromRuns(probeRuns) {
|
|
74
|
+
return [...new Set(probeRuns.flatMap((r) => r.failedTests ?? []).map((t) => t.split('::')[0]).filter(Boolean))];
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
/**
|
|
78
|
+
* The `subject` of an evidence record. `contentHash` is what makes a fault edit
|
|
79
|
+
* visible: weakening a fault after it survived changes this hash, and `status`
|
|
80
|
+
* reports the change with the verdict the fault used to have. It is deliberately
|
|
81
|
+
* NOT part of the fingerprint — repairing a rotted anchor must not churn the
|
|
82
|
+
* baseline — which is exactly why it has to be recorded separately.
|
|
83
|
+
*/
|
|
84
|
+
export function subjectOf(fault, sha256) {
|
|
85
|
+
return {
|
|
86
|
+
kind: 'fault',
|
|
87
|
+
id: fault.id,
|
|
88
|
+
description: fault.description,
|
|
89
|
+
file: fault.file,
|
|
90
|
+
faultClass: fault.faultClass,
|
|
91
|
+
producedBy: fault.producedBy,
|
|
92
|
+
contentHash: sha256(`${fault.find}\n${fault.replace}`),
|
|
93
|
+
};
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
/**
|
|
97
|
+
* May a prior record stand in for this one, or must the fault be probed again?
|
|
98
|
+
*
|
|
99
|
+
* Same source, same defenders, same N — and the SAME FAULT. The last is not a
|
|
100
|
+
* refinement: a verdict is an answer about one specific `find`/`replace`, and
|
|
101
|
+
* once those change the old answer is not an answer to the new question.
|
|
102
|
+
* Without it, weakening a fault that survived keeps its old verdict, and
|
|
103
|
+
* repairing a rotted anchor keeps `unverifiable` — the evidence reporting a
|
|
104
|
+
* result that was never measured against the fault the file now contains.
|
|
105
|
+
* `status` flags the changed hash afterwards, but by then the evidence has
|
|
106
|
+
* already said something untrue.
|
|
107
|
+
*
|
|
108
|
+
* A prior that predates `contentHash` cannot be compared, so it is re-probed.
|
|
109
|
+
* The cost of re-measuring is a run; the cost of the other direction is a
|
|
110
|
+
* verdict nobody measured.
|
|
111
|
+
*/
|
|
112
|
+
export function isReusable(prior, current) {
|
|
113
|
+
if (!prior || !prior.subject?.contentHash) return false;
|
|
114
|
+
// A record that exists because the probe threw is a statement about the run,
|
|
115
|
+
// not about the code. Reusing it would make one disturbed run permanent.
|
|
116
|
+
if (prior.detail?.reason === 'probe-error') return false;
|
|
117
|
+
const same = (x, y) => JSON.stringify(x) === JSON.stringify(y);
|
|
118
|
+
return prior.inputs.targetHash === current.inputs.targetHash
|
|
119
|
+
&& same(prior.inputs.defenderHashes, current.inputs.defenderHashes)
|
|
120
|
+
&& same(prior.defenders.requested, current.requested)
|
|
121
|
+
&& same(prior.defenders.resolved, current.resolved)
|
|
122
|
+
&& prior.subject.contentHash === current.contentHash;
|
|
123
|
+
}
|