testguard-cli 0.4.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +291 -0
- package/README.md +321 -27
- package/package.json +1 -1
- package/spec/GATE-SEMANTICS.md +109 -1
- package/spec/README.md +4 -2
- package/spec/lib/validate.mjs +75 -1
- package/spec/schemas/baseline.schema.json +8 -0
- package/spec/schemas/brief.schema.json +11 -1
- package/spec/schemas/common.schema.json +32 -1
- package/spec/schemas/evidence.schema.json +374 -63
- package/spec/schemas/gate.schema.json +52 -0
- package/spec/schemas/replay.schema.json +85 -0
- package/spec/schemas/status.schema.json +269 -45
- package/src/admit/admit.mjs +17 -0
- package/src/baseline/baseline.mjs +31 -2
- package/src/brief/brief.mjs +96 -8
- package/src/claims/removed.mjs +161 -0
- package/src/cli.mjs +69 -12
- package/src/commands/admit.mjs +116 -0
- package/src/commands/baseline.mjs +17 -3
- package/src/commands/brief.mjs +26 -12
- package/src/commands/claims.mjs +35 -6
- package/src/commands/gate.mjs +47 -0
- package/src/commands/init.mjs +19 -6
- package/src/commands/mcp.mjs +12 -0
- package/src/commands/probe.mjs +12 -3
- package/src/commands/replay.mjs +63 -0
- package/src/commands/status.mjs +4 -1
- package/src/gate/changed.mjs +222 -0
- package/src/git.mjs +5 -0
- package/src/init/init.mjs +182 -29
- package/src/init/templates/SKILL.md +21 -3
- package/src/mcp/server.mjs +104 -0
- package/src/mcp/tools.mjs +114 -0
- package/src/probe/contention.mjs +60 -0
- package/src/probe/discover.mjs +12 -3
- package/src/probe/independence.mjs +59 -0
- package/src/probe/mocks.mjs +120 -0
- package/src/probe/probe.mjs +126 -18
- package/src/probe/rank.mjs +73 -15
- package/src/probe/runners/index.mjs +49 -5
- package/src/probe/runners/jest.mjs +3 -3
- package/src/probe/runners/playwright.mjs +133 -0
- package/src/probe/runners/shared.mjs +70 -13
- package/src/probe/runners/vitest.mjs +4 -3
- package/src/render.mjs +2 -2
- package/src/replay/label.mjs +64 -0
- package/src/replay/replay.mjs +309 -0
- package/src/scaffold/producers.mjs +217 -6
- package/src/scaffold/scaffold.mjs +9 -3
- package/src/status/status.mjs +88 -7
package/src/init/init.mjs
CHANGED
|
@@ -1,68 +1,198 @@
|
|
|
1
|
-
import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
|
|
2
|
-
import { join, dirname } from 'node:path';
|
|
1
|
+
import { existsSync, mkdirSync, readFileSync, writeFileSync, realpathSync } from 'node:fs';
|
|
2
|
+
import { join, dirname, relative, resolve, sep } from 'node:path';
|
|
3
3
|
import { fileURLToPath } from 'node:url';
|
|
4
|
+
import { spawnSync } from 'node:child_process';
|
|
4
5
|
|
|
5
6
|
const TEMPLATES = join(dirname(fileURLToPath(import.meta.url)), 'templates');
|
|
6
|
-
const HOOK_CMD = 'npx -y testguard-cli brief --text';
|
|
7
7
|
const AGENTS_BEGIN = '<!-- testguard:begin -->';
|
|
8
8
|
const AGENTS_END = '<!-- testguard:end -->';
|
|
9
|
-
const
|
|
9
|
+
const GITIGNORE_LINES = ['.testguard/evidence.json', '.testguard/evidence-provisional.json', '.testguard/evidence-partial.json', '.testguard/brief.json', '.testguard/gate.json', '.testguard/scaffold-*.json'];
|
|
10
|
+
|
|
11
|
+
/**
|
|
12
|
+
* The session-start hook command. It runs from the git root (where the agent
|
|
13
|
+
* session lives) and must never fetch from the network: a local install of
|
|
14
|
+
* the project (or of the root) is preferred, then a `testguard` already on
|
|
15
|
+
* PATH, then nothing — the hook exits 0 with no output rather than break a
|
|
16
|
+
* session.
|
|
17
|
+
*
|
|
18
|
+
* No form of `npx` appears here, and `--no-install` is not an exception.
|
|
19
|
+
* Measured: in a project with nothing installed,
|
|
20
|
+
* `npx --no-install --loglevel=http testguard-cli --version` logs
|
|
21
|
+
* `npm http fetch GET 200 https://registry.npmjs.org/testguard-cli`, and
|
|
22
|
+
* against an unreachable registry it exits non-zero. npm resolves the
|
|
23
|
+
* packument before it decides not to install, so `--no-install` is quiet, not
|
|
24
|
+
* offline. A `command -v` lookup covers the global-install case with no
|
|
25
|
+
* network at all, which is the whole point in a repository that pins and
|
|
26
|
+
* audits its dependencies.
|
|
27
|
+
*/
|
|
28
|
+
export function hookCommand(dir) {
|
|
29
|
+
const arg = dir === '.' ? '' : ` ${dir}`;
|
|
30
|
+
const candidates = dir === '.' ? ['node_modules/.bin/testguard'] : [`${dir}/node_modules/.bin/testguard`, 'node_modules/.bin/testguard'];
|
|
31
|
+
// The PATH branch is braced: `a || b && c` binds as `(a || b) && c` in sh,
|
|
32
|
+
// so an unbraced `&&` would run the brief a second time whenever the local
|
|
33
|
+
// binary succeeded.
|
|
34
|
+
const onPath = `{ command -v testguard >/dev/null 2>&1 && testguard brief --text${arg} 2>/dev/null; }`;
|
|
35
|
+
return [...candidates.map((c) => `${c} brief --text${arg} 2>/dev/null`), onPath, 'true'].join(' || ');
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
function agentsBlock(entries) {
|
|
39
|
+
const lines = entries.map((dir) => (dir === '.'
|
|
40
|
+
? '- This project: claims in `testguard.claims.json`; run `testguard status --json` and follow `next`.'
|
|
41
|
+
: `- \`${dir}/\`: claims in \`${dir}/testguard.claims.json\`; run \`testguard status --json ${dir}\` and follow \`next\`.`));
|
|
42
|
+
return `${AGENTS_BEGIN}
|
|
10
43
|
## TestGuard
|
|
11
44
|
|
|
12
|
-
This
|
|
13
|
-
Before writing or changing tests, run \`testguard status --json\` and follow \`next\`.
|
|
45
|
+
This repository's tests are verified by [TestGuard](https://github.com/raccioly/testguard).
|
|
46
|
+
Before writing or changing tests, run \`testguard status --json [dir]\` and follow \`next\`.
|
|
14
47
|
The full operating loop and the verdict table are in \`.claude/skills/testguard/SKILL.md\`.
|
|
15
48
|
Never make a fault die by editing \`testguard.claims.json\`; write the test. Claim edits are recorded in the evidence.
|
|
49
|
+
|
|
50
|
+
${lines.join('\n')}
|
|
16
51
|
${AGENTS_END}`;
|
|
17
|
-
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
/** The git root of `dir`, or null outside a repository. */
|
|
55
|
+
export function gitRoot(dir) {
|
|
56
|
+
const r = spawnSync('git', ['rev-parse', '--show-toplevel'], { cwd: dir, encoding: 'utf8' });
|
|
57
|
+
return r.status === 0 ? r.stdout.trim() : null;
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
/** Is `file` (absolute) ignored by git in `root`? False outside a repository. */
|
|
61
|
+
export function isGitIgnored(root, file) {
|
|
62
|
+
if (!root) return false;
|
|
63
|
+
return spawnSync('git', ['check-ignore', '-q', '--', file], { cwd: root }).status === 0;
|
|
64
|
+
}
|
|
18
65
|
|
|
19
66
|
/**
|
|
20
|
-
* Install the
|
|
21
|
-
*
|
|
22
|
-
*
|
|
67
|
+
* Install the operating layer.
|
|
68
|
+
*
|
|
69
|
+
* Two layers, two places. The AGENT layer — the skill, the session-start
|
|
70
|
+
* hook, the AGENTS.md section — goes to the git root, because that is where
|
|
71
|
+
* an agent session runs; installed in a subdirectory it never fires and is
|
|
72
|
+
* never read (a field report installed it in a backend/ folder and got
|
|
73
|
+
* nothing). The PROJECT layer — the `.gitignore` lines beside the claims
|
|
74
|
+
* file — stays in the project directory. `here: true` keeps everything in
|
|
75
|
+
* the project directory for a subdirectory that is its own agent root.
|
|
76
|
+
*
|
|
77
|
+
* Idempotent per project: a second project in the same repository adds a
|
|
78
|
+
* hook line and an AGENTS.md bullet, never duplicates. Every written file is
|
|
79
|
+
* checked against .gitignore; an ignored file is reported as ignored, never
|
|
80
|
+
* as something to commit.
|
|
23
81
|
*/
|
|
24
|
-
|
|
82
|
+
const CI_EVIDENCE_HELPERS = {
|
|
83
|
+
github: `#!/bin/sh
|
|
84
|
+
# Fetch the evidence CI wrote on a branch, then brief from it. Run it when you
|
|
85
|
+
# want it: the session-start hook never touches the network, and this script
|
|
86
|
+
# exits 0 with a message whenever anything is missing.
|
|
87
|
+
BRANCH="\${1:-main}"
|
|
88
|
+
OUT=.testguard/ci
|
|
89
|
+
command -v gh >/dev/null 2>&1 || { echo "gh is not installed; see README > Read CI's evidence locally" >&2; exit 0; }
|
|
90
|
+
command -v testguard >/dev/null 2>&1 || { echo "testguard is not on PATH; install it or run npx testguard-cli brief --evidence <file>" >&2; exit 0; }
|
|
91
|
+
mkdir -p "$OUT" || exit 0
|
|
92
|
+
gh run download --name "testguard-evidence-$BRANCH" --dir "$OUT" >/dev/null 2>&1 || { echo "no testguard-evidence-$BRANCH artifact to download" >&2; exit 0; }
|
|
93
|
+
exec testguard brief . --text --evidence "$OUT/ci-self-evidence.json"
|
|
94
|
+
`,
|
|
95
|
+
gitlab: `#!/bin/sh
|
|
96
|
+
# Fetch the evidence CI wrote on a branch, then brief from it. Run it when you
|
|
97
|
+
# want it: the session-start hook never touches the network, and this script
|
|
98
|
+
# exits 0 with a message whenever anything is missing.
|
|
99
|
+
BRANCH="\${1:-main}"
|
|
100
|
+
OUT=.testguard/ci
|
|
101
|
+
command -v glab >/dev/null 2>&1 || { echo "glab is not installed; see README > Read CI's evidence locally" >&2; exit 0; }
|
|
102
|
+
command -v testguard >/dev/null 2>&1 || { echo "testguard is not on PATH; install it or run npx testguard-cli brief --evidence <file>" >&2; exit 0; }
|
|
103
|
+
mkdir -p "$OUT" || exit 0
|
|
104
|
+
glab ci artifact "$BRANCH" testguard:probe --path "$OUT/" >/dev/null 2>&1 || { echo "no testguard:probe artifact to download for $BRANCH" >&2; exit 0; }
|
|
105
|
+
exec testguard brief . --text --evidence "$OUT/.testguard/evidence.json"
|
|
106
|
+
`,
|
|
107
|
+
};
|
|
108
|
+
|
|
109
|
+
export function initProject({ projectDir, force = false, here = false, ciEvidence, mcp = false }) {
|
|
110
|
+
projectDir = realpathSync(resolve(projectDir));
|
|
111
|
+
const root = here ? null : gitRoot(projectDir);
|
|
112
|
+
const agentRoot = root ?? projectDir;
|
|
113
|
+
const ignoreRoot = root ?? gitRoot(projectDir);
|
|
114
|
+
const dir = (relative(agentRoot, projectDir).split(sep).join('/')) || '.';
|
|
25
115
|
const done = [];
|
|
116
|
+
const notes = [];
|
|
117
|
+
let mcpConfig;
|
|
26
118
|
const skipped = [];
|
|
119
|
+
const warnings = [];
|
|
120
|
+
const rel = (p) => relative(agentRoot, p).split(sep).join('/');
|
|
121
|
+
const note = (p) => { if (isGitIgnored(ignoreRoot, p)) warnings.push(`${rel(p)} is ignored by .gitignore — it was written, but git will not track it and agents in a fresh clone will never see it; unignore it or move it`); };
|
|
27
122
|
|
|
28
|
-
const skillDir = join(
|
|
123
|
+
const skillDir = join(agentRoot, '.claude', 'skills', 'testguard');
|
|
29
124
|
const skillPath = join(skillDir, 'SKILL.md');
|
|
30
125
|
if (!existsSync(skillPath) || force) {
|
|
126
|
+
const replaced = force && existsSync(skillPath);
|
|
31
127
|
mkdirSync(skillDir, { recursive: true });
|
|
32
128
|
writeFileSync(skillPath, readFileSync(join(TEMPLATES, 'SKILL.md'), 'utf8'));
|
|
33
|
-
done.push(
|
|
129
|
+
done.push(`${rel(skillPath)}${replaced ? ' (replaced)' : ''}`);
|
|
130
|
+
note(skillPath);
|
|
34
131
|
} else {
|
|
35
|
-
skipped.push(
|
|
132
|
+
skipped.push(`${rel(skillPath)} exists (use --force to replace)`);
|
|
36
133
|
}
|
|
37
134
|
|
|
38
|
-
const settingsPath = join(
|
|
135
|
+
const settingsPath = join(agentRoot, '.claude', 'settings.json');
|
|
39
136
|
let settings = {};
|
|
40
137
|
if (existsSync(settingsPath)) {
|
|
41
138
|
try {
|
|
42
139
|
settings = JSON.parse(readFileSync(settingsPath, 'utf8'));
|
|
43
140
|
} catch (e) {
|
|
44
|
-
throw new Error(
|
|
141
|
+
throw new Error(`${rel(settingsPath)} is not valid JSON (${e.message}); fix it before init can add the hook`);
|
|
45
142
|
}
|
|
46
143
|
}
|
|
47
144
|
settings.hooks ??= {};
|
|
48
145
|
settings.hooks.SessionStart ??= [];
|
|
49
|
-
const
|
|
50
|
-
|
|
51
|
-
|
|
146
|
+
const cmd = hookCommand(dir);
|
|
147
|
+
const hasCmd = settings.hooks.SessionStart.some((e) => JSON.stringify(e).includes(cmd));
|
|
148
|
+
// Any earlier hook that reached for npx — `-y` fetched outright, and
|
|
149
|
+
// `--no-install` still resolves the packument from the registry — is replaced.
|
|
150
|
+
// Which project a hook is for is the token right after `brief --text`, if
|
|
151
|
+
// that token is an argument rather than a redirect or a shell operator.
|
|
152
|
+
// Testing for any non-space there mistook `brief --text 2>/dev/null` for a
|
|
153
|
+
// hook belonging to a directory, so a legacy root hook was never replaced.
|
|
154
|
+
const hookDir = (text) => {
|
|
155
|
+
const m = /brief --text(?:\s+([^\s"|;&]+))?/.exec(text);
|
|
156
|
+
const token = m?.[1];
|
|
157
|
+
return !token || token.startsWith('2>') || token.startsWith('>') ? '.' : token;
|
|
158
|
+
};
|
|
159
|
+
const legacy = settings.hooks.SessionStart.findIndex((e) => {
|
|
160
|
+
const text = JSON.stringify(e);
|
|
161
|
+
return /npx\s+(-y|--no-install)[^"]*brief --text/.test(text) && hookDir(text) === dir;
|
|
162
|
+
});
|
|
163
|
+
if (hasCmd) {
|
|
164
|
+
skipped.push(`SessionStart hook for ${dir === '.' ? 'this project' : dir} already present in ${rel(settingsPath)}`);
|
|
52
165
|
} else {
|
|
53
|
-
|
|
166
|
+
if (legacy !== -1) {
|
|
167
|
+
settings.hooks.SessionStart.splice(legacy, 1);
|
|
168
|
+
done.push(`${rel(settingsPath)}: replaced the network-reaching npx hook with a local-then-PATH one`);
|
|
169
|
+
}
|
|
170
|
+
settings.hooks.SessionStart.push({ hooks: [{ type: 'command', command: cmd }] });
|
|
54
171
|
mkdirSync(dirname(settingsPath), { recursive: true });
|
|
55
172
|
writeFileSync(settingsPath, JSON.stringify(settings, null, 2) + '\n');
|
|
56
|
-
done.push(
|
|
173
|
+
done.push(`${rel(settingsPath)}: SessionStart hook → brief --text${dir === '.' ? '' : ` ${dir}`} (local install first, npx --no-install fallback, never a fetch)`);
|
|
174
|
+
note(settingsPath);
|
|
57
175
|
}
|
|
58
176
|
|
|
59
|
-
const agentsPath = join(
|
|
177
|
+
const agentsPath = join(agentRoot, 'AGENTS.md');
|
|
60
178
|
const agents = existsSync(agentsPath) ? readFileSync(agentsPath, 'utf8') : '';
|
|
61
|
-
|
|
62
|
-
|
|
179
|
+
const begin = agents.indexOf(AGENTS_BEGIN);
|
|
180
|
+
const end = agents.indexOf(AGENTS_END);
|
|
181
|
+
if (begin !== -1 && end !== -1) {
|
|
182
|
+
const block = agents.slice(begin, end + AGENTS_END.length);
|
|
183
|
+
const entries = [...block.matchAll(/^- (?:This project|`([^`]+)\/`)/gm)].map((m) => m[1] ?? '.');
|
|
184
|
+
const legacyBlock = entries.length === 0; // a pre-0.6 section with no project list
|
|
185
|
+
if (entries.includes(dir)) {
|
|
186
|
+
skipped.push(`AGENTS.md already has the TestGuard section for ${dir === '.' ? 'this project' : dir}`);
|
|
187
|
+
} else {
|
|
188
|
+
writeFileSync(agentsPath, agents.slice(0, begin) + agentsBlock([...entries, dir]) + agents.slice(end + AGENTS_END.length));
|
|
189
|
+
done.push(legacyBlock ? `AGENTS.md: TestGuard section rewritten to list ${dir}` : `AGENTS.md: TestGuard section now lists ${dir}`);
|
|
190
|
+
note(agentsPath);
|
|
191
|
+
}
|
|
63
192
|
} else {
|
|
64
|
-
writeFileSync(agentsPath, (agents ? agents.replace(/\s*$/, '\n\n') : '# Agent instructions\n\n') +
|
|
193
|
+
writeFileSync(agentsPath, (agents ? agents.replace(/\s*$/, '\n\n') : '# Agent instructions\n\n') + agentsBlock([dir]) + '\n');
|
|
65
194
|
done.push(agents ? 'AGENTS.md: TestGuard section appended' : 'AGENTS.md created with the TestGuard section');
|
|
195
|
+
note(agentsPath);
|
|
66
196
|
}
|
|
67
197
|
|
|
68
198
|
const giPath = join(projectDir, '.gitignore');
|
|
@@ -71,12 +201,35 @@ export function initProject({ projectDir, force = false }) {
|
|
|
71
201
|
const dirIgnored = entries.some((x) => /^\/?\.testguard\/?$/.test(x));
|
|
72
202
|
const missing = dirIgnored ? [] : GITIGNORE_LINES.filter((l) => !entries.includes(l));
|
|
73
203
|
if (dirIgnored) {
|
|
74
|
-
skipped.push(
|
|
204
|
+
skipped.push(`${rel(giPath)} ignores .testguard/ entirely — note that baseline.json should be committed; ignore only the regenerated files if you adopt a baseline`);
|
|
75
205
|
} else if (missing.length) {
|
|
76
206
|
writeFileSync(giPath, (gi ? gi.replace(/\s*$/, '\n') : '') + '# TestGuard: regenerated per run (baseline.json IS committed)\n' + missing.join('\n') + '\n');
|
|
77
|
-
done.push(
|
|
207
|
+
done.push(`${rel(giPath)}: ${missing.length} line${missing.length === 1 ? '' : 's'} added`);
|
|
78
208
|
} else {
|
|
79
|
-
skipped.push(
|
|
209
|
+
skipped.push(`${rel(giPath)} already ignores the regenerated files`);
|
|
210
|
+
}
|
|
211
|
+
if (ciEvidence) {
|
|
212
|
+
const helperPath = join(projectDir, '.testguard', 'fetch-ci-evidence.sh');
|
|
213
|
+
const body = CI_EVIDENCE_HELPERS[ciEvidence];
|
|
214
|
+
if (!body) throw new Error(`--ci-evidence must be github or gitlab, not ${ciEvidence}`);
|
|
215
|
+
if (!existsSync(helperPath) || force) {
|
|
216
|
+
mkdirSync(dirname(helperPath), { recursive: true });
|
|
217
|
+
writeFileSync(helperPath, body, { mode: 0o755 });
|
|
218
|
+
done.push(`.testguard/fetch-ci-evidence.sh (${ciEvidence}) — run it on demand; the session-start hook stays offline`);
|
|
219
|
+
} else {
|
|
220
|
+
skipped.push('.testguard/fetch-ci-evidence.sh exists (use --force to replace)');
|
|
221
|
+
}
|
|
222
|
+
}
|
|
223
|
+
if (mcp) {
|
|
224
|
+
// Printed, never written: a harness's own config is the person's file
|
|
225
|
+
// (and often global), so init shows the snippet and lets them place it.
|
|
226
|
+
const cmd = { command: 'npx', args: ['-y', 'testguard-cli', 'mcp'] };
|
|
227
|
+
mcpConfig = {
|
|
228
|
+
'Claude Code — .mcp.json in the project root, or `claude mcp add`': { mcpServers: { testguard: cmd } },
|
|
229
|
+
'Cursor — .cursor/mcp.json': { mcpServers: { testguard: cmd } },
|
|
230
|
+
'Codex CLI — ~/.codex/config.toml': '[mcp_servers.testguard]\ncommand = "npx"\nargs = ["-y", "testguard-cli", "mcp"]',
|
|
231
|
+
};
|
|
232
|
+
notes.push('MCP: five read-only tools (status, brief, claims, evidence, next_command). Nothing there runs a probe.');
|
|
80
233
|
}
|
|
81
|
-
return { done, skipped };
|
|
234
|
+
return { done, skipped, warnings, agentRoot, projectDir, dir, ...(mcpConfig ? { mcpConfig } : {}) };
|
|
82
235
|
}
|
|
@@ -21,13 +21,31 @@ which files exist. `state` is one of:
|
|
|
21
21
|
| state | meaning | you do |
|
|
22
22
|
|---|---|---|
|
|
23
23
|
| `no-claims` | no `testguard.claims.json` | `testguard scaffold <file>` for a file with guards; replace every `TODO:` statement with what the code guarantees; keep or drop each proposal; move the claims into `testguard.claims.json` |
|
|
24
|
+
| `unclaimed-changes` | files you changed carry no claim (only when a reference is known: `--changed <ref>`, or CI's base branch) | `next.file` names the first; `testguard scaffold <file>` and state the claim, or add a `testguard.ignore.json` path entry with a reason a reviewer will accept. **Before** writing more code. |
|
|
24
25
|
| `unprobed` | claims never probed | `testguard probe` |
|
|
25
26
|
| `evidence-stale` | code, tests or claims changed since the evidence | `testguard probe --include-dirty` (or `--claim <ID>` for one) |
|
|
26
27
|
| `provisional-only` | only `--confirm 1` evidence exists | `testguard probe --confirm 3` |
|
|
27
28
|
| `unproven` | a finding is not covered by the baseline | `next.target` names it; see the verdict table |
|
|
28
29
|
| `clean` | everything killed or baselined | `testguard baseline` if `next` says so; otherwise nothing |
|
|
29
30
|
|
|
30
|
-
Exit codes: `0` clean · `1` unproven claims (or drift) · `2` precondition failed / nothing to do yet · `3` usage.
|
|
31
|
+
Exit codes: `0` clean · `1` unproven claims (or drift, or unclaimed changes) · `2` precondition failed / nothing to do yet · `3` usage.
|
|
32
|
+
|
|
33
|
+
## Every change needs a claim
|
|
34
|
+
|
|
35
|
+
`probe` verifies only the claims that exist; it is silent about unclaimed
|
|
36
|
+
code by construction. So, for every source file you create or change:
|
|
37
|
+
|
|
38
|
+
1. `testguard gate --changed HEAD --include-dirty` (the pre-commit shape; in a
|
|
39
|
+
PR, `--changed origin/<base>`). One unclaimed file exits `1`.
|
|
40
|
+
2. For each `UNCLAIMED` file: `testguard scaffold <file>` proposes faults;
|
|
41
|
+
state what the code guarantees; add the claim, its faults and its
|
|
42
|
+
`defendedBy` to `testguard.claims.json`. A new test file must be named in
|
|
43
|
+
some claim's `defendedBy`, or it is unclaimed too.
|
|
44
|
+
3. Only when a file genuinely carries nothing to claim (generated code, a
|
|
45
|
+
thin wrapper whose logic is claimed elsewhere): a `path` entry in
|
|
46
|
+
`testguard.ignore.json` with a reason and, where possible, an `expires`.
|
|
47
|
+
The gate prints every entry it relied on; a reviewer reads them.
|
|
48
|
+
4. Then `testguard admit <test-file> --claim <ID>` for the new claim's test (or `testguard probe --claim <ID> --include-dirty` for the full report).
|
|
31
49
|
|
|
32
50
|
## Verdict → action
|
|
33
51
|
|
|
@@ -45,8 +63,8 @@ Exit codes: `0` clean · `1` unproven claims (or drift) · `2` precondition fail
|
|
|
45
63
|
|
|
46
64
|
1. `testguard status --json` → take `next.target`.
|
|
47
65
|
2. Write the test. It must **fail when the fault is applied and pass on HEAD**. To check the first half by hand: apply `find` → `replace` in the target file, run the defender, restore.
|
|
48
|
-
3. `testguard
|
|
49
|
-
4. When it is `
|
|
66
|
+
3. `testguard admit <test-file> --claim <ID>` — the two gates as one verb, on your uncommitted test, without touching the tree: `ADMITTED` (exit 0) means the test is green on HEAD and fails on every fault of the claim, 3/3; `NOT ADMITTED` (exit 1) names the first blocking fault and what to do. `--fault <FID>` judges one fault; `--confirm 1` gives a fast **provisional** `ADMITTED?` that must be confirmed at 3 before you commit. (`testguard probe --claim <ID> --include-dirty` is the same run with the full report.)
|
|
67
|
+
4. When it is `ADMITTED` at `--confirm 3`, commit the test.
|
|
50
68
|
5. When everything is killed or baselined, `testguard baseline` if `next` says so, and commit `.testguard/baseline.json`.
|
|
51
69
|
|
|
52
70
|
## Rules that are not yours to bend
|
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
import { TOOLS } from './tools.mjs';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* A Model Context Protocol server over stdio, hand-rolled.
|
|
5
|
+
*
|
|
6
|
+
* Why not the official SDK: this tool has one exact-pinned runtime dependency
|
|
7
|
+
* and intends to keep it that way (AGENTS.md rule 8). The surface here is
|
|
8
|
+
* three methods and five read-only tools and will not grow, so the protocol
|
|
9
|
+
* is written out rather than imported. The version is pinned below and the
|
|
10
|
+
* conformance is covered by tests that speak the wire format.
|
|
11
|
+
*
|
|
12
|
+
* JSON-RPC 2.0 framing: one message per line (newline-delimited JSON), which
|
|
13
|
+
* is what stdio transports use. A notification (no `id`) is never answered.
|
|
14
|
+
*/
|
|
15
|
+
export const PROTOCOL_VERSION = '2025-06-18';
|
|
16
|
+
|
|
17
|
+
const ERROR = { parse: -32700, invalidRequest: -32600, methodNotFound: -32601, invalidParams: -32602, internal: -32603 };
|
|
18
|
+
|
|
19
|
+
const ok = (id, result) => ({ jsonrpc: '2.0', id, result });
|
|
20
|
+
const err = (id, code, message, data) => ({ jsonrpc: '2.0', id, error: { code, message, ...(data ? { data } : {}) } });
|
|
21
|
+
|
|
22
|
+
/** Everything an agent may ask for, with the version stamped onto each call. */
|
|
23
|
+
export function handle(message, { version = '0.0.0' } = {}) {
|
|
24
|
+
if (message === null || typeof message !== 'object' || Array.isArray(message)) return err(null, ERROR.invalidRequest, 'a request must be a JSON object');
|
|
25
|
+
const { id, method, params } = message;
|
|
26
|
+
const isNotification = id === undefined || id === null;
|
|
27
|
+
|
|
28
|
+
switch (method) {
|
|
29
|
+
case 'initialize':
|
|
30
|
+
return isNotification ? null : ok(id, {
|
|
31
|
+
protocolVersion: PROTOCOL_VERSION,
|
|
32
|
+
// Read-only: no prompts, no resources, no sampling. Saying so is part
|
|
33
|
+
// of the contract — nothing here can change a claims file.
|
|
34
|
+
capabilities: { tools: { listChanged: false } },
|
|
35
|
+
serverInfo: { name: 'testguard', version },
|
|
36
|
+
instructions: 'Read testguard_status first: it carries the one next action. Run the command it names in your own terminal — these tools never run a probe, because a probe is long-running, budgeted, and the person should see it happen.',
|
|
37
|
+
});
|
|
38
|
+
|
|
39
|
+
case 'notifications/initialized':
|
|
40
|
+
case 'notifications/cancelled':
|
|
41
|
+
return null;
|
|
42
|
+
|
|
43
|
+
case 'ping':
|
|
44
|
+
return isNotification ? null : ok(id, {});
|
|
45
|
+
|
|
46
|
+
case 'tools/list':
|
|
47
|
+
return isNotification ? null : ok(id, { tools: TOOLS.map(({ name, description, inputSchema }) => ({ name, description, inputSchema, annotations: { readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: false } })) });
|
|
48
|
+
|
|
49
|
+
case 'tools/call': {
|
|
50
|
+
if (isNotification) return null;
|
|
51
|
+
const tool = TOOLS.find((t) => t.name === params?.name);
|
|
52
|
+
if (!tool) return err(id, ERROR.invalidParams, `unknown tool: ${params?.name}`);
|
|
53
|
+
try {
|
|
54
|
+
const result = tool.handler({ ...(params?.arguments ?? {}), __version: version });
|
|
55
|
+
return ok(id, { content: [{ type: 'text', text: JSON.stringify(result, null, 2) }], structuredContent: result, isError: false });
|
|
56
|
+
} catch (e) {
|
|
57
|
+
// A tool failure is a tool result, not a protocol error: the agent
|
|
58
|
+
// should read the reason and act, not lose the connection.
|
|
59
|
+
return ok(id, { content: [{ type: 'text', text: `${tool.name} failed: ${e.message}` }], isError: true });
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
default:
|
|
64
|
+
return isNotification ? null : err(id, ERROR.methodNotFound, `unknown method: ${method}`);
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
/**
|
|
69
|
+
* Speak the protocol on a pair of streams. Resolves when input ends, so the
|
|
70
|
+
* caller can await a clean shutdown.
|
|
71
|
+
*/
|
|
72
|
+
export function serve({ input = process.stdin, output = process.stdout, version = '0.0.0' } = {}) {
|
|
73
|
+
return new Promise((resolveDone) => {
|
|
74
|
+
let buffer = '';
|
|
75
|
+
const write = (msg) => {
|
|
76
|
+
if (msg) output.write(JSON.stringify(msg) + '\n');
|
|
77
|
+
};
|
|
78
|
+
input.setEncoding?.('utf8');
|
|
79
|
+
input.on('data', (chunk) => {
|
|
80
|
+
buffer += chunk;
|
|
81
|
+
let nl;
|
|
82
|
+
while ((nl = buffer.indexOf('\n')) !== -1) {
|
|
83
|
+
const line = buffer.slice(0, nl).trim();
|
|
84
|
+
buffer = buffer.slice(nl + 1);
|
|
85
|
+
if (!line) continue;
|
|
86
|
+
let message;
|
|
87
|
+
try {
|
|
88
|
+
message = JSON.parse(line);
|
|
89
|
+
} catch {
|
|
90
|
+
write(err(null, ERROR.parse, 'invalid JSON'));
|
|
91
|
+
continue;
|
|
92
|
+
}
|
|
93
|
+
try {
|
|
94
|
+
write(handle(message, { version }));
|
|
95
|
+
} catch (e) {
|
|
96
|
+
// The server outlives any single bad request.
|
|
97
|
+
write(err(message?.id ?? null, ERROR.internal, e.message));
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
});
|
|
101
|
+
input.on('end', () => resolveDone(0));
|
|
102
|
+
input.on('close', () => resolveDone(0));
|
|
103
|
+
});
|
|
104
|
+
}
|
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
import { existsSync } from 'node:fs';
|
|
2
|
+
import { join, resolve } from 'node:path';
|
|
3
|
+
import { computeStatus } from '../status/status.mjs';
|
|
4
|
+
import { buildBrief, buildUnclaimedBrief } from '../brief/brief.mjs';
|
|
5
|
+
import { loadClaims, defaultClaimsPath } from '../claims/load.mjs';
|
|
6
|
+
import { scanAnnotations, reconcile } from '../claims/annotations.mjs';
|
|
7
|
+
import { computeRemovedClaims } from '../claims/removed.mjs';
|
|
8
|
+
import { readSpecDoc } from '../evidence/writer.mjs';
|
|
9
|
+
import { resolveChangedRef, withChangedRef } from '../gate/changed.mjs';
|
|
10
|
+
|
|
11
|
+
/**
|
|
12
|
+
* The read-only surface an agent needs, in any harness.
|
|
13
|
+
*
|
|
14
|
+
* TestGuard's operating loop currently lives in a Claude Code skill and a
|
|
15
|
+
* session-start hook, and both vanish the moment the harness is something
|
|
16
|
+
* else. These tools expose the documents the tool already computes, so the
|
|
17
|
+
* loop survives that change.
|
|
18
|
+
*
|
|
19
|
+
* Every tool is read-only and none of them probes. A probe is long-running
|
|
20
|
+
* and budgeted, and the human should see it happen: `next_command` hands back
|
|
21
|
+
* the exact shell line for the agent to run in its own terminal instead.
|
|
22
|
+
*/
|
|
23
|
+
const projectOf = (args) => resolve(args?.dir ?? '.');
|
|
24
|
+
const evidenceOf = (args) => (args?.evidence ? resolve(args.evidence) : undefined);
|
|
25
|
+
|
|
26
|
+
const DIR = { type: 'string', description: 'Project directory (where testguard.claims.json lives). Defaults to the current directory.' };
|
|
27
|
+
const EVIDENCE = { type: 'string', description: "Read this evidence document instead of the project's own — CI's, fetched as an artifact." };
|
|
28
|
+
const CHANGED = { type: 'string', description: 'A git reference to measure the change against, so claim coverage of the change is included.' };
|
|
29
|
+
|
|
30
|
+
const statusFor = (args) => {
|
|
31
|
+
const projectDir = projectOf(args);
|
|
32
|
+
return withChangedRef(resolveChangedRef({ explicit: args?.changed }), (changedRef) =>
|
|
33
|
+
computeStatus({ projectDir, toolVersion: args?.__version ?? '0.0.0', changedRef, evidence: evidenceOf(args) }));
|
|
34
|
+
};
|
|
35
|
+
|
|
36
|
+
export const TOOLS = [
|
|
37
|
+
{
|
|
38
|
+
name: 'testguard_status',
|
|
39
|
+
description: 'Where this project stands and the ONE next action, as the machine-readable status document: state, next {action, command, why}, counts, stale inputs, faults edited since they were probed, and the ranked findings. This is the single source of truth every other rendering derives from — read it before deciding anything.',
|
|
40
|
+
inputSchema: { type: 'object', properties: { dir: DIR, changed: CHANGED, evidence: EVIDENCE }, additionalProperties: false },
|
|
41
|
+
handler: (args) => statusFor(args),
|
|
42
|
+
},
|
|
43
|
+
{
|
|
44
|
+
name: 'testguard_brief',
|
|
45
|
+
description: 'Where the test suite is blind, ranked and capped, with one actionable hint per finding and unclaimed changed files first. Read this before writing or changing code, which is what the session-start hook does for harnesses that have one.',
|
|
46
|
+
inputSchema: { type: 'object', properties: { dir: DIR, changed: CHANGED, evidence: EVIDENCE, max: { type: 'integer', minimum: 1, maximum: 50 } }, additionalProperties: false },
|
|
47
|
+
handler: (args) => {
|
|
48
|
+
const projectDir = projectOf(args);
|
|
49
|
+
const evPath = evidenceOf(args) ?? join(projectDir, '.testguard', 'evidence.json');
|
|
50
|
+
const status = (() => {
|
|
51
|
+
try {
|
|
52
|
+
return statusFor(args);
|
|
53
|
+
} catch {
|
|
54
|
+
return undefined; // a brief must never fail because status could not be computed
|
|
55
|
+
}
|
|
56
|
+
})();
|
|
57
|
+
if (!existsSync(evPath)) {
|
|
58
|
+
if (status?.changes?.uncovered?.length) {
|
|
59
|
+
return buildUnclaimedBrief({ tool: { name: 'testguard', version: args?.__version ?? '0.0.0' }, next: status.next, changes: status.changes });
|
|
60
|
+
}
|
|
61
|
+
return { note: `no evidence at ${evPath}: nothing has been probed yet`, next: status?.next };
|
|
62
|
+
}
|
|
63
|
+
const evidence = readSpecDoc('evidence', evPath);
|
|
64
|
+
const basePath = join(projectDir, '.testguard', 'baseline.json');
|
|
65
|
+
const baseline = existsSync(basePath) ? readSpecDoc('baseline', basePath) : undefined;
|
|
66
|
+
return buildBrief(evidence, baseline, { max: args?.max ?? 20, next: status?.next, changes: status?.changes });
|
|
67
|
+
},
|
|
68
|
+
},
|
|
69
|
+
{
|
|
70
|
+
name: 'testguard_claims',
|
|
71
|
+
description: "What this project claims must be true, each with its faults and defenders, plus drift against `@claim` annotations in source and — when `since` is given — every claim or fault that existed at that reference and does not now. A claim that disappeared is invisible to every other check.",
|
|
72
|
+
inputSchema: { type: 'object', properties: { dir: DIR, since: { type: 'string', description: 'A git reference: report claims and faults removed since it.' } }, additionalProperties: false },
|
|
73
|
+
handler: (args) => {
|
|
74
|
+
const projectDir = projectOf(args);
|
|
75
|
+
const path = defaultClaimsPath(projectDir);
|
|
76
|
+
if (!existsSync(path)) return { path, claims: [], note: 'no claims file in this project yet' };
|
|
77
|
+
const claims = loadClaims(path);
|
|
78
|
+
const drift = reconcile(claims, scanAnnotations(projectDir));
|
|
79
|
+
const removed = args?.since
|
|
80
|
+
? computeRemovedClaims({ projectDir, ref: args.since, claimsPath: path, current: claims, evidencePath: join(projectDir, '.testguard', 'evidence.json'), toolVersion: args?.__version ?? '0.0.0' })
|
|
81
|
+
: undefined;
|
|
82
|
+
return { path, claims, drift, ...(removed ? { removed } : {}) };
|
|
83
|
+
},
|
|
84
|
+
},
|
|
85
|
+
{
|
|
86
|
+
name: 'testguard_evidence',
|
|
87
|
+
description: 'One run\'s findings on disk: every verdict with the runs that produced it. Pass `claim` (and optionally `fault`) for a single record instead of the whole document, which can be large.',
|
|
88
|
+
inputSchema: {
|
|
89
|
+
type: 'object',
|
|
90
|
+
properties: { dir: DIR, evidence: EVIDENCE, claim: { type: 'string', description: 'Return only this claim\'s records.' }, fault: { type: 'string', description: 'With `claim`: only this fault\'s record.' } },
|
|
91
|
+
additionalProperties: false,
|
|
92
|
+
},
|
|
93
|
+
handler: (args) => {
|
|
94
|
+
const projectDir = projectOf(args);
|
|
95
|
+
const evPath = evidenceOf(args) ?? join(projectDir, '.testguard', 'evidence.json');
|
|
96
|
+
if (!existsSync(evPath)) return { note: `no evidence at ${evPath}: run a probe first`, path: evPath };
|
|
97
|
+
const doc = readSpecDoc('evidence', evPath);
|
|
98
|
+
if (!args?.claim) return doc;
|
|
99
|
+
const records = doc.records.filter((r) => r.claim.id === args.claim && (!args.fault || r.subject.id === args.fault));
|
|
100
|
+
return { ...doc, records, filtered: { claim: args.claim, ...(args.fault ? { fault: args.fault } : {}) } };
|
|
101
|
+
},
|
|
102
|
+
},
|
|
103
|
+
{
|
|
104
|
+
name: 'testguard_next_command',
|
|
105
|
+
description: 'The exact shell command to run next, with the reason. Run it yourself in a terminal: a probe is long-running and budgeted, and the person should see it happen. This tool never runs anything.',
|
|
106
|
+
inputSchema: { type: 'object', properties: { dir: DIR, changed: CHANGED, evidence: EVIDENCE }, additionalProperties: false },
|
|
107
|
+
handler: (args) => {
|
|
108
|
+
const doc = statusFor(args);
|
|
109
|
+
return { state: doc.state, action: doc.next.action, command: doc.next.command, why: doc.next.why, ...(doc.next.target ? { target: doc.next.target } : {}), ...(doc.next.file ? { file: doc.next.file } : {}) };
|
|
110
|
+
},
|
|
111
|
+
},
|
|
112
|
+
];
|
|
113
|
+
|
|
114
|
+
export const TOOL_NAMES = TOOLS.map((t) => t.name);
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
import { spawnSync } from 'node:child_process';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Concurrent test runners (issue #26).
|
|
5
|
+
*
|
|
6
|
+
* A probe runs the defenders 3× clean and 3× with each fault applied. Heavy
|
|
7
|
+
* suites time out under parallel CPU load, and a timeout is a verdict —
|
|
8
|
+
* `TIMEOUT` or `FLAKY-DEFENDER` — that says nothing about the claim. A field
|
|
9
|
+
* report serialised everything by hand for exactly this reason. We cannot
|
|
10
|
+
* prevent the contention, but we can name it: warn before the first run, and
|
|
11
|
+
* record it on the evidence so a later reader of a flaky verdict knows the
|
|
12
|
+
* run was contended.
|
|
13
|
+
*
|
|
14
|
+
* Best effort by design: the detector never fails a probe, and a process list
|
|
15
|
+
* it cannot read is simply no detection.
|
|
16
|
+
*/
|
|
17
|
+
|
|
18
|
+
const RUNNER_RE = /\b(vitest|jest|playwright|mocha|ava|karma|cypress)\b/i;
|
|
19
|
+
// Our own child processes and this process are not contention.
|
|
20
|
+
const SELF_RE = /testguard/i;
|
|
21
|
+
|
|
22
|
+
/** Rows of the process table as `{ pid, command }`, or [] when it cannot be read. */
|
|
23
|
+
export function processList({ platform = process.platform, run = spawnSync } = {}) {
|
|
24
|
+
try {
|
|
25
|
+
const r = platform === 'win32'
|
|
26
|
+
? run('wmic', ['process', 'get', 'ProcessId,CommandLine', '/format:csv'], { encoding: 'utf8', timeout: 5000 })
|
|
27
|
+
: run('ps', ['-Ao', 'pid=,args='], { encoding: 'utf8', timeout: 5000 });
|
|
28
|
+
if (r.status !== 0 || !r.stdout) return [];
|
|
29
|
+
return r.stdout.split('\n').map((line) => {
|
|
30
|
+
const m = platform === 'win32' ? /^[^,]*,(.*),(\d+)\s*$/.exec(line) : /^\s*(\d+)\s+(.*)$/.exec(line);
|
|
31
|
+
if (!m) return null;
|
|
32
|
+
const [pid, command] = platform === 'win32' ? [m[2], m[1]] : [m[1], m[2]];
|
|
33
|
+
return { pid: Number(pid), command: command.trim() };
|
|
34
|
+
}).filter(Boolean);
|
|
35
|
+
} catch {
|
|
36
|
+
return [];
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
/**
|
|
41
|
+
* Test runners already running, other than ours. Returns
|
|
42
|
+
* `{ detected, runners: [{ pid, command }] }`; `detected` is false when
|
|
43
|
+
* nothing was found or the process list could not be read.
|
|
44
|
+
*/
|
|
45
|
+
export function detectContention({ self = process.pid, ...opts } = {}) {
|
|
46
|
+
const rows = processList(opts);
|
|
47
|
+
const runners = rows
|
|
48
|
+
.filter((p) => p.pid !== self && RUNNER_RE.test(p.command) && !SELF_RE.test(p.command))
|
|
49
|
+
// `ps` lists the grep/ps itself and any shell wrapper; a runner's command
|
|
50
|
+
// line always names its own binary, so require it to look like an exec.
|
|
51
|
+
.filter((p) => !/^\s*(ps|grep|wmic)\b/.test(p.command))
|
|
52
|
+
.map((p) => ({ pid: p.pid, command: p.command.length > 200 ? p.command.slice(0, 197) + '…' : p.command }));
|
|
53
|
+
return { detected: runners.length > 0, runners };
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
/** One line for the operator: what is running, and what it costs. */
|
|
57
|
+
export const contentionWarning = (c) =>
|
|
58
|
+
`${c.runners.length} test runner${c.runners.length === 1 ? ' is' : 's are'} already running (${c.runners.map((r) => `pid ${r.pid}`).join(', ')}); ` +
|
|
59
|
+
'defenders run N times each here, and a suite that times out under parallel load yields TIMEOUT or FLAKY-DEFENDER — verdicts about the load, not the claim. ' +
|
|
60
|
+
'Wait for it to finish, or pass --serial to run one test file at a time.';
|
package/src/probe/discover.mjs
CHANGED
|
@@ -1,12 +1,21 @@
|
|
|
1
1
|
import { join } from 'node:path';
|
|
2
2
|
import { listTestFiles } from './runners/shared.mjs';
|
|
3
3
|
import { fileImports } from './rank.mjs';
|
|
4
|
+
import { classifyDefenders } from './mocks.mjs';
|
|
4
5
|
|
|
5
6
|
/**
|
|
6
7
|
* When a claim declares no defenders: the test files that import the fault's
|
|
7
|
-
* target file
|
|
8
|
-
*
|
|
8
|
+
* target file — directly, by relative path or through a resolved alias — and
|
|
9
|
+
* do NOT mock it. A file that mocks the target cannot detect any fault in it;
|
|
10
|
+
* counting it would make `nocover` under-report and waste probe runs on files
|
|
11
|
+
* that cannot fail. `nocover` therefore means exactly "no test file imports
|
|
12
|
+
* this source without mocking it".
|
|
9
13
|
*/
|
|
14
|
+
export function discoverDefendersDetailed(projectDir, targetRel) {
|
|
15
|
+
const importing = listTestFiles(projectDir).filter((t) => fileImports(projectDir, join(projectDir, t), targetRel));
|
|
16
|
+
return { importing, ...classifyDefenders(projectDir, targetRel, importing) };
|
|
17
|
+
}
|
|
18
|
+
|
|
10
19
|
export function discoverDefenders(projectDir, targetRel) {
|
|
11
|
-
return
|
|
20
|
+
return discoverDefendersDetailed(projectDir, targetRel).canDetect;
|
|
12
21
|
}
|